Compare commits

...
Author SHA1 Message Date
Nick Craig-Wood 2b1194c57e rc: update docs with new methods 2018-11-05 15:44:40 +00:00
Nick Craig-Wood e6dd121f52 config: add rc operations for config 2018-11-05 15:44:40 +00:00
Nick Craig-Wood e600217666 config: create config directory on save if it is missing 2018-11-05 15:44:40 +00:00
Nick Craig-Wood bc17ca7ed9 rc: implement core/obscure 2018-11-05 15:44:40 +00:00
Nick Craig-Wood 1916410316 rc: add core/version and put definitions next to implementations 2018-11-05 15:44:40 +00:00
Nick Craig-Wood dddfbec92a cmd/version: factor version number parsing routines into fs/version 2018-11-05 15:44:40 +00:00
Nick Craig-Wood 75a88de55c rc/rcserver: with --rc-files if auth set, pass on to URL opened
If `--rc-user` or `--rc-pass` is set then the URL that is opened with
`--rc-files` will have the authorization in the URL in the
`http://user:pass@localhost/` style.
2018-11-05 15:44:40 +00:00
Nick Craig-Wood 2466f4d152 sync: add rc commands for sync/copy/move 2018-11-05 15:44:40 +00:00
Nick Craig-Wood 39283c8a35 operations: implement operations remote control commands 2018-11-05 15:44:40 +00:00
Nick Craig-Wood 46c2f55545 copyurl: factor code into operations and write test 2018-11-04 20:42:57 +00:00
Nick Craig-Wood fc2afcbcbd lsjson: factor internals of lsjson command into operations 2018-11-04 20:42:57 +00:00
Nick Craig-Wood fa0a9653d2 rc: methods marked as AuthRequired need auth unless --rc-no-auth
Methods which can read or mutate external storage will require
authorisation - enforce this.  This can be overidden by `--rc-no-auth`.
2018-11-04 20:42:57 +00:00
Nick Craig-Wood 181267e20e cmd/rc: add --user and --pass flags and interpret --rc-user, --rc-pass, --rc-addr 2018-11-04 20:42:57 +00:00
Nick Craig-Wood 75e8ea383c rc: implement rc.PutCachedFs for prefilling the remote cache 2018-11-04 20:42:57 +00:00
Nick Craig-Wood 8c8b58a7de rc: expire remote cache and fix tests under race detector 2018-11-04 20:42:57 +00:00
Nick Craig-Wood b961e07c57 rc: ensure rclone fails to start up if the --rc port is in use already 2018-11-04 15:11:51 +00:00
Nick Craig-Wood 0b80d1481a cache: make tests not start an rc but use the internal framework 2018-11-04 15:11:51 +00:00
Nick Craig-Wood 89550e7121 rcserver: serve directories as well as files 2018-11-04 15:11:51 +00:00
Nick Craig-Wood 370c218c63 cmd/http: factor directory serving routines into httplib/serve and write tests 2018-11-04 12:46:44 +00:00
Nick Craig-Wood b972dcb0ae rc: implement options/blocks,get,set and register options 2018-11-03 11:32:00 +00:00
Nick Craig-Wood 0bfa9811f7 rc: factor server code into rcserver and implement serving objects
If a GET or HEAD request is receivied with a URL parameter of fs then
it will be served from that remote.
2018-11-03 11:32:00 +00:00
Nick Craig-Wood aa9b2c31f4 serve/restic: factor object serving into cmd/httplib/serve 2018-11-03 11:32:00 +00:00
Nick Craig-Wood cff75db6a4 rcd: implement new command just to serve the remote control API 2018-11-03 11:32:00 +00:00
Nick Craig-Wood 75252e4a89 rc: add --rc-files flag to serve files on the rc http server
This enables building a browser based UI for rclone
2018-11-03 11:32:00 +00:00
Nick Craig-Wood 2089405e1b fs/rc: add more infrastructure to help writing rc functions
- Fs cache for rc commands
- Helper functions for parsing the input
- Reshape command for manipulating JSON blobs
- Background Job starting, control, query and expiry
2018-11-02 17:32:20 +00:00
Nick Craig-Wood a379eec9d9 fstest/mockfs: create mock fs.Fs for testing 2018-11-02 17:32:20 +00:00
Nick Craig-Wood 45d5339fcb cmd/rc: add --json flag for structured JSON input 2018-11-02 17:32:20 +00:00
Nick Craig-Wood bb5637d46a serve http, webdav, restic: ensure rclone exits if the port is in use 2018-11-02 17:32:20 +00:00
Nick Craig-Wood 1f05d5bf4a delete: clarify that it only deletes files not directories 2018-11-02 17:07:45 +00:00
HerrH ff87da9c3b Added some more links for easier finding
Expanded the Installation & Docs section with links to the website and added a link to the full list of storage providers and features.
2018-11-02 16:56:20 +00:00
ssaqua 3d81b75f44 dedupe: check for existing filename before renaming a dupe file 2018-11-02 16:51:52 +00:00
Nick Craig-Wood baba6d67e6 s3: set ACL for server side copies to that provided by the user - fixes #2691
Before this change the ACL for objects which were server side copied
was left at the default "private" settings. S3 doesn't copy the ACL
from the source when you copy an object, you have to set it afresh
which is what this does.
2018-11-02 16:22:31 +00:00
Nick Craig-Wood 04c0564fe2 Add Ralf Hemberger to contributors 2018-11-02 09:53:23 +00:00
Ralf Hemberger 91cfdb81f5 change spaces to tab 2018-11-02 09:50:34 +00:00
Ralf Hemberger deae7bf33c WebDav - Add RFC3339 date format - fixes #2712 2018-11-02 09:50:34 +00:00
Henning Surmeier 04a0da1f92 ncdu: remove option ('d' key)
delete files by pressing 'd' in the ncdu listing

GUI Improvements:
Boxes now have a border around them
Boxes can ask questions and allow the selection of options. The
selected option will be given to the UI.boxMenuHandler function.

Fixes #2571
2018-10-28 20:44:03 +00:00
Henning Surmeier 9486df0226 ncdu/scan: remove option for memory representation
Remove files/directories from the in memory structs of the cloud
directory. Size and Count will be recalculated and populated upwards
to the parent directories.
2018-10-28 20:44:03 +00:00
Nick Craig-Wood 948a5d25c2 operations: Fix Purge and Rmdirs when dir is not empty
Before this change, Purge on the fallback path would try to delete
directories starting from the root rather than the dir passed in.
Rmdirs would also attempt to delete the root.
2018-10-27 11:51:17 +01:00
Nick Craig-Wood f7c31cd210 Add Florian Gamboeck to contributors 2018-10-27 00:28:11 +01:00
Florian Gamboeck 696e7b2833 backend/cache: Print correct info about Cache Writes 2018-10-27 00:27:47 +01:00
Anagh Kumar Baranwal e76cf1217f Added docs to check for key generation on Mega
Signed-off-by: Anagh Kumar Baranwal <anaghk.dos@gmail.com>
2018-10-25 22:49:21 +01:00
Nick Craig-Wood 543e37f662 Require go1.8 for compilation 2018-10-25 17:06:33 +01:00
Nick Craig-Wood c514cb752d vendor: update to latest versions of everything 2018-10-25 17:06:33 +01:00
Nick Craig-Wood c0ca93ae6f opendrive: fix retries of upload chunks - fixes #2646
Before this change, upload chunks were being emptied on retry.  This
change introduces a RepeatableReader to fix the problem.
2018-10-25 11:50:38 +01:00
Nick Craig-Wood 38a89d49ae fstest/test_all: tidy HTML report
- link test number to online copy
- style links
- attempt to make a nicer colour scheme
2018-10-25 11:33:17 +01:00
Anagh Kumar Baranwal 6531126eb2 Fixes the rc docs creation
Signed-off-by: Anagh Kumar Baranwal <anaghk.dos@gmail.com>
2018-10-25 11:29:59 +01:00
Nick Craig-Wood 25d0e59ef8 fstest/test_all: make sure Version is correct in build 2018-10-25 08:36:09 +01:00
Nick Craig-Wood b0db08fd2b fstest/test_all: constrain to go1.10 and above 2018-10-24 21:33:42 +01:00
Nick Craig-Wood 07addf74fd fstest/test_all: upload a copy of the report to "current" 2018-10-24 12:21:07 +01:00
Nick Craig-Wood 52c7c738ca fstest/test_all: limit concurrency and run tests in random order 2018-10-24 10:46:58 +01:00
Nick Craig-Wood 5c32b32011 fstest/test_all: fix directories that tests are run in
- Don't build a binary for backend tests
- Run tests in their relevant directories
2018-10-23 17:31:11 +01:00
Nick Craig-Wood fe61cff079 crypt: ensure integration tests run correctly when -remote is set 2018-10-23 17:12:38 +01:00
Nick Craig-Wood fbab1e55bb fstest/test_all: adapt to nested test definitions 2018-10-23 16:56:35 +01:00
Nick Craig-Wood 1bfd07567e fstest/test_all: add oneonly flag to only run one test per backend if required 2018-10-23 14:07:48 +01:00
Nick Craig-Wood f97c4c8d9d fstest/test_all: rework integration tests to improve output
- Make integration tests use a config file
- Output individual logs for each test
- Make HTML report and open browser
- Optionally email and upload results
2018-10-23 14:07:48 +01:00
Anagh Kumar Baranwal a3c55462a8 Set python version explicitly to 2 to avoid issues on systems where
the default python version is `3`

Signed-off-by: Anagh Kumar Baranwal <anaghk.dos@gmail.com>
2018-10-23 12:14:52 +01:00
Anagh Kumar Baranwal bbb9a504a8 Added docs to use the -P/--progress flag for real time statistics
Signed-off-by: Anagh Kumar Baranwal <anaghk.dos@gmail.com>
2018-10-23 12:14:52 +01:00
Jon Fautley dedc7d885c sftp: Ensure file hash checking is really disabled 2018-10-23 12:03:50 +01:00
Nick Craig-Wood c5ac96e9e7 Make --files-from only read the objects specified and don't scan directories
Before this change using --files-from would scan all the directories
that the files could possibly be in causing rclone to do more work
that was necessary.

After this change, rclone constructs an in memory tree using the
--fast-list mechanism but from all of the files in the --files-from
list and without scanning any directories.

Any objects that are not found in the --files-from list are ignored
silently.

This mechanism is used for sync/copy/move (march) and all of the
listing commands ls/lsf/md5sum/etc (walk).
2018-10-20 18:13:31 +01:00
Nick Craig-Wood 9959c5f17f webdav: add Content-Type to PUT requests - fixes #2664 2018-10-18 13:18:24 +01:00
Nick Craig-Wood e8d0a363fc opendrive: fix transfer of files with + and & in - fixes #2657 2018-10-17 14:22:04 +01:00
albertony 935b7c1c0f jottacloud: fix bug in --fast-list handing of empty folders - fixes #2650 2018-10-17 13:58:36 +01:00
Fabian Möller 15ce0ae57c fstests: fix maximum tested size in TestFsPutChunked
Before this it was possible hat maxChunkSize was incorrectly set to 200.
2018-10-16 11:50:47 +02:00
Nick Craig-Wood 67703a73de Start v1.44-DEV development 2018-10-15 12:33:27 +01:00
Nick Craig-Wood f96ce5674b Version v1.44 2018-10-15 11:03:08 +01:00
Nick Craig-Wood 7f0b204292 azureblob: work around SDK bug which causes errors for chunk-sized files (again)
Until https://github.com/Azure/azure-storage-blob-go/pull/75 is merged
the SDK can't upload a single blob of exactly the chunk size, so
upload files of this size with a multpart upload as a work around.

The previous fix for this 6a773289e7 turned out to cause problems
uploading files with maximum chunk size so needed to be redone.

Fixes #2653
2018-10-15 09:05:34 +01:00
Nick Craig-Wood 83b1ae4833 Add buergi to contributors 2018-10-14 17:20:34 +01:00
buergi 753cc63d96 webdav: add workaround for missing mtime - fixes #2420 2018-10-14 17:19:23 +01:00
Nick Craig-Wood 5dac8e055f union: fix --backup-dir on union backend
Before this fix --backup-dir would fail.

This is fixed by wrapping objects returned so that they belong to the
union Fs rather than the underlying Fs.
2018-10-14 15:19:02 +01:00
Nick Craig-Wood c3a8eb1c10 fstests: make findObject() sleep a bit longer to fix b2 largePut tests 2018-10-14 14:45:23 +01:00
Nick Craig-Wood 0f2a5403db acd,box,onedrive,opendrive,ploud: fix Features() retaining the original receiver
Before this change the Features() method would return a different Fs
to that the Features() method was called on if the remote was
instantiated on a file.

The practical effect of this is that optional features, eg `rclone
about` wouldn't work properly when called on a file, and likely this
has been causing low level problems for users of these backends for
ages.

Ideally there would be a test for this, but it turns out that this is
really hard, so instead of that all the backends have been converted
to not copy the Fs and a big warning comment inserted for future
readers.

Fixes #2182
2018-10-14 14:41:26 +01:00
Nick Craig-Wood dcce84714e onedrive: fix link command for non root 2018-10-14 14:17:53 +01:00
Nick Craig-Wood eb8130f48a fstests: update TestPublicLink comment to show how to run solo 2018-10-14 14:17:05 +01:00
Nick Craig-Wood aa58f66806 build: add longer timeout to integration tests 2018-10-14 14:16:33 +01:00
Nick Craig-Wood a3dc591b8e Add teresy to contributors 2018-10-14 00:19:49 +01:00
teresy 5ee1bd7ba4 Remove redundant nil checks 2018-10-14 00:19:35 +01:00
Nick Craig-Wood dbedf33b9f s3: fix v2 signer on files with spaces - fixes #2438
Before this fix the v2 signer was failing for files with spaces in.
2018-10-14 00:10:29 +01:00
Nick Craig-Wood 0f02c9540c s3: make --s3-v2-auth flag
This is an alternative to setting the region to "other-v2-signature"
which is inconvenient for multi-region providers.
2018-10-14 00:10:29 +01:00
Nick Craig-Wood 06922674c8 drive, s3: review hidden config items 2018-10-13 23:30:13 +01:00
Nick Craig-Wood 8ad7da066c drive: when listing team drives, continue on failure
This means that if the team drive listing returns a 500 error (which
seems reasonably common) rclone will continue to the point where it
asks for the team drive ID.

https://forum.rclone.org/t/many-team-drives-causes-rclone-to-fail/7159
2018-10-13 23:30:13 +01:00
Nick Craig-Wood e1503add41 azureblob, b2, drive: implement set upload cutoff for chunked upload tests 2018-10-13 22:49:12 +01:00
Nick Craig-Wood 6fea75afde fstests: fix upload offsets not being set and redownload test files
In chunked upload tests:

- Add ability to set upload offset
- Read back the uploaded file to check it is OK
2018-10-13 22:49:12 +01:00
Nick Craig-Wood 6a773289e7 azureblob: work around SDK bug which causes errors for chunk-sized files
See https://github.com/Azure/azure-storage-blob-go/pull/75 for details
2018-10-13 22:49:12 +01:00
Nick Craig-Wood ade252f13b build: fixup code formatting after goimports change 2018-10-13 22:47:12 +01:00
Nick Craig-Wood bb2e361004 jottacloud: Fix socket leak on Object.Remove - fixes #2637 2018-10-13 22:47:12 +01:00
Nick Craig-Wood b24facb73d rest: Fix documentation so it is clearer when resp.Body is closed 2018-10-13 22:47:12 +01:00
Nick Craig-Wood 014d58a757 Add David Haguenauer to contributors 2018-10-13 12:56:21 +01:00
David Haguenauer 1d16e16b30 docs: replace "Github" with "GitHub"
This the way GitHub refer to themselves.
2018-10-13 12:55:45 +01:00
Nick Craig-Wood 249a523dd3 build: fix golint install with new path 2018-10-12 11:35:35 +01:00
Nick Craig-Wood 8d72ef8d1e cmd: Don't print non-ASCII characters with --progress on windows - fixes #2501
This bug causes lots of strange behaviour with non-ASCII characters and --progress

https://github.com/Azure/go-ansiterm/issues/26
2018-10-11 21:25:04 +01:00
Nick Craig-Wood bc8f0208aa rest: Remove auth headers on HTTP redirect
Before this change the rest package would forward all the headers on
an HTTP redirect, including the Authorization: header.  This caused
problems when forwarded to a signed S3 URL ("Only one auth mechanism
allowed") as well as being a potential security risk.

After we use the go1.8+ mechanism for doing this instead of using our
own which does it correctly removing the Authorization: header when
redirecting to a different host.

This hasn't fixed the behaviour for rclone compiled with go1.7.

Fixes #2635
2018-10-11 21:20:33 +01:00
Nick Craig-Wood ee25b6106a Add jackyzy823 to contributors 2018-10-11 14:50:33 +01:00
Nick Craig-Wood 5c1b135304 Add dcpu to contributors 2018-10-11 14:50:33 +01:00
HerrH 2f2029fed5 Improved and updated the readme
Updated providers list, added links to docs, improved readability
2018-10-11 14:50:21 +01:00
Fabian Möller 57273d364b fstests: add TestFsPutChunked 2018-10-11 14:47:58 +01:00
Fabian Möller 84289d1d69 readers: add NewPatternReader 2018-10-11 14:47:58 +01:00
Fabian Möller 98e2746e31 backend: add fstests.ChunkedUploadConfig
- azureblob
- b2
- drive
- dropbox
- onedrive
- s3
- swift
2018-10-11 14:47:58 +01:00
Fabian Möller c00ec0cbe4 fstests: add ChunkedUploadConfig 2018-10-11 14:47:58 +01:00
Fabian Möller 1a40bceb1d backend: unify NewFs path handling for wrapping remotes
Use the same function to join the root paths for the wrapping remotes
alias, cache and crypt.
The new function fspath.JoinRootPath is equivalent to path.Join, but if
the first non empty element starts with "//", this is preserved to allow
Windows network path to be used in these remotes.
2018-10-10 17:50:27 +01:00
jackyzy823 411a6cc472 onedrive: add link sharing support #2178 2018-10-09 20:11:48 +08:00
Fabian Möller 1e2676df26 union: fix ChangeNotify to support multiple remotes
To correctly support multiple remotes, each remote has to receive a
value on the input channel.
2018-10-07 11:13:37 +02:00
Nick Craig-Wood 364fca5cea union: implement optional interfaces (Move, DirMove, Copy etc) - fixes #2619
Implement optional interfaces
- Purge
- PutStream
- Copy
- Move
- DirMove
- DirCacheFlush
- ChangeNotify
- About

Make Hashes() return the intersection of all the hashes supported by the remotes
2018-10-07 00:06:29 +01:00
Nick Craig-Wood 87e1efa997 mount, vfs: Remove EXPERIMENTAL tags
rclone mount and the --vfs-cache-mode has been tested extensively by
users now so removing the EXPERIMENTAL tag is appropriate.
2018-10-06 11:47:46 +01:00
Nick Craig-Wood 6709084e2f config: Show URL of backend help page when starting config 2018-10-06 11:47:46 +01:00
Nick Craig-Wood 6b1f915ebc fs: Implement RegInfo.FileName to return the on disk filename for a backend
Use it in make_backend_docs.py
2018-10-06 11:47:46 +01:00
Nick Craig-Wood 78b9bd77f5 docs: auto generate backend options documentation
This inserts the output of "rclone help backend xxx" into the help
pages for each backend.
2018-10-06 11:47:46 +01:00
Nick Craig-Wood a9273c5da5 docs: move documentation for options from docs/content into backends
In the following commit, the documentation will be autogenerated.
2018-10-06 11:47:46 +01:00
Nick Craig-Wood 14128656db cmd: Implement specialised help for flags and backends - fixes #2541
Instead of showing all flags/backends all the time, you can type

    rclone help flags
    rclone help flags <regexp>
    rclone help backends
    rclone help backend <name>
2018-10-06 11:47:45 +01:00
Nick Craig-Wood 1557287c64 fs: Make Option.GetValue() public #2541 2018-10-06 11:47:45 +01:00
Nick Craig-Wood e7e467fb3a cmd: factor FlagName into fs.Option #2541 2018-10-06 11:47:45 +01:00
Nick Craig-Wood 5fde7d8b12 cmd: split flags up into global and backend flags #2541 2018-10-06 11:47:45 +01:00
Nick Craig-Wood 3c086f5f7f cmd: Make default help less verbose #2541
This stops the default help showing all the flags, backends, commands
2018-10-06 11:47:45 +01:00
dcpu c0084f43dd cache: Remove entries that no longer exist in the source
list directory with 25k files

before(1.43.1)
5m24s

after
3m21s
2018-10-06 11:23:33 +01:00
Nick Craig-Wood ddbd4fd881 Add Paul Kohout to contributors 2018-10-04 08:25:39 +01:00
Paul Kohout 7826e39fcf s3: use configured server-side-encryption and storace class options when calling CopyObject() - fixes #2610 2018-10-04 08:25:20 +01:00
Nick Craig-Wood 06ae4258be cmd: Fix -P not ending with a new line
Before this fix rclone didn't wait for the stats to be finished before
exiting, so the final new line was never printed.

After this change rclone will wait for the stats routine to cease
before exiting.
2018-10-03 21:46:18 +01:00
Alex Chen d9037fe2be onedrive: ignore OneNote files by default - fixes #211 2018-10-03 12:46:25 +08:00
Fabian Möller 1d14972e41 vfs: reduce directory cache cleared by poll-interval
Reduce the number of nodes purged from the dir-cache when ForgetPath is
called. This is done by only forgetting the cache of the received path
and invalidating the parent folder cache by resetting *Dir.read.

The parent will read the listing on the next access and reuse the
dir-cache of entries in *Dir.items.
2018-10-02 10:21:14 +01:00
Fabian Möller 05fa9cb379 drive: improve directory notifications in ChangeNotify
When moving a directory in drive, most of the time only a notification
for the directory itself is created, not the old or new parents.

This tires to find the old path in the dirCache and the new path with
the dirCache of the new parent, which can result in two notifications
for a moved directory.
2018-10-02 10:14:14 +01:00
Nick Craig-Wood 59e14c25df vfs: enable rename for nearly all remotes using server side Move or Copy
Before this change remotes without server side Move (eg swift, s3,
gcs) would not be able to rename files.

After it means nearly all remotes will be able to rename files on
rclone mount with the notable exceptions of b2 and yandex.

This changes checks to see if the remote can do Move or Copy then
calls `operations.Move` to do the actual move.  This will do a server
side Move or Copy but won't download and re-upload the file.

It also checks to see if the destination exists first which avoids
conflicts or duplicates.

Fixes #1965
Fixes #2569
2018-09-29 14:56:20 +01:00
Nick Craig-Wood fc640d3a09 Add Frantisek Fuka to contributors 2018-09-29 14:55:11 +01:00
Frantisek Fuka e1f67295b4 b2: add note about cleanup in docs
Added: "Note that `cleanup` does not remove partially uploaded filesfrom the bucket."
2018-09-29 14:47:31 +01:00
Henning Surmeier 22ac80e83a webdav/sharepoint: renew cookies after 12hrs 2018-09-26 13:04:41 +01:00
Nick Craig-Wood c7aa6b587b Add xnaas to contributors 2018-09-26 10:07:13 +01:00
xnaas 8d1848bebe docs: note --track-renames doesn't work with crypt 2018-09-26 10:06:19 +01:00
Fabian Möller 527c0af1c3 drive: cleanup changeNotifyRunner 2018-09-25 17:54:48 +02:00
Fabian Möller a20fae0364 drive: code cleanup 2018-09-25 15:20:23 +01:00
Fabian Möller 15b1a1f909 drive: add support for apps-script to json export 2018-09-25 15:20:23 +01:00
Fabian Möller 80b25daac7 drive: add support for multipart document extensions 2018-09-25 15:20:23 +01:00
Fabian Möller 70b30d5ca4 drive: add document links 2018-09-25 15:20:23 +01:00
Fabian Möller 0b2fc621fc drive: restructure Object type 2018-09-25 15:20:23 +01:00
Fabian Möller 171e39b230 drive: add --drive-import-formats
Add a new flag to the drive backend to allow document conversions oni upload.
The existing --drive-formats flag has been renamed to --drive-export-formats.
The old flag is still working to be backward compatible.
2018-09-25 15:20:23 +01:00
Fabian Möller 690a44e40e drive: rewrite mime type and extension handling
Make use of the mime package to find matching extensions and mime types.
For simplicity, all extensions are now prefixed with "." to match the
mime package requirements.

Parsed extensions get converted if needed.
2018-09-25 15:20:23 +01:00
Fabian Möller d9a3b26e47 vfs: add vfs/poll-interval rc command
This command can be used to query the current status of the
poll-interval option and also update the value.
2018-09-25 14:01:13 +02:00
Fabian Möller 1eec59e091 fs: update ChangeNotifier interface
This introduces a channel to the ChangeNotify function, which can be
used to update the poll-interval and cleanly exit the polling function.
2018-09-25 14:01:13 +02:00
Nick Craig-Wood 96ce49ec4e Add ssaqua to contributors 2018-09-24 17:08:47 +01:00
ssaqua ae63e4b4f0 list: change debug logs for excluded items 2018-09-24 17:08:35 +01:00
Nick Craig-Wood e2fb588eb9 Add frenos to contributors 2018-09-24 17:05:03 +01:00
frenos 382a6863b5 rc: add support for OPTIONS and basic CORS - #2575 2018-09-24 17:04:47 +01:00
Nick Craig-Wood 7b975bc1ff alias: Fix handling of Windows network paths
Before this fix, the alias backend would mangle Windows paths like
//server/drive as it was treating them as unix paths.

See https://forum.rclone.org/t/smb-share-alias/6857
2018-09-21 18:24:21 +01:00
Nick Craig-Wood 467fe30a5e vendor: update to latest versions of everything 2018-09-21 18:23:37 +01:00
Nick Craig-Wood 4415aa5c2e build: fix make update 2018-09-21 18:23:37 +01:00
Nick Craig-Wood 17ab38502d Revamp issue and PR templates and CONTRIBUTING guide
Thanks to @fd0 of the restic project for a very useful blog post and
something to plagiarise :-)

https://restic.net/blog/2018-09-09/GitHub-issue-templates
2018-09-21 18:17:32 +01:00
Nick Craig-Wood 9fa8c959ee local: preallocate files on linux with fallocate(2) 2018-09-19 16:04:57 +01:00
Nick Craig-Wood f29c6049fc local: preallocate files on Windows to reduce fragmentation #2469
Before this change on Windows, files copied locally could become
heavily fragmented (300+ fragments for maybe 100 MB), no matter how
much contiguous free space there was (even if it's over 1TiB). This
can needlessly yet severely adversely affect performance on hard
disks.

This changes uses NtSetInformationFile to pre-allocate the space to
avoid this.

It does nothing on other OSes other than Windows.
2018-09-19 16:04:57 +01:00
Nick Craig-Wood e44fa5db8e build: update git bisect scripts 2018-09-19 16:04:57 +01:00
Fabian Möller 03ea05b860 drive: add workaround for slow downloads
Add --drive-v2-download-min-size flag to allow downloading files via the
drive v2 API. If files are greater than this flag, a download link is
generated when needed. The flag is disabled by default.
2018-09-18 15:55:50 +01:00
Fabian Möller b8678c9d4b vendor: add google.golang.org/api/drive/v2 2018-09-18 15:55:50 +01:00
Fabian Möller 13823a7743 drive: fix escaped chars in documents during list
Fixes #2591
2018-09-18 15:53:44 +01:00
sandeepkru b94d87ae2d azureblob and fstests - Modify integration tests to include new
optional setting to test SetTier on only few supported tiers.

Remove unused optional interface ListTiers and backend and internal tests
2018-09-18 13:56:09 +01:00
sandeepkru e0c5f7ff1b fs - Remove unreferenced ListTierer optional interface 2018-09-18 13:56:09 +01:00
Nick Craig-Wood b22ecbe174 Add Joanna Marek to contributors 2018-09-18 10:27:33 +01:00
Nick Craig-Wood c41be436c6 Add Antoine GIRARD to contributors 2018-09-18 10:27:33 +01:00
Joanna Marek e022ffce0f accounting: change too long names cutting mechanism - fixes #2490 2018-09-18 10:27:23 +01:00
albertony cfe65f1e72 jottacloud: minor update in docs 2018-09-18 10:25:30 +01:00
Sebastian Bünger b18595ae07 jottacloud: Fix handling of reserved characters. fixes #2531 2018-09-17 12:42:35 +01:00
Nick Craig-Wood d27630626a webdav: add a small pause after failed upload before deleting file #2517
This fixes the integration tests for `serve webdav` which uses the
webdav backend tests.
2018-09-17 08:51:50 +01:00
Nick Craig-Wood c473c7cb53 ftp: add a small pause after failed upload before deleting file #2517
This fixes the integration tests for `serve ftp` which uses the ftp
backend tests.
2018-09-17 08:51:50 +01:00
Nick Craig-Wood ef3526b3b8 vfs: fix race condition detected by serve ftp tests 2018-09-17 08:50:34 +01:00
Nick Craig-Wood d4ee7277c0 serve ftp: disable on plan9 since it doesn't compile 2018-09-17 08:50:34 +01:00
Antoine GIRARD 4a3efa5d45 cmd/serve: add ftp server - implement #2151 2018-09-17 08:50:34 +01:00
Nick Craig-Wood a14f0d46d7 vendor: add github.com/goftp/server 2018-09-17 08:50:34 +01:00
705 changed files with 118929 additions and 38843 deletions

No files matched your search

+31
View File
@@ -0,0 +1,31 @@
<!--
Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
https://forum.rclone.org/
instead of filing an issue for a quick response.
If you are reporting a bug or asking for a new feature then please use one of the templates here:
https://github.com/ncw/rclone/issues/new
otherwise fill in the form below.
Thank you
The Rclone Developers
-->
#### Output of `rclone version`
#### Describe the issue
@@ -1,14 +1,17 @@
---
name: Bug report
about: Report a problem with rclone
---
<!--
Hi!
Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
We understand you are having a problem with rclone or have an idea for an improvement - we want to help you with that!
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
https://forum.rclone.org/
instead of filing an issue. We'll reply quickly and it won't increase our massive issue backlog.
instead of filing an issue for a quick response.
If you think you might have found a bug, please can you try to replicate it with the latest beta?
@@ -16,9 +19,7 @@ If you think you might have found a bug, please can you try to replicate it with
If you can still replicate it with the latest beta, then please fill in the info below which makes our lives much easier. A log with -vv will make our day :-)
If you have an idea for an improvement, then please search the old issues first and if you don't find your idea, make a new issue.
Thanks
Thank you
The Rclone Developers
@@ -27,17 +28,23 @@ The Rclone Developers
#### What is the problem you are having with rclone?
#### What is your rclone version (eg output from `rclone -V`)
#### What is your rclone version (output from `rclone version`)
#### Which OS you are using and how many bits (eg Windows 7, 64 bit)
#### Which cloud storage system are you using? (eg Google Drive)
#### The command you were trying to run (eg `rclone copy /tmp remote:tmp`)
#### A log from the command with the `-vv` flag (eg output from `rclone -vv copy /tmp remote:tmp`)
+36
View File
@@ -0,0 +1,36 @@
---
name: Feature request
about: Suggest a new feature or enhancement for rclone
---
<!--
Welcome :-)
So you've got an idea to improve rclone? We love that! You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
Here is a checklist of things to do:
1. Please search the old issues first for your idea and +1 or comment on an existing issue if possible.
2. Discuss on the forum first: https://forum.rclone.org/
3. Make a feature request issue (this is the right place!).
4. Be prepared to get involved making the feature :-)
Looking forward to your great idea!
The Rclone Developers
-->
#### What is your current rclone version (output from `rclone version`)?
#### What problem are you are trying to solve?
#### How do you think rclone should be changed to solve that?
+29
View File
@@ -0,0 +1,29 @@
<!--
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.
You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->
#### What is the purpose of this change?
<!--
Describe the changes here
-->
#### Was the change discussed in an issue or in the forum before?
<!--
Link issues and relevant forum posts here.
-->
#### Checklist
- [ ] I have read the [contribution guidelines](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#submitting-a-pull-request).
- [ ] I have added tests for all changes in this PR if appropriate.
- [ ] I have added documentation for the changes if appropriate.
- [ ] All commit messages are in [house style](https://github.com/ncw/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [ ] I'm done, this Pull Request is ready for review :-)
-1
View File
@@ -4,7 +4,6 @@ dist: trusty
os:
- linux
go:
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
+38 -22
View File
@@ -21,14 +21,14 @@ with the [latest beta of rclone](https://beta.rclone.org/):
## Submitting a pull request ##
If you find a bug that you'd like to fix, or a new feature that you'd
like to implement then please submit a pull request via Github.
like to implement then please submit a pull request via GitHub.
If it is a big feature then make an issue first so it can be discussed.
You'll need a Go environment set up with GOPATH set. See [the Go
getting started docs](https://golang.org/doc/install) for more info.
First in your web browser press the fork button on [rclone's Github
First in your web browser press the fork button on [rclone's GitHub
page](https://github.com/ncw/rclone).
Now in your terminal
@@ -64,22 +64,23 @@ packages which you can install with
Make sure you
* Add documentation for a new feature (see below for where)
* Add unit tests for a new feature
* Add [documentation](#writing-documentation) for a new feature.
* Follow the [commit message guidelines](#commit-messages).
* Add [unit tests](#testing) for a new feature
* squash commits down to one per feature
* rebase to master `git rebase master`
* rebase to master with `git rebase master`
When you are done with that
git push origin my-new-feature
Go to the Github website and click [Create pull
Go to the GitHub website and click [Create pull
request](https://help.github.com/articles/creating-a-pull-request/).
You patch will get reviewed and you might get asked to fix some stuff.
If so, then make the changes in the same branch, squash the commits,
rebase it to master then push it to Github with `--force`.
rebase it to master then push it to GitHub with `--force`.
## Enabling CI for your fork ##
@@ -122,6 +123,13 @@ but they can be run against any of the remotes.
cd fs/operations
go test -v -remote TestDrive:
If you want to use the integration test framework to run these tests
all together with an HTML report and test retries then from the
project root:
go install github.com/ncw/rclone/fstest/test_all
test_all -backend drive
If you want to run all the integration tests against all the remotes,
then change into the project root and run
@@ -181,10 +189,14 @@ with modules beneath.
If you are adding a new feature then please update the documentation.
If you add a new flag, then if it is a general flag, document it in
If you add a new general flag (not for a backend), then document it in
`docs/content/docs.md` - the flags there are supposed to be in
alphabetical order. If it is a remote specific flag, then document it
in `docs/content/remote.md`.
alphabetical order.
If you add a new backend option/flag, then it should be documented in
the source file in the `Help:` field. The first line of this is used
for the flag help, the remainder is shown to the user in `rclone
config` and is added to the docs with `make backenddocs`.
The only documentation you need to edit are the `docs/content/*.md`
files. The MANUAL.*, rclone.1, web site etc are all auto generated
@@ -203,14 +215,20 @@ file.
## Commit messages ##
Please make the first line of your commit message a summary of the
change, and prefix it with the directory of the change followed by a
colon. The changelog gets made by looking at just these first lines
so make it good!
change that a user (not a developer) of rclone would like to read, and
prefix it with the directory of the change followed by a colon. The
changelog gets made by looking at just these first lines so make it
good!
If you have more to say about the commit, then enter a blank line and
carry on the description. Remember to say why the change was needed -
the commit itself shows what was changed.
Writing more is better than less. Comparing the behaviour before the
change to that after the change is very useful. Imagine you are
writing to yourself in 12 months time when you've forgotten everything
about what you just did and you need to get up to speed quickly.
If the change fixes an issue then write `Fixes #1234` in the commit
message. This can be on the subject line if it will fit. If you
don't want to close the associated issue just put `#1234` and the
@@ -258,9 +276,8 @@ To add a dependency `github.com/ncw/new_dependency` see the
instructions below. These will fetch the dependency, add it to
`go.mod` and `go.sum` and vendor it for older go versions.
export GO111MODULE=on
go get github.com/ncw/new_dependency
go mod vendor
GO111MODULE=on go get github.com/ncw/new_dependency
GO111MODULE=on go mod vendor
You can add constraints on that package when doing `go get` (see the
go docs linked above), but don't unless you really need to.
@@ -275,9 +292,8 @@ in `vendor`.
If you need to update a dependency then run
export GO111MODULE=on
go get -u github.com/pkg/errors
go mod vendor
GO111MODULE=on go get -u github.com/pkg/errors
GO111MODULE=on go mod vendor
Check in in a single commit as above.
@@ -334,7 +350,7 @@ Unit tests
Integration tests
* Add your fs to `fstest/test_all/test_all.go`
* Add your backend to `fstest/test_all/config.yaml`
* Make sure integration tests pass with
* `cd fs/operations`
* `go test -v -remote TestRemote:`
@@ -349,8 +365,8 @@ See the [testing](#testing) section for more information on integration tests.
Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in alphabetical order but with the local file system last.
* `README.md` - main Github page
* `docs/content/remote.md` - main docs page
* `README.md` - main GitHub page
* `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
* `docs/content/overview.md` - overview docs
* `docs/content/docs.md` - list of remotes in config section
* `docs/content/about.md` - front page of rclone.org
+1 -1
View File
@@ -58,7 +58,7 @@ Close tickets as soon as you can - make sure they are tagged with a release. Po
Try to process pull requests promptly!
Merging pull requests on Github itself works quite well now-a-days so you can squash and rebase or rebase pull requests. rclone doesn't use merge commits. Use the squash and rebase option if you need to edit the commit message.
Merging pull requests on GitHub itself works quite well now-a-days so you can squash and rebase or rebase pull requests. rclone doesn't use merge commits. Use the squash and rebase option if you need to edit the commit message.
After merging the commit, in your local master branch, do `git pull` then run `bin/update-authors.py` to update the authors file then `git push`.
+3423 -395
View File
File diff suppressed because it is too large. Load diff
+3340 -448
View File
File diff suppressed because it is too large. Load diff
+3278 -395
View File
File diff suppressed because it is too large. Load diff
+10 -9
View File
@@ -50,10 +50,9 @@ version:
# Full suite of integration tests
test: rclone
go install github.com/ncw/rclone/fstest/test_all
-go test -v -count 1 $(BUILDTAGS) $(GO_FILES) 2>&1 | tee test.log
-test_all github.com/ncw/rclone/fs/operations github.com/ncw/rclone/fs/sync 2>&1 | tee fs/test_all.log
@echo "Written logs in test.log and fs/test_all.log"
go install --ldflags "-s -X github.com/ncw/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/ncw/rclone/fstest/test_all
-test_all 2>&1 | tee test_all.log
@echo "Written logs in test_all.log"
# Quick test
quicktest:
@@ -88,7 +87,7 @@ build_dep:
ifdef FULL_TESTS
go get -u github.com/kisielk/errcheck
go get -u golang.org/x/tools/cmd/goimports
go get -u github.com/golang/lint/golint
go get -u golang.org/x/lint/golint
endif
# Get the release dependencies
@@ -99,15 +98,15 @@ release_dep:
# Update dependencies
update:
GO111MODULE=on go get -u ./...
GO111MODULE=on go tidy
GO111MODULE=on go vendor
GO111MODULE=on go mod tidy
GO111MODULE=on go mod vendor
doc: rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
rclone.1: MANUAL.md
pandoc -s --from markdown --to man MANUAL.md -o rclone.1
MANUAL.md: bin/make_manual.py docs/content/*.md commanddocs
MANUAL.md: bin/make_manual.py docs/content/*.md commanddocs backenddocs
./bin/make_manual.py
MANUAL.html: MANUAL.md
@@ -119,6 +118,9 @@ MANUAL.txt: MANUAL.md
commanddocs: rclone
rclone gendocs docs/content/commands/
backenddocs: rclone bin/make_backend_docs.py
./bin/make_backend_docs.py
rcdocs: rclone
bin/make_rc_docs.sh
@@ -229,4 +231,3 @@ startdev:
winzip:
zip -9 rclone-$(TAG).zip rclone.exe
+66 -34
View File
@@ -2,10 +2,11 @@
[Website](https://rclone.org) |
[Documentation](https://rclone.org/docs/) |
[Download](https://rclone.org/downloads/) |
[Contributing](CONTRIBUTING.md) |
[Changelog](https://rclone.org/changelog/) |
[Installation](https://rclone.org/install/) |
[Forum](https://forum.rclone.org/)
[Forum](https://forum.rclone.org/) |
[G+](https://google.com/+RcloneOrg)
[![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
@@ -13,50 +14,81 @@
[![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
[![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone)
Rclone is a command line program to sync files and directories to and from
# Rclone
* Amazon Drive ([See note](https://rclone.org/amazonclouddrive/#status))
* Amazon S3 / Dreamhost / Ceph / Minio / Wasabi
* Backblaze B2
* Box
* Dropbox
* FTP
* Google Cloud Storage
* Google Drive
* HTTP
* Hubic
* Jottacloud
* Mega
* Microsoft Azure Blob Storage
* Microsoft OneDrive
* OpenDrive
* Openstack Swift / Rackspace cloud files / Memset Memstore / OVH / Oracle Cloud Storage
* pCloud
* QingStor
* SFTP
* Webdav / Owncloud / Nextcloud
* Yandex Disk
* The local filesystem
Rclone *("rsync for cloud storage")* is a command line program to sync files and directories to and from different cloud storage providers.
Features
## Storage providers
* Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
* Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
* Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
* Box [:page_facing_up:](https://rclone.org/box/)
* Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
* DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
* Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
* Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
* FTP [:page_facing_up:](https://rclone.org/ftp/)
* Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
* Google Drive [:page_facing_up:](https://rclone.org/drive/)
* HTTP [:page_facing_up:](https://rclone.org/http/)
* Hubic [:page_facing_up:](https://rclone.org/hubic/)
* Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
* IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
* Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
* Mega [:page_facing_up:](https://rclone.org/mega/)
* Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
* Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
* Minio [:page_facing_up:](https://rclone.org/s3/#minio)
* Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
* OVH [:page_facing_up:](https://rclone.org/swift/)
* OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
* Openstack Swift [:page_facing_up:](https://rclone.org/swift/)
* Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
* ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
* pCloud [:page_facing_up:](https://rclone.org/pcloud/)
* put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
* QingStor [:page_facing_up:](https://rclone.org/qingstor/)
* Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
* SFTP [:page_facing_up:](https://rclone.org/sftp/)
* Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
* WebDAV [:page_facing_up:](https://rclone.org/webdav/)
* Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
* The local filesystem [:page_facing_up:](https://rclone.org/local/)
Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
## Features
* MD5/SHA1 hashes checked at all times for file integrity
* Timestamps preserved on files
* Partial syncs supported on a whole file basis
* Copy mode to just copy new/changed files
* Sync (one way) mode to make a directory identical
* Check mode to check for file hash equality
* [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
* [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
* [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
* Can sync to and from network, eg two different cloud accounts
* Optional encryption (Crypt)
* Optional FUSE mount
* Optional encryption ([Crypt](https://rclone.org/crypt/))
* Optional cache ([Cache](https://rclone.org/cache/))
* Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
See the home page for installation, usage, documentation, changelog
and configuration walkthroughs.
## Installation & documentation
* https://rclone.org/
Please see the [rclone website](https://rclone.org/) for:
* [Installation](https://rclone.org/install/)
* [Documentation & configuration](https://rclone.org/docs/)
* [Changelog](https://rclone.org/changelog/)
* [FAQ](https://rclone.org/faq/)
* [Storage providers](https://rclone.org/overview/)
* [Forum](https://forum.rclone.org/)
* ...and more
## Downloads
* https://rclone.org/downloads/
License
-------
This is free software under the terms of MIT the license (check the
COPYING file included in this package).
[COPYING file](/rclone/COPYING) included in this package).
+15
View File
@@ -32,6 +32,21 @@ Early in the next release cycle update the vendored dependencies
* git add new files
* git commit -a -v
If `make update` fails with errors like this:
```
# github.com/cpuguy83/go-md2man/md2man
../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:11:16: undefined: blackfriday.EXTENSION_NO_INTRA_EMPHASIS
../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:12:16: undefined: blackfriday.EXTENSION_TABLES
```
Can be fixed with
* GO111MODULE=on go get -u github.com/russross/blackfriday@v1.5.2
* GO111MODULE=on go mod tidy
* GO111MODULE=on go mod vendor
Making a point release. If rclone needs a point release due to some
horrendous bug, then
* git branch v1.XX v1.XX-fixes
+3 -8
View File
@@ -2,13 +2,12 @@ package alias
import (
"errors"
"path"
"path/filepath"
"strings"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config/configmap"
"github.com/ncw/rclone/fs/config/configstruct"
"github.com/ncw/rclone/fs/fspath"
)
// Register with Fs
@@ -47,13 +46,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if strings.HasPrefix(opt.Remote, name+":") {
return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
}
_, configName, fsPath, err := fs.ParseRemote(opt.Remote)
fsInfo, configName, fsPath, config, err := fs.ConfigFs(opt.Remote)
if err != nil {
return nil, err
}
root = path.Join(fsPath, filepath.ToSlash(root))
if configName == "local" {
return fs.NewFs(root)
}
return fs.NewFs(configName + ":" + root)
return fsInfo.NewFs(configName, fspath.JoinRootPath(fsPath, root), config)
}
+68 -20
View File
@@ -97,13 +97,42 @@ func init() {
Hide: fs.OptionHideBoth,
Advanced: true,
}, {
Name: "upload_wait_per_gb",
Help: "Additional time per GB to wait after a failed complete upload to see if it appears.",
Name: "upload_wait_per_gb",
Help: `Additional time per GB to wait after a failed complete upload to see if it appears.
Sometimes Amazon Drive gives an error when a file has been fully
uploaded but the file appears anyway after a little while. This
happens sometimes for files over 1GB in size and nearly every time for
files bigger than 10GB. This parameter controls the time rclone waits
for the file to appear.
The default value for this parameter is 3 minutes per GB, so by
default it will wait 3 minutes for every GB uploaded to see if the
file appears.
You can disable this feature by setting it to 0. This may cause
conflict errors as rclone retries the failed upload but the file will
most likely appear correctly eventually.
These values were determined empirically by observing lots of uploads
of big files for a range of file sizes.
Upload with the "-v" flag to see more info about what rclone is doing
in this situation.`,
Default: fs.Duration(180 * time.Second),
Advanced: true,
}, {
Name: "templink_threshold",
Help: "Files >= this size will be downloaded via their tempLink.",
Name: "templink_threshold",
Help: `Files >= this size will be downloaded via their tempLink.
Files this size or more will be downloaded via their "tempLink". This
is to work around a problem with Amazon Drive which blocks downloads
of files bigger than about 10GB. The default for this is 9GB which
shouldn't need to be changed.
To download files above this threshold, rclone requests a "tempLink"
which downloads the file through a temporary URL directly from the
underlying S3 storage.`,
Default: defaultTempLinkThreshold,
Advanced: true,
}},
@@ -283,16 +312,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
newF := *f
newF.dirCache = dircache.New(newRoot, f.trueRootID, &newF)
newF.root = newRoot
tempF := *f
tempF.dirCache = dircache.New(newRoot, f.trueRootID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = newF.dirCache.FindRoot(false)
err = tempF.dirCache.FindRoot(false)
if err != nil {
// No root so return old f
return f, nil
}
_, err := newF.newObjectWithInfo(remote, nil)
_, err := tempF.newObjectWithInfo(remote, nil)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
@@ -300,8 +329,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
return nil, err
}
// XXX: update the old f here instead of returning tempF, since
// `features` were already filled with functions having *f as a receiver.
// See https://github.com/ncw/rclone/issues/2182
f.dirCache = tempF.dirCache
f.root = tempF.root
// return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile
return f, fs.ErrorIsFile
}
return f, nil
}
@@ -1240,24 +1274,38 @@ func (o *Object) MimeType() string {
// Automatically restarts itself in case of unexpected behaviour of the remote.
//
// Close the returned channel to stop being notified.
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
checkpoint := f.opt.Checkpoint
quit := make(chan bool)
go func() {
var ticker *time.Ticker
var tickerC <-chan time.Time
for {
checkpoint = f.changeNotifyRunner(notifyFunc, checkpoint)
if err := config.SetValueAndSave(f.name, "checkpoint", checkpoint); err != nil {
fs.Debugf(f, "Unable to save checkpoint: %v", err)
}
select {
case <-quit:
return
case <-time.After(pollInterval):
case pollInterval, ok := <-pollIntervalChan:
if !ok {
if ticker != nil {
ticker.Stop()
}
return
}
if pollInterval == 0 {
if ticker != nil {
ticker.Stop()
ticker, tickerC = nil, nil
}
} else {
ticker = time.NewTicker(pollInterval)
tickerC = ticker.C
}
case <-tickerC:
checkpoint = f.changeNotifyRunner(notifyFunc, checkpoint)
if err := config.SetValueAndSave(f.name, "checkpoint", checkpoint); err != nil {
fs.Debugf(f, "Unable to save checkpoint: %v", err)
}
}
}
}()
return quit
}
func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), checkpoint string) string {
+84 -37
View File
@@ -22,7 +22,7 @@ import (
"sync"
"time"
"github.com/Azure/azure-storage-blob-go/2018-03-28/azblob"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/fs/config/configmap"
@@ -45,10 +45,10 @@ const (
maxTotalParts = 50000 // in multipart upload
storageDefaultBaseURL = "blob.core.windows.net"
// maxUncommittedSize = 9 << 30 // can't upload bigger than this
defaultChunkSize = 4 * 1024 * 1024
maxChunkSize = 100 * 1024 * 1024
defaultUploadCutoff = 256 * 1024 * 1024
maxUploadCutoff = 256 * 1024 * 1024
defaultChunkSize = 4 * fs.MebiByte
maxChunkSize = 100 * fs.MebiByte
defaultUploadCutoff = 256 * fs.MebiByte
maxUploadCutoff = 256 * fs.MebiByte
defaultAccessTier = azblob.AccessTierNone
)
@@ -73,23 +73,44 @@ func init() {
Advanced: true,
}, {
Name: "upload_cutoff",
Help: "Cutoff for switching to chunked upload.",
Help: "Cutoff for switching to chunked upload (<= 256MB).",
Default: fs.SizeSuffix(defaultUploadCutoff),
Advanced: true,
}, {
Name: "chunk_size",
Help: "Upload chunk size. Must fit in memory.",
Name: "chunk_size",
Help: `Upload chunk size (<= 100MB).
Note that this is stored in memory and there may be up to
"--transfers" chunks stored at once in memory.`,
Default: fs.SizeSuffix(defaultChunkSize),
Advanced: true,
}, {
Name: "list_chunk",
Help: "Size of blob list.",
Name: "list_chunk",
Help: `Size of blob list.
This sets the number of blobs requested in each listing chunk. Default
is the maximum, 5000. "List blobs" requests are permitted 2 minutes
per megabyte to complete. If an operation is taking longer than 2
minutes per megabyte on average, it will time out (
[source](https://docs.microsoft.com/en-us/rest/api/storageservices/setting-timeouts-for-blob-service-operations#exceptions-to-default-timeout-interval)
). This can be used to limit the number of blobs items to return, to
avoid the time out.`,
Default: maxListChunkSize,
Advanced: true,
}, {
Name: "access_tier",
Help: "Access tier of blob, supports hot, cool and archive tiers.\nArchived blobs can be restored by setting access tier to hot or cool." +
" Leave blank if you intend to use default access tier, which is set at account level",
Help: `Access tier of blob: hot, cool or archive.
Archived blobs can be restored by setting access tier to hot or
cool. Leave blank if you intend to use default access tier, which is
set at account level
If there is no "access tier" specified, rclone doesn't apply any tier.
rclone performs "Set Tier" operation on blobs while uploading, if objects
are not modified, specifying "access tier" to new one will have no effect.
If blobs are in "archive tier" at remote, trying to perform data transfer
operations from remote will not be allowed. User should first restore by
tiering blob to "Hot" or "Cool".`,
Advanced: true,
}},
})
@@ -191,19 +212,6 @@ func validateAccessTier(tier string) bool {
}
}
// validAccessTiers returns list of supported storage tiers on azureblob fs
func validAccessTiers() []string {
validTiers := [...]azblob.AccessTierType{azblob.AccessTierHot, azblob.AccessTierCool,
azblob.AccessTierArchive}
var tiers [len(validTiers)]string
for i, tier := range validTiers {
tiers[i] = string(tier)
}
return tiers[:]
}
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
401, // Unauthorized (eg "Token has expired")
@@ -229,6 +237,40 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
return fserrors.ShouldRetry(err), err
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.Byte
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
if cs > maxChunkSize {
return errors.Errorf("%s is greater than %s", cs, maxChunkSize)
}
return nil
}
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
}
return
}
func checkUploadCutoff(cs fs.SizeSuffix) error {
if cs > maxUploadCutoff {
return errors.Errorf("%v must be less than or equal to %v", cs, maxUploadCutoff)
}
return nil
}
func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadCutoff(cs)
if err == nil {
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
}
return
}
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -238,11 +280,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
return nil, err
}
if opt.UploadCutoff > maxUploadCutoff {
return nil, errors.Errorf("azure: upload cutoff (%v) must be less than or equal to %v", opt.UploadCutoff, maxUploadCutoff)
err = checkUploadCutoff(opt.UploadCutoff)
if err != nil {
return nil, errors.Wrap(err, "azure: upload cutoff")
}
if opt.ChunkSize > maxChunkSize {
return nil, errors.Errorf("azure: chunk size can't be greater than %v - was %v", maxChunkSize, opt.ChunkSize)
err = checkUploadChunkSize(opt.ChunkSize)
if err != nil {
return nil, errors.Wrap(err, "azure: chunk size")
}
if opt.ListChunkSize > maxListChunkSize {
return nil, errors.Errorf("azure: blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
@@ -321,7 +365,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
BucketBased: true,
SetTier: true,
GetTier: true,
ListTiers: true,
}).Fill(f)
if f.root != "" {
f.root += "/"
@@ -1247,11 +1290,20 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
Metadata: o.meta,
BlobHTTPHeaders: httpHeaders,
}
// FIXME Until https://github.com/Azure/azure-storage-blob-go/pull/75
// is merged the SDK can't upload a single blob of exactly the chunk
// size, so upload with a multpart upload to work around.
// See: https://github.com/ncw/rclone/issues/2653
multipartUpload := size >= int64(o.fs.opt.UploadCutoff)
if size == int64(o.fs.opt.ChunkSize) {
multipartUpload = true
fs.Debugf(o, "Setting multipart upload for file of chunk size (%d) to work around SDK bug", size)
}
ctx := context.Background()
// Don't retry, return a retry error instead
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
if size >= int64(o.fs.opt.UploadCutoff) {
if multipartUpload {
// If a large file upload in chunks
err = o.uploadMultipart(in, size, &blob, &httpHeaders)
} else {
@@ -1316,7 +1368,7 @@ func (o *Object) SetTier(tier string) error {
blob := o.getBlobReference()
ctx := context.Background()
err := o.fs.pacer.Call(func() (bool, error) {
_, err := blob.SetTier(ctx, desiredAccessTier)
_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{})
return o.fs.shouldRetry(err)
})
@@ -1337,11 +1389,6 @@ func (o *Object) GetTier() string {
return string(o.accessTier)
}
// ListTiers returns list of storage tiers supported on this object
func (o *Object) ListTiers() []string {
return validAccessTiers()
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
+1 -3
View File
@@ -11,9 +11,7 @@ import (
func (f *Fs) InternalTest(t *testing.T) {
// Check first feature flags are set on this
// remote
enabled := f.Features().ListTiers
assert.True(t, enabled)
enabled = f.Features().SetTier
enabled := f.Features().SetTier
assert.True(t, enabled)
enabled = f.Features().GetTier
assert.True(t, enabled)
+21 -4
View File
@@ -2,19 +2,36 @@
// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
package azureblob_test
package azureblob
import (
"testing"
"github.com/ncw/rclone/backend/azureblob"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestAzureBlob:",
NilObject: (*azureblob.Object)(nil),
RemoteName: "TestAzureBlob:",
NilObject: (*Object)(nil),
TiersToTest: []string{"Hot", "Cool"},
ChunkedUpload: fstests.ChunkedUploadConfig{
MaxChunkSize: maxChunkSize,
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadCutoff(cs)
}
var (
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
_ fstests.SetUploadCutoffer = (*Fs)(nil)
)
+81 -25
View File
@@ -48,9 +48,9 @@ const (
decayConstant = 1 // bigger for slower decay, exponential
maxParts = 10000
maxVersions = 100 // maximum number of versions we search in --b2-versions mode
minChunkSize = 5E6
defaultChunkSize = 96 * 1024 * 1024
defaultUploadCutoff = 200E6
minChunkSize = 5 * fs.MebiByte
defaultChunkSize = 96 * fs.MebiByte
defaultUploadCutoff = 200 * fs.MebiByte
)
// Globals
@@ -77,14 +77,24 @@ func init() {
Help: "Endpoint for the service.\nLeave blank normally.",
Advanced: true,
}, {
Name: "test_mode",
Help: "A flag string for X-Bz-Test-Mode header for debugging.",
Name: "test_mode",
Help: `A flag string for X-Bz-Test-Mode header for debugging.
This is for debugging purposes only. Setting it to one of the strings
below will cause b2 to return specific errors:
* "fail_some_uploads"
* "expire_some_account_authorization_tokens"
* "force_cap_exceeded"
These will be set in the "X-Bz-Test-Mode" header which is documented
in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration_checklist.html).`,
Default: "",
Hide: fs.OptionHideConfigurator,
Advanced: true,
}, {
Name: "versions",
Help: "Include old versions in directory listings.",
Help: "Include old versions in directory listings.\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
Default: false,
Advanced: true,
}, {
@@ -92,13 +102,22 @@ func init() {
Help: "Permanently delete files on remote removal, otherwise hide files.",
Default: false,
}, {
Name: "upload_cutoff",
Help: "Cutoff for switching to chunked upload.",
Name: "upload_cutoff",
Help: `Cutoff for switching to chunked upload.
Files above this size will be uploaded in chunks of "--b2-chunk-size".
This value should be set no larger than 4.657GiB (== 5GB).`,
Default: fs.SizeSuffix(defaultUploadCutoff),
Advanced: true,
}, {
Name: "chunk_size",
Help: "Upload chunk size. Must fit in memory.",
Name: "chunk_size",
Help: `Upload chunk size. Must fit in memory.
When uploading large files, chunk the file into this size. Note that
these chunks are buffered in memory and there might a maximum of
"--transfers" chunks in progress at once. 5,000,000 Bytes is the
minimim size.`,
Default: fs.SizeSuffix(defaultChunkSize),
Advanced: true,
}},
@@ -263,6 +282,37 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
return nil
}
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
f.fillBufferTokens() // reset the buffer tokens
}
return
}
func checkUploadCutoff(opt *Options, cs fs.SizeSuffix) error {
if cs < opt.ChunkSize {
return errors.Errorf("%v is less than chunk size %v", cs, opt.ChunkSize)
}
return nil
}
func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadCutoff(&f.opt, cs)
if err == nil {
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
}
return
}
// NewFs contstructs an Fs from the path, bucket:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -271,11 +321,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
if opt.UploadCutoff < opt.ChunkSize {
return nil, errors.Errorf("b2: upload cutoff (%v) must be greater than or equal to chunk size (%v)", opt.UploadCutoff, opt.ChunkSize)
err = checkUploadCutoff(opt, opt.UploadCutoff)
if err != nil {
return nil, errors.Wrap(err, "b2: upload cutoff")
}
if opt.ChunkSize < minChunkSize {
return nil, errors.Errorf("b2: chunk size can't be less than %v - was %v", minChunkSize, opt.ChunkSize)
err = checkUploadChunkSize(opt.ChunkSize)
if err != nil {
return nil, errors.Wrap(err, "b2: chunk size")
}
bucket, directory, err := parsePath(root)
if err != nil {
@@ -291,13 +343,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
opt.Endpoint = defaultEndpoint
}
f := &Fs{
name: name,
opt: *opt,
bucket: bucket,
root: directory,
srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
bufferTokens: make(chan []byte, fs.Config.Transfers),
name: name,
opt: *opt,
bucket: bucket,
root: directory,
srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
}
f.features = (&fs.Features{
ReadMimeType: true,
@@ -310,10 +361,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
f.srv.SetHeader(testModeHeader, testMode)
fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
}
// Fill up the buffer tokens
for i := 0; i < fs.Config.Transfers; i++ {
f.bufferTokens <- nil
}
f.fillBufferTokens()
err = f.authorizeAccount()
if err != nil {
return nil, errors.Wrap(err, "failed to authorize account")
@@ -421,6 +469,14 @@ func (f *Fs) clearUploadURL() {
f.uploadMu.Unlock()
}
// Fill up (or reset) the buffer tokens
func (f *Fs) fillBufferTokens() {
f.bufferTokens = make(chan []byte, fs.Config.Transfers)
for i := 0; i < fs.Config.Transfers; i++ {
f.bufferTokens <- nil
}
}
// getUploadBlock gets a block from the pool of size chunkSize
func (f *Fs) getUploadBlock() []byte {
buf := <-f.bufferTokens
+20 -3
View File
@@ -1,10 +1,10 @@
// Test B2 filesystem interface
package b2_test
package b2
import (
"testing"
"github.com/ncw/rclone/backend/b2"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
@@ -12,6 +12,23 @@ import (
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestB2:",
NilObject: (*b2.Object)(nil),
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: minChunkSize,
NeedMultipleChunks: true,
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadCutoff(cs)
}
var (
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
_ fstests.SetUploadCutoffer = (*Fs)(nil)
)
+13 -8
View File
@@ -85,7 +85,7 @@ func init() {
Help: "Box App Client Secret\nLeave blank normally.",
}, {
Name: "upload_cutoff",
Help: "Cutoff for switching to multipart upload.",
Help: "Cutoff for switching to multipart upload (>= 50MB).",
Default: fs.SizeSuffix(defaultUploadCutoff),
Advanced: true,
}, {
@@ -283,16 +283,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
newF := *f
newF.dirCache = dircache.New(newRoot, rootID, &newF)
newF.root = newRoot
tempF := *f
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = newF.dirCache.FindRoot(false)
err = tempF.dirCache.FindRoot(false)
if err != nil {
// No root so return old f
return f, nil
}
_, err := newF.newObjectWithInfo(remote, nil)
_, err := tempF.newObjectWithInfo(remote, nil)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
@@ -300,9 +300,14 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
return nil, err
}
f.features.Fill(&newF)
f.features.Fill(&tempF)
// XXX: update the old f here instead of returning tempF, since
// `features` were already filled with functions having *f as a receiver.
// See https://github.com/ncw/rclone/issues/2182
f.dirCache = tempF.dirCache
f.root = tempF.root
// return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile
return f, fs.ErrorIsFile
}
return f, nil
}
+166 -56
View File
@@ -11,6 +11,7 @@ import (
"os/signal"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
@@ -23,6 +24,7 @@ import (
"github.com/ncw/rclone/fs/config/configmap"
"github.com/ncw/rclone/fs/config/configstruct"
"github.com/ncw/rclone/fs/config/obscure"
"github.com/ncw/rclone/fs/fspath"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/fs/rc"
"github.com/ncw/rclone/fs/walk"
@@ -86,8 +88,12 @@ func init() {
Help: "Skip all certificate verifications when connecting to the Plex server",
Advanced: true,
}, {
Name: "chunk_size",
Help: "The size of a chunk. Lower value good for slow connections but can affect seamless reading.",
Name: "chunk_size",
Help: `The size of a chunk (partial file data).
Use lower numbers for slower connections. If the chunk size is
changed, any downloaded chunks will be invalid and cache-chunk-path
will need to be cleared or unexpected EOF errors will occur.`,
Default: DefCacheChunkSize,
Examples: []fs.OptionExample{{
Value: "1m",
@@ -100,8 +106,10 @@ func init() {
Help: "10 MB",
}},
}, {
Name: "info_age",
Help: "How much time should object info (file size, file hashes etc) be stored in cache.\nUse a very high value if you don't plan on changing the source FS from outside the cache.\nAccepted units are: \"s\", \"m\", \"h\".",
Name: "info_age",
Help: `How long to cache file structure information (directory listings, file size, times etc).
If all write operations are done through the cache then you can safely make
this value very large as the cache store will also be updated in real time.`,
Default: DefCacheInfoAge,
Examples: []fs.OptionExample{{
Value: "1h",
@@ -114,8 +122,11 @@ func init() {
Help: "48 hours",
}},
}, {
Name: "chunk_total_size",
Help: "The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted.",
Name: "chunk_total_size",
Help: `The total size that the chunks can take up on the local disk.
If the cache exceeds this value then it will start to delete the
oldest chunks until it goes under this value.`,
Default: DefCacheTotalChunkSize,
Examples: []fs.OptionExample{{
Value: "500M",
@@ -130,63 +141,143 @@ func init() {
}, {
Name: "db_path",
Default: filepath.Join(config.CacheDir, "cache-backend"),
Help: "Directory to cache DB",
Help: "Directory to store file structure metadata DB.\nThe remote name is used as the DB file name.",
Advanced: true,
}, {
Name: "chunk_path",
Default: filepath.Join(config.CacheDir, "cache-backend"),
Help: "Directory to cache chunk files",
Name: "chunk_path",
Default: filepath.Join(config.CacheDir, "cache-backend"),
Help: `Directory to cache chunk files.
Path to where partial file data (chunks) are stored locally. The remote
name is appended to the final path.
This config follows the "--cache-db-path". If you specify a custom
location for "--cache-db-path" and don't specify one for "--cache-chunk-path"
then "--cache-chunk-path" will use the same path as "--cache-db-path".`,
Advanced: true,
}, {
Name: "db_purge",
Default: false,
Help: "Purge the cache DB before",
Help: "Clear all the cached data for this remote on start.",
Hide: fs.OptionHideConfigurator,
Advanced: true,
}, {
Name: "chunk_clean_interval",
Default: DefCacheChunkCleanInterval,
Help: "Interval at which chunk cleanup runs",
Name: "chunk_clean_interval",
Default: DefCacheChunkCleanInterval,
Help: `How often should the cache perform cleanups of the chunk storage.
The default value should be ok for most people. If you find that the
cache goes over "cache-chunk-total-size" too often then try to lower
this value to force it to perform cleanups more often.`,
Advanced: true,
}, {
Name: "read_retries",
Default: DefCacheReadRetries,
Help: "How many times to retry a read from a cache storage",
Name: "read_retries",
Default: DefCacheReadRetries,
Help: `How many times to retry a read from a cache storage.
Since reading from a cache stream is independent from downloading file
data, readers can get to a point where there's no more data in the
cache. Most of the times this can indicate a connectivity issue if
cache isn't able to provide file data anymore.
For really slow connections, increase this to a point where the stream is
able to provide data but your experience will be very stuttering.`,
Advanced: true,
}, {
Name: "workers",
Default: DefCacheTotalWorkers,
Help: "How many workers should run in parallel to download chunks",
Name: "workers",
Default: DefCacheTotalWorkers,
Help: `How many workers should run in parallel to download chunks.
Higher values will mean more parallel processing (better CPU needed)
and more concurrent requests on the cloud provider. This impacts
several aspects like the cloud provider API limits, more stress on the
hardware that rclone runs on but it also means that streams will be
more fluid and data will be available much more faster to readers.
**Note**: If the optional Plex integration is enabled then this
setting will adapt to the type of reading performed and the value
specified here will be used as a maximum number of workers to use.`,
Advanced: true,
}, {
Name: "chunk_no_memory",
Default: DefCacheChunkNoMemory,
Help: "Disable the in-memory cache for storing chunks during streaming",
Name: "chunk_no_memory",
Default: DefCacheChunkNoMemory,
Help: `Disable the in-memory cache for storing chunks during streaming.
By default, cache will keep file data during streaming in RAM as well
to provide it to readers as fast as possible.
This transient data is evicted as soon as it is read and the number of
chunks stored doesn't exceed the number of workers. However, depending
on other settings like "cache-chunk-size" and "cache-workers" this footprint
can increase if there are parallel streams too (multiple files being read
at the same time).
If the hardware permits it, use this feature to provide an overall better
performance during streaming but it can also be disabled if RAM is not
available on the local machine.`,
Advanced: true,
}, {
Name: "rps",
Default: int(DefCacheRps),
Help: "Limits the number of requests per second to the source FS. -1 disables the rate limiter",
Name: "rps",
Default: int(DefCacheRps),
Help: `Limits the number of requests per second to the source FS (-1 to disable)
This setting places a hard limit on the number of requests per second
that cache will be doing to the cloud provider remote and try to
respect that value by setting waits between reads.
If you find that you're getting banned or limited on the cloud
provider through cache and know that a smaller number of requests per
second will allow you to work with it then you can use this setting
for that.
A good balance of all the other settings should make this setting
useless but it is available to set for more special cases.
**NOTE**: This will limit the number of requests during streams but
other API calls to the cloud provider like directory listings will
still pass.`,
Advanced: true,
}, {
Name: "writes",
Default: DefCacheWrites,
Help: "Will cache file data on writes through the FS",
Name: "writes",
Default: DefCacheWrites,
Help: `Cache file data on writes through the FS
If you need to read files immediately after you upload them through
cache you can enable this flag to have their data stored in the
cache store at the same time during upload.`,
Advanced: true,
}, {
Name: "tmp_upload_path",
Default: "",
Help: "Directory to keep temporary files until they are uploaded to the cloud storage",
Name: "tmp_upload_path",
Default: "",
Help: `Directory to keep temporary files until they are uploaded.
This is the path where cache will use as a temporary storage for new
files that need to be uploaded to the cloud provider.
Specifying a value will enable this feature. Without it, it is
completely disabled and files will be uploaded directly to the cloud
provider`,
Advanced: true,
}, {
Name: "tmp_wait_time",
Default: DefCacheTmpWaitTime,
Help: "How long should files be stored in local cache before being uploaded",
Name: "tmp_wait_time",
Default: DefCacheTmpWaitTime,
Help: `How long should files be stored in local cache before being uploaded
This is the duration that a file must wait in the temporary location
_cache-tmp-upload-path_ before it is selected for upload.
Note that only one file is uploaded at a time and it can take longer
to start the upload if a queue formed for this purpose.`,
Advanced: true,
}, {
Name: "db_wait_time",
Default: DefCacheDbWaitTime,
Help: "How long to wait for the DB to be available - 0 is unlimited",
Name: "db_wait_time",
Default: DefCacheDbWaitTime,
Help: `How long to wait for the DB to be available - 0 is unlimited
Only one process can have the DB open at any one time, so rclone waits
for this duration for the DB to become available before it gives an
error.
If you set it to 0 then it will wait forever.`,
Advanced: true,
}},
})
@@ -273,7 +364,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", opt.Remote)
}
remotePath := path.Join(wPath, rootPath)
remotePath := fspath.JoinRootPath(wPath, rootPath)
wrappedFs, wrapErr := wInfo.NewFs(wName, remotePath, wConfig)
if wrapErr != nil && wrapErr != fs.ErrorIsFile {
return nil, errors.Wrapf(wrapErr, "failed to make remote %s:%s to wrap", wName, remotePath)
@@ -380,7 +471,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
fs.Infof(name, "Chunk Clean Interval: %v", f.opt.ChunkCleanInterval)
fs.Infof(name, "Workers: %v", f.opt.TotalWorkers)
fs.Infof(name, "File Age: %v", f.opt.InfoAge)
if !f.opt.StoreWrites {
if f.opt.StoreWrites {
fs.Infof(name, "Cache Writes: enabled")
}
@@ -415,7 +506,9 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
}()
if doChangeNotify := wrappedFs.Features().ChangeNotify; doChangeNotify != nil {
doChangeNotify(f.receiveChangeNotify, time.Duration(f.opt.ChunkCleanInterval))
pollInterval := make(chan time.Duration, 1)
pollInterval <- time.Duration(f.opt.ChunkCleanInterval)
doChangeNotify(f.receiveChangeNotify, pollInterval)
}
f.features = (&fs.Features{
@@ -780,12 +873,15 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
// ChangeNotify can subsribe multiple callers
// this is coupled with the wrapped fs ChangeNotify (if it supports it)
// and also notifies other caches (i.e VFS) to clear out whenever something changes
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
f.parentsForgetMu.Lock()
defer f.parentsForgetMu.Unlock()
fs.Debugf(f, "subscribing to ChangeNotify")
f.parentsForgetFn = append(f.parentsForgetFn, notifyFunc)
return make(chan bool)
go func() {
for range pollInterval {
}
}()
}
// Name of the remote (as passed into NewFs)
@@ -886,7 +982,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
fs.Debugf(dir, "list: cached entries: %v", entries)
return entries, nil
}
// FIXME need to clean existing cached listing
// we first search any temporary files stored locally
var cachedEntries fs.DirEntries
@@ -912,27 +1007,42 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
}
// search from the source
entries, err = f.Fs.List(dir)
sourceEntries, err := f.Fs.List(dir)
if err != nil {
return nil, err
}
fs.Debugf(dir, "list: read %v from source", len(entries))
fs.Debugf(dir, "list: source entries: %v", entries)
fs.Debugf(dir, "list: read %v from source", len(sourceEntries))
fs.Debugf(dir, "list: source entries: %v", sourceEntries)
sort.Sort(sourceEntries)
for _, entry := range entries {
entryRemote := entry.Remote()
i := sort.Search(len(sourceEntries), func(i int) bool { return sourceEntries[i].Remote() >= entryRemote })
if i < len(sourceEntries) && sourceEntries[i].Remote() == entryRemote {
continue
}
fp := path.Join(f.Root(), entryRemote)
switch entry.(type) {
case fs.Object:
_ = f.cache.RemoveObject(fp)
case fs.Directory:
_ = f.cache.RemoveDir(fp)
}
fs.Debugf(dir, "list: remove entry: %v", entryRemote)
}
entries = nil
// and then iterate over the ones from source (temp Objects will override source ones)
var batchDirectories []*Directory
for _, entry := range entries {
sort.Sort(cachedEntries)
tmpCnt := len(cachedEntries)
for _, entry := range sourceEntries {
switch o := entry.(type) {
case fs.Object:
// skip over temporary objects (might be uploading)
found := false
for _, t := range cachedEntries {
if t.Remote() == o.Remote() {
found = true
break
}
}
if found {
oRemote := o.Remote()
i := sort.Search(tmpCnt, func(i int) bool { return cachedEntries[i].Remote() >= oRemote })
if i < tmpCnt && cachedEntries[i].Remote() == oRemote {
continue
}
co := ObjectFromOriginal(f, o).persist()
+12 -27
View File
@@ -4,6 +4,9 @@ package cache_test
import (
"bytes"
"encoding/base64"
goflag "flag"
"fmt"
"io"
"io/ioutil"
"log"
@@ -12,21 +15,12 @@ import (
"path"
"path/filepath"
"runtime"
"runtime/debug"
"strconv"
"strings"
"testing"
"time"
"github.com/pkg/errors"
"encoding/base64"
goflag "flag"
"fmt"
"runtime/debug"
"encoding/json"
"net/http"
"github.com/ncw/rclone/backend/cache"
"github.com/ncw/rclone/backend/crypt"
_ "github.com/ncw/rclone/backend/drive"
@@ -36,10 +30,11 @@ import (
"github.com/ncw/rclone/fs/config/configmap"
"github.com/ncw/rclone/fs/object"
"github.com/ncw/rclone/fs/rc"
"github.com/ncw/rclone/fs/rc/rcflags"
"github.com/ncw/rclone/fstest"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -695,8 +690,8 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
}
func TestInternalChangeSeenAfterRc(t *testing.T) {
rcflags.Opt.Enabled = true
rc.Start(&rcflags.Opt)
cacheExpire := rc.Calls.Get("cache/expire")
assert.NotNil(t, cacheExpire)
id := fmt.Sprintf("ticsarc%v", time.Now().Unix())
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
@@ -729,13 +724,8 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
require.NoError(t, err)
require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
m := make(map[string]string)
res, err := http.Post(fmt.Sprintf("http://localhost:5572/cache/expire?remote=%s", "data.bin"), "application/json; charset=utf-8", strings.NewReader(""))
require.NoError(t, err)
defer func() {
_ = res.Body.Close()
}()
_ = json.NewDecoder(res.Body).Decode(&m)
// Call the rc function
m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
require.Contains(t, m, "status")
require.Contains(t, m, "message")
require.Equal(t, "ok", m["status"])
@@ -755,13 +745,8 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
li1, err = runInstance.list(t, rootFs, "")
require.Len(t, li1, 1)
m = make(map[string]string)
res2, err := http.Post("http://localhost:5572/cache/expire?remote=/", "application/json; charset=utf-8", strings.NewReader(""))
require.NoError(t, err)
defer func() {
_ = res2.Body.Close()
}()
_ = json.NewDecoder(res2.Body).Decode(&m)
// Call the rc function
m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
require.Contains(t, m, "status")
require.Contains(t, m, "message")
require.Equal(t, "ok", m["status"])
+1 -2
View File
@@ -3,6 +3,7 @@
package cache_test
import (
"fmt"
"math/rand"
"os"
"path"
@@ -10,8 +11,6 @@ import (
"testing"
"time"
"fmt"
"github.com/ncw/rclone/backend/cache"
_ "github.com/ncw/rclone/backend/drive"
"github.com/ncw/rclone/fs"
+1 -2
View File
@@ -3,9 +3,8 @@
package cache
import (
"time"
"path"
"time"
"github.com/ncw/rclone/fs"
)
+2 -3
View File
@@ -5,12 +5,11 @@ package cache
import (
"fmt"
"io"
"sync"
"time"
"path"
"runtime"
"strings"
"sync"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/operations"
+3 -5
View File
@@ -3,18 +3,16 @@
package cache
import (
"bytes"
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
"sync"
"bytes"
"io/ioutil"
"time"
"github.com/ncw/rclone/fs"
"github.com/patrickmn/go-cache"
+3 -6
View File
@@ -3,20 +3,17 @@
package cache
import (
"time"
"bytes"
"encoding/binary"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"strings"
"sync"
"io/ioutil"
"fmt"
"time"
bolt "github.com/coreos/bbolt"
"github.com/ncw/rclone/fs"
+1 -3
View File
@@ -17,11 +17,9 @@ import (
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/pkg/errors"
"github.com/rfjakob/eme"
"golang.org/x/crypto/nacl/secretbox"
"golang.org/x/crypto/scrypt"
"github.com/rfjakob/eme"
)
// Constants
+15 -7
View File
@@ -4,7 +4,6 @@ package crypt
import (
"fmt"
"io"
"path"
"strings"
"time"
@@ -13,6 +12,7 @@ import (
"github.com/ncw/rclone/fs/config/configmap"
"github.com/ncw/rclone/fs/config/configstruct"
"github.com/ncw/rclone/fs/config/obscure"
"github.com/ncw/rclone/fs/fspath"
"github.com/ncw/rclone/fs/hash"
"github.com/pkg/errors"
)
@@ -67,8 +67,16 @@ func init() {
Help: "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
IsPassword: true,
}, {
Name: "show_mapping",
Help: "For all files listed show how the names encrypt.",
Name: "show_mapping",
Help: `For all files listed show how the names encrypt.
If this flag is set then for each file that the remote is asked to
list, it will log (at level INFO) a line stating the decrypted file
name and the encrypted file name.
This is so you can work out which encrypted names are which decrypted
names just in case you need to do something with the encrypted file
names, or for debugging purposes.`,
Default: false,
Hide: fs.OptionHideConfigurator,
Advanced: true,
@@ -135,11 +143,11 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
}
// Look for a file first
remotePath := path.Join(wPath, cipher.EncryptFileName(rpath))
remotePath := fspath.JoinRootPath(wPath, cipher.EncryptFileName(rpath))
wrappedFs, err := wInfo.NewFs(wName, remotePath, wConfig)
// if that didn't produce a file, look for a directory
if err != fs.ErrorIsFile {
remotePath = path.Join(wPath, cipher.EncryptDirName(rpath))
remotePath = fspath.JoinRootPath(wPath, cipher.EncryptDirName(rpath))
wrappedFs, err = wInfo.NewFs(wName, remotePath, wConfig)
}
if err != fs.ErrorIsFile && err != nil {
@@ -165,7 +173,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
doChangeNotify := wrappedFs.Features().ChangeNotify
if doChangeNotify != nil {
f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
decrypted, err := f.DecryptFileName(path)
if err != nil {
@@ -174,7 +182,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
}
notifyFunc(decrypted, entryType)
}
return doChangeNotify(wrappedNotifyFunc, pollInterval)
doChangeNotify(wrappedNotifyFunc, pollInterval)
}
}
+23
View File
@@ -7,13 +7,30 @@ import (
"testing"
"github.com/ncw/rclone/backend/crypt"
_ "github.com/ncw/rclone/backend/drive" // for integration tests
_ "github.com/ncw/rclone/backend/local"
_ "github.com/ncw/rclone/backend/swift" // for integration tests
"github.com/ncw/rclone/fs/config/obscure"
"github.com/ncw/rclone/fstest"
"github.com/ncw/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
if *fstest.RemoteName == "" {
t.Skip("Skipping as -remote not set")
}
fstests.Run(t, &fstests.Opt{
RemoteName: *fstest.RemoteName,
NilObject: (*crypt.Object)(nil),
})
}
// TestStandard runs integration tests against the remote
func TestStandard(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skipping as -remote set")
}
tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
name := "TestCrypt"
fstests.Run(t, &fstests.Opt{
@@ -30,6 +47,9 @@ func TestStandard(t *testing.T) {
// TestOff runs integration tests against the remote
func TestOff(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skipping as -remote set")
}
tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-off")
name := "TestCrypt2"
fstests.Run(t, &fstests.Opt{
@@ -46,6 +66,9 @@ func TestOff(t *testing.T) {
// TestObfuscate runs integration tests against the remote
func TestObfuscate(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skipping as -remote set")
}
tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
name := "TestCrypt3"
fstests.Run(t, &fstests.Opt{
+983 -450
View File
File diff suppressed because it is too large. Load diff
+198 -65
View File
@@ -1,62 +1,55 @@
package drive
import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"mime"
"path/filepath"
"strings"
"testing"
"google.golang.org/api/drive/v3"
_ "github.com/ncw/rclone/backend/local"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/operations"
"github.com/ncw/rclone/fstest/fstests"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/api/drive/v3"
)
const exampleExportFormats = `{
"application/vnd.google-apps.document": [
"application/rtf",
"application/vnd.oasis.opendocument.text",
"text/html",
"application/pdf",
"application/epub+zip",
"application/zip",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"text/plain"
],
"application/vnd.google-apps.spreadsheet": [
"application/x-vnd.oasis.opendocument.spreadsheet",
"text/tab-separated-values",
"application/pdf",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"text/csv",
"application/zip",
"application/vnd.oasis.opendocument.spreadsheet"
],
"application/vnd.google-apps.jam": [
"application/pdf"
],
"application/vnd.google-apps.script": [
"application/vnd.google-apps.script+json"
],
"application/vnd.google-apps.presentation": [
"application/vnd.oasis.opendocument.presentation",
"application/pdf",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain"
],
"application/vnd.google-apps.form": [
"application/zip"
],
"application/vnd.google-apps.drawing": [
"image/svg+xml",
"image/png",
"application/pdf",
"image/jpeg"
]
}`
/*
var additionalMimeTypes = map[string]string{
"application/vnd.ms-excel.sheet.macroenabled.12": ".xlsm",
"application/vnd.ms-excel.template.macroenabled.12": ".xltm",
"application/vnd.ms-powerpoint.presentation.macroenabled.12": ".pptm",
"application/vnd.ms-powerpoint.slideshow.macroenabled.12": ".ppsm",
"application/vnd.ms-powerpoint.template.macroenabled.12": ".potm",
"application/vnd.ms-powerpoint": ".ppt",
"application/vnd.ms-word.document.macroenabled.12": ".docm",
"application/vnd.ms-word.template.macroenabled.12": ".dotm",
"application/vnd.openxmlformats-officedocument.presentationml.template": ".potx",
"application/vnd.openxmlformats-officedocument.spreadsheetml.template": ".xltx",
"application/vnd.openxmlformats-officedocument.wordprocessingml.template": ".dotx",
"application/vnd.sun.xml.writer": ".sxw",
"text/richtext": ".rtf",
}
*/
// Load the example export formats into exportFormats for testing
func TestInternalLoadExampleExportFormats(t *testing.T) {
exportFormatsOnce.Do(func() {})
assert.NoError(t, json.Unmarshal([]byte(exampleExportFormats), &_exportFormats))
func TestInternalLoadExampleFormats(t *testing.T) {
fetchFormatsOnce.Do(func() {})
buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
var about struct {
ExportFormats map[string][]string `json:"exportFormats,omitempty"`
ImportFormats map[string][]string `json:"importFormats,omitempty"`
}
require.NoError(t, err)
require.NoError(t, json.Unmarshal(buf, &about))
_exportFormats = fixMimeTypeMap(about.ExportFormats)
_importFormats = fixMimeTypeMap(about.ImportFormats)
}
func TestInternalParseExtensions(t *testing.T) {
@@ -65,27 +58,24 @@ func TestInternalParseExtensions(t *testing.T) {
want []string
wantErr error
}{
{"doc", []string{"doc"}, nil},
{" docx ,XLSX, pptx,svg", []string{"docx", "xlsx", "pptx", "svg"}, nil},
{"docx,svg,Docx", []string{"docx", "svg"}, nil},
{"docx,potato,docx", []string{"docx"}, errors.New(`couldn't find mime type for extension "potato"`)},
{"doc", []string{".doc"}, nil},
{" docx ,XLSX, pptx,svg", []string{".docx", ".xlsx", ".pptx", ".svg"}, nil},
{"docx,svg,Docx", []string{".docx", ".svg"}, nil},
{"docx,potato,docx", []string{".docx"}, errors.New(`couldn't find MIME type for extension ".potato"`)},
} {
f := new(Fs)
gotErr := f.parseExtensions(test.in)
extensions, _, gotErr := parseExtensions(test.in)
if test.wantErr == nil {
assert.NoError(t, gotErr)
} else {
assert.EqualError(t, gotErr, test.wantErr.Error())
}
assert.Equal(t, test.want, f.extensions)
assert.Equal(t, test.want, extensions)
}
// Test it is appending
f := new(Fs)
assert.Nil(t, f.parseExtensions("docx,svg"))
assert.Nil(t, f.parseExtensions("docx,svg,xlsx"))
assert.Equal(t, []string{"docx", "svg", "xlsx"}, f.extensions)
extensions, _, gotErr := parseExtensions("docx,svg", "docx,svg,xlsx")
assert.NoError(t, gotErr)
assert.Equal(t, []string{".docx", ".svg", ".xlsx"}, extensions)
}
func TestInternalFindExportFormat(t *testing.T) {
@@ -99,17 +89,17 @@ func TestInternalFindExportFormat(t *testing.T) {
wantMimeType string
}{
{[]string{}, "", ""},
{[]string{"pdf"}, "pdf", "application/pdf"},
{[]string{"pdf", "rtf", "xls"}, "pdf", "application/pdf"},
{[]string{"xls", "rtf", "pdf"}, "rtf", "application/rtf"},
{[]string{"xls", "csv", "svg"}, "", ""},
{[]string{".pdf"}, ".pdf", "application/pdf"},
{[]string{".pdf", ".rtf", ".xls"}, ".pdf", "application/pdf"},
{[]string{".xls", ".rtf", ".pdf"}, ".rtf", "application/rtf"},
{[]string{".xls", ".csv", ".svg"}, "", ""},
} {
f := new(Fs)
f.extensions = test.extensions
f.exportExtensions = test.extensions
gotExtension, gotFilename, gotMimeType, gotIsDocument := f.findExportFormat(item)
assert.Equal(t, test.wantExtension, gotExtension)
if test.wantExtension != "" {
assert.Equal(t, item.Name+"."+gotExtension, gotFilename)
assert.Equal(t, item.Name+gotExtension, gotFilename)
} else {
assert.Equal(t, "", gotFilename)
}
@@ -117,3 +107,146 @@ func TestInternalFindExportFormat(t *testing.T) {
assert.Equal(t, true, gotIsDocument)
}
}
func TestMimeTypesToExtension(t *testing.T) {
for mimeType, extension := range _mimeTypeToExtension {
extensions, err := mime.ExtensionsByType(mimeType)
assert.NoError(t, err)
assert.Contains(t, extensions, extension)
}
}
func TestExtensionToMimeType(t *testing.T) {
for mimeType, extension := range _mimeTypeToExtension {
gotMimeType := mime.TypeByExtension(extension)
mediatype, _, err := mime.ParseMediaType(gotMimeType)
assert.NoError(t, err)
assert.Equal(t, mimeType, mediatype)
}
}
func TestExtensionsForExportFormats(t *testing.T) {
if _exportFormats == nil {
t.Error("exportFormats == nil")
}
for fromMT, toMTs := range _exportFormats {
for _, toMT := range toMTs {
if !isInternalMimeType(toMT) {
extensions, err := mime.ExtensionsByType(toMT)
assert.NoError(t, err, "invalid MIME type %q", toMT)
assert.NotEmpty(t, extensions, "No extension found for %q (from: %q)", fromMT, toMT)
}
}
}
}
func TestExtensionsForImportFormats(t *testing.T) {
t.Skip()
if _importFormats == nil {
t.Error("_importFormats == nil")
}
for fromMT := range _importFormats {
if !isInternalMimeType(fromMT) {
extensions, err := mime.ExtensionsByType(fromMT)
assert.NoError(t, err, "invalid MIME type %q", fromMT)
assert.NotEmpty(t, extensions, "No extension found for %q", fromMT)
}
}
}
func (f *Fs) InternalTestDocumentImport(t *testing.T) {
oldAllow := f.opt.AllowImportNameChange
f.opt.AllowImportNameChange = true
defer func() {
f.opt.AllowImportNameChange = oldAllow
}()
testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
require.NoError(t, err)
testFilesFs, err := fs.NewFs(testFilesPath)
require.NoError(t, err)
_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
require.NoError(t, err)
err = operations.CopyFile(f, testFilesFs, "example2.doc", "example2.doc")
require.NoError(t, err)
}
func (f *Fs) InternalTestDocumentUpdate(t *testing.T) {
testFilesPath, err := filepath.Abs(filepath.FromSlash("test/files"))
require.NoError(t, err)
testFilesFs, err := fs.NewFs(testFilesPath)
require.NoError(t, err)
_, f.importMimeTypes, err = parseExtensions("odt,ods,doc")
require.NoError(t, err)
err = operations.CopyFile(f, testFilesFs, "example2.xlsx", "example1.ods")
require.NoError(t, err)
}
func (f *Fs) InternalTestDocumentExport(t *testing.T) {
var buf bytes.Buffer
var err error
f.exportExtensions, _, err = parseExtensions("txt")
require.NoError(t, err)
obj, err := f.NewObject("example2.txt")
require.NoError(t, err)
rc, err := obj.Open()
require.NoError(t, err)
defer func() { require.NoError(t, rc.Close()) }()
_, err = io.Copy(&buf, rc)
require.NoError(t, err)
text := buf.String()
for _, excerpt := range []string{
"Lorem ipsum dolor sit amet, consectetur",
"porta at ultrices in, consectetur at augue.",
} {
require.Contains(t, text, excerpt)
}
}
func (f *Fs) InternalTestDocumentLink(t *testing.T) {
var buf bytes.Buffer
var err error
f.exportExtensions, _, err = parseExtensions("link.html")
require.NoError(t, err)
obj, err := f.NewObject("example2.link.html")
require.NoError(t, err)
rc, err := obj.Open()
require.NoError(t, err)
defer func() { require.NoError(t, rc.Close()) }()
_, err = io.Copy(&buf, rc)
require.NoError(t, err)
text := buf.String()
require.True(t, strings.HasPrefix(text, "<html>"))
require.True(t, strings.HasSuffix(text, "</html>\n"))
for _, excerpt := range []string{
`<meta http-equiv="refresh"`,
`Loading <a href="`,
} {
require.Contains(t, text, excerpt)
}
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("DocumentImport", f.InternalTestDocumentImport)
t.Run("DocumentUpdate", f.InternalTestDocumentUpdate)
t.Run("DocumentExport", f.InternalTestDocumentExport)
t.Run("DocumentLink", f.InternalTestDocumentLink)
}
var _ fstests.InternalTester = (*Fs)(nil)
+20 -3
View File
@@ -1,10 +1,10 @@
// Test Drive filesystem interface
package drive_test
package drive
import (
"testing"
"github.com/ncw/rclone/backend/drive"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
@@ -12,6 +12,23 @@ import (
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestDrive:",
NilObject: (*drive.Object)(nil),
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: minChunkSize,
CeilChunkSize: fstests.NextPowerOfTwo,
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadCutoff(cs)
}
var (
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
_ fstests.SetUploadCutoffer = (*Fs)(nil)
)
+178
View File
@@ -0,0 +1,178 @@
{
"importFormats": {
"text/tab-separated-values": [
"application/vnd.google-apps.spreadsheet"
],
"application/x-vnd.oasis.opendocument.presentation": [
"application/vnd.google-apps.presentation"
],
"image/jpeg": [
"application/vnd.google-apps.document"
],
"image/bmp": [
"application/vnd.google-apps.document"
],
"image/gif": [
"application/vnd.google-apps.document"
],
"application/vnd.ms-excel.sheet.macroenabled.12": [
"application/vnd.google-apps.spreadsheet"
],
"application/vnd.openxmlformats-officedocument.wordprocessingml.template": [
"application/vnd.google-apps.document"
],
"application/vnd.ms-powerpoint.presentation.macroenabled.12": [
"application/vnd.google-apps.presentation"
],
"application/vnd.ms-word.template.macroenabled.12": [
"application/vnd.google-apps.document"
],
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
"application/vnd.google-apps.document"
],
"image/pjpeg": [
"application/vnd.google-apps.document"
],
"application/vnd.google-apps.script+text/plain": [
"application/vnd.google-apps.script"
],
"application/vnd.ms-excel": [
"application/vnd.google-apps.spreadsheet"
],
"application/vnd.sun.xml.writer": [
"application/vnd.google-apps.document"
],
"application/vnd.ms-word.document.macroenabled.12": [
"application/vnd.google-apps.document"
],
"application/vnd.ms-powerpoint.slideshow.macroenabled.12": [
"application/vnd.google-apps.presentation"
],
"text/rtf": [
"application/vnd.google-apps.document"
],
"text/plain": [
"application/vnd.google-apps.document"
],
"application/vnd.oasis.opendocument.spreadsheet": [
"application/vnd.google-apps.spreadsheet"
],
"application/x-vnd.oasis.opendocument.spreadsheet": [
"application/vnd.google-apps.spreadsheet"
],
"image/png": [
"application/vnd.google-apps.document"
],
"application/x-vnd.oasis.opendocument.text": [
"application/vnd.google-apps.document"
],
"application/msword": [
"application/vnd.google-apps.document"
],
"application/pdf": [
"application/vnd.google-apps.document"
],
"application/json": [
"application/vnd.google-apps.script"
],
"application/x-msmetafile": [
"application/vnd.google-apps.drawing"
],
"application/vnd.openxmlformats-officedocument.spreadsheetml.template": [
"application/vnd.google-apps.spreadsheet"
],
"application/vnd.ms-powerpoint": [
"application/vnd.google-apps.presentation"
],
"application/vnd.ms-excel.template.macroenabled.12": [
"application/vnd.google-apps.spreadsheet"
],
"image/x-bmp": [
"application/vnd.google-apps.document"
],
"application/rtf": [
"application/vnd.google-apps.document"
],
"application/vnd.openxmlformats-officedocument.presentationml.template": [
"application/vnd.google-apps.presentation"
],
"image/x-png": [
"application/vnd.google-apps.document"
],
"text/html": [
"application/vnd.google-apps.document"
],
"application/vnd.oasis.opendocument.text": [
"application/vnd.google-apps.document"
],
"application/vnd.openxmlformats-officedocument.presentationml.presentation": [
"application/vnd.google-apps.presentation"
],
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [
"application/vnd.google-apps.spreadsheet"
],
"application/vnd.google-apps.script+json": [
"application/vnd.google-apps.script"
],
"application/vnd.openxmlformats-officedocument.presentationml.slideshow": [
"application/vnd.google-apps.presentation"
],
"application/vnd.ms-powerpoint.template.macroenabled.12": [
"application/vnd.google-apps.presentation"
],
"text/csv": [
"application/vnd.google-apps.spreadsheet"
],
"application/vnd.oasis.opendocument.presentation": [
"application/vnd.google-apps.presentation"
],
"image/jpg": [
"application/vnd.google-apps.document"
],
"text/richtext": [
"application/vnd.google-apps.document"
]
},
"exportFormats": {
"application/vnd.google-apps.document": [
"application/rtf",
"application/vnd.oasis.opendocument.text",
"text/html",
"application/pdf",
"application/epub+zip",
"application/zip",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"text/plain"
],
"application/vnd.google-apps.spreadsheet": [
"application/x-vnd.oasis.opendocument.spreadsheet",
"text/tab-separated-values",
"application/pdf",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"text/csv",
"application/zip",
"application/vnd.oasis.opendocument.spreadsheet"
],
"application/vnd.google-apps.jam": [
"application/pdf"
],
"application/vnd.google-apps.script": [
"application/vnd.google-apps.script+json"
],
"application/vnd.google-apps.presentation": [
"application/vnd.oasis.opendocument.presentation",
"application/pdf",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain"
],
"application/vnd.google-apps.form": [
"application/zip"
],
"application/vnd.google-apps.drawing": [
"image/svg+xml",
"image/png",
"application/pdf",
"image/jpeg"
]
}
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
+6 -5
View File
@@ -50,11 +50,12 @@ type resumableUpload struct {
}
// Upload the io.Reader in of size bytes with contentType and info
func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string, info *drive.File, remote string) (*drive.File, error) {
params := make(url.Values)
params.Set("alt", "json")
params.Set("uploadType", "resumable")
params.Set("fields", partialFields)
func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
params := url.Values{
"alt": {"json"},
"uploadType": {"resumable"},
"fields": {partialFields},
}
if f.isTeamDrive {
params.Set("supportsTeamDrives", "true")
}
+33 -6
View File
@@ -79,8 +79,8 @@ const (
// Choose 48MB which is 91% of Maximum speed. rclone by
// default does 4 transfers so this should use 4*48MB = 192MB
// by default.
defaultChunkSize = 48 * 1024 * 1024
maxChunkSize = 150 * 1024 * 1024
defaultChunkSize = 48 * fs.MebiByte
maxChunkSize = 150 * fs.MebiByte
)
var (
@@ -120,8 +120,15 @@ func init() {
Name: config.ConfigClientSecret,
Help: "Dropbox App Client Secret\nLeave blank normally.",
}, {
Name: "chunk_size",
Help: fmt.Sprintf("Upload chunk size. Max %v.", fs.SizeSuffix(maxChunkSize)),
Name: "chunk_size",
Help: fmt.Sprintf(`Upload chunk size. (< %v).
Any files larger than this will be uploaded in chunks of this size.
Note that chunks are buffered in memory (one at a time) so rclone can
deal with retries. Setting this larger will increase the speed
slightly (at most 10%% for 128MB in tests) at the cost of using more
memory. It can be set smaller if you are tight on memory.`, fs.SizeSuffix(maxChunkSize)),
Default: fs.SizeSuffix(defaultChunkSize),
Advanced: true,
}},
@@ -195,6 +202,25 @@ func shouldRetry(err error) (bool, error) {
return fserrors.ShouldRetry(err), err
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.Byte
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
if cs > maxChunkSize {
return errors.Errorf("%s is greater than %s", cs, maxChunkSize)
}
return nil
}
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
}
return
}
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -203,8 +229,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
if opt.ChunkSize > maxChunkSize {
return nil, errors.Errorf("chunk size too big, must be < %v", maxChunkSize)
err = checkUploadChunkSize(opt.ChunkSize)
if err != nil {
return nil, errors.Wrap(err, "dropbox: chunk size")
}
// Convert the old token if it exists. The old token was just
+12 -3
View File
@@ -1,10 +1,10 @@
// Test Dropbox filesystem interface
package dropbox_test
package dropbox
import (
"testing"
"github.com/ncw/rclone/backend/dropbox"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
@@ -12,6 +12,15 @@ import (
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestDropbox:",
NilObject: (*dropbox.Object)(nil),
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
MaxChunkSize: maxChunkSize,
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
var _ fstests.SetUploadChunkSizer = (*Fs)(nil)
+6 -1
View File
@@ -704,6 +704,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
path := path.Join(o.fs.root, o.remote)
// remove the file if upload failed
remove := func() {
// Give the FTP server a chance to get its internal state in order after the error.
// The error may have been local in which case we closed the connection. The server
// may still be dealing with it for a moment. A sleep isn't ideal but I haven't been
// able to think of a better method to find out if the server has finished - ncw
time.Sleep(1 * time.Second)
removeErr := o.Remove()
if removeErr != nil {
fs.Debugf(o, "Failed to remove: %v", removeErr)
@@ -717,7 +722,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
}
err = c.Stor(path, in)
if err != nil {
_ = c.Quit()
_ = c.Quit() // toss this connection to avoid sync errors
remove()
return errors.Wrap(err, "update stor")
}
+15 -9
View File
@@ -80,7 +80,7 @@ func init() {
Advanced: true,
}, {
Name: "unlink",
Help: "Remove existing public link to file/folder with link command rather than creating.",
Help: "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
Default: false,
Advanced: true,
}},
@@ -199,7 +199,7 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) {
func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
opts := rest.Opts{
Method: "GET",
Path: rest.URLPathEscape(f.user),
Path: urlPathEscape(f.user),
}
var resp *http.Response
@@ -220,7 +220,7 @@ func (f *Fs) setEndpointURL(mountpoint string) (err error) {
if err != nil {
return errors.Wrap(err, "failed to get endpoint url")
}
f.endpointURL = rest.URLPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
f.endpointURL = urlPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
return nil
}
@@ -241,6 +241,11 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
// Jottacloud want's '+' to be URL encoded even though the RFC states it's not reserved
func urlPathEscape(in string) string {
return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
}
// filePathRaw returns an unescaped file path (f.root, file)
func (f *Fs) filePathRaw(file string) string {
return path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file)))
@@ -248,7 +253,7 @@ func (f *Fs) filePathRaw(file string) string {
// filePath returns a escaped file path (f.root, file)
func (f *Fs) filePath(file string) string {
return rest.URLPathEscape(f.filePathRaw(file))
return urlPathEscape(f.filePathRaw(file))
}
// filePath returns a escaped file path (f.root, remote)
@@ -459,12 +464,12 @@ func (f *Fs) listFileDir(remoteStartPath string, startFolder *api.JottaFolder, f
if folder.Deleted {
return nil
}
folderPath := path.Join(folder.Path, folder.Name)
remoteDirLength := len(folderPath) - pathPrefixLength
folderPath := restoreReservedChars(path.Join(folder.Path, folder.Name))
folderPathLength := len(folderPath)
var remoteDir string
if remoteDirLength > 0 {
remoteDir = restoreReservedChars(folderPath[pathPrefixLength+1:])
if remoteDirLength > startPathLength {
if folderPathLength > pathPrefixLength {
remoteDir = folderPath[pathPrefixLength+1:]
if folderPathLength > startPathLength {
d := fs.NewDir(remoteDir, time.Time(folder.ModifiedAt))
err := fn(d)
if err != nil {
@@ -1080,6 +1085,7 @@ func (o *Object) Remove() error {
Method: "POST",
Path: o.filePath(),
Parameters: url.Values{},
NoResponse: true,
}
if o.fs.opt.HardDelete {
+3 -28
View File
@@ -4,50 +4,25 @@ import (
"crypto/md5"
"fmt"
"io"
"io/ioutil"
"testing"
"github.com/ncw/rclone/lib/readers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A test reader to return a test pattern of size
type testReader struct {
size int64
c byte
}
// Reader is the interface that wraps the basic Read method.
func (r *testReader) Read(p []byte) (n int, err error) {
for i := range p {
if r.size <= 0 {
return n, io.EOF
}
p[i] = r.c
r.c = (r.c + 1) % 253
r.size--
n++
}
return
}
func TestReadMD5(t *testing.T) {
// smoke test the reader
b, err := ioutil.ReadAll(&testReader{size: 10})
require.NoError(t, err)
assert.Equal(t, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, b)
// Check readMD5 for different size and threshold
for _, size := range []int64{0, 1024, 10 * 1024, 100 * 1024} {
t.Run(fmt.Sprintf("%d", size), func(t *testing.T) {
hasher := md5.New()
n, err := io.Copy(hasher, &testReader{size: size})
n, err := io.Copy(hasher, readers.NewPatternReader(size))
require.NoError(t, err)
assert.Equal(t, n, size)
wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
for _, threshold := range []int64{512, 1024, 10 * 1024, 20 * 1024} {
t.Run(fmt.Sprintf("%d", threshold), func(t *testing.T) {
in := &testReader{size: size}
in := readers.NewPatternReader(size)
gotMD5, out, cleanup, err := readMD5(in, size, threshold)
defer cleanup()
require.NoError(t, err)
-7
View File
@@ -27,21 +27,14 @@ import (
var (
charMap = map[rune]rune{
'\\': '\', // FULLWIDTH REVERSE SOLIDUS
'+': '+', // FULLWIDTH PLUS SIGN
'*': '*', // FULLWIDTH ASTERISK
'<': '<', // FULLWIDTH LESS-THAN SIGN
'>': '>', // FULLWIDTH GREATER-THAN SIGN
'?': '?', // FULLWIDTH QUESTION MARK
'!': '!', // FULLWIDTH EXCLAMATION MARK
'&': '&', // FULLWIDTH AMPERSAND
':': ':', // FULLWIDTH COLON
';': ';', // FULLWIDTH SEMICOLON
'|': '|', // FULLWIDTH VERTICAL LINE
'#': '#', // FULLWIDTH NUMBER SIGN
'%': '%', // FULLWIDTH PERCENT SIGN
'"': '"', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
'\'': ''', // FULLWIDTH APOSTROPHE
'~': '~', // FULLWIDTH TILDE
' ': '␠', // SYMBOL FOR SPACE
}
invCharMap map[rune]rune
+2 -2
View File
@@ -9,8 +9,8 @@ func TestReplace(t *testing.T) {
}{
{"", ""},
{"abc 123", "abc 123"},
{`\+*<>?!&:;|#%"'~`, `\+*<>?!&:;|#%"'~`},
{`\+*<>?!&:;|#%"'~\+*<>?!&:;|#%"'~`, `\+*<>?!&:;|#%"'~\+*<>?!&:;|#%"'~`},
{`\*<>?:;|"`, `\*<>?:;|"`},
{`\*<>?:;|"\*<>?:;|"`, `\*<>?:;|"\*<>?:;|"`},
{" leading space", "␠leading space"},
{"trailing space ", "trailing space␠"},
{" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},
+26 -6
View File
@@ -49,19 +49,33 @@ func init() {
ShortOpt: "L",
Advanced: true,
}, {
Name: "skip_links",
Help: "Don't warn about skipped symlinks.",
Name: "skip_links",
Help: `Don't warn about skipped symlinks.
This flag disables warning messages on skipped symlinks or junction
points, as you explicitly acknowledge that they should be skipped.`,
Default: false,
NoPrefix: true,
Advanced: true,
}, {
Name: "no_unicode_normalization",
Help: "Don't apply unicode normalization to paths and filenames",
Name: "no_unicode_normalization",
Help: `Don't apply unicode normalization to paths and filenames (Deprecated)
This flag is deprecated now. Rclone no longer normalizes unicode file
names, but it compares them with unicode normalization in the sync
routine instead.`,
Default: false,
Advanced: true,
}, {
Name: "no_check_updated",
Help: "Don't check to see if the files change during upload",
Name: "no_check_updated",
Help: `Don't check to see if the files change during upload
Normally rclone checks the size and modification time of files as they
are being uploaded and aborts with a message which starts "can't copy
- source file is being updated" if the file changes during upload.
However on some file systems this modification time check may fail (eg
[Glusterfs #2206](https://github.com/ncw/rclone/issues/2206)) so this
check can be disabled with this flag.`,
Default: false,
Advanced: true,
}, {
@@ -817,6 +831,12 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
return err
}
// Pre-allocate the file for performance reasons
err = preAllocate(src.Size(), out)
if err != nil {
fs.Debugf(o, "Failed to pre-allocate: %v", err)
}
// Calculate the hash of the object we are reading as we go along
hash, err := hash.NewMultiHasherTypes(hashes)
if err != nil {
+10
View File
@@ -0,0 +1,10 @@
//+build !windows,!linux
package local
import "os"
// preAllocate the file for performance reasons
func preAllocate(size int64, out *os.File) error {
return nil
}
+22
View File
@@ -0,0 +1,22 @@
//+build linux
package local
import (
"os"
"golang.org/x/sys/unix"
)
// preAllocate the file for performance reasons
func preAllocate(size int64, out *os.File) error {
if size <= 0 {
return nil
}
err := unix.Fallocate(int(out.Fd()), unix.FALLOC_FL_KEEP_SIZE, 0, size)
// FIXME could be doing something here
// if err == unix.ENOSPC {
// log.Printf("No space")
// }
return err
}
+79
View File
@@ -0,0 +1,79 @@
//+build windows
package local
import (
"os"
"syscall"
"unsafe"
"github.com/pkg/errors"
"golang.org/x/sys/windows"
)
var (
ntdll = windows.NewLazySystemDLL("ntdll.dll")
ntQueryVolumeInformationFile = ntdll.NewProc("NtQueryVolumeInformationFile")
ntSetInformationFile = ntdll.NewProc("NtSetInformationFile")
)
type fileAllocationInformation struct {
AllocationSize uint64
}
type fileFsSizeInformation struct {
TotalAllocationUnits uint64
AvailableAllocationUnits uint64
SectorsPerAllocationUnit uint32
BytesPerSector uint32
}
type ioStatusBlock struct {
Status, Information uintptr
}
// preAllocate the file for performance reasons
func preAllocate(size int64, out *os.File) error {
if size <= 0 {
return nil
}
var (
iosb ioStatusBlock
fsSizeInfo fileFsSizeInformation
allocInfo fileAllocationInformation
)
// Query info about the block sizes on the file system
_, _, e1 := ntQueryVolumeInformationFile.Call(
uintptr(out.Fd()),
uintptr(unsafe.Pointer(&iosb)),
uintptr(unsafe.Pointer(&fsSizeInfo)),
uintptr(unsafe.Sizeof(fsSizeInfo)),
uintptr(3), // FileFsSizeInformation
)
if e1 != nil && e1 != syscall.Errno(0) {
return errors.Wrap(e1, "preAllocate NtQueryVolumeInformationFile failed")
}
// Calculate the allocation size
clusterSize := uint64(fsSizeInfo.BytesPerSector) * uint64(fsSizeInfo.SectorsPerAllocationUnit)
if clusterSize <= 0 {
return errors.Errorf("preAllocate clusterSize %d <= 0", clusterSize)
}
allocInfo.AllocationSize = (1 + uint64(size-1)/clusterSize) * clusterSize
// Ask for the allocation
_, _, e1 = ntSetInformationFile.Call(
uintptr(out.Fd()),
uintptr(unsafe.Pointer(&iosb)),
uintptr(unsafe.Pointer(&allocInfo)),
uintptr(unsafe.Sizeof(allocInfo)),
uintptr(19), // FileAllocationInformation
)
if e1 != nil && e1 != syscall.Errno(0) {
return errors.Wrap(e1, "preAllocate NtSetInformationFile failed")
}
return nil
}
+11 -4
View File
@@ -63,13 +63,20 @@ func init() {
Required: true,
IsPassword: true,
}, {
Name: "debug",
Help: "Output more debug from Mega.",
Name: "debug",
Help: `Output more debug from Mega.
If this flag is set (along with -vv) it will print further debugging
information from the mega backend.`,
Default: false,
Advanced: true,
}, {
Name: "hard_delete",
Help: "Delete files permanently rather than putting them into the trash.",
Name: "hard_delete",
Help: `Delete files permanently rather than putting them into the trash.
Normally the mega backend will put all deletions into the trash rather
than permanently deleting them. If you specify this then rclone will
permanently delete objects instead.`,
Default: false,
Advanced: true,
}},
+52
View File
@@ -9,6 +9,9 @@ import (
const (
timeFormat = `"` + time.RFC3339 + `"`
// PackageTypeOneNote is the package type value for OneNote files
PackageTypeOneNote = "oneNote"
)
// Error is returned from one drive when things go wrong
@@ -107,6 +110,7 @@ type RemoteItemFacet struct {
LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only.
Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only.
File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only.
Package *PackageFacet `json:"package"` // If present, indicates that this item is a package instead of a folder or file. Packages are treated like files in some contexts and folders in others. Read-only.
FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write.
ParentReference *ItemReference `json:"parentReference"` // Parent information, if the item has a parent. Read-write.
Size int64 `json:"size"` // Size of the item in bytes. Read-only.
@@ -147,6 +151,13 @@ type FileSystemInfoFacet struct {
type DeletedFacet struct {
}
// PackageFacet indicates that a DriveItem is the top level item
// in a "package" or a collection of items that should be treated as a collection instead of individual items.
// `oneNote` is the only currently defined value.
type PackageFacet struct {
Type string `json:"type"`
}
// Item represents metadata for an item in OneDrive
type Item struct {
ID string `json:"id"` // The unique identifier of the item within the Drive. Read-only.
@@ -170,6 +181,7 @@ type Item struct {
// Audio *AudioFacet `json:"audio"` // Audio metadata, if the item is an audio file. Read-only.
// Video *VideoFacet `json:"video"` // Video metadata, if the item is a video. Read-only.
// Location *LocationFacet `json:"location"` // Location metadata, if the item has location data. Read-only.
Package *PackageFacet `json:"package"` // If present, indicates that this item is a package instead of a folder or file. Packages are treated like files in some contexts and folders in others. Read-only.
Deleted *DeletedFacet `json:"deleted"` // Information about the deleted state of the item. Read-only.
}
@@ -238,6 +250,28 @@ type MoveItemRequest struct {
FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"` // File system information on client. Read-write.
}
//CreateShareLinkRequest is the request to create a sharing link
//Always Type:view and Scope:anonymous for public sharing
type CreateShareLinkRequest struct {
Type string `json:"type"` //Link type in View, Edit or Embed
Scope string `json:"scope,omitempty"` //Optional. Scope in anonymousi, organization
}
//CreateShareLinkResponse is the response from CreateShareLinkRequest
type CreateShareLinkResponse struct {
ID string `json:"id"`
Roles []string `json:"roles"`
Link struct {
Type string `json:"type"`
Scope string `json:"scope"`
WebURL string `json:"webUrl"`
Application struct {
ID string `json:"id"`
DisplayName string `json:"displayName"`
} `json:"application"`
} `json:"link"`
}
// AsyncOperationStatus provides information on the status of a asynchronous job progress.
//
// The following API calls return AsyncOperationStatus resources:
@@ -281,6 +315,24 @@ func (i *Item) GetFolder() *FolderFacet {
return i.Folder
}
// GetPackage returns a normalized Package of the item
func (i *Item) GetPackage() *PackageFacet {
if i.IsRemote() && i.RemoteItem.Package != nil {
return i.RemoteItem.Package
}
return i.Package
}
// GetPackageType returns the package type of the item if available,
// otherwise ""
func (i *Item) GetPackageType() string {
pack := i.GetPackage()
if pack == nil {
return ""
}
return pack.Type
}
// GetFile returns a normalized File of the item
func (i *Item) GetFile() *FileFacet {
if i.IsRemote() && i.RemoteItem.File != nil {
+130 -24
View File
@@ -43,6 +43,8 @@ const (
driveTypePersonal = "personal"
driveTypeBusiness = "business"
driveTypeSharepoint = "documentLibrary"
defaultChunkSize = 10 * fs.MebiByte
chunkSizeMultiple = 320 * fs.KibiByte
)
// Globals
@@ -212,9 +214,12 @@ func init() {
Name: config.ConfigClientSecret,
Help: "Microsoft App Client Secret\nLeave blank normally.",
}, {
Name: "chunk_size",
Help: "Chunk size to upload files with - must be multiple of 320k.",
Default: fs.SizeSuffix(10 * 1024 * 1024),
Name: "chunk_size",
Help: `Chunk size to upload files with - must be multiple of 320k.
Above this size files will be chunked - must be multiple of 320k. Note
that the chunks will be buffered into memory.`,
Default: defaultChunkSize,
Advanced: true,
}, {
Name: "drive_id",
@@ -226,15 +231,27 @@ func init() {
Help: "The type of the drive ( personal | business | documentLibrary )",
Default: "",
Advanced: true,
}, {
Name: "expose_onenote_files",
Help: `Set to make OneNote files show up in directory listings.
By default rclone will hide OneNote files in directory listings because
operations like "Open" and "Update" won't work on them. But this
behaviour may also prevent you from deleting them. If you want to
delete OneNote files or otherwise want them to show up in directory
listing, set this option.`,
Default: false,
Advanced: true,
}},
})
}
// Options defines the configuration for this backend
type Options struct {
ChunkSize fs.SizeSuffix `config:"chunk_size"`
DriveID string `config:"drive_id"`
DriveType string `config:"drive_type"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
DriveID string `config:"drive_id"`
DriveType string `config:"drive_type"`
ExposeOneNoteFiles bool `config:"expose_onenote_files"`
}
// Fs represents a remote one drive
@@ -255,15 +272,16 @@ type Fs struct {
//
// Will definitely have info but maybe not meta
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
hasMetaData bool // whether info below has been set
size int64 // size of the object
modTime time.Time // modification time of the object
id string // ID of the object
sha1 string // SHA-1 of the object content
quickxorhash string // QuickXorHash of the object content
mimeType string // Content-Type of object from server (may not be as uploaded)
fs *Fs // what this object is part of
remote string // The remote path
hasMetaData bool // whether info below has been set
isOneNoteFile bool // Whether the object is a OneNote file
size int64 // size of the object
modTime time.Time // modification time of the object
id string // ID of the object
sha1 string // SHA-1 of the object content
quickxorhash string // QuickXorHash of the object content
mimeType string // Content-Type of object from server (may not be as uploaded)
}
// ------------------------------------------------------------
@@ -352,6 +370,25 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.Byte
if cs%chunkSizeMultiple != 0 {
return errors.Errorf("%s is not a multiple of %s", cs, chunkSizeMultiple)
}
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
return nil
}
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
}
return
}
// NewFs constructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -360,8 +397,10 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
if opt.ChunkSize%(320*1024) != 0 {
return nil, errors.Errorf("chunk size %d is not a multiple of 320k", opt.ChunkSize)
err = checkUploadChunkSize(opt.ChunkSize)
if err != nil {
return nil, errors.Wrap(err, "onedrive: chunk size")
}
if opt.DriveID == "" || opt.DriveType == "" {
@@ -409,16 +448,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
newF := *f
newF.dirCache = dircache.New(newRoot, rootInfo.ID, &newF)
newF.root = newRoot
tempF := *f
tempF.dirCache = dircache.New(newRoot, rootInfo.ID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = newF.dirCache.FindRoot(false)
err = tempF.dirCache.FindRoot(false)
if err != nil {
// No root so return old f
return f, nil
}
_, err := newF.newObjectWithInfo(remote, nil)
_, err := tempF.newObjectWithInfo(remote, nil)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
@@ -426,8 +465,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
return nil, err
}
// XXX: update the old f here instead of returning tempF, since
// `features` were already filled with functions having *f as a receiver.
// See https://github.com/ncw/rclone/issues/2182
f.dirCache = tempF.dirCache
f.root = tempF.root
// return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile
return f, fs.ErrorIsFile
}
return f, nil
}
@@ -488,6 +532,9 @@ func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err er
}
return "", false, err
}
if info.GetPackageType() == api.PackageTypeOneNote {
return "", false, errors.New("found OneNote file when looking for folder")
}
if info.GetFolder() == nil {
return "", false, errors.New("found file when looking for folder")
}
@@ -596,6 +643,11 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
}
var iErr error
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool {
if !f.opt.ExposeOneNoteFiles && info.GetPackageType() == api.PackageTypeOneNote {
fs.Debugf(info.Name, "OneNote file not shown in directory listing")
return false
}
remote := path.Join(dir, info.GetName())
folder := info.GetFolder()
if folder != nil {
@@ -1064,6 +1116,32 @@ func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.QuickXorHash)
}
// PublicLink returns a link for downloading without accout.
func (f *Fs) PublicLink(remote string) (link string, err error) {
info, _, err := f.readMetaDataForPath(f.srvPath(remote))
if err != nil {
return "", err
}
opts := newOptsCall(info.ID, "POST", "/createLink")
share := api.CreateShareLinkRequest{
Type: "view",
Scope: "anonymous",
}
var resp *http.Response
var result api.CreateShareLinkResponse
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, &share, &result)
return shouldRetry(resp, err)
})
if err != nil {
fmt.Println(err)
return "", err
}
return result.Link.WebURL, nil
}
// ------------------------------------------------------------
// Fs returns the parent Fs
@@ -1084,9 +1162,14 @@ func (o *Object) Remote() string {
return o.remote
}
// srvPath returns a path for use in server given a remote
func (f *Fs) srvPath(remote string) string {
return replaceReservedChars(f.rootSlash() + remote)
}
// srvPath returns a path for use in server
func (o *Object) srvPath() string {
return replaceReservedChars(o.fs.rootSlash() + o.remote)
return o.fs.srvPath(o.remote)
}
// Hash returns the SHA-1 of an object returning a lowercase hex string
@@ -1121,6 +1204,8 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
o.hasMetaData = true
o.size = info.GetSize()
o.isOneNoteFile = info.GetPackageType() == api.PackageTypeOneNote
// Docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes
//
// We use SHA1 for onedrive personal and QuickXorHash for onedrive for business
@@ -1232,6 +1317,10 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
if o.id == "" {
return nil, errors.New("can't download - no id")
}
if o.isOneNoteFile {
return nil, errors.New("can't open a OneNote file")
}
fs.FixRangeOption(options, o.size)
var resp *http.Response
opts := newOptsCall(o.id, "GET", "/content")
@@ -1275,6 +1364,12 @@ func (o *Object) createUploadSession(modTime time.Time) (response *api.CreateUpl
var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, &createRequest, &response)
if apiErr, ok := err.(*api.Error); ok {
if apiErr.ErrorInfo.Code == "nameAlreadyExists" {
// Make the error more user-friendly
err = errors.New(err.Error() + " (is it a OneNote file?)")
}
}
return shouldRetry(resp, err)
})
return response, err
@@ -1407,6 +1502,12 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &info)
if apiErr, ok := err.(*api.Error); ok {
if apiErr.ErrorInfo.Code == "nameAlreadyExists" {
// Make the error more user-friendly
err = errors.New(err.Error() + " (is it a OneNote file?)")
}
}
return shouldRetry(resp, err)
})
if err != nil {
@@ -1425,6 +1526,10 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
//
// The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
if o.hasMetaData && o.isOneNoteFile {
return errors.New("can't upload content to a OneNote file")
}
o.fs.tokenRenewer.Start()
defer o.fs.tokenRenewer.Stop()
@@ -1494,6 +1599,7 @@ var (
_ fs.DirMover = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.PublicLinker = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
_ fs.IDer = &Object{}
+12 -3
View File
@@ -1,10 +1,10 @@
// Test OneDrive filesystem interface
package onedrive_test
package onedrive
import (
"testing"
"github.com/ncw/rclone/backend/onedrive"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
@@ -12,6 +12,15 @@ import (
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestOneDrive:",
NilObject: (*onedrive.Object)(nil),
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
CeilChunkSize: fstests.NextMultipleOf(chunkSizeMultiple),
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
var _ fstests.SetUploadChunkSizer = (*Fs)(nil)
+22 -9
View File
@@ -6,6 +6,7 @@ import (
"io"
"mime/multipart"
"net/http"
"net/url"
"path"
"strconv"
"strings"
@@ -20,6 +21,7 @@ import (
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/lib/dircache"
"github.com/ncw/rclone/lib/pacer"
"github.com/ncw/rclone/lib/readers"
"github.com/ncw/rclone/lib/rest"
"github.com/pkg/errors"
)
@@ -177,17 +179,17 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
newF := *f
newF.dirCache = dircache.New(newRoot, "0", &newF)
newF.root = newRoot
tempF := *f
tempF.dirCache = dircache.New(newRoot, "0", &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = newF.dirCache.FindRoot(false)
err = tempF.dirCache.FindRoot(false)
if err != nil {
// No root so return old f
return f, nil
}
_, err := newF.newObjectWithInfo(remote, nil)
_, err := tempF.newObjectWithInfo(remote, nil)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
@@ -195,8 +197,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
return nil, err
}
// XXX: update the old f here instead of returning tempF, since
// `features` were already filled with functions having *f as a receiver.
// See https://github.com/ncw/rclone/issues/2182
f.dirCache = tempF.dirCache
f.root = tempF.root
// return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile
return f, fs.ErrorIsFile
}
return f, nil
}
@@ -925,8 +932,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
// resp.Body.Close()
// fs.Debugf(nil, "PostOpen: %#v", openResponse)
// 1 MB chunks size
// 10 MB chunks size
chunkSize := int64(1024 * 1024 * 10)
buf := make([]byte, int(chunkSize))
chunkOffset := int64(0)
remainingBytes := size
chunkCounter := 0
@@ -939,14 +947,19 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
remainingBytes -= currentChunkSize
fs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes)
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, currentChunkSize)
err = o.fs.pacer.Call(func() (bool, error) {
// seek to the start in case this is a retry
if _, err = chunk.Seek(0, io.SeekStart); err != nil {
return false, err
}
var formBody bytes.Buffer
w := multipart.NewWriter(&formBody)
fw, err := w.CreateFormFile("file_data", o.remote)
if err != nil {
return false, err
}
if _, err = io.CopyN(fw, in, currentChunkSize); err != nil {
if _, err = io.Copy(fw, chunk); err != nil {
return false, err
}
// Add session_id
@@ -1077,7 +1090,7 @@ func (o *Object) readMetaData() (err error) {
err = o.fs.pacer.Call(func() (bool, error) {
opts := rest.Opts{
Method: "GET",
Path: "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + rest.URLPathEscape(replaceReservedChars(leaf)),
Path: "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)),
}
resp, err = o.fs.srv.CallJSON(&opts, nil, &folderList)
return o.fs.shouldRetry(resp, err)
+11 -6
View File
@@ -276,16 +276,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
newF := *f
newF.dirCache = dircache.New(newRoot, rootID, &newF)
newF.root = newRoot
tempF := *f
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = newF.dirCache.FindRoot(false)
err = tempF.dirCache.FindRoot(false)
if err != nil {
// No root so return old f
return f, nil
}
_, err := newF.newObjectWithInfo(remote, nil)
_, err := tempF.newObjectWithInfo(remote, nil)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
@@ -293,8 +293,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
return nil, err
}
// XXX: update the old f here instead of returning tempF, since
// `features` were already filled with functions having *f as a receiver.
// See https://github.com/ncw/rclone/issues/2182
f.dirCache = tempF.dirCache
f.root = tempF.root
// return an error with an fs which points to the parent
return &newF, fs.ErrorIsFile
return f, fs.ErrorIsFile
}
return f, nil
}
+88 -14
View File
@@ -448,7 +448,12 @@ func init() {
Provider: "!AWS,IBMCOS",
}, {
Name: "acl",
Help: "Canned ACL used when creating buckets and/or storing objects in S3.\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Note that this ACL is applied when server side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.`,
Examples: []fs.OptionExample{{
Value: "private",
Help: "Owner gets FULL_CONTROL. No one else has access rights (default).",
@@ -517,7 +522,7 @@ func init() {
}},
}, {
Name: "storage_class",
Help: "The storage class to use when storing objects in S3.",
Help: "The storage class to use when storing new objects in S3.",
Provider: "AWS",
Examples: []fs.OptionExample{{
Value: "",
@@ -536,9 +541,18 @@ func init() {
Help: "One Zone Infrequent Access storage class",
}},
}, {
Name: "chunk_size",
Help: "Chunk size to use for uploading",
Default: fs.SizeSuffix(s3manager.MinUploadPartSize),
Name: "chunk_size",
Help: `Chunk size to use for uploading.
Any files larger than this will be uploaded in chunks of this
size. The default is 5MB. The minimum is 5MB.
Note that "--s3-upload-concurrency" chunks of this size are buffered
in memory per transfer.
If you are transferring large files over high speed links and you have
enough memory, then increasing this will speed up the transfers.`,
Default: minChunkSize,
Advanced: true,
}, {
Name: "disable_checksum",
@@ -548,18 +562,41 @@ func init() {
}, {
Name: "session_token",
Help: "An AWS session token",
Hide: fs.OptionHideBoth,
Advanced: true,
}, {
Name: "upload_concurrency",
Help: "Concurrency for multipart uploads.",
Name: "upload_concurrency",
Help: `Concurrency for multipart uploads.
This is the number of chunks of the same file that are uploaded
concurrently.
If you are uploading small numbers of large file over high speed link
and these uploads do not fully utilize your bandwidth, then increasing
this may help to speed up the transfers.`,
Default: 2,
Advanced: true,
}, {
Name: "force_path_style",
Help: "If true use path style access if false use virtual hosted style.\nSome providers (eg Aliyun OSS or Netease COS) require this.",
Name: "force_path_style",
Help: `If true use path style access if false use virtual hosted style.
If this is true (the default) then rclone will use path style access,
if false then rclone will use virtual path style. See [the AWS S3
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
for more info.
Some providers (eg Aliyun OSS or Netease COS) require this set to false.`,
Default: true,
Advanced: true,
}, {
Name: "v2_auth",
Help: `If true use v2 authentication.
If this is false (the default) then rclone will use v4 authentication.
If it is set then rclone will use v2 authentication.
Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
Default: false,
Advanced: true,
}},
})
}
@@ -572,7 +609,8 @@ const (
maxRetries = 10 // number of retries to make of operations
maxSizeForCopy = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
maxFileSize = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
minSleep = 10 * time.Millisecond // In case of error, start at 10ms sleep.
minChunkSize = fs.SizeSuffix(s3manager.MinUploadPartSize)
minSleep = 10 * time.Millisecond // In case of error, start at 10ms sleep.
)
// Options defines the configuration for this backend
@@ -593,6 +631,7 @@ type Options struct {
SessionToken string `config:"session_token"`
UploadConcurrency int `config:"upload_concurrency"`
ForcePathStyle bool `config:"force_path_style"`
V2Auth bool `config:"v2_auth"`
}
// Fs represents a remote s3 server
@@ -767,7 +806,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
ses := session.New()
c := s3.New(ses, awsConfig)
if opt.Region == "other-v2-signature" {
if opt.V2Auth || opt.Region == "other-v2-signature" {
fs.Debugf(nil, "Using v2 auth")
signer := func(req *request.Request) {
// Ignore AnonymousCredentials object
@@ -783,6 +822,21 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
return c, ses, nil
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
return nil
}
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
}
return
}
// NewFs constructs an Fs from the path, bucket:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -791,8 +845,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
if opt.ChunkSize < fs.SizeSuffix(s3manager.MinUploadPartSize) {
return nil, errors.Errorf("s3 chunk size (%v) must be >= %v", opt.ChunkSize, fs.SizeSuffix(s3manager.MinUploadPartSize))
err = checkUploadChunkSize(opt.ChunkSize)
if err != nil {
return nil, errors.Wrap(err, "s3: chunk size")
}
bucket, directory, err := s3ParsePath(root)
if err != nil {
@@ -1236,10 +1291,20 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
source := pathEscape(srcFs.bucket + "/" + srcFs.root + srcObj.remote)
req := s3.CopyObjectInput{
Bucket: &f.bucket,
ACL: &f.opt.ACL,
Key: &key,
CopySource: &source,
MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
}
if f.opt.ServerSideEncryption != "" {
req.ServerSideEncryption = &f.opt.ServerSideEncryption
}
if f.opt.SSEKMSKeyID != "" {
req.SSEKMSKeyId = &f.opt.SSEKMSKeyID
}
if f.opt.StorageClass != "" {
req.StorageClass = &f.opt.StorageClass
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.c.CopyObject(&req)
return shouldRetry(err)
@@ -1409,6 +1474,15 @@ func (o *Object) SetModTime(modTime time.Time) error {
Metadata: o.meta,
MetadataDirective: &directive,
}
if o.fs.opt.ServerSideEncryption != "" {
req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
}
if o.fs.opt.SSEKMSKeyID != "" {
req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
}
if o.fs.opt.StorageClass != "" {
req.StorageClass = &o.fs.opt.StorageClass
}
err = o.fs.pacer.Call(func() (bool, error) {
_, err := o.fs.c.CopyObject(&req)
return shouldRetry(err)
+12 -3
View File
@@ -1,10 +1,10 @@
// Test S3 filesystem interface
package s3_test
package s3
import (
"testing"
"github.com/ncw/rclone/backend/s3"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
@@ -12,6 +12,15 @@ import (
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestS3:",
NilObject: (*s3.Object)(nil),
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: minChunkSize,
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
var _ fstests.SetUploadChunkSizer = (*Fs)(nil)
+1 -10
View File
@@ -44,16 +44,7 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
req.Header.Set("Date", date)
// Sort out URI
uri := req.URL.Opaque
if uri != "" {
if strings.HasPrefix(uri, "//") {
// Strip off //host/uri
uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/")
req.URL.Opaque = uri // reset to plain URI otherwise Ceph gets confused
}
} else {
uri = req.URL.Path
}
uri := req.URL.EscapedPath()
if uri == "" {
uri = "/"
}
+18 -3
View File
@@ -90,9 +90,20 @@ func init() {
Help: "Allow asking for SFTP password when needed.",
Advanced: true,
}, {
Name: "path_override",
Default: "",
Help: "Override path used by SSH connection.",
Name: "path_override",
Default: "",
Help: `Override path used by SSH connection.
This allows checksum calculation when SFTP and SSH paths are
different. This issue affects among others Synology NAS boxes.
Shared folders can be found in directories representing volumes
rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory
Home directory can be found in a shared folder called "home"
rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory`,
Advanced: true,
}, {
Name: "set_modtime",
@@ -758,6 +769,10 @@ func (o *Object) Hash(r hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
if o.fs.opt.DisableHashCheck {
return "", nil
}
c, err := o.fs.getSftpConnection()
if err != nil {
return "", errors.Wrap(err, "Hash get SFTP connection")
+34 -5
View File
@@ -29,13 +29,17 @@ import (
const (
directoryMarkerContentType = "application/directory" // content type of directory marker objects
listChunks = 1000 // chunk size to read directory listings
defaultChunkSize = 5 * fs.GibiByte
)
// SharedOptions are shared between swift and hubic
var SharedOptions = []fs.Option{{
Name: "chunk_size",
Help: "Above this size files will be chunked into a _segments container.",
Default: fs.SizeSuffix(5 * 1024 * 1024 * 1024),
Name: "chunk_size",
Help: `Above this size files will be chunked into a _segments container.
Above this size files will be chunked into a _segments container. The
default for this is 5GB which is its maximum value.`,
Default: defaultChunkSize,
Advanced: true,
}}
@@ -129,8 +133,13 @@ func init() {
Value: "admin",
}},
}, {
Name: "storage_policy",
Help: "The storage policy to use when creating a new container",
Name: "storage_policy",
Help: `The storage policy to use when creating a new container
This applies the specified storage policy when creating a new
container. The policy cannot be changed afterwards. The allowed
configuration values and their meaning depend on your Swift storage
provider.`,
Default: "",
Examples: []fs.OptionExample{{
Help: "Default",
@@ -294,6 +303,22 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
return c, nil
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.Byte
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
return nil
}
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
}
return
}
// NewFsWithConnection constructs an Fs from the path, container:path
// and authenticated connection.
//
@@ -344,6 +369,10 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
err = checkUploadChunkSize(opt.ChunkSize)
if err != nil {
return nil, errors.Wrap(err, "swift: chunk size")
}
c, err := swiftConnection(opt, name)
if err != nil {
+9 -3
View File
@@ -1,10 +1,10 @@
// Test Swift filesystem interface
package swift_test
package swift
import (
"testing"
"github.com/ncw/rclone/backend/swift"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fstest/fstests"
)
@@ -12,6 +12,12 @@ import (
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestSwift:",
NilObject: (*swift.Object)(nil),
NilObject: (*Object)(nil),
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
var _ fstests.SetUploadChunkSizer = (*Fs)(nil)
+208 -12
View File
@@ -19,7 +19,7 @@ import (
func init() {
fsi := &fs.RegInfo{
Name: "union",
Description: "Builds a stackable unification remote, which can appear to merge the contents of several remotes",
Description: "A stackable unification remote, which can appear to merge the contents of several remotes",
NewFs: NewFs,
Options: []fs.Option{{
Name: "remotes",
@@ -35,13 +35,36 @@ type Options struct {
Remotes fs.SpaceSepList `config:"remotes"`
}
// Fs represents a remote acd server
// Fs represents a union of remotes
type Fs struct {
name string // name of this remote
features *fs.Features // optional features
opt Options // options for this Fs
root string // the path we are working on
remotes []fs.Fs // slice of remotes
wr fs.Fs // writable remote
hashSet hash.Set // intersection of hash types
}
// Object describes a union Object
//
// This is a wrapped object which returns the Union Fs as its parent
type Object struct {
fs.Object
fs *Fs // what this object is part of
}
// Wrap an existing object in the union Object
func (f *Fs) wrapObject(o fs.Object) *Object {
return &Object{
Object: o,
fs: f,
}
}
// Fs returns the union Fs as the parent
func (o *Object) Fs() fs.Info {
return o.fs
}
// Name of the remote (as passed into NewFs)
@@ -66,18 +89,146 @@ func (f *Fs) Features() *fs.Features {
// Rmdir removes the root directory of the Fs object
func (f *Fs) Rmdir(dir string) error {
return f.remotes[len(f.remotes)-1].Rmdir(dir)
return f.wr.Rmdir(dir)
}
// Hashes returns hash.HashNone to indicate remote hashing is unavailable
func (f *Fs) Hashes() hash.Set {
// This could probably be set if all remotes share the same hashing algorithm
return hash.Set(hash.None)
return f.hashSet
}
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(dir string) error {
return f.remotes[len(f.remotes)-1].Mkdir(dir)
return f.wr.Mkdir(dir)
}
// Purge all files in the root and the root directory
//
// Implement this if you have a way of deleting all the files
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
func (f *Fs) Purge() error {
return f.wr.Features().Purge()
}
// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
if src.Fs() != f.wr {
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
o, err := f.wr.Features().Copy(src, remote)
if err != nil {
return nil, err
}
return f.wrapObject(o), nil
}
// Move src to this remote using server side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
if src.Fs() != f.wr {
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
o, err := f.wr.Features().Move(src, remote)
if err != nil {
return nil, err
}
return f.wrapObject(o), err
}
// DirMove moves src, srcRemote to this remote at dstRemote
// using server side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
srcFs, ok := src.(*Fs)
if !ok {
fs.Debugf(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
return f.wr.Features().DirMove(srcFs.wr, srcRemote, dstRemote)
}
// ChangeNotify calls the passed function with a path
// that has had changes. If the implementation
// uses polling, it should adhere to the given interval.
// At least one value will be written to the channel,
// specifying the initial value and updated values might
// follow. A 0 Duration should pause the polling.
// The ChangeNotify implemantion must empty the channel
// regulary. When the channel gets closed, the implemantion
// should stop polling and release resources.
func (f *Fs) ChangeNotify(fn func(string, fs.EntryType), ch <-chan time.Duration) {
var remoteChans []chan time.Duration
for _, remote := range f.remotes {
if ChangeNotify := remote.Features().ChangeNotify; ChangeNotify != nil {
ch := make(chan time.Duration)
remoteChans = append(remoteChans, ch)
ChangeNotify(fn, ch)
}
}
go func() {
for i := range ch {
for _, c := range remoteChans {
c <- i
}
}
for _, c := range remoteChans {
close(c)
}
}()
}
// DirCacheFlush resets the directory cache - used in testing
// as an optional interface
func (f *Fs) DirCacheFlush() {
for _, remote := range f.remotes {
if DirCacheFlush := remote.Features().DirCacheFlush; DirCacheFlush != nil {
DirCacheFlush()
}
}
}
// PutStream uploads to the remote path with the modTime given of indeterminate size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
o, err := f.wr.Features().PutStream(in, src, options...)
if err != nil {
return nil, err
}
return f.wrapObject(o), err
}
// About gets quota information from the Fs
func (f *Fs) About() (*fs.Usage, error) {
return f.wr.Features().About()
}
// Put in to the remote path with the modTime given of the given size
@@ -86,7 +237,11 @@ func (f *Fs) Mkdir(dir string) error {
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return f.remotes[len(f.remotes)-1].Put(in, src, options...)
o, err := f.wr.Put(in, src, options...)
if err != nil {
return nil, err
}
return f.wrapObject(o), err
}
// List the objects and directories in dir into entries. The
@@ -117,8 +272,11 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
if !found {
return nil, fs.ErrorDirNotFound
}
for key := range set {
entries = append(entries, set[key])
for _, entry := range set {
if o, ok := entry.(fs.Object); ok {
entry = f.wrapObject(o)
}
entries = append(entries, entry)
}
return entries, nil
}
@@ -134,7 +292,7 @@ func (f *Fs) NewObject(path string) (fs.Object, error) {
if err != nil {
return nil, errors.Wrapf(err, "NewObject failed on %v", remote)
}
return obj, nil
return f.wrapObject(obj), nil
}
return nil, fs.ErrorObjectNotFound
}
@@ -204,6 +362,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
root: root,
opt: *opt,
remotes: remotes,
wr: remotes[len(remotes)-1],
}
var features = (&fs.Features{
CaseInsensitive: true,
@@ -212,16 +371,53 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
WriteMimeType: true,
CanHaveEmptyDirectories: true,
BucketBased: true,
SetTier: true,
GetTier: true,
}).Fill(f)
features = features.Mask(f.wr) // mask the features just on the writable fs
// FIXME maybe should be masking the bools here?
// Clear ChangeNotify and DirCacheFlush if all are nil
clearChangeNotify := true
clearDirCacheFlush := true
for _, remote := range f.remotes {
features = features.Mask(remote)
remoteFeatures := remote.Features()
if remoteFeatures.ChangeNotify != nil {
clearChangeNotify = false
}
if remoteFeatures.DirCacheFlush != nil {
clearDirCacheFlush = false
}
}
if clearChangeNotify {
features.ChangeNotify = nil
}
if clearDirCacheFlush {
features.DirCacheFlush = nil
}
f.features = features
// Get common intersection of hashes
hashSet := f.remotes[0].Hashes()
for _, remote := range f.remotes[1:] {
hashSet = hashSet.Overlap(remote.Hashes())
}
f.hashSet = hashSet
return f, nil
}
// Check the interfaces are satisfied
var (
_ fs.Fs = &Fs{}
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.PutStreamer = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ fs.ChangeNotifier = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
)
+7
View File
@@ -145,6 +145,7 @@ var timeFormats = []string{
time.RFC1123Z, // Fri, 05 Jan 2018 14:14:38 +0000 (as used by mydrive.ch)
time.UnixDate, // Wed May 17 15:31:58 UTC 2017 (as used in an internal server)
noZerosRFC1123, // Fri, 7 Sep 2018 08:49:58 GMT (as used by server in #2574)
time.RFC3339, // Wed, 31 Oct 2018 13:57:11 CET (as used by komfortcloud.de)
}
// UnmarshalXML turns XML into a Time
@@ -155,6 +156,12 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
return err
}
// If time is missing then return the epoch
if v == "" {
*t = Time(time.Unix(0, 0))
return nil
}
// Parse the time format in multiple possible ways
var newT time.Time
for _, timeFormat := range timeFormats {
+32
View File
@@ -0,0 +1,32 @@
package odrvcookie
import (
"time"
"github.com/ncw/rclone/lib/rest"
)
// CookieRenew holds information for the renew
type CookieRenew struct {
srv *rest.Client
timer *time.Ticker
renewFn func()
}
// NewRenew returns and starts a CookieRenew
func NewRenew(interval time.Duration, renewFn func()) *CookieRenew {
renew := CookieRenew{
timer: time.NewTicker(interval),
renewFn: renewFn,
}
go renew.Renew()
return &renew
}
// Renew calls the renewFn for every tick
func (c *CookieRenew) Renew() {
for {
<-c.timer.C // wait for tick
c.renewFn()
}
}
+18
View File
@@ -372,6 +372,17 @@ func (f *Fs) setQuirks(vendor string) error {
if err != nil {
return err
}
odrvcookie.NewRenew(12*time.Hour, func() {
spCookies, err := spCk.Cookies()
if err != nil {
fs.Errorf("could not renew cookies: %s", err.Error())
return
}
f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa)
fs.Debugf(spCookies, "successfully renewed sharepoint cookies")
})
f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa)
// sharepoint, unlike the other vendors, only lists files if the depth header is set to 0
@@ -957,6 +968,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
Body: in,
NoResponse: true,
ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
ContentType: fs.MimeType(src),
}
if o.fs.useOCMtime {
opts.ExtraHeaders = map[string]string{
@@ -968,6 +980,12 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
return shouldRetry(resp, err)
})
if err != nil {
// Give the WebDAV server a chance to get its internal state in order after the
// error. The error may have been local in which case we closed the connection.
// The server may still be dealing with it for a moment. A sleep isn't ideal but I
// haven't been able to think of a better method to find out if the server has
// finished - ncw
time.Sleep(1 * time.Second)
// Remove failed upload
_ = o.Remove()
return err
+2 -4
View File
@@ -67,10 +67,8 @@ var defaultErrorHandler ErrorHandler = func(resp *http.Response) error {
func (HTTPRequest *HTTPRequest) run(client *Client) ([]byte, error) {
var err error
values := make(url.Values)
if HTTPRequest.Parameters != nil {
for k, v := range HTTPRequest.Parameters {
values.Set(k, fmt.Sprintf("%v", v))
}
for k, v := range HTTPRequest.Parameters {
values.Set(k, fmt.Sprintf("%v", v))
}
var req *http.Request
+1 -1
View File
@@ -9,7 +9,7 @@ set -e
# Compile the go version
cd src
./make.bash
./make.bash || exit 125
# Make sure we are using it
source ~/bin/use-go1.11
+25 -7
View File
@@ -1,15 +1,33 @@
#!/bin/bash
# Example script for git-bisect -run
# Example script for git bisect run
#
# Copy this file into /tmp say before running as it will be
# overwritten by the bisect as it is checked in.
#
# Change the test below to find out whether rclone is working or not
#
# Run from the project root
#
# git bisect start
# git checkout master
# git bisect bad
# git checkout v1.41 (or whatever is the first good one)
# git bisect good
# git bisect run /tmp/bisect-rclone.sh
set -e
# Compile
make
# Compile notifying git on compile failure
make || exit 125
rclone version
# Test whatever it is that is going wrong
truncate -s 10M /tmp/10M
rclone delete azure:rclone-test1/10M || true
rclone --retries 1 copyto -vv /tmp/10M azure:rclone-test1/10M --azureblob-upload-cutoff 1M
# Test whatever it is that is going wrong - exit with non zero exit code on failure
# commented out examples follow
# truncate -s 10M /tmp/10M
# rclone delete azure:rclone-test1/10M || true
# rclone --retries 1 copyto -vv /tmp/10M azure:rclone-test1/10M --azureblob-upload-cutoff 1M
# rm -f "/tmp/tests's.docx" || true
# rclone -vv --retries 1 copy "drive:test/tests's.docx" /tmp
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python2
"""
Make backend documentation
"""
import os
import subprocess
marker = "<!--- autogenerated options"
start = marker + " start"
stop = marker + " stop"
def find_backends():
"""Return a list of all backends"""
return [ x for x in os.listdir("backend") if x not in ("all",) ]
def output_docs(backend, out):
"""Output documentation for backend options to out"""
out.flush()
subprocess.check_call(["rclone", "help", "backend", backend], stdout=out)
def alter_doc(backend):
"""Alter the documentation for backend"""
doc_file = "docs/content/"+backend+".md"
if not os.path.exists(doc_file):
raise ValueError("Didn't find doc file %s" % (doc_file,))
new_file = doc_file+"~new~"
altered = False
with open(doc_file, "r") as in_file, open(new_file, "w") as out_file:
in_docs = False
for line in in_file:
if not in_docs:
if start in line:
in_docs = True
start_full = start + " - DO NOT EDIT, instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs -->\n" % (backend, backend)
out_file.write(start_full)
output_docs(backend, out_file)
out_file.write(stop+" -->\n")
altered = True
if not in_docs:
out_file.write(line)
if in_docs:
if stop in line:
in_docs = False
os.rename(doc_file, doc_file+"~")
os.rename(new_file, doc_file)
if not altered:
raise ValueError("Didn't find '%s' markers for in %s" % (start, doc_file))
if __name__ == "__main__":
failed, success = 0, 0
for backend in find_backends():
try:
alter_doc(backend)
except Exception, e:
print "Failed adding docs for %s backend: %s" % (backend, e)
failed += 1
else:
success += 1
print "Added docs for %d backends with %d failures" % (success, failed)
+1 -1
View File
@@ -165,7 +165,7 @@ def main():
%s
* Bug Fixes
%s
%s""" % (version, datetime.date.today(), "\n".join(new_features_lines), "\n".join(bugfix_lines), "\n".join(backend_lines)))
%s""" % (next_version, datetime.date.today(), "\n".join(new_features_lines), "\n".join(bugfix_lines), "\n".join(backend_lines)))
sys.stdout.write(old_tail)
+1 -1
View File
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python2
"""
Make single page versions of the documentation for release and
conversion into man pages etc.
+9 -7
View File
@@ -4,18 +4,20 @@
set -e
go install
mkdir -p /tmp/rclone_cache_test
mkdir -p /tmp/rclone/cache_test
mkdir -p /tmp/rclone/rc_mount
export RCLONE_CONFIG_RCDOCS_TYPE=cache
export RCLONE_CONFIG_RCDOCS_REMOTE=/tmp/rclone/cache_test
rclone -q --rc mount rcdocs: /mnt/tmp/ &
rclone -q --rc mount rcdocs: /tmp/rclone/rc_mount &
sleep 0.5
rclone rc > /tmp/z.md
fusermount -z -u /mnt/tmp/
rclone rc > /tmp/rclone/z.md
fusermount -u -z /tmp/rclone/rc_mount > /dev/null 2>&1 || umount /tmp/rclone/rc_mount
awk '
BEGIN {p=1}
/^<!--- autogenerated start/ {print;system("cat /tmp/z.md");p=0}
/^<!--- autogenerated start/ {print;system("cat /tmp/rclone/z.md");p=0}
/^<!--- autogenerated stop/ {p=1}
p' docs/content/rc.md > /tmp/rc.md
p' docs/content/rc.md > /tmp/rclone/rc.md
mv /tmp/rc.md docs/content/rc.md
mv /tmp/rclone/rc.md docs/content/rc.md
rm -rf /tmp/rclone
+1
View File
@@ -43,6 +43,7 @@ import (
_ "github.com/ncw/rclone/cmd/purge"
_ "github.com/ncw/rclone/cmd/rc"
_ "github.com/ncw/rclone/cmd/rcat"
_ "github.com/ncw/rclone/cmd/rcd"
_ "github.com/ncw/rclone/cmd/reveal"
_ "github.com/ncw/rclone/cmd/rmdir"
_ "github.com/ncw/rclone/cmd/rmdirs"
+40 -105
View File
@@ -17,12 +17,9 @@ import (
"runtime/pprof"
"strconv"
"strings"
"sync"
"time"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/fs/config/configflags"
@@ -32,9 +29,12 @@ import (
"github.com/ncw/rclone/fs/fserrors"
"github.com/ncw/rclone/fs/fspath"
fslog "github.com/ncw/rclone/fs/log"
"github.com/ncw/rclone/fs/rc"
"github.com/ncw/rclone/fs/rc/rcflags"
"github.com/ncw/rclone/fs/rc/rcserver"
"github.com/ncw/rclone/lib/atexit"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Globals
@@ -66,81 +66,6 @@ const (
exitCodeTransferExceeded
)
// Root is the main rclone command
var Root = &cobra.Command{
Use: "rclone",
Short: "Sync files and directories to and from local and remote object stores - " + fs.Version,
Long: `
Rclone is a command line program to sync files and directories to and
from various cloud storage systems and using file transfer services, such as:
* Amazon Drive
* Amazon S3
* Backblaze B2
* Box
* Dropbox
* FTP
* Google Cloud Storage
* Google Drive
* HTTP
* Hubic
* Jottacloud
* Mega
* Microsoft Azure Blob Storage
* Microsoft OneDrive
* OpenDrive
* Openstack Swift / Rackspace cloud files / Memset Memstore
* pCloud
* QingStor
* SFTP
* Webdav / Owncloud / Nextcloud
* Yandex Disk
* The local filesystem
Features
* MD5/SHA1 hashes checked at all times for file integrity
* Timestamps preserved on files
* Partial syncs supported on a whole file basis
* Copy mode to just copy new/changed files
* Sync (one way) mode to make a directory identical
* Check mode to check for file hash equality
* Can sync to and from network, eg two different cloud accounts
See the home page for installation, usage, documentation, changelog
and configuration walkthroughs.
* https://rclone.org/
`,
PersistentPostRun: func(cmd *cobra.Command, args []string) {
fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
atexit.Run()
},
}
// runRoot implements the main rclone command with no subcommands
func runRoot(cmd *cobra.Command, args []string) {
if version {
ShowVersion()
resolveExitCode(nil)
} else {
_ = Root.Usage()
_, _ = fmt.Fprintf(os.Stderr, "Command not found.\n")
resolveExitCode(errorCommandNotFound)
}
}
func init() {
// Add global flags
configflags.AddFlags(pflag.CommandLine)
filterflags.AddFlags(pflag.CommandLine)
rcflags.AddFlags(pflag.CommandLine)
Root.Run = runRoot
Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number")
cobra.OnInitialize(initConfig)
}
// ShowVersion prints the version to stdout
func ShowVersion() {
fmt.Printf("rclone %s\n", fs.Version)
@@ -293,7 +218,7 @@ func ShowStats() bool {
// Run the function with stats and retries if required
func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
var err error
var stopStats chan struct{}
stopStats := func() {}
if !showStats && ShowStats() {
showStats = true
}
@@ -331,9 +256,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
time.Sleep(*retriesInterval)
}
}
if showStats {
close(stopStats)
}
stopStats()
if err != nil {
log.Printf("Failed to %s: %v", cmd.Name(), err)
resolveExitCode(err)
@@ -384,24 +307,31 @@ func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) {
// StartStats prints the stats every statsInterval
//
// It returns a channel which should be closed to stop the stats.
func StartStats() chan struct{} {
// It returns a func which should be called to stop the stats.
func StartStats() func() {
if *statsInterval <= 0 {
return func() {}
}
stopStats := make(chan struct{})
if *statsInterval > 0 {
go func() {
ticker := time.NewTicker(*statsInterval)
for {
select {
case <-ticker.C:
accounting.Stats.Log()
case <-stopStats:
ticker.Stop()
return
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
ticker := time.NewTicker(*statsInterval)
for {
select {
case <-ticker.C:
accounting.Stats.Log()
case <-stopStats:
ticker.Stop()
return
}
}()
}
}()
return func() {
close(stopStats)
wg.Wait()
}
return stopStats
}
// initConfig is run by cobra after initialising the flags
@@ -422,8 +352,11 @@ func initConfig() {
// Write the args for debug purposes
fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args)
// Start the remote control if configured
rc.Start(&rcflags.Opt)
// Start the remote control server if configured
_, err = rcserver.Start(&rcflags.Opt)
if err != nil {
log.Fatalf("Failed to start remote control: %v", err)
}
// Setup CPU profiling if desired
if *cpuProfile != "" {
@@ -501,8 +434,11 @@ func resolveExitCode(err error) {
}
}
var backendFlags map[string]struct{}
// AddBackendFlags creates flags for all the backend options
func AddBackendFlags() {
backendFlags = map[string]struct{}{}
for _, fsInfo := range fs.Registry {
done := map[string]struct{}{}
for i := range fsInfo.Options {
@@ -513,10 +449,7 @@ func AddBackendFlags() {
}
done[opt.Name] = struct{}{}
// Make a flag from each option
name := strings.Replace(opt.Name, "_", "-", -1) // convert snake_case to kebab-case
if !opt.NoPrefix {
name = fsInfo.Prefix + "-" + name
}
name := opt.FlagName(fsInfo.Prefix)
found := pflag.CommandLine.Lookup(name) != nil
if !found {
// Take first line of help only
@@ -533,6 +466,7 @@ func AddBackendFlags() {
if opt.Hide&fs.OptionHideCommandLine != 0 {
flag.Hidden = true
}
backendFlags[name] = struct{}{}
} else {
fs.Errorf(nil, "Not adding duplicate flag --%s", name)
}
@@ -543,6 +477,7 @@ func AddBackendFlags() {
// Main runs rclone interpreting flags and commands out of os.Args
func Main() {
setupRootCommand(Root)
AddBackendFlags()
if err := Root.Execute(); err != nil {
log.Fatalf("Fatal error: %v", err)
+47 -3
View File
@@ -1,8 +1,11 @@
package config
import (
"errors"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs/config"
"github.com/ncw/rclone/fs/rc"
"github.com/spf13/cobra"
)
@@ -93,7 +96,16 @@ you would do:
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(2, 256, command, args)
return config.CreateRemote(args[0], args[1], args[2:])
in, err := argsToMap(args[2:])
if err != nil {
return err
}
err = config.CreateRemote(args[0], args[1], in)
if err != nil {
return err
}
config.ShowRemote(args[0])
return nil
},
}
@@ -110,7 +122,16 @@ For example to update the env_auth field of a remote of name myremote you would
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(3, 256, command, args)
return config.UpdateRemote(args[0], args[1:])
in, err := argsToMap(args[1:])
if err != nil {
return err
}
err = config.UpdateRemote(args[0], in)
if err != nil {
return err
}
config.ShowRemote(args[0])
return nil
},
}
@@ -136,6 +157,29 @@ For example to set password of a remote of name myremote you would do:
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(3, 256, command, args)
return config.PasswordRemote(args[0], args[1:])
in, err := argsToMap(args[1:])
if err != nil {
return err
}
err = config.PasswordRemote(args[0], in)
if err != nil {
return err
}
config.ShowRemote(args[0])
return nil
},
}
// This takes a list of arguments in key value key value form and
// converts it into a map
func argsToMap(args []string) (out rc.Params, err error) {
if len(args)%2 != 0 {
return nil, errors.New("found key without value")
}
out = rc.Params{}
// Set the config
for i := 0; i < len(args); i += 2 {
out[args[i]] = args[i+1]
}
return out, nil
}
+2
View File
@@ -50,6 +50,8 @@ If you are familiar with ` + "`rsync`" + `, rclone always works as if you had
written a trailing / - meaning "copy the contents of this directory".
This applies to all commands and whether you are talking about the
source or destination.
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
+2
View File
@@ -40,6 +40,8 @@ This will:
This doesn't transfer unchanged files, testing by size and
modification time or MD5SUM. It doesn't delete files from the
destination.
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
+1 -11
View File
@@ -1,9 +1,6 @@
package copyurl
import (
"net/http"
"time"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs/operations"
"github.com/spf13/cobra"
@@ -25,14 +22,7 @@ without saving it in tmp storage.
fsdst, dstFileName := cmd.NewFsDstFile(args[1:])
cmd.Run(true, true, command, func() error {
resp, err := http.Get(args[0])
if err != nil {
return err
}
_, err = operations.RcatSize(fsdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
_, err := operations.CopyURL(fsdst, dstFileName, args[0])
return err
})
},
+5 -1
View File
@@ -14,9 +14,13 @@ var commandDefintion = &cobra.Command{
Use: "delete remote:path",
Short: `Remove the contents of path.`,
Long: `
Remove the contents of path. Unlike ` + "`" + `purge` + "`" + ` it obeys include/exclude
Remove the files in path. Unlike ` + "`" + `purge` + "`" + ` it obeys include/exclude
filters so can be used to selectively delete files.
` + "`" + `rclone delete` + "`" + ` only deletes objects but leaves the directory structure
alone. If you want to delete a directory and all of its contents use
` + "`" + `rclone purge` + "`" + `
Eg delete all files bigger than 100MBytes
Check what would be deleted first (use either)
-1
View File
@@ -3,7 +3,6 @@ package deletefile
import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs/operations"
"github.com/pkg/errors"
"github.com/spf13/cobra"
)
+250
View File
@@ -0,0 +1,250 @@
package cmd
import (
"fmt"
"log"
"os"
"regexp"
"strings"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config/configflags"
"github.com/ncw/rclone/fs/filter/filterflags"
"github.com/ncw/rclone/fs/rc/rcflags"
"github.com/ncw/rclone/lib/atexit"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
// Root is the main rclone command
var Root = &cobra.Command{
Use: "rclone",
Short: "Show help for rclone commands, flags and backends.",
Long: `
Rclone syncs files to and from cloud storage providers as well as
mounting them, listing them in lots of different ways.
See the home page (https://rclone.org/) for installation, usage,
documentation, changelog and configuration walkthroughs.
`,
PersistentPostRun: func(cmd *cobra.Command, args []string) {
fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
atexit.Run()
},
}
// root help command
var helpCommand = &cobra.Command{
Use: "help",
Short: Root.Short,
Long: Root.Long,
Run: func(command *cobra.Command, args []string) {
Root.SetOutput(os.Stdout)
_ = Root.Usage()
},
}
// to filter the flags with
var flagsRe *regexp.Regexp
// Show the flags
var helpFlags = &cobra.Command{
Use: "flags [<regexp to match>]",
Short: "Show the global flags for rclone",
Run: func(command *cobra.Command, args []string) {
if len(args) > 0 {
re, err := regexp.Compile(args[0])
if err != nil {
log.Fatalf("Failed to compile flags regexp: %v", err)
}
flagsRe = re
}
Root.SetOutput(os.Stdout)
_ = command.Usage()
},
}
// Show the backends
var helpBackends = &cobra.Command{
Use: "backends",
Short: "List the backends available",
Run: func(command *cobra.Command, args []string) {
showBackends()
},
}
// Show a single backend
var helpBackend = &cobra.Command{
Use: "backend <name>",
Short: "List full info about a backend",
Run: func(command *cobra.Command, args []string) {
if len(args) == 0 {
Root.SetOutput(os.Stdout)
_ = command.Usage()
return
}
showBackend(args[0])
},
}
// runRoot implements the main rclone command with no subcommands
func runRoot(cmd *cobra.Command, args []string) {
if version {
ShowVersion()
resolveExitCode(nil)
} else {
_ = cmd.Usage()
if len(args) > 0 {
_, _ = fmt.Fprintf(os.Stderr, "Command not found.\n")
}
resolveExitCode(errorCommandNotFound)
}
}
// setupRootCommand sets default usage, help, and error handling for
// the root command.
//
// Helpful example: http://rtfcode.com/xref/moby-17.03.2-ce/cli/cobra.go
func setupRootCommand(rootCmd *cobra.Command) {
// Add global flags
configflags.AddFlags(pflag.CommandLine)
filterflags.AddFlags(pflag.CommandLine)
rcflags.AddFlags(pflag.CommandLine)
Root.Run = runRoot
Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number")
cobra.AddTemplateFunc("showGlobalFlags", func(cmd *cobra.Command) bool {
return cmd.CalledAs() == "flags"
})
cobra.AddTemplateFunc("showCommands", func(cmd *cobra.Command) bool {
return cmd.CalledAs() != "flags"
})
cobra.AddTemplateFunc("showLocalFlags", func(cmd *cobra.Command) bool {
// Don't show local flags (which are the global ones on the root) on "rclone" and
// "rclone help" (which shows the global help)
return cmd.CalledAs() != "rclone" && cmd.CalledAs() != ""
})
cobra.AddTemplateFunc("backendFlags", func(cmd *cobra.Command, include bool) *pflag.FlagSet {
backendFlagSet := pflag.NewFlagSet("Backend Flags", pflag.ExitOnError)
cmd.InheritedFlags().VisitAll(func(flag *pflag.Flag) {
matched := flagsRe == nil || flagsRe.MatchString(flag.Name)
if _, ok := backendFlags[flag.Name]; matched && ok == include {
backendFlagSet.AddFlag(flag)
}
})
return backendFlagSet
})
rootCmd.SetUsageTemplate(usageTemplate)
// rootCmd.SetHelpTemplate(helpTemplate)
// rootCmd.SetFlagErrorFunc(FlagErrorFunc)
rootCmd.SetHelpCommand(helpCommand)
// rootCmd.PersistentFlags().BoolP("help", "h", false, "Print usage")
// rootCmd.PersistentFlags().MarkShorthandDeprecated("help", "please use --help")
rootCmd.AddCommand(helpCommand)
helpCommand.AddCommand(helpFlags)
helpCommand.AddCommand(helpBackends)
helpCommand.AddCommand(helpBackend)
cobra.OnInitialize(initConfig)
}
var usageTemplate = `Usage:{{if .Runnable}}
{{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}
{{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}}
Aliases:
{{.NameAndAliases}}{{end}}{{if .HasExample}}
Examples:
{{.Example}}{{end}}{{if and (showCommands .) .HasAvailableSubCommands}}
Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}}
{{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if and (showLocalFlags .) .HasAvailableLocalFlags}}
Flags:
{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if and (showGlobalFlags .) .HasAvailableInheritedFlags}}
Global Flags:
{{(backendFlags . false).FlagUsages | trimTrailingWhitespaces}}
Backend Flags:
{{(backendFlags . true).FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}}
{{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}
Use "rclone [command] --help" for more information about a command.
Use "rclone help flags" for to see the global flags.
Use "rclone help backends" for a list of supported services.
`
// show all the backends
func showBackends() {
fmt.Printf("All rclone backends:\n\n")
for _, backend := range fs.Registry {
fmt.Printf(" %-12s %s\n", backend.Prefix, backend.Description)
}
fmt.Printf("\nTo see more info about a particular backend use:\n")
fmt.Printf(" rclone help backend <name>\n")
}
func quoteString(v interface{}) string {
switch v.(type) {
case string:
return fmt.Sprintf("%q", v)
}
return fmt.Sprint(v)
}
// show a single backend
func showBackend(name string) {
backend, err := fs.Find(name)
if err != nil {
log.Fatal(err)
}
var standardOptions, advancedOptions fs.Options
done := map[string]struct{}{}
for _, opt := range backend.Options {
// Skip if done already (eg with Provider options)
if _, doneAlready := done[opt.Name]; doneAlready {
continue
}
if opt.Advanced {
advancedOptions = append(advancedOptions, opt)
} else {
standardOptions = append(standardOptions, opt)
}
}
optionsType := "standard"
for _, opts := range []fs.Options{standardOptions, advancedOptions} {
if len(opts) == 0 {
continue
}
fmt.Printf("### %s Options\n\n", strings.Title(optionsType))
fmt.Printf("Here are the %s options specific to %s (%s).\n\n", optionsType, backend.Name, backend.Description)
optionsType = "advanced"
for _, opt := range opts {
done[opt.Name] = struct{}{}
fmt.Printf("#### --%s\n\n", opt.FlagName(backend.Prefix))
fmt.Printf("%s\n\n", opt.Help)
fmt.Printf("- Config: %s\n", opt.Name)
fmt.Printf("- Env Var: %s\n", opt.EnvVarName(backend.Prefix))
fmt.Printf("- Type: %s\n", opt.Type())
fmt.Printf("- Default: %s\n", quoteString(opt.GetValue()))
if len(opt.Examples) > 0 {
fmt.Printf("- Examples:\n")
for _, ex := range opt.Examples {
fmt.Printf(" - %s\n", quoteString(ex.Value))
for _, line := range strings.Split(ex.Help, "\n") {
fmt.Printf(" - %s\n", line)
}
}
}
fmt.Printf("\n")
}
}
}
+1 -2
View File
@@ -4,13 +4,12 @@ import (
"bytes"
"testing"
_ "github.com/ncw/rclone/backend/local"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/list"
"github.com/ncw/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
_ "github.com/ncw/rclone/backend/local"
)
func TestDefaultLsf(t *testing.T) {
+18 -134
View File
@@ -3,62 +3,26 @@ package lsjson
import (
"encoding/json"
"fmt"
"log"
"os"
"path"
"time"
"github.com/ncw/rclone/backend/crypt"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/ls/lshelp"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/operations"
"github.com/ncw/rclone/fs/walk"
"github.com/pkg/errors"
"github.com/spf13/cobra"
)
var (
recurse bool
showHash bool
showEncrypted bool
showOrigIDs bool
noModTime bool
opt operations.ListJSONOpt
)
func init() {
cmd.Root.AddCommand(commandDefintion)
commandDefintion.Flags().BoolVarP(&recurse, "recursive", "R", false, "Recurse into the listing.")
commandDefintion.Flags().BoolVarP(&showHash, "hash", "", false, "Include hashes in the output (may take longer).")
commandDefintion.Flags().BoolVarP(&noModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
commandDefintion.Flags().BoolVarP(&showEncrypted, "encrypted", "M", false, "Show the encrypted names.")
commandDefintion.Flags().BoolVarP(&showOrigIDs, "original", "", false, "Show the ID of the underlying Object.")
}
// lsJSON in the struct which gets marshalled for each line
type lsJSON struct {
Path string
Name string
Encrypted string `json:",omitempty"`
Size int64
MimeType string `json:",omitempty"`
ModTime Timestamp //`json:",omitempty"`
IsDir bool
Hashes map[string]string `json:",omitempty"`
ID string `json:",omitempty"`
OrigID string `json:",omitempty"`
}
// Timestamp a time in RFC3339 format with Nanosecond precision secongs
type Timestamp time.Time
// MarshalJSON turns a Timestamp into JSON
func (t Timestamp) MarshalJSON() (out []byte, err error) {
tt := time.Time(t)
if tt.IsZero() {
return []byte(`""`), nil
}
return []byte(`"` + tt.Format(time.RFC3339Nano) + `"`), nil
commandDefintion.Flags().BoolVarP(&opt.Recurse, "recursive", "R", false, "Recurse into the listing.")
commandDefintion.Flags().BoolVarP(&opt.ShowHash, "hash", "", false, "Include hashes in the output (may take longer).")
commandDefintion.Flags().BoolVarP(&opt.NoModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
commandDefintion.Flags().BoolVarP(&opt.ShowEncrypted, "encrypted", "M", false, "Show the encrypted names.")
commandDefintion.Flags().BoolVarP(&opt.ShowOrigIDs, "original", "", false, "Show the ID of the underlying Object.")
}
var commandDefintion = &cobra.Command{
@@ -104,107 +68,27 @@ can be processed line by line as each item is written one to a line.
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
var cipher crypt.Cipher
if showEncrypted {
fsInfo, _, _, config, err := fs.ConfigFs(args[0])
if err != nil {
log.Fatalf(err.Error())
}
if fsInfo.Name != "crypt" {
log.Fatalf("The remote needs to be of type \"crypt\"")
}
cipher, err = crypt.NewCipher(config)
if err != nil {
log.Fatalf(err.Error())
}
}
cmd.Run(false, false, command, func() error {
fmt.Println("[")
first := true
err := walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(dirPath string, entries fs.DirEntries, err error) error {
err := operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error {
out, err := json.Marshal(item)
if err != nil {
fs.CountError(err)
fs.Errorf(dirPath, "error listing: %v", err)
return nil
return errors.Wrap(err, "failed to marshal list object")
}
for _, entry := range entries {
item := lsJSON{
Path: entry.Remote(),
Name: path.Base(entry.Remote()),
Size: entry.Size(),
MimeType: fs.MimeTypeDirEntry(entry),
}
if !noModTime {
item.ModTime = Timestamp(entry.ModTime())
}
if cipher != nil {
switch entry.(type) {
case fs.Directory:
item.Encrypted = cipher.EncryptDirName(path.Base(entry.Remote()))
case fs.Object:
item.Encrypted = cipher.EncryptFileName(path.Base(entry.Remote()))
default:
fs.Errorf(nil, "Unknown type %T in listing", entry)
}
}
if do, ok := entry.(fs.IDer); ok {
item.ID = do.ID()
}
if showOrigIDs {
cur := entry
for {
u, ok := cur.(fs.ObjectUnWrapper)
if !ok {
break // not a wrapped object, use current id
}
next := u.UnWrap()
if next == nil {
break // no base object found, use current id
}
cur = next
}
if do, ok := cur.(fs.IDer); ok {
item.OrigID = do.ID()
}
}
switch x := entry.(type) {
case fs.Directory:
item.IsDir = true
case fs.Object:
item.IsDir = false
if showHash {
item.Hashes = make(map[string]string)
for _, hashType := range x.Fs().Hashes().Array() {
hash, err := x.Hash(hashType)
if err != nil {
fs.Errorf(x, "Failed to read hash: %v", err)
} else if hash != "" {
item.Hashes[hashType.String()] = hash
}
}
}
default:
fs.Errorf(nil, "Unknown type %T in listing", entry)
}
out, err := json.Marshal(item)
if err != nil {
return errors.Wrap(err, "failed to marshal list object")
}
if first {
first = false
} else {
fmt.Print(",\n")
}
_, err = os.Stdout.Write(out)
if err != nil {
return errors.Wrap(err, "failed to write to output")
}
if first {
first = false
} else {
fmt.Print(",\n")
}
_, err = os.Stdout.Write(out)
if err != nil {
return errors.Wrap(err, "failed to write to output")
}
return nil
})
if err != nil {
return errors.Wrap(err, "error listing JSON")
return err
}
if !first {
fmt.Println()
+3 -6
View File
@@ -66,14 +66,12 @@ func checkMountEmpty(mountpoint string) error {
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
var commandDefintion = &cobra.Command{
Use: commandName + " remote:path /path/to/mountpoint",
Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`,
Short: `Mount the remote as file system on a mountpoint.`,
Long: `
rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to
mount any of Rclone's cloud storage systems as a file system with
FUSE.
This is **EXPERIMENTAL** - use with care.
First set up your remote using ` + "`rclone config`" + `. Check it works with ` + "`rclone ls`" + ` etc.
Start the mount like this
@@ -148,7 +146,7 @@ File systems expect things to be 100% reliable, whereas cloud storage
systems are a long way from 100% reliable. The rclone sync/copy
commands cope with this with lots of retries. However rclone ` + commandName + `
can't use retries in the same way without making local copies of the
uploads. Look at the **EXPERIMENTAL** [file caching](#file-caching)
uploads. Look at the [file caching](#file-caching)
for solutions to make ` + commandName + ` mount more reliable.
### Attribute caching
@@ -226,8 +224,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
// Show stats if the user has specifically requested them
if cmd.ShowStats() {
stopStats := cmd.StartStats()
defer close(stopStats)
defer cmd.StartStats()()
}
// Skip checkMountEmpty if --allow-non-empty flag is used or if
+2
View File
@@ -39,6 +39,8 @@ If you want to delete empty source directories after move, use the --delete-empt
**Important**: Since this can cause data loss, test first with the
--dry-run flag.
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
+2
View File
@@ -43,6 +43,8 @@ transfer.
**Important**: Since this can cause data loss, test first with the
--dry-run flag.
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
+209 -22
View File
@@ -13,6 +13,7 @@ import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/ncdu/scan"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/operations"
termbox "github.com/nsf/termbox-go"
"github.com/pkg/errors"
"github.com/spf13/cobra"
@@ -42,8 +43,11 @@ Here are the keys - press '?' to toggle the help on and off
` + strings.Join(helpText[1:], "\n ") + `
This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
rclone remotes. It is missing lots of features at the moment, most
importantly deleting files, but is useful as it stands.
rclone remotes. It is missing lots of features at the moment
but is useful as it stands.
Note that it might take some time to delete big files/folders. The
UI won't respond in the meantime since the deletion is done synchronously.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
@@ -63,6 +67,7 @@ var helpText = []string{
" c toggle counts",
" g toggle graph",
" n,s,C sort by name,size,count",
" d delete file/directory",
" ^L refresh screen",
" ? to toggle help on and off",
" q/ESC/c-C to quit",
@@ -70,24 +75,27 @@ var helpText = []string{
// UI contains the state of the user interface
type UI struct {
f fs.Fs // fs being displayed
fsName string // human name of Fs
root *scan.Dir // root directory
d *scan.Dir // current directory being displayed
path string // path of current directory
showBox bool // whether to show a box
boxText []string // text to show in box
entries fs.DirEntries // entries of current directory
sortPerm []int // order to display entries in after sorting
invSortPerm []int // inverse order
dirListHeight int // height of listing
listing bool // whether listing is in progress
showGraph bool // toggle showing graph
showCounts bool // toggle showing counts
sortByName int8 // +1 for normal, 0 for off, -1 for reverse
sortBySize int8
sortByCount int8
dirPosMap map[string]dirPos // store for directory positions
f fs.Fs // fs being displayed
fsName string // human name of Fs
root *scan.Dir // root directory
d *scan.Dir // current directory being displayed
path string // path of current directory
showBox bool // whether to show a box
boxText []string // text to show in box
boxMenu []string // box menu options
boxMenuButton int
boxMenuHandler func(fs fs.Fs, path string, option int) (string, error)
entries fs.DirEntries // entries of current directory
sortPerm []int // order to display entries in after sorting
invSortPerm []int // inverse order
dirListHeight int // height of listing
listing bool // whether listing is in progress
showGraph bool // toggle showing graph
showCounts bool // toggle showing counts
sortByName int8 // +1 for normal, 0 for off, -1 for reverse
sortBySize int8
sortByCount int8
dirPosMap map[string]dirPos // store for directory positions
}
// Where we have got to in the directory listing
@@ -130,6 +138,54 @@ func Linef(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, format string,
Line(x, y, xmax, fg, bg, spacer, s)
}
// LineOptions Print line of selectable options
func LineOptions(x, y, xmax int, fg, bg termbox.Attribute, options []string, selected int) {
defaultBg := bg
defaultFg := fg
// Print left+right whitespace to center the options
xoffset := ((xmax - x) - lineOptionLength(options)) / 2
for j := x; j < x+xoffset; j++ {
termbox.SetCell(j, y, ' ', fg, bg)
}
for j := xmax - xoffset; j < xmax; j++ {
termbox.SetCell(j, y, ' ', fg, bg)
}
x += xoffset
for i, o := range options {
termbox.SetCell(x, y, ' ', fg, bg)
if i == selected {
bg = termbox.ColorBlack
fg = termbox.ColorWhite
}
termbox.SetCell(x+1, y, '<', fg, bg)
x += 2
// print option text
for _, c := range o {
termbox.SetCell(x, y, c, fg, bg)
x++
}
termbox.SetCell(x, y, '>', fg, bg)
bg = defaultBg
fg = defaultFg
termbox.SetCell(x+1, y, ' ', fg, bg)
x += 2
}
}
func lineOptionLength(o []string) int {
count := 0
for _, i := range o {
count += len(i)
}
return count + 4*len(o) // spacer and arrows <entry>
}
// Box the u.boxText onto the screen
func (u *UI) Box() {
w, h := termbox.Size()
@@ -147,6 +203,15 @@ func (u *UI) Box() {
x := (w - boxWidth) / 2
y := (h - boxHeight) / 2
xmax := x + boxWidth
if len(u.boxMenu) != 0 {
count := lineOptionLength(u.boxMenu)
if x+boxWidth > x+count {
xmax = x + boxWidth
} else {
xmax = x + count
}
}
ymax := y + len(u.boxText)
// draw text
fg, bg := termbox.ColorRed, termbox.ColorWhite
@@ -155,7 +220,43 @@ func (u *UI) Box() {
fg = termbox.ColorBlack
}
// FIXME draw a box around
if len(u.boxMenu) != 0 {
ymax++
LineOptions(x, ymax-1, xmax, fg, bg, u.boxMenu, u.boxMenuButton)
}
// draw top border
for i := y; i < ymax; i++ {
termbox.SetCell(x-1, i, '│', fg, bg)
termbox.SetCell(xmax, i, '│', fg, bg)
}
for j := x; j < xmax; j++ {
termbox.SetCell(j, y-1, '─', fg, bg)
termbox.SetCell(j, ymax, '─', fg, bg)
}
termbox.SetCell(x-1, y-1, '┌', fg, bg)
termbox.SetCell(xmax, y-1, '┐', fg, bg)
termbox.SetCell(x-1, ymax, '└', fg, bg)
termbox.SetCell(xmax, ymax, '┘', fg, bg)
}
func (u *UI) moveBox(to int) {
if len(u.boxMenu) == 0 {
return
}
if to > 0 { // move right
u.boxMenuButton++
} else { // move left
u.boxMenuButton--
}
if u.boxMenuButton >= len(u.boxMenu) {
u.boxMenuButton = len(u.boxMenu) - 1
} else if u.boxMenuButton < 0 {
u.boxMenuButton = 0
}
}
// find the biggest entry in the current listing
@@ -314,6 +415,57 @@ func (u *UI) move(d int) {
u.dirPosMap[u.path] = dirPos
}
func (u *UI) removeEntry(pos int) {
u.d.Remove(pos)
u.setCurrentDir(u.d)
}
// delete the entry at the current position
func (u *UI) delete() {
dirPos := u.sortPerm[u.dirPosMap[u.path].entry]
entry := u.entries[dirPos]
file := false
d, _ := u.d.GetDir(dirPos)
if d == nil {
file = true
}
u.boxMenu = []string{"cancel", "confirm"}
if file {
u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
if o != 1 {
return "Aborted!", nil
}
err := f.Rmdir(entry.String())
if err != nil {
return "", err
}
u.removeEntry(dirPos)
return "Successfully deleted file!", nil
}
u.popupBox([]string{
"Delete this file?",
u.fsName + entry.String()})
} else {
u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
if o != 1 {
return "Aborted!", nil
}
err := operations.Purge(f, entry.String())
if err != nil {
return "", err
}
u.removeEntry(dirPos)
return "Successfully purged folder!", nil
}
u.popupBox([]string{
"Purge this directory?",
"ALL files in it will be deleted",
u.fsName + entry.String()})
}
}
// Sort by the configured sort method
type ncduSort struct {
sortPerm []int
@@ -405,6 +557,25 @@ func (u *UI) enter() {
u.setCurrentDir(d)
}
// handles a box option that was selected
func (u *UI) handleBoxOption() {
msg, err := u.boxMenuHandler(u.f, u.path, u.boxMenuButton)
// reset
u.boxMenuButton = 0
u.boxMenu = []string{}
u.boxMenuHandler = nil
if err != nil {
u.popupBox([]string{
"error:",
err.Error(),
})
return
}
u.popupBox([]string{"Finished:", msg})
}
// up goes up to the parent directory
func (u *UI) up() {
if u.d == nil {
@@ -524,8 +695,22 @@ outer:
case termbox.KeyPgup, '=', '+':
u.move(-u.dirListHeight)
case termbox.KeyArrowLeft, 'h':
if u.showBox {
u.moveBox(-1)
break
}
u.up()
case termbox.KeyArrowRight, 'l', termbox.KeyEnter:
case termbox.KeyEnter:
if len(u.boxMenu) > 0 {
u.handleBoxOption()
break
}
u.enter()
case termbox.KeyArrowRight, 'l':
if u.showBox {
u.moveBox(1)
break
}
u.enter()
case 'c':
u.showCounts = !u.showCounts
@@ -537,6 +722,8 @@ outer:
u.toggleSort(&u.sortBySize)
case 'C':
u.toggleSort(&u.sortByCount)
case 'd':
u.delete()
case '?':
u.togglePopupBox(helpText)
+39
View File
@@ -70,6 +70,45 @@ func (d *Dir) Entries() fs.DirEntries {
return append(fs.DirEntries(nil), d.entries...)
}
// Remove removes the i-th entry from the
// in-memory representation of the remote directory
func (d *Dir) Remove(i int) {
d.mu.Lock()
defer d.mu.Unlock()
d.remove(i)
}
// removes the i-th entry from the
// in-memory representation of the remote directory
//
// Call with d.mu held
func (d *Dir) remove(i int) {
size := d.entries[i].Size()
count := int64(1)
subDir, ok := d.getDir(i)
if ok {
size = subDir.size
count = subDir.count
delete(d.dirs, path.Base(subDir.path))
}
d.size -= size
d.count -= count
d.entries = append(d.entries[:i], d.entries[i+1:]...)
dir := d
// populate changed size and count to parent(s)
for parent := d.parent; parent != nil; parent = parent.parent {
parent.mu.Lock()
parent.dirs[path.Base(dir.path)] = dir
parent.size -= size
parent.count -= count
dir = parent
parent.mu.Unlock()
}
}
// gets the directory of the i-th entry
//
// returns nil if it is a file
+9 -3
View File
@@ -25,8 +25,8 @@ const (
// startProgress starts the progress bar printing
//
// It returns a channel which should be closed to stop the stats.
func startProgress() chan struct{} {
// It returns a func which should be called to stop the stats.
func startProgress() func() {
stopStats := make(chan struct{})
oldLogPrint := fs.LogPrint
if !log.Redirected() {
@@ -36,7 +36,10 @@ func startProgress() chan struct{} {
}
}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
progressInterval := defaultProgressInterval
if ShowStats() && *statsInterval > 0 {
progressInterval = *statsInterval
@@ -54,7 +57,10 @@ func startProgress() chan struct{} {
}
}
}()
return stopStats
return func() {
close(stopStats)
wg.Wait()
}
}
// VT100 codes
+9
View File
@@ -21,6 +21,15 @@ func writeToTerminal(b []byte) {
winEventHandler := winterm.CreateWinEventHandler(os.Stdout.Fd(), os.Stdout)
ansiParser = ansiterm.CreateParser("Ground", winEventHandler)
})
// Remove all non-ASCII characters until this is fixed
// https://github.com/Azure/go-ansiterm/issues/26
r := []rune(string(b))
for i := range r {
if r[i] >= 127 {
r[i] = '.'
}
}
b = []byte(string(r))
_, err := ansiParser.Parse(b)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "\n*** Error from ANSI parser: %v\n", err)
+84 -23
View File
@@ -19,31 +19,50 @@ import (
)
var (
noOutput = false
url = "http://localhost:5572/"
noOutput = false
url = "http://localhost:5572/"
jsonInput = ""
authUser = ""
authPass = ""
)
func init() {
cmd.Root.AddCommand(commandDefintion)
commandDefintion.Flags().BoolVarP(&noOutput, "no-output", "", noOutput, "If set don't output the JSON result.")
commandDefintion.Flags().StringVarP(&url, "url", "", url, "URL to connect to rclone remote control.")
commandDefintion.Flags().StringVarP(&jsonInput, "json", "", jsonInput, "Input JSON - use instead of key=value args.")
commandDefintion.Flags().StringVarP(&authUser, "user", "", "", "Username to use to rclone remote control.")
commandDefintion.Flags().StringVarP(&authPass, "pass", "", "", "Password to use to connect to rclone remote control.")
}
var commandDefintion = &cobra.Command{
Use: "rc commands parameter",
Short: `Run a command against a running rclone.`,
Long: `
This runs a command against a running rclone. By default it will use
that specified in the --rc-addr command.
This runs a command against a running rclone. Use the --url flag to
specify an non default URL to connect on. This can be either a
":port" which is taken to mean "http://localhost:port" or a
"host:port" which is taken to mean "http://host:port"
A username and password can be passed in with --user and --pass.
Note that --rc-addr, --rc-user, --rc-pass will be read also for --url,
--user, --pass.
Arguments should be passed in as parameter=value.
The result will be returned as a JSON object by default.
The --json parameter can be used to pass in a JSON blob as an input
instead of key=value arguments. This is the only way of passing in
more complicated values.
Use "rclone rc" to see a list of all possible commands.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 1E9, command, args)
cmd.Run(false, false, command, func() error {
parseFlags()
if len(args) == 0 {
return list()
}
@@ -52,30 +71,56 @@ Use "rclone rc" to see a list of all possible commands.`,
},
}
// Parse the flags
func parseFlags() {
// set alternates from alternate flags
setAlternateFlag("rc-addr", &url)
setAlternateFlag("rc-user", &authUser)
setAlternateFlag("rc-pass", &authPass)
// If url is just :port then fix it up
if strings.HasPrefix(url, ":") {
url = "localhost" + url
}
// if url is just host:port add http://
if !strings.HasPrefix(url, "http:") && !strings.HasPrefix(url, "https:") {
url = "http://" + url
}
// if url doesn't end with / add it
if !strings.HasSuffix(url, "/") {
url += "/"
}
}
// If the user set flagName set the output to its value
func setAlternateFlag(flagName string, output *string) {
if rcFlag := pflag.Lookup(flagName); rcFlag != nil && rcFlag.Changed {
*output = rcFlag.Value.String()
}
}
// do a call from (path, in) to (out, err).
//
// if err is set, out may be a valid error return or it may be nil
func doCall(path string, in rc.Params) (out rc.Params, err error) {
// Do HTTP request
client := fshttp.NewClient(fs.Config)
url := url
// set the user use --rc-addr as well as --url
if rcAddrFlag := pflag.Lookup("rc-addr"); rcAddrFlag != nil && rcAddrFlag.Changed {
url = rcAddrFlag.Value.String()
if strings.HasPrefix(url, ":") {
url = "localhost" + url
}
url = "http://" + url + "/"
}
if !strings.HasSuffix(url, "/") {
url += "/"
}
url += path
data, err := json.Marshal(in)
if err != nil {
return nil, errors.Wrap(err, "failed to encode JSON")
}
resp, err := client.Post(url, "application/json", bytes.NewBuffer(data))
req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
if err != nil {
return nil, errors.Wrap(err, "failed to make request")
}
req.Header.Set("Content-Type", "application/json")
if authUser != "" || authPass != "" {
req.SetBasicAuth(authUser, authPass)
}
resp, err := client.Do(req)
if err != nil {
return nil, errors.Wrap(err, "connection failed")
}
@@ -115,13 +160,24 @@ func run(args []string) (err error) {
// parse input
in := make(rc.Params)
for _, param := range args[1:] {
equals := strings.IndexRune(param, '=')
if equals < 0 {
return errors.Errorf("No '=' found in parameter %q", param)
params := args[1:]
if jsonInput == "" {
for _, param := range params {
equals := strings.IndexRune(param, '=')
if equals < 0 {
return errors.Errorf("no '=' found in parameter %q", param)
}
key, value := param[:equals], param[equals+1:]
in[key] = value
}
} else {
if len(params) > 0 {
return errors.New("can't use --json and parameters together")
}
err = json.Unmarshal([]byte(jsonInput), &in)
if err != nil {
return errors.Wrap(err, "bad --json input")
}
key, value := param[:equals], param[equals+1:]
in[key] = value
}
// Do the call
@@ -155,6 +211,11 @@ func list() error {
}
fmt.Printf("### %s: %s\n\n", info["Path"], info["Title"])
fmt.Printf("%s\n\n", info["Help"])
if authRequired := info["AuthRequired"]; authRequired != nil {
if authRequired.(bool) {
fmt.Printf("Authentication is required for this call.\n\n")
}
}
}
return nil
}
+49
View File
@@ -0,0 +1,49 @@
package rcd
import (
"log"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs/rc/rcflags"
"github.com/ncw/rclone/fs/rc/rcserver"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(commandDefintion)
}
var commandDefintion = &cobra.Command{
Use: "rcd <path to files to serve>*",
Short: `Run rclone listening to remote control commands only.`,
Long: `
This runs rclone so that it only listents to remote control commands.
This is useful if you are controlling rclone via the rc API.
If you pass in a path to a directory, rclone will serve that directory
for GET requests on the URL passed in. It will also open the URL in
the browser when rclone is run.
See the [rc documentation](/rc/) for more info on the rc flags.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 1, command, args)
if rcflags.Opt.Enabled {
log.Fatalf("Don't supply --rc flag when using rcd")
}
// Start the rc
rcflags.Opt.Enabled = true
if len(args) > 0 {
rcflags.Opt.Files = args[0]
}
s, err := rcserver.Start(&rcflags.Opt)
if err != nil {
log.Fatalf("Failed to start remote control: %v", err)
}
if s == nil {
log.Fatal("rc server not configured")
}
s.Wait()
},
}
+420
View File
@@ -0,0 +1,420 @@
// Package ftp implements an FTP server for rclone
//+build !plan9
package ftp
import (
"errors"
"fmt"
"io"
"net"
"os"
"os/user"
"strconv"
"sync"
ftp "github.com/goftp/server"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/ftp/ftpflags"
"github.com/ncw/rclone/cmd/serve/ftp/ftpopt"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/fs/log"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
)
func init() {
ftpflags.AddFlags(Command.Flags())
vfsflags.AddFlags(Command.Flags())
}
// Command definition for cobra
var Command = &cobra.Command{
Use: "ftp remote:path",
Short: `Serve remote:path over FTP.`,
Long: `
rclone serve ftp implements a basic ftp server to serve the
remote over FTP protocol. This can be viewed with a ftp client
or you can make a remote of type ftp to read and write it.
` + ftpopt.Help + vfs.Help,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
s, err := newServer(f, &ftpflags.Opt)
if err != nil {
return err
}
return s.serve()
})
},
}
// server contains everything to run the server
type server struct {
f fs.Fs
srv *ftp.Server
}
// Make a new FTP to serve the remote
func newServer(f fs.Fs, opt *ftpopt.Options) (*server, error) {
host, port, err := net.SplitHostPort(opt.ListenAddr)
if err != nil {
return nil, errors.New("Failed to parse host:port")
}
portNum, err := strconv.Atoi(port)
if err != nil {
return nil, errors.New("Failed to parse host:port")
}
ftpopt := &ftp.ServerOpts{
Name: "Rclone FTP Server",
WelcomeMessage: "Welcome on Rclone FTP Server",
Factory: &DriverFactory{
vfs: vfs.New(f, &vfsflags.Opt),
},
Hostname: host,
Port: portNum,
PassivePorts: opt.PassivePorts,
Auth: &Auth{
BasicUser: opt.BasicUser,
BasicPass: opt.BasicPass,
},
Logger: &Logger{},
//TODO implement a maximum of https://godoc.org/github.com/goftp/server#ServerOpts
}
return &server{
f: f,
srv: ftp.NewServer(ftpopt),
}, nil
}
// serve runs the ftp server
func (s *server) serve() error {
fs.Logf(s.f, "Serving FTP on %s", s.srv.Hostname+":"+strconv.Itoa(s.srv.Port))
return s.srv.ListenAndServe()
}
// serve runs the ftp server
func (s *server) close() error {
fs.Logf(s.f, "Stopping FTP on %s", s.srv.Hostname+":"+strconv.Itoa(s.srv.Port))
return s.srv.Shutdown()
}
//Logger ftp logger output formatted message
type Logger struct{}
//Print log simple text message
func (l *Logger) Print(sessionID string, message interface{}) {
fs.Infof(sessionID, "%s", message)
}
//Printf log formatted text message
func (l *Logger) Printf(sessionID string, format string, v ...interface{}) {
fs.Infof(sessionID, format, v...)
}
//PrintCommand log formatted command execution
func (l *Logger) PrintCommand(sessionID string, command string, params string) {
if command == "PASS" {
fs.Infof(sessionID, "> PASS ****")
} else {
fs.Infof(sessionID, "> %s %s", command, params)
}
}
//PrintResponse log responses
func (l *Logger) PrintResponse(sessionID string, code int, message string) {
fs.Infof(sessionID, "< %d %s", code, message)
}
//Auth struct to handle ftp auth (temporary simple for POC)
type Auth struct {
BasicUser string
BasicPass string
}
//CheckPasswd handle auth based on configuration
func (a *Auth) CheckPasswd(user, pass string) (bool, error) {
return a.BasicUser == user && (a.BasicPass == "" || a.BasicPass == pass), nil
}
//DriverFactory factory of ftp driver for each session
type DriverFactory struct {
vfs *vfs.VFS
}
//NewDriver start a new session
func (f *DriverFactory) NewDriver() (ftp.Driver, error) {
log.Trace("", "Init driver")("")
return &Driver{
vfs: f.vfs,
}, nil
}
//Driver impletation of ftp server
type Driver struct {
vfs *vfs.VFS
lock sync.Mutex
}
//Init a connection
func (d *Driver) Init(*ftp.Conn) {
defer log.Trace("", "Init session")("")
}
//Stat get information on file or folder
func (d *Driver) Stat(path string) (fi ftp.FileInfo, err error) {
defer log.Trace(path, "")("fi=%+v, err = %v", &fi, &err)
n, err := d.vfs.Stat(path)
if err != nil {
return nil, err
}
return &FileInfo{n, n.Mode(), d.vfs.Opt.UID, d.vfs.Opt.GID}, err
}
//ChangeDir move current folder
func (d *Driver) ChangeDir(path string) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(path, "")("err = %v", &err)
n, err := d.vfs.Stat(path)
if err != nil {
return err
}
if !n.IsDir() {
return errors.New("Not a directory")
}
return nil
}
//ListDir list content of a folder
func (d *Driver) ListDir(path string, callback func(ftp.FileInfo) error) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(path, "")("err = %v", &err)
node, err := d.vfs.Stat(path)
if err == vfs.ENOENT {
return errors.New("Directory not found")
} else if err != nil {
return err
}
if !node.IsDir() {
return errors.New("Not a directory")
}
dir := node.(*vfs.Dir)
dirEntries, err := dir.ReadDirAll()
if err != nil {
return err
}
// Account the transfer
accounting.Stats.Transferring(path)
defer accounting.Stats.DoneTransferring(path, true)
for _, file := range dirEntries {
err = callback(&FileInfo{file, file.Mode(), d.vfs.Opt.UID, d.vfs.Opt.GID})
if err != nil {
return err
}
}
return nil
}
//DeleteDir delete a folder and his content
func (d *Driver) DeleteDir(path string) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(path, "")("err = %v", &err)
node, err := d.vfs.Stat(path)
if err != nil {
return err
}
if !node.IsDir() {
return errors.New("Not a directory")
}
err = node.Remove()
if err != nil {
return err
}
return nil
}
//DeleteFile delete a file
func (d *Driver) DeleteFile(path string) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(path, "")("err = %v", &err)
node, err := d.vfs.Stat(path)
if err != nil {
return err
}
if !node.IsFile() {
return errors.New("Not a file")
}
err = node.Remove()
if err != nil {
return err
}
return nil
}
//Rename rename a file or folder
func (d *Driver) Rename(oldName, newName string) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(oldName, "newName=%q", newName)("err = %v", &err)
return d.vfs.Rename(oldName, newName)
}
//MakeDir create a folder
func (d *Driver) MakeDir(path string) (err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(path, "")("err = %v", &err)
dir, leaf, err := d.vfs.StatParent(path)
if err != nil {
return err
}
_, err = dir.Mkdir(leaf)
return err
}
//GetFile download a file
func (d *Driver) GetFile(path string, offset int64) (size int64, fr io.ReadCloser, err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(path, "offset=%v", offset)("err = %v", &err)
node, err := d.vfs.Stat(path)
if err == vfs.ENOENT {
fs.Infof(path, "File not found")
return 0, nil, errors.New("File not found")
} else if err != nil {
return 0, nil, err
}
if !node.IsFile() {
return 0, nil, errors.New("Not a file")
}
handle, err := node.Open(os.O_RDONLY)
if err != nil {
return 0, nil, err
}
_, err = handle.Seek(offset, os.SEEK_SET)
if err != nil {
return 0, nil, err
}
// Account the transfer
accounting.Stats.Transferring(path)
defer accounting.Stats.DoneTransferring(path, true)
return node.Size(), handle, nil
}
//PutFile upload a file
func (d *Driver) PutFile(path string, data io.Reader, appendData bool) (n int64, err error) {
d.lock.Lock()
defer d.lock.Unlock()
defer log.Trace(path, "append=%v", appendData)("err = %v", &err)
var isExist bool
node, err := d.vfs.Stat(path)
if err == nil {
isExist = true
if node.IsDir() {
return 0, errors.New("A dir has the same name")
}
} else {
if os.IsNotExist(err) {
isExist = false
} else {
return 0, err
}
}
if appendData && !isExist {
appendData = false
}
if !appendData {
if isExist {
err = node.Remove()
if err != nil {
return 0, err
}
}
f, err := d.vfs.OpenFile(path, os.O_RDWR|os.O_CREATE, 0660)
if err != nil {
return 0, err
}
defer closeIO(path, f)
bytes, err := io.Copy(f, data)
if err != nil {
return 0, err
}
return bytes, nil
}
of, err := d.vfs.OpenFile(path, os.O_APPEND|os.O_RDWR, 0660)
if err != nil {
return 0, err
}
defer closeIO(path, of)
_, err = of.Seek(0, os.SEEK_END)
if err != nil {
return 0, err
}
bytes, err := io.Copy(of, data)
if err != nil {
return 0, err
}
return bytes, nil
}
//FileInfo struct ot hold file infor for ftp server
type FileInfo struct {
os.FileInfo
mode os.FileMode
owner uint32
group uint32
}
//Mode return êrm mode of file.
func (f *FileInfo) Mode() os.FileMode {
return f.mode
}
//Owner return owner of file. Try to find the username if possible
func (f *FileInfo) Owner() string {
str := fmt.Sprint(f.owner)
u, err := user.LookupId(str)
if err != nil {
return str //User not found
}
return u.Username
}
//Group return group of file. Try to find the group name if possible
func (f *FileInfo) Group() string {
str := fmt.Sprint(f.group)
g, err := user.LookupGroupId(str)
if err != nil {
return str //Group not found default to numrical value
}
return g.Name
}
func closeIO(path string, c io.Closer) {
err := c.Close()
if err != nil {
log.Trace(path, "")("err = %v", &err)
}
}
+89
View File
@@ -0,0 +1,89 @@
// Serve ftp tests set up a server and run the integration tests
// for the ftp remote against it.
//
// We skip tests on platforms with troublesome character mappings
//+build !windows,!darwin,!plan9
package ftp
import (
"fmt"
"os"
"os/exec"
"testing"
ftp "github.com/goftp/server"
_ "github.com/ncw/rclone/backend/local"
"github.com/ncw/rclone/cmd/serve/ftp/ftpopt"
"github.com/ncw/rclone/fstest"
"github.com/stretchr/testify/assert"
)
const (
testHOST = "localhost"
testPORT = "51780"
testPASSIVEPORTRANGE = "30000-32000"
)
// TestFTP runs the ftp server then runs the unit tests for the
// ftp remote against it.
func TestFTP(t *testing.T) {
opt := ftpopt.DefaultOpt
opt.ListenAddr = testHOST + ":" + testPORT
opt.PassivePorts = testPASSIVEPORTRANGE
opt.BasicUser = "rclone"
opt.BasicPass = "password"
fstest.Initialise()
fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
assert.NoError(t, err)
defer clean()
err = fremote.Mkdir("")
assert.NoError(t, err)
// Start the server
w, err := newServer(fremote, &opt)
assert.NoError(t, err)
go func() {
err := w.serve()
if err != ftp.ErrServerClosed {
assert.NoError(t, err)
}
}()
defer func() {
err := w.close()
assert.NoError(t, err)
}()
// Change directory to run the tests
err = os.Chdir("../../../backend/ftp")
assert.NoError(t, err, "failed to cd to ftp remote")
// Run the ftp tests with an on the fly remote
args := []string{"test"}
if testing.Verbose() {
args = append(args, "-v")
}
if *fstest.Verbose {
args = append(args, "-verbose")
}
args = append(args, "-list-retries", fmt.Sprint(*fstest.ListRetries))
args = append(args, "-remote", "ftptest:")
cmd := exec.Command("go", args...)
cmd.Env = append(os.Environ(),
"RCLONE_CONFIG_FTPTEST_TYPE=ftp",
"RCLONE_CONFIG_FTPTEST_HOST="+testHOST,
"RCLONE_CONFIG_FTPTEST_PORT="+testPORT,
"RCLONE_CONFIG_FTPTEST_USER=rclone",
"RCLONE_CONFIG_FTPTEST_PASS=0HU5Hx42YiLoNGJxppOOP3QTbr-KB_MP", // ./rclone obscure password
)
out, err := cmd.CombinedOutput()
if len(out) != 0 {
t.Logf("\n----------\n%s----------\n", string(out))
}
assert.NoError(t, err, "Running ftp integration tests")
}
+11
View File
@@ -0,0 +1,11 @@
// Build for unsupported platforms to stop go complaining
// about "no buildable Go source files "
// +build plan9
package ftp
import "github.com/spf13/cobra"
// Command definition is nil to show not implemented
var Command *cobra.Command = nil
+27
View File
@@ -0,0 +1,27 @@
package ftpflags
import (
"github.com/ncw/rclone/cmd/serve/ftp/ftpopt"
"github.com/ncw/rclone/fs/config/flags"
"github.com/ncw/rclone/fs/rc"
"github.com/spf13/pflag"
)
// Options set by command line flags
var (
Opt = ftpopt.DefaultOpt
)
// AddFlagsPrefix adds flags for the ftpopt
func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *ftpopt.Options) {
rc.AddOption("ftp", &Opt)
flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.")
flags.StringVarP(flagSet, &Opt.PassivePorts, prefix+"passive-port", "", Opt.PassivePorts, "Passive port range to use.")
flags.StringVarP(flagSet, &Opt.BasicUser, prefix+"user", "", Opt.BasicUser, "User name for authentication.")
flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication. (empty value allow every password)")
}
// AddFlags adds flags for the httplib
func AddFlags(flagSet *pflag.FlagSet) {
AddFlagsPrefix(flagSet, "", &Opt)
}
+38
View File
@@ -0,0 +1,38 @@
package ftpopt
// Help contains text describing the http server to add to the command
// help.
var Help = `
### Server options
Use --addr to specify which IP address and port the server should
listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
IPs. By default it only listens on localhost. You can use port
:0 to let the OS choose an available port.
If you set --addr to listen on a public or LAN accessible IP address
then using Authentication is advised - see the next section for info.
#### Authentication
By default this will serve files without needing a login.
You can set a single username and password with the --user and --pass flags.
`
// Options contains options for the http Server
type Options struct {
//TODO add more options
ListenAddr string // Port to listen on
PassivePorts string // Passive ports range
BasicUser string // single username for basic auth if not using Htpasswd
BasicPass string // password for BasicUser
}
// DefaultOpt is the default values used for Options
var DefaultOpt = Options{
ListenAddr: "localhost:2121",
PassivePorts: "30000-32000",
BasicUser: "anonymous",
BasicPass: "",
}
+26 -89
View File
@@ -1,8 +1,6 @@
package http
import (
"fmt"
"html/template"
"net/http"
"os"
"path"
@@ -12,9 +10,9 @@ import (
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
"github.com/ncw/rclone/cmd/serve/httplib/serve"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/accounting"
"github.com/ncw/rclone/lib/rest"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
@@ -46,7 +44,11 @@ control the stats printing.
f := cmd.NewFsSrc(args)
cmd.Run(false, true, command, func() error {
s := newServer(f, &httpflags.Opt)
s.serve()
err := s.Serve()
if err != nil {
return err
}
s.Wait()
return nil
})
},
@@ -54,30 +56,32 @@ control the stats printing.
// server contains everything to run the server
type server struct {
*httplib.Server
f fs.Fs
vfs *vfs.VFS
srv *httplib.Server
}
func newServer(f fs.Fs, opt *httplib.Options) *server {
mux := http.NewServeMux()
s := &server{
f: f,
vfs: vfs.New(f, &vfsflags.Opt),
srv: httplib.NewServer(mux, opt),
Server: httplib.NewServer(mux, opt),
f: f,
vfs: vfs.New(f, &vfsflags.Opt),
}
mux.HandleFunc("/", s.handler)
return s
}
// serve runs the http server - doesn't return
func (s *server) serve() {
err := s.srv.Serve()
// Serve runs the http server in the background.
//
// Use s.Close() and s.Wait() to shutdown server
func (s *server) Serve() error {
err := s.Server.Serve()
if err != nil {
fs.Errorf(s.f, "Opening listener: %v", err)
return err
}
fs.Logf(s.f, "Serving on %s", s.srv.URL())
s.srv.Wait()
fs.Logf(s.f, "Serving on %s", s.URL())
return nil
}
// handler reads incoming requests and dispatches them
@@ -99,62 +103,6 @@ func (s *server) handler(w http.ResponseWriter, r *http.Request) {
}
}
// entry is a directory entry
type entry struct {
remote string
URL string
Leaf string
}
// entries represents a directory
type entries []entry
// addEntry adds an entry to that directory
func (es *entries) addEntry(node interface {
Path() string
Name() string
IsDir() bool
}) {
remote := node.Path()
leaf := node.Name()
urlRemote := leaf
if node.IsDir() {
leaf += "/"
urlRemote += "/"
}
*es = append(*es, entry{remote: remote, URL: rest.URLPathEscape(urlRemote), Leaf: leaf})
}
// indexPage is a directory listing template
var indexPage = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
</head>
<body>
<h1>{{ .Title }}</h1>
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
{{ end }}</body>
</html>
`
// indexTemplate is the instantiated indexPage
var indexTemplate = template.Must(template.New("index").Parse(indexPage))
// indexData is used to fill in the indexTemplate
type indexData struct {
Title string
Entries entries
}
// error returns an http.StatusInternalServerError and logs the error
func internalError(what interface{}, w http.ResponseWriter, text string, err error) {
fs.CountError(err)
fs.Errorf(what, "%s: %v", text, err)
http.Error(w, text+".", http.StatusInternalServerError)
}
// serveDir serves a directory index at dirRemote
func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
// List the directory
@@ -163,7 +111,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
http.Error(w, "Directory not found", http.StatusNotFound)
return
} else if err != nil {
internalError(dirRemote, w, "Failed to list directory", err)
serve.Error(dirRemote, w, "Failed to list directory", err)
return
}
if !node.IsDir() {
@@ -173,28 +121,17 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
dir := node.(*vfs.Dir)
dirEntries, err := dir.ReadDirAll()
if err != nil {
internalError(dirRemote, w, "Failed to list directory", err)
serve.Error(dirRemote, w, "Failed to list directory", err)
return
}
var out entries
// Make the entries for display
directory := serve.NewDirectory(dirRemote)
for _, node := range dirEntries {
out.addEntry(node)
directory.AddEntry(node.Path(), node.IsDir())
}
// Account the transfer
accounting.Stats.Transferring(dirRemote)
defer accounting.Stats.DoneTransferring(dirRemote, true)
fs.Infof(dirRemote, "%s: Serving directory", r.RemoteAddr)
err = indexTemplate.Execute(w, indexData{
Entries: out,
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
})
if err != nil {
internalError(dirRemote, w, "Failed to render template", err)
return
}
directory.Serve(w, r)
}
// serveFile serves a file object at remote
@@ -205,7 +142,7 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
http.Error(w, "File not found", http.StatusNotFound)
return
} else if err != nil {
internalError(remote, w, "Failed to find file", err)
serve.Error(remote, w, "Failed to find file", err)
return
}
if !node.IsFile() {
@@ -239,7 +176,7 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
// open the object
in, err := file.Open(os.O_RDONLY)
if err != nil {
internalError(remote, w, "Failed to open file", err)
serve.Error(remote, w, "Failed to open file", err)
return
}
defer func() {
+3 -33
View File
@@ -7,7 +7,6 @@ import (
"io/ioutil"
"net"
"net/http"
"path"
"strings"
"testing"
"time"
@@ -35,7 +34,7 @@ func startServer(t *testing.T, f fs.Fs) {
opt := httplib.DefaultOpt
opt.ListenAddr = testBindAddress
httpServer = newServer(f, &opt)
go httpServer.serve()
assert.NoError(t, httpServer.Serve())
// try to connect to the test server
pause := time.Millisecond
@@ -202,36 +201,7 @@ func TestGET(t *testing.T) {
}
}
type mockNode struct {
path string
isdir bool
}
func (n mockNode) Path() string { return n.path }
func (n mockNode) Name() string {
if n.path == "" {
return ""
}
return path.Base(n.path)
}
func (n mockNode) IsDir() bool { return n.isdir }
func TestAddEntry(t *testing.T) {
var es entries
es.addEntry(mockNode{path: "", isdir: true})
es.addEntry(mockNode{path: "dir", isdir: true})
es.addEntry(mockNode{path: "a/b/c/d.txt", isdir: false})
es.addEntry(mockNode{path: "a/b/c/colon:colon.txt", isdir: false})
es.addEntry(mockNode{path: "\"quotes\".txt", isdir: false})
assert.Equal(t, entries{
{remote: "", URL: "/", Leaf: "/"},
{remote: "dir", URL: "dir/", Leaf: "dir/"},
{remote: "a/b/c/d.txt", URL: "d.txt", Leaf: "d.txt"},
{remote: "a/b/c/colon:colon.txt", URL: "./colon:colon.txt", Leaf: "colon:colon.txt"},
{remote: "\"quotes\".txt", URL: "%22quotes%22.txt", Leaf: "\"quotes\".txt"},
}, es)
}
func TestFinalise(t *testing.T) {
httpServer.srv.Close()
httpServer.Close()
httpServer.Wait()
}
Loaded 100 of 705 files, more files were not shown because too many files have changed in this diff. Show more