Compare commits

..
Author SHA1 Message Date
Nick Craig-Wood c741c02fb6 dropbox: make sure we wait for previous batch to finish before starting a new one 2020-09-08 15:03:31 +01:00
Nick Craig-Wood 56e8d75cab dropbox: add --dropbox-batch flag to speed up uploading of lots of small files 2020-09-08 15:03:31 +01:00
Nick Craig-Wood e74f5b8906 dropbox: raise priority of rate limited message to INFO to make it more noticeable
If you exceed rate limits, dropbox tells you to wait for 300 seconds -
this is rather a long time for the user to be waiting for rclone to
finish, so emit a NOTICE level log instead of a DEBUG.
2020-09-08 15:03:31 +01:00
Nick Craig-Wood e4a7686444 accounting: remove new line from end of --stats-one-line display 2020-09-08 15:03:23 +01:00
Tim Gallant c3884aafd9 drive: adds special oauth help test - fixes #4555 2020-09-07 12:48:46 +01:00
Nick Craig-Wood 0a9785a4ff build: don't explicitly set ARM version to fix ARMv5 build #4553
This partially reverts commit f71f6c57d7.
2020-09-07 12:39:26 +01:00
Nick Craig-Wood 8140f67092 check: fix docs
See: https://forum.rclone.org/t/possible-issue-with-documention/18926
2020-09-07 12:10:52 +01:00
Nick Craig-Wood 4a001b8a02 check: add back missing --download flag - fixes #4565
This was accidentally removed when refactoring check and cryptcheck in

8b6f2bbb4b check,cryptcheck: add reporting of filenames for same/missing/changed #3264
2020-09-05 09:29:35 +01:00
Nick Craig-Wood 525433e6dd build: fix "Illegal instruction" error for ARMv6 builds - fixes #4553
Before this change we used `go build -i` to build the releases in parallel.

However this causes the ARMv6 and ARMv7 build to get mixed up somehow,
causing an illegal instruction when running rclone binaries on ARMv6.

See go bug: https://github.com/golang/go/issues/41223

This removes the -i which should have no effect on build times on the
CI and appears to fix the problem.
2020-09-04 16:30:50 +01:00
Nick Craig-Wood f71f6c57d7 build: explicitly set ARM version to fix build #4553 2020-09-04 16:30:50 +01:00
albertony e35623c72e docs/jottacloud: mention that uploads from local disk will not need to cache files to disk for md5 calculation 2020-09-04 00:04:09 +02:00
Nick Craig-Wood 344bce7e2a docs: fix formatting of rc docs page
See: https://forum.rclone.org/t/rclone-1-53-release/18880/24
2020-09-03 11:53:24 +01:00
Nick Craig-Wood 3a4322a7ba build: update build for stable branch 2020-09-03 11:30:23 +01:00
Nick Craig-Wood 27b9ae4fc3 vfs: fix spurious error "vfs cache: failed to _ensure cache EOF"
Before this change the error message was produced for every file which
was confusing users.

After this change we check for EOF and return from ReadAt at that
point.

See: https://forum.rclone.org/t/rclone-1-53-release/18880/10
2020-09-03 10:25:00 +01:00
Nick Craig-Wood 7e2488af10 build: include vendor tar ball in release and fix startdev 2020-09-02 17:53:05 +01:00
Nick Craig-Wood 41ecb586c4 Start v1.54.0-DEV development 2020-09-02 17:52:58 +01:00
103 changed files with 25728 additions and 35795 deletions

No files matched your search

+9 -12
View File
@@ -46,7 +46,6 @@ jobs:
go: '1.15.x'
gotags: cmount
build_flags: '-include "^windows/amd64" -cgo'
build_args: '-buildmode exe'
quicktest: true
racequicktest: true
deploy: true
@@ -58,7 +57,6 @@ jobs:
goarch: '386'
cgo: '1'
build_flags: '-include "^windows/386" -cgo'
build_args: '-buildmode exe'
quicktest: true
deploy: true
@@ -109,11 +107,10 @@ jobs:
- name: Set environment variables
shell: bash
run: |
echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV
if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi
if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
echo '::set-env name=GOTAGS::${{ matrix.gotags }}'
echo '::set-env name=BUILD_FLAGS::${{ matrix.build_flags }}'
if [[ "${{ matrix.goarch }}" != "" ]]; then echo '::set-env name=GOARCH::${{ matrix.goarch }}' ; fi
if [[ "${{ matrix.cgo }}" != "" ]]; then echo '::set-env name=CGO_ENABLED::${{ matrix.cgo }}' ; fi
- name: Install Libraries on Linux
shell: bash
@@ -128,7 +125,7 @@ jobs:
shell: bash
run: |
brew update
brew install --cask osxfuse
brew cask install osxfuse
if: matrix.os == 'macOS-latest'
- name: Install Libraries on Windows
@@ -136,10 +133,10 @@ jobs:
run: |
$ProgressPreference = 'SilentlyContinue'
choco install -y winfsp zip
echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
Write-Host "::set-env name=CPATH::C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse"
if ($env:GOARCH -eq "386") {
choco install -y mingw --forcex86 --force
echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
Write-Host "::add-path::C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin"
}
# Copy mingw32-make.exe to make.exe so the same command line
# can be used on Windows as on macOS and Linux
@@ -226,8 +223,8 @@ jobs:
- name: Set environment variables
shell: bash
run: |
echo 'GOPATH=${{ runner.workspace }}' >> $GITHUB_ENV
echo '${{ runner.workspace }}/bin' >> $GITHUB_PATH
echo '::set-env name=GOPATH::${{ runner.workspace }}'
echo '::add-path::${{ runner.workspace }}/bin'
- name: Cross-compile rclone
run: |
@@ -15,7 +15,7 @@ jobs:
with:
fetch-depth: 0
- name: Build and publish image
uses: ilteoood/docker_buildx@1.1.0
uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
with:
tag: beta
imageName: rclone/rclone
@@ -23,7 +23,7 @@ jobs:
id: actual_major_version
run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
- name: Build and publish image
uses: ilteoood/docker_buildx@1.1.0
uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
with:
tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
imageName: rclone/rclone
Generated
+8086 -9478
View File
File diff suppressed because it is too large. Load diff
Generated
+131 -576
View File
File diff suppressed because it is too large. Load diff
Generated
+7713 -9149
View File
File diff suppressed because it is too large. Load diff
+8 -8
View File
@@ -46,13 +46,13 @@ endif
.PHONY: rclone test_all vars version
rclone:
go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS)
go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS)
mkdir -p `go env GOPATH`/bin/
cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
test_all:
go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/rclone/rclone/fstest/test_all
vars:
@echo SHELL="'$(SHELL)'"
@@ -188,10 +188,10 @@ upload_github:
./bin/upload-github $(TAG)
cross: doc
go run bin/cross-compile.go -release current $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
go run bin/cross-compile.go -release current $(BUILDTAGS) $(TAG)
beta:
go run bin/cross-compile.go $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
go run bin/cross-compile.go $(BUILDTAGS) $(TAG)
rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
@echo Beta release ready at https://pub.rclone.org/$(TAG)/
@@ -199,23 +199,23 @@ log_since_last_release:
git log $(LAST_TAG)..
compile_all:
go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(TAG)
ci_upload:
sudo chown -R $$USER build
find build -type l -delete
gzip -r9v build
./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
ifndef BRANCH_PATH
./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
endif
@echo Beta release ready at $(BETA_URL)/testbuilds
ci_beta:
git log $(LAST_TAG).. > /tmp/git-log.txt
go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
ifndef BRANCH_PATH
rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
endif
@echo Beta release ready at $(BETA_URL)
-1
View File
@@ -64,7 +64,6 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
* StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
* SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
* Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/)
* Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
* Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
* WebDAV [:page_facing_up:](https://rclone.org/webdav/)
* Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
+2 -1
View File
@@ -65,8 +65,9 @@ Now
* git cherry-pick any fixes
* Do the steps as above
* make startstable
* NB this overwrites the current beta so we need to do this - FIXME is this true any more?
* git co master
* `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
* # cherry pick the changes to the changelog
* git checkout ${BASE_TAG}-stable docs/content/changelog.md
* git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
* git push
+1 -1
View File
@@ -1 +1 @@
v1.53.4
v1.54.0
+87 -155
View File
@@ -121,8 +121,6 @@ const maxTransactionProbes = 100
// standard chunker errors
var (
ErrChunkOverflow = errors.New("chunk number overflow")
ErrMetaTooBig = errors.New("metadata is too big")
ErrMetaUnknown = errors.New("unknown metadata, please upgrade rclone")
)
// variants of baseMove's parameter delMode
@@ -298,8 +296,6 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
ServerSideAcrossConfigs: true,
}).Fill(f).Mask(baseFs).WrapsFs(f, baseFs)
f.features.Disable("ListR") // Recursive listing may cause chunker skip files
return f, err
}
@@ -695,47 +691,43 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
switch entry := dirOrObject.(type) {
case fs.Object:
remote := entry.Remote()
mainRemote, chunkNo, ctrlType, xactID := f.parseChunkName(remote)
if mainRemote == "" {
// this is meta object or standalone file
object := f.newObject("", entry, nil)
byRemote[remote] = object
tempEntries = append(tempEntries, object)
break
}
// this is some kind of chunk
// metobject should have been created above if present
isSpecial := xactID != "" || ctrlType != ""
mainObject := byRemote[mainRemote]
if mainObject == nil && f.useMeta && !isSpecial {
fs.Debugf(f, "skip orphan data chunk %q", remote)
break
}
if mainObject == nil && !f.useMeta {
// this is the "nometa" case
// create dummy chunked object without metadata
mainObject = f.newObject(mainRemote, nil, nil)
byRemote[mainRemote] = mainObject
if !badEntry[mainRemote] {
tempEntries = append(tempEntries, mainObject)
if mainRemote, chunkNo, ctrlType, xactID := f.parseChunkName(remote); mainRemote != "" {
if xactID != "" {
if revealHidden {
fs.Infof(f, "ignore temporary chunk %q", remote)
}
break
}
}
if isSpecial {
if revealHidden {
fs.Infof(f, "ignore non-data chunk %q", remote)
if ctrlType != "" {
if revealHidden {
fs.Infof(f, "ignore control chunk %q", remote)
}
break
}
// need to read metadata to ensure actual object type
if f.useMeta && mainObject != nil && mainObject.size <= maxMetadataSize {
mainObject.unsure = true
mainObject := byRemote[mainRemote]
if mainObject == nil && f.useMeta {
fs.Debugf(f, "skip chunk %q without meta object", remote)
break
}
if mainObject == nil {
// useMeta is false - create chunked object without metadata
mainObject = f.newObject(mainRemote, nil, nil)
byRemote[mainRemote] = mainObject
if !badEntry[mainRemote] {
tempEntries = append(tempEntries, mainObject)
}
}
if err := mainObject.addChunk(entry, chunkNo); err != nil {
if f.opt.FailHard {
return nil, err
}
badEntry[mainRemote] = true
}
break
}
if err := mainObject.addChunk(entry, chunkNo); err != nil {
if f.opt.FailHard {
return nil, err
}
badEntry[mainRemote] = true
}
object := f.newObject("", entry, nil)
byRemote[remote] = object
tempEntries = append(tempEntries, object)
case fs.Directory:
isSubdir[entry.Remote()] = true
wrapDir := fs.NewDirCopy(ctx, entry)
@@ -790,13 +782,6 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
// but opening even a small file can be slow on some backends.
//
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return f.scanObject(ctx, remote, false)
}
// scanObject is like NewObject with optional quick scan mode.
// The quick mode avoids directory requests other than `List`,
// ignores non-chunked objects and skips chunk size checks.
func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.Object, error) {
if err := f.forbidChunk(false, remote); err != nil {
return nil, errors.Wrap(err, "can't access")
}
@@ -857,15 +842,8 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
continue // bypass regexp to save cpu
}
mainRemote, chunkNo, ctrlType, xactID := f.parseChunkName(entryRemote)
if mainRemote == "" || mainRemote != remote {
continue // skip non-conforming chunks
}
if ctrlType != "" || xactID != "" {
if f.useMeta {
// temporary/control chunk calls for lazy metadata read
o.unsure = true
}
continue
if mainRemote == "" || mainRemote != remote || ctrlType != "" || xactID != "" {
continue // skip non-conforming, temporary and control chunks
}
//fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
if err := o.addChunk(entry, chunkNo); err != nil {
@@ -875,7 +853,7 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
if o.main == nil && (o.chunks == nil || len(o.chunks) == 0) {
// Scanning hasn't found data chunks with conforming names.
if f.useMeta || quickScan {
if f.useMeta {
// Metadata is required but absent and there are no chunks.
return nil, fs.ErrorObjectNotFound
}
@@ -898,10 +876,8 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
// file without metadata. Validate it and update the total data size.
// As an optimization, skip metadata reading here - we will call
// readMetadata lazily when needed (reading can be expensive).
if !quickScan {
if err := o.validate(); err != nil {
return nil, err
}
if err := o.validate(); err != nil {
return nil, err
}
return o, nil
}
@@ -910,24 +886,13 @@ func (o *Object) readMetadata(ctx context.Context) error {
if o.isFull {
return nil
}
if !o.f.useMeta || (!o.isComposite() && !o.unsure) {
if !o.isComposite() || !o.f.useMeta {
o.isFull = true
return nil
}
// validate metadata
metaObject := o.main
if metaObject.Size() > maxMetadataSize {
if o.unsure {
// this is not metadata but a foreign object
o.unsure = false
o.chunks = nil // make isComposite return false
o.isFull = true // cache results
return nil
}
return ErrMetaTooBig
}
reader, err := metaObject.Open(ctx)
if err != nil {
return err
@@ -940,22 +905,8 @@ func (o *Object) readMetadata(ctx context.Context) error {
switch o.f.opt.MetaFormat {
case "simplejson":
metaInfo, madeByChunker, err := unmarshalSimpleJSON(ctx, metaObject, metadata)
if o.unsure {
o.unsure = false
if !madeByChunker {
// this is not metadata but a foreign object
o.chunks = nil // make isComposite return false
o.isFull = true // cache results
return nil
}
}
switch err {
case nil:
// fall thru
case ErrMetaTooBig, ErrMetaUnknown:
return err // return these errors unwrapped for unit tests
default:
metaInfo, err := unmarshalSimpleJSON(ctx, metaObject, metadata, true)
if err != nil {
return errors.Wrap(err, "invalid metadata")
}
if o.size != metaInfo.Size() || len(o.chunks) != metaInfo.nChunks {
@@ -970,27 +921,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
}
// put implements Put, PutStream, PutUnchecked, Update
func (f *Fs) put(
ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
basePut putFn, action string, target fs.Object) (obj fs.Object, err error) {
if err := f.forbidChunk(src, remote); err != nil {
return nil, errors.Wrap(err, action+" refused")
}
if target == nil {
// Get target object with a quick directory scan
if obj, err := f.scanObject(ctx, remote, true); err == nil {
target = obj
}
}
if target != nil {
obj := target.(*Object)
if err := obj.readMetadata(ctx); err == ErrMetaUnknown {
// refuse to update a file of unsupported format
return nil, errors.Wrap(err, "refusing to "+action)
}
}
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption, basePut putFn) (obj fs.Object, err error) {
c := f.newChunkingReader(src)
wrapIn := c.wrapStream(ctx, in, src)
@@ -1027,8 +958,6 @@ func (f *Fs) put(
}
info := f.wrapInfo(src, chunkRemote, size)
// Refill chunkLimit and let basePut repeatedly call chunkingReader.Read()
c.chunkLimit = c.chunkSize
// TODO: handle range/limit options
chunk, errChunk := basePut(ctx, wrapIn, info, options...)
if errChunk != nil {
@@ -1080,8 +1009,8 @@ func (f *Fs) put(
// Check for input that looks like valid metadata
needMeta := len(c.chunks) > 1
if c.readCount <= maxMetadataSize && len(c.chunks) == 1 {
_, madeByChunker, _ := unmarshalSimpleJSON(ctx, c.chunks[0], c.smallHead)
needMeta = madeByChunker
_, err := unmarshalSimpleJSON(ctx, c.chunks[0], c.smallHead, false)
needMeta = err == nil
}
// Finalize small object as non-chunked.
@@ -1237,14 +1166,10 @@ func (c *chunkingReader) updateHashes() {
func (c *chunkingReader) Read(buf []byte) (bytesRead int, err error) {
if c.chunkLimit <= 0 {
// Chunk complete - switch to next one.
// Note #1:
// We might not get here because some remotes (eg. box multi-uploader)
// read the specified size exactly and skip the concluding EOF Read.
// Then a check in the put loop will kick in.
// Note #2:
// The crypt backend after receiving EOF here will call Read again
// and we must insist on returning EOF, so we postpone refilling
// chunkLimit to the main loop.
c.chunkLimit = c.chunkSize
return 0, io.EOF
}
if int64(len(buf)) > c.chunkLimit {
@@ -1328,16 +1253,29 @@ func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return f.put(ctx, in, src, src.Remote(), options, f.base.Put, "put", nil)
if err := f.forbidChunk(src, src.Remote()); err != nil {
return nil, errors.Wrap(err, "refusing to put")
}
return f.put(ctx, in, src, src.Remote(), options, f.base.Put)
}
// PutStream uploads to the remote path with the modTime given of indeterminate size
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return f.put(ctx, in, src, src.Remote(), options, f.base.Features().PutStream, "upload", nil)
if err := f.forbidChunk(src, src.Remote()); err != nil {
return nil, errors.Wrap(err, "refusing to upload")
}
return f.put(ctx, in, src, src.Remote(), options, f.base.Features().PutStream)
}
// Update in to the object with the modTime given of the given size
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
if err := o.f.forbidChunk(o, o.Remote()); err != nil {
return errors.Wrap(err, "update refused")
}
if err := o.readMetadata(ctx); err != nil {
// refuse to update a file of unsupported format
return errors.Wrap(err, "refusing to update")
}
basePut := o.f.base.Put
if src.Size() < 0 {
basePut = o.f.base.Features().PutStream
@@ -1345,7 +1283,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return errors.New("wrapped file system does not support streaming uploads")
}
}
oNew, err := o.f.put(ctx, in, src, o.Remote(), options, basePut, "update", o)
oNew, err := o.f.put(ctx, in, src, o.Remote(), options, basePut)
if err == nil {
*o = *oNew.(*Object)
}
@@ -1459,7 +1397,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
// to corrupt file in hard mode. Hence, refuse to Remove, too.
return errors.Wrap(err, "refuse to corrupt")
}
if err := o.readMetadata(ctx); err == ErrMetaUnknown {
if err := o.readMetadata(ctx); err != nil {
// Proceed but warn user that unexpected things can happen.
fs.Errorf(o, "Removing a file with unsupported metadata: %v", err)
}
@@ -1487,11 +1425,6 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
if err := f.forbidChunk(o, remote); err != nil {
return nil, errors.Wrapf(err, "can't %s", opName)
}
if err := o.readMetadata(ctx); err != nil {
// Refuse to copy/move composite files with invalid or future
// metadata format which might involve unsupported chunk types.
return nil, errors.Wrapf(err, "can't %s this file", opName)
}
if !o.isComposite() {
fs.Debugf(o, "%s non-chunked object...", opName)
oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
@@ -1500,6 +1433,11 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
}
return f.newObject("", oResult, nil), nil
}
if err := o.readMetadata(ctx); err != nil {
// Refuse to copy/move composite files with invalid or future
// metadata format which might involve unsupported chunk types.
return nil, errors.Wrapf(err, "can't %s this file", opName)
}
fs.Debugf(o, "%s %d data chunks...", opName, len(o.chunks))
mainRemote := o.remote
@@ -1590,10 +1528,6 @@ func (f *Fs) okForServerSide(ctx context.Context, src fs.Object, opName string)
return
}
if obj.unsure {
// ensure object is composite if need to re-read metadata
_ = obj.readMetadata(ctx)
}
requireMetaHash := obj.isComposite() && f.opt.MetaFormat == "simplejson"
if !requireMetaHash && !f.hashAll {
ok = true // hash is not required for metadata
@@ -1777,7 +1711,6 @@ type Object struct {
chunks []fs.Object // active data chunks if file is composite, or wrapped file as a single chunk if meta format is 'none'
size int64 // cached total size of chunks in a composite file or -1 for non-chunked files
isFull bool // true if metadata has been read
unsure bool // true if need to read metadata to detect object type
md5 string
sha1 string
f *Fs
@@ -1928,16 +1861,15 @@ func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
// on the level of wrapped remote but chunker is unaware of that.
//
func (o *Object) Hash(ctx context.Context, hashType hash.Type) (string, error) {
if err := o.readMetadata(ctx); err != nil {
return "", err // valid metadata is required to get hash, abort
}
if !o.isComposite() {
// First, chain to the wrapped non-chunked file if possible.
if value, err := o.mainChunk().Hash(ctx, hashType); err == nil && value != "" {
return value, nil
}
}
if err := o.readMetadata(ctx); err != nil {
return "", err // valid metadata is required to get hash, abort
}
// Try hash from metadata if the file is composite or if wrapped remote fails.
switch hashType {
case hash.MD5:
@@ -1962,13 +1894,13 @@ func (o *Object) UnWrap() fs.Object {
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
if !o.isComposite() {
return o.mainChunk().Open(ctx, options...) // chain to wrapped non-chunked file
}
if err := o.readMetadata(ctx); err != nil {
// refuse to open unsupported format
return nil, errors.Wrap(err, "can't open")
}
if !o.isComposite() {
return o.mainChunk().Open(ctx, options...) // chain to wrapped non-chunked file
}
var openOptions []fs.OpenOption
var offset, limit int64 = 0, -1
@@ -2241,57 +2173,57 @@ func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1 s
// handled by current implementation.
// The version check below will then explicitly ask user to upgrade rclone.
//
func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte) (info *ObjectInfo, madeByChunker bool, err error) {
func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte, strictChecks bool) (info *ObjectInfo, err error) {
// Be strict about JSON format
// to reduce possibility that a random small file resembles metadata.
if data != nil && len(data) > maxMetadataSize {
return nil, false, ErrMetaTooBig
return nil, errors.New("too big")
}
if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' {
return nil, false, errors.New("invalid json")
return nil, errors.New("invalid json")
}
var metadata metaSimpleJSON
err = json.Unmarshal(data, &metadata)
if err != nil {
return nil, false, err
return nil, err
}
// Basic fields are strictly required
// to reduce possibility that a random small file resembles metadata.
if metadata.Version == nil || metadata.Size == nil || metadata.ChunkNum == nil {
return nil, false, errors.New("missing required field")
return nil, errors.New("missing required field")
}
// Perform strict checks, avoid corruption of future metadata formats.
if *metadata.Version < 1 {
return nil, false, errors.New("wrong version")
return nil, errors.New("wrong version")
}
if *metadata.Size < 0 {
return nil, false, errors.New("negative file size")
return nil, errors.New("negative file size")
}
if *metadata.ChunkNum < 0 {
return nil, false, errors.New("negative number of chunks")
return nil, errors.New("negative number of chunks")
}
if *metadata.ChunkNum > maxSafeChunkNumber {
return nil, true, ErrChunkOverflow // produced by incompatible version of rclone
return nil, ErrChunkOverflow
}
if metadata.MD5 != "" {
_, err = hex.DecodeString(metadata.MD5)
if len(metadata.MD5) != 32 || err != nil {
return nil, false, errors.New("wrong md5 hash")
return nil, errors.New("wrong md5 hash")
}
}
if metadata.SHA1 != "" {
_, err = hex.DecodeString(metadata.SHA1)
if len(metadata.SHA1) != 40 || err != nil {
return nil, false, errors.New("wrong sha1 hash")
return nil, errors.New("wrong sha1 hash")
}
}
// ChunkNum is allowed to be 0 in future versions
if *metadata.ChunkNum < 1 && *metadata.Version <= metadataVersion {
return nil, false, errors.New("wrong number of chunks")
return nil, errors.New("wrong number of chunks")
}
// Non-strict mode also accepts future metadata versions
if *metadata.Version > metadataVersion {
return nil, true, ErrMetaUnknown // produced by incompatible version of rclone
if *metadata.Version > metadataVersion && strictChecks {
return nil, fmt.Errorf("version %d is not supported, please upgrade rclone", metadata.Version)
}
var nilFs *Fs // nil object triggers appropriate type method
@@ -2299,7 +2231,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte)
info.nChunks = *metadata.ChunkNum
info.md5 = metadata.MD5
info.sha1 = metadata.SHA1
return info, true, nil
return info, nil
}
func silentlyRemove(ctx context.Context, o fs.Object) {
+257 -17
View File
@@ -22,6 +22,7 @@ of path_display and all will be well.
*/
import (
"bytes"
"context"
"fmt"
"io"
@@ -29,9 +30,11 @@ import (
"path"
"regexp"
"strings"
"sync"
"time"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/async"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/auth"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/common"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
@@ -47,6 +50,7 @@ import (
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
@@ -61,6 +65,7 @@ const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
maxBatchSize = 1000
// Upload chunk size - setting too small makes uploads slow.
// Chunks are buffered into memory for retries.
//
@@ -142,6 +147,23 @@ memory. It can be set smaller if you are tight on memory.`, maxChunkSize),
Help: "Impersonate this user when using a business account.",
Default: "",
Advanced: true,
}, {
Name: "batch",
Help: `Enable batching of files if non-zero.
This sets the batch size of files to upload. It has to be less than 1000. A
sensible setting is probably 1000 if you are using this feature.
Rclone will close any outstanding batches when it exits.
Setting this is a great idea if you are uploading lots of small files as it will
make them a lot quicker. You can use --transfers 32 to maximise throughput.
It has the downside that rclone can't check the hash of the file after upload,
so using "rclone check" after the transfer completes is recommended.
`,
Default: 0,
Advanced: true,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
@@ -163,6 +185,7 @@ memory. It can be set smaller if you are tight on memory.`, maxChunkSize),
type Options struct {
ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"`
Batch int `config:"batch"`
Enc encoder.MultiEncoder `config:"encoding"`
}
@@ -180,6 +203,7 @@ type Fs struct {
slashRootSlash string // root with "/" prefix and postfix, lowercase
pacer *fs.Pacer // To pace the API calls
ns string // The namespace we are using or "" for none
batcher *batcher // batch builder
}
// Object describes a dropbox object
@@ -195,6 +219,165 @@ type Object struct {
// ------------------------------------------------------------
// batcher holds info about the current items waiting for upload
type batcher struct {
f *Fs // Fs this batch is part of
mu sync.Mutex // lock for vars below
commitMu sync.Mutex // lock for waiting for batch
maxBatch int // maximum size for batch
active int // number of batches being sent
items []*files.UploadSessionFinishArg // current uncommitted files
atexit atexit.FnHandle // atexit handle
}
// newBatcher creates a new batcher structure
func newBatcher(f *Fs, maxBatch int) *batcher {
return &batcher{
f: f,
maxBatch: maxBatch,
}
}
// Start starts adding an item to a batch returning true if it was
// successfully started
//
// This should be paired with End
func (b *batcher) Start() bool {
if b.maxBatch <= 0 {
return false
}
b.mu.Lock()
defer b.mu.Unlock()
b.active++
// FIXME set a timer or something
return true
}
// End ends adding an item
func (b *batcher) End(started bool) error {
if !started {
return nil
}
b.mu.Lock()
defer b.mu.Unlock()
b.active--
if len(b.items) < b.maxBatch {
return nil
}
return b._commit(false)
}
// Waits for the batch to complete - call with batchMu held
func (b *batcher) _waitForBatchResult(res *files.UploadSessionFinishBatchLaunch) (batchResult *files.UploadSessionFinishBatchResult, err error) {
if res.AsyncJobId == "" {
return res.Complete, nil
}
var batchStatus *files.UploadSessionFinishBatchJobStatus
sleepTime := time.Second
const maxTries = 120
for try := 1; try <= maxTries; try++ {
err = b.f.pacer.Call(func() (bool, error) {
batchStatus, err = b.f.srv.UploadSessionFinishBatchCheck(&async.PollArg{
AsyncJobId: res.AsyncJobId,
})
return shouldRetry(err)
})
if err != nil {
fs.Errorf(b.f, "failed to wait for batch: %v", err)
break
}
if batchStatus.Tag == "complete" {
break
}
fs.Debugf(b.f, "sleeping for %v to wait for batch to complete, try %d/%d", sleepTime, try, maxTries)
time.Sleep(sleepTime)
}
return batchStatus.Complete, nil
}
// commit a batch - call with batchMu held
//
// if finalizing is true then it doesn't unregister Finalize as this
// causes a deadlock during finalization.
func (b *batcher) _commit(finalizing bool) (err error) {
b.commitMu.Lock()
batch := "batch"
if finalizing {
batch = "last batch"
}
fs.Debugf(b.f, "comitting %s length %d", batch, len(b.items))
var arg = &files.UploadSessionFinishBatchArg{
Entries: b.items,
}
var res *files.UploadSessionFinishBatchLaunch
err = b.f.pacer.Call(func() (bool, error) {
res, err = b.f.srv.UploadSessionFinishBatch(arg)
// If error is insufficient space then don't retry
if e, ok := err.(files.UploadSessionFinishAPIError); ok {
if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
err = fserrors.NoRetryError(err)
return false, err
}
}
// after the first chunk is uploaded, we retry everything
return err != nil, err
})
if err != nil {
b.commitMu.Unlock()
return err
}
// Clear batch
b.items = nil
// If finalizing, don't unregister or get result
if finalizing {
b.commitMu.Unlock()
return nil
}
// Unregister the atexit since queue is empty
atexit.Unregister(b.atexit)
b.atexit = nil
// Wait for the batch to finish before we proceed in the background
go func() {
defer b.commitMu.Unlock()
_, err = b._waitForBatchResult(res)
if err != nil {
fs.Errorf(b.f, "Error waiting for batch to finish: %v", err)
}
}()
return nil
}
// Add adds a finished item to the batch
func (b *batcher) Add(commitInfo *files.UploadSessionFinishArg) {
fs.Debugf(b.f, "adding %q to batch", commitInfo.Commit.Path)
b.mu.Lock()
defer b.mu.Unlock()
b.items = append(b.items, commitInfo)
if b.atexit == nil {
b.atexit = atexit.Register(b.Finalize)
}
}
// Finalize finishes any pending batches
func (b *batcher) Finalize() {
b.mu.Lock()
defer b.mu.Unlock()
if len(b.items) == 0 {
return
}
err := b._commit(true)
if err != nil {
fs.Errorf(b.f, "Failed to finalize last batch: %v", err)
}
}
// ------------------------------------------------------------
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
@@ -230,7 +413,7 @@ func shouldRetry(err error) (bool, error) {
switch e := err.(type) {
case auth.RateLimitAPIError:
if e.RateLimitError.RetryAfter > 0 {
fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
fs.Logf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
err = pacer.RetryAfterError(err, time.Duration(e.RateLimitError.RetryAfter)*time.Second)
}
return true, err
@@ -273,6 +456,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, errors.Wrap(err, "dropbox: chunk size")
}
if opt.Batch > maxBatchSize || opt.Batch < 0 {
return nil, errors.Errorf("dropbox: batch must be < %d and >= 0 - it is currently %d", maxBatchSize, opt.Batch)
}
// Convert the old token if it exists. The old token was just
// just a string, the new one is a JSON blob
@@ -297,6 +483,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
opt: *opt,
pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.batcher = newBatcher(f, f.opt.Batch)
config := dropbox.Config{
LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
Client: oAuthClient, // maybe???
@@ -1044,6 +1231,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
// avoidable request to the Dropbox API that does not carry payload.
func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
batching := o.fs.batcher.Start()
defer func() {
batchErr := o.fs.batcher.End(batching)
if err != nil {
err = batchErr
}
}()
chunkSize := int64(o.fs.opt.ChunkSize)
chunks := 0
if size != -1 {
@@ -1057,11 +1251,15 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
fs.Debugf(o, "Streaming chunk %d/%d", cur, cur)
} else if chunks == 0 {
fs.Debugf(o, "Streaming chunk %d/unknown", cur)
} else {
} else if chunks != 1 {
fs.Debugf(o, "Uploading chunk %d/%d", cur, chunks)
}
}
appendArg := files.UploadSessionAppendArg{
Close: chunks == 1,
}
// write the first chunk
fmtChunk(1, false)
var res *files.UploadSessionStartResult
@@ -1071,7 +1269,10 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
if _, err = chunk.Seek(0, io.SeekStart); err != nil {
return false, nil
}
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
arg := files.UploadSessionStartArg{
Close: appendArg.Close,
}
res, err = o.fs.srv.UploadSessionStart(&arg, chunk)
return shouldRetry(err)
})
if err != nil {
@@ -1082,22 +1283,34 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
SessionId: res.SessionId,
Offset: 0,
}
appendArg := files.UploadSessionAppendArg{
Cursor: &cursor,
Close: false,
}
appendArg.Cursor = &cursor
// write more whole chunks (if any)
// write more whole chunks (if any, and if !batching), if
// batching write the last chunk also.
currentChunk := 2
for {
if chunks > 0 && currentChunk >= chunks {
// if the size is known, only upload full chunks. Remaining bytes are uploaded with
// the UploadSessionFinish request.
break
} else if chunks == 0 && in.BytesRead()-cursor.Offset < uint64(chunkSize) {
// if the size is unknown, upload as long as we can read full chunks from the reader.
// The UploadSessionFinish request will not contain any payload.
break
if chunks > 0 {
// Size known
if currentChunk == chunks {
// Last chunk
if !batching {
// if the size is known, only upload full chunks. Remaining bytes are uploaded with
// the UploadSessionFinish request.
break
}
appendArg.Close = true
} else if currentChunk > chunks {
break
}
} else {
// Size unknown
lastReadWasShort := in.BytesRead()-cursor.Offset < uint64(chunkSize)
if lastReadWasShort {
// if the size is unknown, upload as long as we can read full chunks from the reader.
// The UploadSessionFinish request will not contain any payload.
// This is also what we want if batching
break
}
}
cursor.Offset = in.BytesRead()
fmtChunk(currentChunk, false)
@@ -1123,6 +1336,26 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
Cursor: &cursor,
Commit: commitInfo,
}
// If we are batching then we should have written all the data now
// store the commit info now for a batch commit
if batching {
// If we haven't closed the session then we need to
if !appendArg.Close {
fs.Debugf(o, "Closing session")
var empty bytes.Buffer
err = o.fs.pacer.Call(func() (bool, error) {
err = o.fs.srv.UploadSessionAppendV2(&appendArg, &empty)
// after the first chunk is uploaded, we retry everything
return err != nil, err
})
if err != nil {
return nil, err
}
}
o.fs.batcher.Add(args)
return nil, nil
}
fmtChunk(currentChunk, true)
chunk = readers.NewRepeatableReaderBuffer(in, buf)
err = o.fs.pacer.Call(func() (bool, error) {
@@ -1165,7 +1398,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
size := src.Size()
var err error
var entry *files.FileMetadata
if size > int64(o.fs.opt.ChunkSize) || size == -1 {
if size > int64(o.fs.opt.ChunkSize) || size == -1 || o.fs.opt.Batch > 0 {
entry, err = o.uploadChunked(in, commitInfo, size)
} else {
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1176,6 +1409,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if err != nil {
return errors.Wrap(err, "upload failed")
}
// If we haven't received data back from batch upload then fake it
if entry == nil {
o.bytes = size
o.modTime = commitInfo.ClientModified
o.hash = "" // we don't have this
return nil
}
return o.setMetadataFromEntry(entry)
}
+1 -1
View File
@@ -323,7 +323,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
// This will create a duplicate if we upload a new file without
// checking to see if there is one already - use Put() for that.
func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
if size > int64(300e9) {
if size > int64(100e9) {
return nil, errors.New("File too big, cant upload")
} else if size == 0 {
return nil, fs.ErrorCantUploadEmptyFiles
@@ -841,27 +841,20 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
remote: remote,
}
rewriteRequest := f.svc.Objects.Rewrite(srcBucket, srcPath, dstBucket, dstPath, nil)
if !f.opt.BucketPolicyOnly {
rewriteRequest.DestinationPredefinedAcl(f.opt.ObjectACL)
}
var rewriteResponse *storage.RewriteResponse
for {
err = f.pacer.Call(func() (bool, error) {
rewriteResponse, err = rewriteRequest.Context(ctx).Do()
return shouldRetry(err)
})
if err != nil {
return nil, err
var newObject *storage.Object
err = f.pacer.Call(func() (bool, error) {
copyObject := f.svc.Objects.Copy(srcBucket, srcPath, dstBucket, dstPath, nil)
if !f.opt.BucketPolicyOnly {
copyObject.DestinationPredefinedAcl(f.opt.ObjectACL)
}
if rewriteResponse.Done {
break
}
rewriteRequest.RewriteToken(rewriteResponse.RewriteToken)
fs.Debugf(dstObj, "Continuing rewrite %d bytes done", rewriteResponse.TotalBytesRewritten)
newObject, err = copyObject.Context(ctx).Do()
return shouldRetry(err)
})
if err != nil {
return nil, err
}
// Set the metadata for the new object while we have it
dstObj.setMetaData(rewriteResponse.Resource)
dstObj.setMetaData(newObject)
return dstObj, nil
}
+16 -10
View File
@@ -373,9 +373,6 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) {
fmt.Printf("Login Token> ")
loginToken := config.ReadLine()
m.Set(configClientID, "jottacli")
m.Set(configClientSecret, "")
token, err := doAuthV2(ctx, srv, loginToken, m)
if err != nil {
log.Fatalf("Failed to get oauth token: %s", err)
@@ -387,6 +384,7 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) {
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
if config.Confirm(false) {
oauthConfig.ClientID = "jottacli"
oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
if err != nil {
log.Fatalf("Failed to load oAuthClient: %s", err)
@@ -553,7 +551,7 @@ func (f *Fs) setEndpointURL() {
if f.opt.Mountpoint == "" {
f.opt.Mountpoint = defaultMountpoint
}
f.endpointURL = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
f.endpointURL = urlPathEscape(path.Join(f.user, f.opt.Device, f.opt.Mountpoint))
}
// readMetaDataForPath reads the metadata from the path
@@ -730,9 +728,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Renew the token in the background
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
_, err := f.readMetaDataForPath(ctx, "")
if err == fs.ErrorNotAFile {
err = nil
}
return err
})
@@ -1092,7 +1087,8 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
return shouldRetry(resp, err)
retry, _ := shouldRetry(resp, err)
return (retry && resp.StatusCode != 500), err
})
if err != nil {
return nil, err
@@ -1196,6 +1192,18 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
// surprise! jottacloud fucked up dirmove - the api spits out an error but
// dir gets moved regardless
if apiErr, ok := err.(*api.Error); ok {
if apiErr.StatusCode == 500 {
_, err := f.NewObject(ctx, dstRemote)
if err == fs.ErrorNotAFile {
log.Printf("FIXME: ignoring DirMove error - move succeeded anyway\n")
return nil
}
return err
}
}
if err != nil {
return errors.Wrap(err, "couldn't move directory")
}
@@ -1469,8 +1477,6 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
//
// The new object may have been created if an error is returned
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
o.fs.tokenRenewer.Start()
defer o.fs.tokenRenewer.Stop()
size := src.Size()
md5String, err := src.Hash(ctx, hash.MD5)
if err != nil || md5String == "" {
+1 -10
View File
@@ -1213,7 +1213,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
// Set the file to be a sparse file (important on Windows)
err = file.SetSparse(out)
if err != nil {
fs.Errorf(o, "Failed to set sparse: %v", err)
fs.Debugf(o, "Failed to set sparse: %v", err)
}
}
@@ -1231,15 +1231,6 @@ func (o *Object) setMetadata(info os.FileInfo) {
o.modTime = info.ModTime()
o.mode = info.Mode()
o.fs.objectMetaMu.Unlock()
// On Windows links read as 0 size so set the correct size here
if runtime.GOOS == "windows" && o.translatedLink {
linkdst, err := os.Readlink(o.path)
if err != nil {
fs.Errorf(o, "Failed to read link size: %v", err)
} else {
o.size = int64(len(linkdst))
}
}
}
// Stat an Object into info
+10 -1
View File
@@ -6,6 +6,7 @@ import (
"os"
"path"
"path/filepath"
"runtime"
"testing"
"time"
@@ -88,6 +89,9 @@ func TestSymlink(t *testing.T) {
// Object viewed as symlink
file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
if runtime.GOOS == "windows" {
file2.Size = 0 // symlinks are 0 length under Windows
}
// Object viewed as destination
file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
@@ -117,6 +121,9 @@ func TestSymlink(t *testing.T) {
// Create a symlink
modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
if runtime.GOOS == "windows" {
file3.Size = 0 // symlinks are 0 length under Windows
}
fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
if haveLChtimes {
fstest.CheckItems(t, r.Flocal, file1, file2, file3)
@@ -135,7 +142,9 @@ func TestSymlink(t *testing.T) {
o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+linkSuffix)
require.NoError(t, err)
assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
assert.Equal(t, int64(8), o.Size())
if runtime.GOOS != "windows" {
assert.Equal(t, int64(8), o.Size())
}
// Check that NewObject doesn't see the non suffixed version
_, err = r.Flocal.NewObject(ctx, "symlink2.txt")
+66 -1
View File
@@ -117,7 +117,7 @@ type ListItem struct {
Name string `json:"name"`
Home string `json:"home"`
Size int64 `json:"size"`
Mtime uint64 `json:"mtime,omitempty"`
Mtime int64 `json:"mtime,omitempty"`
Hash string `json:"hash,omitempty"`
VirusScan string `json:"virus_scan,omitempty"`
Tree string `json:"tree,omitempty"`
@@ -159,6 +159,71 @@ type FolderInfoResponse struct {
Email string `json:"email"`
}
// ShardInfoResponse ...
type ShardInfoResponse struct {
Email string `json:"email"`
Body struct {
Video []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"video"`
ViewDirect []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"view_direct"`
WeblinkView []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"weblink_view"`
WeblinkVideo []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"weblink_video"`
WeblinkGet []struct {
Count int `json:"count"`
URL string `json:"url"`
} `json:"weblink_get"`
Stock []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"stock"`
WeblinkThumbnails []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"weblink_thumbnails"`
PublicUpload []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"public_upload"`
Auth []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"auth"`
Web []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"web"`
View []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"view"`
Upload []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"upload"`
Get []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"get"`
Thumbnails []struct {
Count string `json:"count"`
URL string `json:"url"`
} `json:"thumbnails"`
} `json:"body"`
Time int64 `json:"time"`
Status int `json:"status"`
}
// CleanupResponse ...
type CleanupResponse struct {
Email string `json:"email"`
+30 -62
View File
@@ -37,7 +37,6 @@ import (
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/pkg/errors"
@@ -656,14 +655,9 @@ func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.D
if err != nil {
return nil, -1, err
}
mTime := int64(item.Mtime)
if mTime < 0 {
fs.Debugf(f, "Fixing invalid timestamp %d on mailru file %q", mTime, remote)
mTime = 0
}
switch item.Kind {
case "folder":
dir := fs.NewDir(remote, time.Unix(mTime, 0)).SetSize(item.Size)
dir := fs.NewDir(remote, time.Unix(item.Mtime, 0)).SetSize(item.Size)
dirSize := item.Count.Files + item.Count.Folders
return dir, dirSize, nil
case "file":
@@ -677,7 +671,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.D
hasMetaData: true,
size: item.Size,
mrHash: binHash,
modTime: time.Unix(mTime, 0),
modTime: time.Unix(item.Mtime, 0),
}
return file, -1, nil
default:
@@ -1867,30 +1861,30 @@ func (f *Fs) uploadShard(ctx context.Context) (string, error) {
return f.shardURL, nil
}
opts := rest.Opts{
RootURL: api.DispatchServerURL,
Method: "GET",
Path: "/u",
}
var (
res *http.Response
url string
err error
)
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.Call(ctx, &opts)
if err == nil {
url, err = readBodyWord(res)
}
return fserrors.ShouldRetry(err), err
})
token, err := f.accessToken()
if err != nil {
closeBody(res)
return "", err
}
f.shardURL = url
opts := rest.Opts{
Method: "GET",
Path: "/api/m1/dispatcher",
Parameters: url.Values{
"client_id": {api.OAuthClientID},
"access_token": {token},
},
}
var info api.ShardInfoResponse
err = f.pacer.Call(func() (bool, error) {
res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(res, err, f, &opts)
})
if err != nil {
return "", err
}
f.shardURL = info.Body.Upload[0].URL
f.shardExpiry = time.Now().Add(shardExpirySec * time.Second)
fs.Debugf(f, "new upload shard: %s", f.shardURL)
@@ -2122,18 +2116,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return nil, err
}
start, end, partialRequest := getTransferRange(o.size, options...)
headers := map[string]string{
"Accept": "*/*",
"Content-Type": "application/octet-stream",
}
if partialRequest {
rangeStr := fmt.Sprintf("bytes=%d-%d", start, end-1)
headers["Range"] = rangeStr
// headers["Content-Range"] = rangeStr
headers["Accept-Ranges"] = "bytes"
}
start, end, partial := getTransferRange(o.size, options...)
// TODO: set custom timeouts
opts := rest.Opts{
@@ -2144,7 +2127,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
"client_id": {api.OAuthClientID},
"token": {token},
},
ExtraHeaders: headers,
ExtraHeaders: map[string]string{
"Accept": "*/*",
"Range": fmt.Sprintf("bytes=%d-%d", start, end-1),
},
}
var res *http.Response
@@ -2165,36 +2151,18 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return nil, err
}
// Server should respond with Status 206 and Content-Range header to a range
// request. Status 200 (and no Content-Range) means a full-content response.
partialResponse := res.StatusCode == 206
var (
hasher gohash.Hash
wrapStream io.ReadCloser
)
if !partialResponse {
var hasher gohash.Hash
if !partial {
// Cannot check hash of partial download
hasher = mrhash.New()
}
wrapStream = &endHandler{
wrapStream := &endHandler{
ctx: ctx,
stream: res.Body,
hasher: hasher,
o: o,
server: server,
}
if partialRequest && !partialResponse {
fs.Debugf(o, "Server returned full content instead of range")
if start > 0 {
// Discard the beginning of the data
_, err = io.CopyN(ioutil.Discard, wrapStream, start)
if err != nil {
return nil, err
}
}
wrapStream = readers.NewLimitedReadCloser(wrapStream, end-start)
}
return wrapStream, nil
}
-4
View File
@@ -1247,10 +1247,6 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return nil, errors.Wrap(err, "about failed")
}
q := drive.Quota
// On (some?) Onedrive sharepoints these are all 0 so return unknown in that case
if q.Total == 0 && q.Used == 0 && q.Deleted == 0 && q.Remaining == 0 {
return &fs.Usage{}, nil
}
usage = &fs.Usage{
Total: fs.NewUsageValue(q.Total), // quota of bytes that can be used
Used: fs.NewUsageValue(q.Used), // bytes in use
+1
View File
@@ -646,6 +646,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
400, // Bad request (seen in "Next token is expired")
401, // Unauthorized (seen in "Token has expired")
408, // Request Timeout
423, // Locked - get this on folders sometimes
+2 -3
View File
@@ -104,9 +104,8 @@ type ItemResult struct {
// Hashes contains the supported hashes
type Hashes struct {
SHA1 string `json:"sha1"`
MD5 string `json:"md5"`
SHA256 string `json:"sha256"`
SHA1 string `json:"sha1"`
MD5 string `json:"md5"`
}
// UploadFileResponse is the response from /uploadfile
-7
View File
@@ -885,13 +885,6 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
// EU region supports SHA1 and SHA256 (but rclone doesn't
// support SHA256 yet).
//
// https://forum.rclone.org/t/pcloud-to-local-no-hashes-in-common/19440
if f.opt.Hostname == "eapi.pcloud.com" {
return hash.Set(hash.SHA1)
}
return hash.Set(hash.MD5 | hash.SHA1)
}
+18 -163
View File
@@ -58,7 +58,7 @@ import (
func init() {
fs.Register(&fs.RegInfo{
Name: "s3",
Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)",
Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
NewFs: NewFs,
CommandHelp: commandHelp,
Options: []fs.Option{{
@@ -94,9 +94,6 @@ func init() {
}, {
Value: "StackPath",
Help: "StackPath Object Storage",
}, {
Value: "TencentCOS",
Help: "Tencent Cloud Object Storage (COS)",
}, {
Value: "Wasabi",
Help: "Wasabi Object Storage",
@@ -122,9 +119,6 @@ func init() {
Name: "secret_access_key",
Help: "AWS Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
}, {
// References:
// 1. https://docs.aws.amazon.com/general/latest/gr/rande.html
// 2. https://docs.aws.amazon.com/general/latest/gr/s3.html
Name: "region",
Help: "Region to connect to.",
Provider: "AWS",
@@ -134,12 +128,12 @@ func init() {
}, {
Value: "us-east-2",
Help: "US East (Ohio) Region\nNeeds location constraint us-east-2.",
}, {
Value: "us-west-1",
Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.",
}, {
Value: "us-west-2",
Help: "US West (Oregon) Region\nNeeds location constraint us-west-2.",
}, {
Value: "us-west-1",
Help: "US West (Northern California) Region\nNeeds location constraint us-west-1.",
}, {
Value: "ca-central-1",
Help: "Canada (Central) Region\nNeeds location constraint ca-central-1.",
@@ -149,15 +143,9 @@ func init() {
}, {
Value: "eu-west-2",
Help: "EU (London) Region\nNeeds location constraint eu-west-2.",
}, {
Value: "eu-west-3",
Help: "EU (Paris) Region\nNeeds location constraint eu-west-3.",
}, {
Value: "eu-north-1",
Help: "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
}, {
Value: "eu-south-1",
Help: "EU (Milan) Region\nNeeds location constraint eu-south-1.",
}, {
Value: "eu-central-1",
Help: "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -173,36 +161,15 @@ func init() {
}, {
Value: "ap-northeast-2",
Help: "Asia Pacific (Seoul)\nNeeds location constraint ap-northeast-2.",
}, {
Value: "ap-northeast-3",
Help: "Asia Pacific (Osaka-Local)\nNeeds location constraint ap-northeast-3.",
}, {
Value: "ap-south-1",
Help: "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.",
}, {
Value: "ap-east-1",
Help: "Asia Pacific (Hong Kong) Region\nNeeds location constraint ap-east-1.",
Help: "Asia Patific (Hong Kong) Region\nNeeds location constraint ap-east-1.",
}, {
Value: "sa-east-1",
Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
}, {
Value: "me-south-1",
Help: "Middle East (Bahrain) Region\nNeeds location constraint me-south-1.",
}, {
Value: "af-south-1",
Help: "Africa (Cape Town) Region\nNeeds location constraint af-south-1.",
}, {
Value: "cn-north-1",
Help: "China (Beijing) Region\nNeeds location constraint cn-north-1.",
}, {
Value: "cn-northwest-1",
Help: "China (Ningxia) Region\nNeeds location constraint cn-northwest-1.",
}, {
Value: "us-gov-east-1",
Help: "AWS GovCloud (US-East) Region\nNeeds location constraint us-gov-east-1.",
}, {
Value: "us-gov-west-1",
Help: "AWS GovCloud (US) Region\nNeeds location constraint us-gov-west-1.",
}},
}, {
Name: "region",
@@ -218,7 +185,7 @@ func init() {
}, {
Name: "region",
Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
Provider: "!AWS,Alibaba,Scaleway,TencentCOS",
Provider: "!AWS,Alibaba,Scaleway",
Examples: []fs.OptionExample{{
Value: "",
Help: "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -509,73 +476,10 @@ func init() {
Value: "s3.eu-central-1.stackpathstorage.com",
Help: "EU Endpoint",
}},
}, {
// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
Name: "endpoint",
Help: "Endpoint for Tencent COS API.",
Provider: "TencentCOS",
Examples: []fs.OptionExample{{
Value: "cos.ap-beijing.myqcloud.com",
Help: "Beijing Region.",
}, {
Value: "cos.ap-nanjing.myqcloud.com",
Help: "Nanjing Region.",
}, {
Value: "cos.ap-shanghai.myqcloud.com",
Help: "Shanghai Region.",
}, {
Value: "cos.ap-guangzhou.myqcloud.com",
Help: "Guangzhou Region.",
}, {
Value: "cos.ap-nanjing.myqcloud.com",
Help: "Nanjing Region.",
}, {
Value: "cos.ap-chengdu.myqcloud.com",
Help: "Chengdu Region.",
}, {
Value: "cos.ap-chongqing.myqcloud.com",
Help: "Chongqing Region.",
}, {
Value: "cos.ap-hongkong.myqcloud.com",
Help: "Hong Kong (China) Region.",
}, {
Value: "cos.ap-singapore.myqcloud.com",
Help: "Singapore Region.",
}, {
Value: "cos.ap-mumbai.myqcloud.com",
Help: "Mumbai Region.",
}, {
Value: "cos.ap-seoul.myqcloud.com",
Help: "Seoul Region.",
}, {
Value: "cos.ap-bangkok.myqcloud.com",
Help: "Bangkok Region.",
}, {
Value: "cos.ap-tokyo.myqcloud.com",
Help: "Tokyo Region.",
}, {
Value: "cos.na-siliconvalley.myqcloud.com",
Help: "Silicon Valley Region.",
}, {
Value: "cos.na-ashburn.myqcloud.com",
Help: "Virginia Region.",
}, {
Value: "cos.na-toronto.myqcloud.com",
Help: "Toronto Region.",
}, {
Value: "cos.eu-frankfurt.myqcloud.com",
Help: "Frankfurt Region.",
}, {
Value: "cos.eu-moscow.myqcloud.com",
Help: "Moscow Region.",
}, {
Value: "cos.accelerate.myqcloud.com",
Help: "Use Tencent COS Accelerate Endpoint.",
}},
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.\nRequired when using an S3 clone.",
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath",
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
Examples: []fs.OptionExample{{
Value: "objects-us-east-1.dream.io",
Help: "Dream Objects endpoint",
@@ -615,12 +519,12 @@ func init() {
}, {
Value: "us-east-2",
Help: "US East (Ohio) Region.",
}, {
Value: "us-west-1",
Help: "US West (Northern California) Region.",
}, {
Value: "us-west-2",
Help: "US West (Oregon) Region.",
}, {
Value: "us-west-1",
Help: "US West (Northern California) Region.",
}, {
Value: "ca-central-1",
Help: "Canada (Central) Region.",
@@ -630,15 +534,9 @@ func init() {
}, {
Value: "eu-west-2",
Help: "EU (London) Region.",
}, {
Value: "eu-west-3",
Help: "EU (Paris) Region.",
}, {
Value: "eu-north-1",
Help: "EU (Stockholm) Region.",
}, {
Value: "eu-south-1",
Help: "EU (Milan) Region.",
}, {
Value: "EU",
Help: "EU Region.",
@@ -653,37 +551,16 @@ func init() {
Help: "Asia Pacific (Tokyo) Region.",
}, {
Value: "ap-northeast-2",
Help: "Asia Pacific (Seoul) Region.",
}, {
Value: "ap-northeast-3",
Help: "Asia Pacific (Osaka-Local) Region.",
Help: "Asia Pacific (Seoul)",
}, {
Value: "ap-south-1",
Help: "Asia Pacific (Mumbai) Region.",
Help: "Asia Pacific (Mumbai)",
}, {
Value: "ap-east-1",
Help: "Asia Pacific (Hong Kong) Region.",
Help: "Asia Pacific (Hong Kong)",
}, {
Value: "sa-east-1",
Help: "South America (Sao Paulo) Region.",
}, {
Value: "me-south-1",
Help: "Middle East (Bahrain) Region.",
}, {
Value: "af-south-1",
Help: "Africa (Cape Town) Region.",
}, {
Value: "cn-north-1",
Help: "China (Beijing) Region",
}, {
Value: "cn-northwest-1",
Help: "China (Ningxia) Region.",
}, {
Value: "us-gov-east-1",
Help: "AWS GovCloud (US-East) Region.",
}, {
Value: "us-gov-west-1",
Help: "AWS GovCloud (US) Region.",
}},
}, {
Name: "location_constraint",
@@ -789,7 +666,7 @@ func init() {
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath,TencentCOS",
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
}, {
Name: "acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -801,13 +678,9 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
Note that this ACL is applied when server side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.`,
Examples: []fs.OptionExample{{
Value: "default",
Help: "Owner gets Full_CONTROL. No one else has access rights (default).",
Provider: "TencentCOS",
}, {
Value: "private",
Help: "Owner gets FULL_CONTROL. No one else has access rights (default).",
Provider: "!IBMCOS,TencentCOS",
Provider: "!IBMCOS",
}, {
Value: "public-read",
Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
@@ -969,24 +842,6 @@ isn't set then "acl" is used instead.`,
Value: "STANDARD_IA",
Help: "Infrequent access storage mode.",
}},
}, {
// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
Name: "storage_class",
Help: "The storage class to use when storing new objects in Tencent COS.",
Provider: "TencentCOS",
Examples: []fs.OptionExample{{
Value: "",
Help: "Default",
}, {
Value: "STANDARD",
Help: "Standard storage class",
}, {
Value: "ARCHIVE",
Help: "Archive storage mode.",
}, {
Value: "STANDARD_IA",
Help: "Infrequent access storage mode.",
}},
}, {
// Mapping from here: https://www.scaleway.com/en/docs/object-storage-glacier/#-Scaleway-Storage-Classes
Name: "storage_class",
@@ -1120,7 +975,7 @@ if false then rclone will use virtual path style. See [the AWS S3
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
for more info.
Some providers (eg AWS, Aliyun OSS, Netease COS or Tencent COS) require this set to
Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to
false - rclone will do this automatically based on the provider
setting.`,
Default: true,
@@ -1450,7 +1305,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
if opt.Region == "" {
opt.Region = "us-east-1"
}
if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.Provider == "TencentCOS" || opt.UseAccelerateEndpoint {
if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.UseAccelerateEndpoint {
opt.ForcePathStyle = false
}
if opt.Provider == "Scaleway" && opt.MaxUploadParts > 1000 {
@@ -1732,7 +1587,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
//
// So we enable only on providers we know supports it properly, all others can retry when a
// XML Syntax error is detected.
var urlEncodeListings = (f.opt.Provider == "AWS" || f.opt.Provider == "Wasabi" || f.opt.Provider == "Alibaba" || f.opt.Provider == "Minio" || f.opt.Provider == "TencentCOS")
var urlEncodeListings = (f.opt.Provider == "AWS" || f.opt.Provider == "Wasabi" || f.opt.Provider == "Alibaba" || f.opt.Provider == "Minio")
for {
// FIXME need to implement ALL loop
req := s3.ListObjectsInput{
+1 -1
View File
@@ -46,7 +46,7 @@ type Library struct {
Encrypted bool `json:"encrypted"`
Owner string `json:"owner"`
ID string `json:"id"`
Size int64 `json:"size"`
Size int `json:"size"`
Name string `json:"name"`
Modified int64 `json:"mtime"`
}
+1 -1
View File
@@ -1004,7 +1004,7 @@ func (f *Fs) listLibraries(ctx context.Context) (entries fs.DirEntries, err erro
for _, library := range libraries {
d := fs.NewDir(library.Name, time.Unix(library.Modified, 0))
d.SetSize(library.Size)
d.SetSize(int64(library.Size))
entries = append(entries, d)
}
+1 -1
View File
@@ -1087,7 +1087,7 @@ func shellEscape(str string) string {
func parseHash(bytes []byte) string {
// For strings with backslash *sum writes a leading \
// https://unix.stackexchange.com/q/313733/94054
return strings.ToLower(strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0]) // Split at hash / filename separator / all convert to lowercase
return strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0] // Split at hash / filename separator
}
// Parses the byte array output from the SSH session
+3
View File
@@ -3,6 +3,7 @@ package policy
import (
"context"
"math/rand"
"time"
"github.com/rclone/rclone/backend/union/upstream"
"github.com/rclone/rclone/fs"
@@ -19,10 +20,12 @@ type EpRand struct {
}
func (p *EpRand) rand(upstreams []*upstream.Fs) *upstream.Fs {
rand.Seed(time.Now().Unix())
return upstreams[rand.Intn(len(upstreams))]
}
func (p *EpRand) randEntries(entries []upstream.Entry) upstream.Entry {
rand.Seed(time.Now().Unix())
return entries[rand.Intn(len(entries))]
}
+4 -10
View File
@@ -145,16 +145,11 @@ func (f *Fs) Hashes() hash.Set {
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
upstreams, err := f.create(ctx, dir)
if err == fs.ErrorObjectNotFound {
if dir != parentDir(dir) {
if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
return err
}
upstreams, err = f.create(ctx, dir)
} else if dir == "" {
// If root dirs not created then create them
upstreams, err = f.upstreams, nil
if err == fs.ErrorObjectNotFound && dir != parentDir(dir) {
if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
return err
}
upstreams, err = f.create(ctx, dir)
}
if err != nil {
return err
@@ -823,7 +818,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
fs.Debugf(f, "actionPolicy = %T, createPolicy = %T, searchPolicy = %T", f.actionPolicy, f.createPolicy, f.searchPolicy)
var features = (&fs.Features{
CaseInsensitive: true,
DuplicateFiles: false,
-4
View File
@@ -1129,14 +1129,10 @@ func (o *Object) Storable() bool {
// Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
var resp *http.Response
fs.FixRangeOption(options, o.size)
opts := rest.Opts{
Method: "GET",
Path: o.filePath(),
Options: options,
ExtraHeaders: map[string]string{
"Depth": "0",
},
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.Call(ctx, &opts)
+2 -10
View File
@@ -36,7 +36,6 @@ var (
cgo = flag.Bool("cgo", false, "Use cgo for the build")
noClean = flag.Bool("no-clean", false, "Don't clean the build directory before running.")
tags = flag.String("tags", "", "Space separated list of build tags")
buildmode = flag.String("buildmode", "", "Passed to go build -buildmode flag")
compileOnly = flag.Bool("compile-only", false, "Just build the binary, not the zip.")
)
@@ -301,15 +300,8 @@ func compileArch(version, goos, goarch, dir string) bool {
"-trimpath",
"-o", output,
"-tags", *tags,
}
if *buildmode != "" {
args = append(args,
"-buildmode", *buildmode,
)
}
args = append(args,
"..",
)
}
env := []string{
"GOOS=" + goos,
"GOARCH=" + stripVersion(goarch),
@@ -332,7 +324,7 @@ func compileArch(version, goos, goarch, dir string) bool {
artifacts := []string{buildZip(dir)}
// build a .deb and .rpm if appropriate
if goos == "linux" {
artifacts = append(artifacts, buildDebAndRpm(dir, version, stripVersion(goarch))...)
artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...)
}
if *copyAs != "" {
for _, artifact := range artifacts {
+7 -9
View File
@@ -15,12 +15,10 @@ description: |
vendor: "rclone"
homepage: "https://rclone.org"
license: "MIT"
contents:
- src: ./rclone
dst: /usr/bin/rclone
- src: ./README.html
dst: /usr/share/doc/rclone/README.html
- src: ./README.txt
dst: /usr/share/doc/rclone/README.txt
- src: ./rclone.1
dst: /usr/share/man/man1/rclone.1
# No longer supported? See https://github.com/goreleaser/nfpm/issues/144
# bindir: "/usr/bin"
files:
./rclone: "/usr/bin/rclone"
./README.html: "/usr/share/doc/rclone/README.html"
./README.txt: "/usr/share/doc/rclone/README.txt"
./rclone.1: "/usr/share/man/man1/rclone.1"
+1 -1
View File
@@ -14,7 +14,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "cleanup remote:path",
Short: `Clean up the remote if possible.`,
Short: `Clean up the remote if possible`,
Long: `
Clean up the remote if possible. Empty the trash or delete old file
versions. Not supported by all remotes.
+2 -4
View File
@@ -9,6 +9,7 @@ package cmd
import (
"fmt"
"log"
"math/rand"
"os"
"os/exec"
"path"
@@ -34,7 +35,6 @@ import (
"github.com/rclone/rclone/fs/rc/rcflags"
"github.com/rclone/rclone/fs/rc/rcserver"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/random"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
@@ -512,9 +512,7 @@ func AddBackendFlags() {
// Main runs rclone interpreting flags and commands out of os.Args
func Main() {
if err := random.Seed(); err != nil {
log.Fatalf("Fatal error: %v", err)
}
rand.Seed(time.Now().Unix())
setupRootCommand(Root)
AddBackendFlags()
if err := Root.Execute(); err != nil {
-32
View File
@@ -1,32 +0,0 @@
// Build for macos with the brew tag to handle the absence
// of fuse and print an appropriate error message
// +build brew
// +build darwin
package cmount
import (
"github.com/pkg/errors"
"github.com/rclone/rclone/cmd/mountlib"
"github.com/rclone/rclone/vfs"
)
func init() {
name := "mount"
cmd := mountlib.NewMountCommand(name, false, mount)
cmd.Aliases = append(cmd.Aliases, "cmount")
mountlib.AddRc("cmount", mount)
}
// mount the file system
//
// The mount point will be ready when this returns.
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(_ *vfs.VFS, _ string, _ *mountlib.Options) (<-chan error, func() error, error) {
return nil, nil, errors.New("mount is not supported on MacOS when installed via Homebrew. " +
"Please install the binaries available at https://rclone." +
"org/downloads/ instead if you want to use the mount command")
}
+1 -1
View File
@@ -1,6 +1,6 @@
// Build for cmount for unsupported platforms to stop go complaining
// about "no buildable Go source files "
// +build !linux,!darwin,!freebsd,!windows !brew !cgo !cmount
// +build !linux,!darwin,!freebsd,!windows !cgo !cmount
package cmount
+1 -1
View File
@@ -22,7 +22,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "copy source:path dest:path",
Short: `Copy files from source to dest, skipping already copied.`,
Short: `Copy files from source to dest, skipping already copied`,
Long: `
Copy the source to the destination. Doesn't transfer
unchanged files, testing by size and modification time or
+1 -1
View File
@@ -15,7 +15,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "copyto source:path dest:path",
Short: `Copy files from source to dest, skipping already copied.`,
Short: `Copy files from source to dest, skipping already copied`,
Long: `
If source:path is a file or directory then it copies it to a file or
directory named dest:path.
+1 -1
View File
@@ -44,7 +44,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "lsf remote:path",
Short: `List directories and objects in remote:path formatted for parsing.`,
Short: `List directories and objects in remote:path formatted for parsing`,
Long: `
List the contents of the source path (directories and objects) to
standard output in a form which is easy to parse by scripts. By
-7
View File
@@ -107,13 +107,6 @@ func (d *Dir) ReadDirAll(ctx context.Context) (dirents []fuse.Dirent, err error)
if err != nil {
return nil, translateError(err)
}
dirents = append(dirents, fuse.Dirent{
Type: fuse.DT_Dir,
Name: ".",
}, fuse.Dirent{
Type: fuse.DT_Dir,
Name: "..",
})
for _, node := range items {
name := node.Name()
if len(name) > mountlib.MaxLeafSize {
+1 -1
View File
@@ -1,7 +1,7 @@
// Build for mount for unsupported platforms to stop go complaining
// about "no buildable Go source files "
// Invert the build constraint: linux,go1.13 freebsd,go1.13
// Invert the build constraint: linux,go1.13 darwin,go1.13 freebsd,go1.13
//
// !((linux&&go1.13) || (darwin&&go1.13) || (freebsd&&go1.13))
// == !(linux&&go1.13) && !(darwin&&go1.13) && !(freebsd&&go1.13))
+2 -2
View File
@@ -67,8 +67,8 @@ func setAttr(node vfs.Node, attr *fuse.Attr) {
modTime := node.ModTime()
// set attributes
vfs := node.VFS()
attr.Owner.Gid = vfs.Opt.GID
attr.Owner.Uid = vfs.Opt.UID
attr.Owner.Gid = vfs.Opt.UID
attr.Owner.Uid = vfs.Opt.GID
attr.Mode = getMode(node)
attr.Size = Size
attr.Nlink = 1
+3 -3
View File
@@ -192,9 +192,6 @@ Stopping the mount manually:
# OS X
umount /path/to/local/mount
**Note**: As of ` + "`rclone` 1.52.2, `rclone mount`" + ` now requires Go version 1.13
or newer on some platforms depending on the underlying FUSE library in use.
### Installing on Windows
To run rclone ` + commandName + ` on Windows, you will need to
@@ -336,6 +333,9 @@ With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following
parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
When --vfs-read-chunk-size-limit 500M is specified, the result would be
0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.
Chunked reading will only work with --vfs-cache-mode < full, as the file will always
be copied to the vfs cache before opening with --vfs-cache-mode full.
` + vfs.Help,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
+1 -1
View File
@@ -17,7 +17,7 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "obscure password",
Short: `Obscure password for use in the rclone config file.`,
Short: `Obscure password for use in the rclone config file`,
Long: `In the rclone config file, human readable passwords are
obscured. Obscuring them is done by encrypting them and writing them
out in base64. This is **not** a secure way of encrypting these
+3 -21
View File
@@ -1,7 +1,6 @@
package http
import (
"io"
"net/http"
"os"
"path"
@@ -173,11 +172,8 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
obj := entry.(fs.Object)
file := node.(*vfs.File)
// Set content length if we know how long the object is
knownSize := obj.Size() >= 0
if knownSize {
w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
}
// Set content length since we know how long the object is
w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
// Set content type
mimeType := fs.MimeType(r.Context(), obj)
@@ -214,19 +210,5 @@ func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string
// FIXME in = fs.NewAccount(in, obj).WithBuffer() // account the transfer
// Serve the file
if knownSize {
http.ServeContent(w, r, remote, node.ModTime(), in)
} else {
// http.ServeContent can't serve unknown length files
if rangeRequest := r.Header.Get("Range"); rangeRequest != "" {
http.Error(w, "Can't use Range: on files of unknown length", http.StatusRequestedRangeNotSatisfiable)
return
}
n, err := io.Copy(w, in)
if err != nil {
fs.Errorf(obj, "Didn't finish writing GET request (wrote %d/unknown bytes): %v", n, err)
return
}
}
http.ServeContent(w, r, remote, node.ModTime(), in)
}
+27 -34
View File
@@ -75,39 +75,6 @@ func (s *server) getVFS(what string, sshConn *ssh.ServerConn) (VFS *vfs.VFS) {
return VFS
}
// Accept a single connection - run in a go routine as the ssh
// authentication can block
func (s *server) acceptConnection(nConn net.Conn) {
what := describeConn(nConn)
// Before use, a handshake must be performed on the incoming net.Conn.
sshConn, chans, reqs, err := ssh.NewServerConn(nConn, s.config)
if err != nil {
fs.Errorf(what, "SSH login failed: %v", err)
return
}
fs.Infof(what, "SSH login from %s using %s", sshConn.User(), sshConn.ClientVersion())
// Discard all global out-of-band Requests
go ssh.DiscardRequests(reqs)
c := &conn{
what: what,
vfs: s.getVFS(what, sshConn),
}
if c.vfs == nil {
fs.Infof(what, "Closing unauthenticated connection (couldn't find VFS)")
_ = nConn.Close()
return
}
c.handlers = newVFSHandler(c.vfs)
// Accept all channels
go c.handleChannels(chans)
}
// Accept connections and call them in a go routine
func (s *server) acceptConnections() {
for {
nConn, err := s.listener.Accept()
@@ -118,7 +85,33 @@ func (s *server) acceptConnections() {
fs.Errorf(nil, "Failed to accept incoming connection: %v", err)
continue
}
go s.acceptConnection(nConn)
what := describeConn(nConn)
// Before use, a handshake must be performed on the incoming net.Conn.
sshConn, chans, reqs, err := ssh.NewServerConn(nConn, s.config)
if err != nil {
fs.Errorf(what, "SSH login failed: %v", err)
continue
}
fs.Infof(what, "SSH login from %s using %s", sshConn.User(), sshConn.ClientVersion())
// Discard all global out-of-band Requests
go ssh.DiscardRequests(reqs)
c := &conn{
what: what,
vfs: s.getVFS(what, sshConn),
}
if c.vfs == nil {
fs.Infof(what, "Closing unauthenticated connection (couldn't find VFS)")
_ = nConn.Close()
continue
}
c.handlers = newVFSHandler(c.vfs)
// Accept all channels
go c.handleChannels(chans)
}
}
-1
View File
@@ -148,7 +148,6 @@ WebDAV or S3, that work out of the box.)
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
{{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}}
{{< provider name="Tardigrade" home="https://tardigrade.io/" config="/tardigrade/" >}}
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
{{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
{{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}
+1 -2
View File
@@ -404,7 +404,6 @@ Note that Box is case insensitive so you can't have a file called
"Hello.doc" and one called "hello.doc".
Box file names can't have the `\` character in. rclone maps this to
and from an identical looking unicode equivalent `\` (U+FF3C Fullwidth
Reverse Solidus).
and from an identical looking unicode equivalent `\`.
Box only supports filenames up to 255 characters in length.
-138
View File
@@ -5,144 +5,6 @@ description: "Rclone Changelog"
# Changelog
## v1.53.4 - 2021-01-20
[See commits](https://github.com/rclone/rclone/compare/v1.53.3...v1.53.4)
* Bug Fixes
* accounting: Fix data race in Transferred() (Maciej Zimnoch)
* build
* Stop tagged releases making a current beta (Nick Craig-Wood)
* Upgrade docker buildx action (Matteo Pietro Dazzi)
* Add -buildmode to cross-compile.go (Nick Craig-Wood)
* Fix docker build by upgrading ilteoood/docker_buildx (Nick Craig-Wood)
* Revert GitHub actions brew fix since this is now fixed (Nick Craig-Wood)
* Fix brew install --cask syntax for macOS build (Nick Craig-Wood)
* Update nfpm syntax to fix build of .deb/.rpm packages (Nick Craig-Wood)
* Fix for Windows build errors (Ivan Andreev)
* fs: Parseduration: fixed tests to use UTC time (Ankur Gupta)
* fshttp: Prevent overlap of HTTP headers in logs (Nathan Collins)
* rc
* Fix core/command giving 500 internal error (Nick Craig-Wood)
* Add Copy method to rc.Params (Nick Craig-Wood)
* Fix 500 error when marshalling errors from core/command (Nick Craig-Wood)
* plugins: Create plugins files only if webui is enabled. (negative0)
* serve http: Fix serving files of unknown length (Nick Craig-Wood)
* serve sftp: Fix authentication on one connection blocking others (Nick Craig-Wood)
* Mount
* Add optional `brew` tag to throw an error when using mount in the binaries installed via Homebrew (Anagh Kumar Baranwal)
* Add "." and ".." to directories to match cmount and expectations (Nick Craig-Wood)
* VFS
* Make cache dir absolute before using it to fix path too long errors (Nick Craig-Wood)
* Chunker
* Improve detection of incompatible metadata (Ivan Andreev)
* Google Cloud Storage
* Fix server side copy of large objects (Nick Craig-Wood)
* Jottacloud
* Fix token renewer to fix long uploads (Nick Craig-Wood)
* Fix token refresh failed: is not a regular file error (Nick Craig-Wood)
* Pcloud
* Only use SHA1 hashes in EU region (Nick Craig-Wood)
* Sharefile
* Undo Fix backend due to API swapping integers for strings (Nick Craig-Wood)
* WebDAV
* Fix Open Range requests to fix 4shared mount (Nick Craig-Wood)
* Add "Depth: 0" to GET requests to fix bitrix (Nick Craig-Wood)
## v1.53.3 - 2020-11-19
[See commits](https://github.com/rclone/rclone/compare/v1.53.2...v1.53.3)
* Bug Fixes
* random: Fix incorrect use of math/rand instead of crypto/rand CVE-2020-28924 (Nick Craig-Wood)
* Passwords you have generated with `rclone config` may be insecure
* See [issue #4783](https://github.com/rclone/rclone/issues/4783) for more details and a checking tool
* random: Seed math/rand in one place with crypto strong seed (Nick Craig-Wood)
* VFS
* Fix vfs/refresh calls with fs= parameter (Nick Craig-Wood)
* Sharefile
* Fix backend due to API swapping integers for strings (Nick Craig-Wood)
## v1.53.2 - 2020-10-26
[See commits](https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2)
* Bug Fixes
* acounting
* Fix incorrect speed and transferTime in core/stats (Nick Craig-Wood)
* Stabilize display order of transfers on Windows (Nick Craig-Wood)
* operations
* Fix use of --suffix without --backup-dir (Nick Craig-Wood)
* Fix spurious "--checksum is in use but the source and destination have no hashes in common" (Nick Craig-Wood)
* build
* Work around GitHub actions brew problem (Nick Craig-Wood)
* Stop using set-env and set-path in the GitHub actions (Nick Craig-Wood)
* Mount
* mount2: Fix the swapped UID / GID values (Russell Cattelan)
* VFS
* Detect and recover from a file being removed externally from the cache (Nick Craig-Wood)
* Fix a deadlock vulnerability in downloaders.Close (Leo Luan)
* Fix a race condition in retryFailedResets (Leo Luan)
* Fix missed concurrency control between some item operations and reset (Leo Luan)
* Add exponential backoff during ENOSPC retries (Leo Luan)
* Add a missed update of used cache space (Leo Luan)
* Fix --no-modtime to not attempt to set modtimes (as documented) (Nick Craig-Wood)
* Local
* Fix sizes and syncing with --links option on Windows (Nick Craig-Wood)
* Chunker
* Disable ListR to fix missing files on GDrive (workaround) (Ivan Andreev)
* Fix upload over crypt (Ivan Andreev)
* Fichier
* Increase maximum file size from 100GB to 300GB (gyutw)
* Jottacloud
* Remove clientSecret from config when upgrading to token based authentication (buengese)
* Avoid double url escaping of device/mountpoint (albertony)
* Remove DirMove workaround as it's not required anymore - also (buengese)
* Mailru
* Fix uploads after recent changes on server (Ivan Andreev)
* Fix range requests after june changes on server (Ivan Andreev)
* Fix invalid timestamp on corrupted files (fixes) (Ivan Andreev)
* Onedrive
* Fix disk usage for sharepoint (Nick Craig-Wood)
* S3
* Add missing regions for AWS (Anagh Kumar Baranwal)
* Seafile
* Fix accessing libraries > 2GB on 32 bit systems (Muffin King)
* SFTP
* Always convert the checksum to lower case (buengese)
* Union
* Create root directories if none exist (Nick Craig-Wood)
## v1.53.1 - 2020-09-13
[See commits](https://github.com/rclone/rclone/compare/v1.53.0...v1.53.1)
* Bug Fixes
* accounting: Remove new line from end of --stats-one-line display (Nick Craig-Wood)
* check
* Add back missing --download flag (Nick Craig-Wood)
* Fix docs (Nick Craig-Wood)
* docs
* Note --log-file does append (Nick Craig-Wood)
* Add full stops for consistency in rclone --help (edwardxml)
* Add Tencent COS to s3 provider list (wjielai)
* Updated mount command to reflect that it requires Go 1.13 or newer (Evan Harris)
* jottacloud: Mention that uploads from local disk will not need to cache files to disk for md5 calculation (albertony)
* Fix formatting of rc docs page (Nick Craig-Wood)
* build
* Include vendor tar ball in release and fix startdev (Nick Craig-Wood)
* Fix "Illegal instruction" error for ARMv6 builds (Nick Craig-Wood)
* Fix architecture name in ARMv7 build (Nick Craig-Wood)
* VFS
* Fix spurious error "vfs cache: failed to _ensure cache EOF" (Nick Craig-Wood)
* Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
* Local
* Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
* Drive
* Re-adds special oauth help text (Tim Gallant)
* Opendrive
* Do not retry 400 errors (Evan Harris)
## v1.53.0 - 2020-09-02
[See commits](https://github.com/rclone/rclone/compare/v1.52.0...v1.53.0)
+5 -5
View File
@@ -39,10 +39,10 @@ See the [global flags page](/flags/) for global options not listed here.
* [rclone backend](/commands/rclone_backend/) - Run a backend specific command.
* [rclone cat](/commands/rclone_cat/) - Concatenates any files and sends them to stdout.
* [rclone check](/commands/rclone_check/) - Checks the files in the source and destination match.
* [rclone cleanup](/commands/rclone_cleanup/) - Clean up the remote if possible.
* [rclone cleanup](/commands/rclone_cleanup/) - Clean up the remote if possible
* [rclone config](/commands/rclone_config/) - Enter an interactive configuration session.
* [rclone copy](/commands/rclone_copy/) - Copy files from source to dest, skipping already copied.
* [rclone copyto](/commands/rclone_copyto/) - Copy files from source to dest, skipping already copied.
* [rclone copy](/commands/rclone_copy/) - Copy files from source to dest, skipping already copied
* [rclone copyto](/commands/rclone_copyto/) - Copy files from source to dest, skipping already copied
* [rclone copyurl](/commands/rclone_copyurl/) - Copy url content to dest.
* [rclone cryptcheck](/commands/rclone_cryptcheck/) - Cryptcheck checks the integrity of a crypted remote.
* [rclone cryptdecode](/commands/rclone_cryptdecode/) - Cryptdecode returns unencrypted file names.
@@ -56,7 +56,7 @@ See the [global flags page](/flags/) for global options not listed here.
* [rclone listremotes](/commands/rclone_listremotes/) - List all the remotes in the config file.
* [rclone ls](/commands/rclone_ls/) - List the objects in the path with size and path.
* [rclone lsd](/commands/rclone_lsd/) - List all directories/containers/buckets in the path.
* [rclone lsf](/commands/rclone_lsf/) - List directories and objects in remote:path formatted for parsing.
* [rclone lsf](/commands/rclone_lsf/) - List directories and objects in remote:path formatted for parsing
* [rclone lsjson](/commands/rclone_lsjson/) - List directories and objects in the path in JSON format.
* [rclone lsl](/commands/rclone_lsl/) - List the objects in path with modification time, size and path.
* [rclone md5sum](/commands/rclone_md5sum/) - Produces an md5sum file for all the objects in the path.
@@ -65,7 +65,7 @@ See the [global flags page](/flags/) for global options not listed here.
* [rclone move](/commands/rclone_move/) - Move files from source to dest.
* [rclone moveto](/commands/rclone_moveto/) - Move file or directory from source to dest.
* [rclone ncdu](/commands/rclone_ncdu/) - Explore a remote with a text based user interface.
* [rclone obscure](/commands/rclone_obscure/) - Obscure password for use in the rclone config file.
* [rclone obscure](/commands/rclone_obscure/) - Obscure password for use in the rclone config file
* [rclone purge](/commands/rclone_purge/) - Remove the path and all of its contents.
* [rclone rc](/commands/rclone_rc/) - Run a command against a running rclone.
* [rclone rcat](/commands/rclone_rcat/) - Copies standard input to file on remote.
+1 -2
View File
@@ -29,7 +29,7 @@ the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--src-only`
and `--error` flags write paths, one per line, to the file name (or
stdout if it is `-`) supplied. What they write is described in the
help below. For example `--differ` will write all paths which are
@@ -55,7 +55,6 @@ rclone check source:path dest:path [flags]
```
--combined string Make a combined report of changes to this file
--differ string Report all non-matching files to this file
--download Check by downloading rather than with hash.
--error string Report all files with errors (hashing or reading) to this file
-h, --help help for check
--match string Report all matching files to this file
+2 -2
View File
@@ -1,13 +1,13 @@
---
title: "rclone cleanup"
description: "Clean up the remote if possible."
description: "Clean up the remote if possible"
slug: rclone_cleanup
url: /commands/rclone_cleanup/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/cleanup/ and as part of making a release run "make commanddocs"
---
# rclone cleanup
Clean up the remote if possible.
Clean up the remote if possible
## Synopsis
+2 -2
View File
@@ -1,13 +1,13 @@
---
title: "rclone copy"
description: "Copy files from source to dest, skipping already copied."
description: "Copy files from source to dest, skipping already copied"
slug: rclone_copy
url: /commands/rclone_copy/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/copy/ and as part of making a release run "make commanddocs"
---
# rclone copy
Copy files from source to dest, skipping already copied.
Copy files from source to dest, skipping already copied
## Synopsis
+2 -2
View File
@@ -1,13 +1,13 @@
---
title: "rclone copyto"
description: "Copy files from source to dest, skipping already copied."
description: "Copy files from source to dest, skipping already copied"
slug: rclone_copyto
url: /commands/rclone_copyto/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/copyto/ and as part of making a release run "make commanddocs"
---
# rclone copyto
Copy files from source to dest, skipping already copied.
Copy files from source to dest, skipping already copied
## Synopsis
+1 -1
View File
@@ -40,7 +40,7 @@ the source match the files in the destination, not the other way
around. This means that extra files in the destination that are not in
the source will not be detected.
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--src-only`
and `--error` flags write paths, one per line, to the file name (or
stdout if it is `-`) supplied. What they write is described in the
help below. For example `--differ` will write all paths which are
+2 -2
View File
@@ -1,13 +1,13 @@
---
title: "rclone lsf"
description: "List directories and objects in remote:path formatted for parsing."
description: "List directories and objects in remote:path formatted for parsing"
slug: rclone_lsf
url: /commands/rclone_lsf/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/lsf/ and as part of making a release run "make commanddocs"
---
# rclone lsf
List directories and objects in remote:path formatted for parsing.
List directories and objects in remote:path formatted for parsing
## Synopsis
+3 -8
View File
@@ -49,9 +49,6 @@ Stopping the mount manually:
# OS X
umount /path/to/local/mount
**Note**: As of `rclone` 1.52.2, `rclone mount` now requires Go version 1.13
or newer on some platforms depending on the underlying FUSE library in use.
## Installing on Windows
To run rclone mount on Windows, you will need to
@@ -194,6 +191,9 @@ parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
When --vfs-read-chunk-size-limit 500M is specified, the result would be
0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.
Chunked reading will only work with --vfs-cache-mode < full, as the file will always
be copied to the vfs cache before opening with --vfs-cache-mode full.
## VFS - Virtual File System
This command uses the VFS layer. This adapts the cloud storage objects
@@ -357,11 +357,6 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
+2 -2
View File
@@ -1,13 +1,13 @@
---
title: "rclone obscure"
description: "Obscure password for use in the rclone config file."
description: "Obscure password for use in the rclone config file"
slug: rclone_obscure
url: /commands/rclone_obscure/
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/obscure/ and as part of making a release run "make commanddocs"
---
# rclone obscure
Obscure password for use in the rclone config file.
Obscure password for use in the rclone config file
## Synopsis
@@ -196,11 +196,6 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -195,11 +195,6 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -267,11 +267,6 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -206,11 +206,6 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
@@ -275,11 +275,6 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
## VFS Performance
These flags may be used to enable/disable features of the VFS for
+63 -60
View File
@@ -6,26 +6,23 @@ description: "Encryption overlay remote"
{{< icon "fa fa-lock" >}}Crypt
----------------------------------------
Rclone `crypt` remotes encrypt and decrypt other remotes.
The `crypt` remote encrypts and decrypts another remote.
To use `crypt`, first set up the underlying remote. Follow the `rclone
config` instructions for that remote.
To use it first set up the underlying remote following the config
instructions for that remote. You can also use a local pathname
instead of a remote which will encrypt and decrypt from that directory
which might be useful for encrypting onto a USB stick for example.
`crypt` applied to a local pathname instead of a remote will
encrypt and decrypt that directory, and can be used to encrypt USB
removable drives.
First check your chosen remote is working - we'll call it
`remote:path` in these docs. Note that anything inside `remote:path`
will be encrypted and anything outside won't. This means that if you
are using a bucket based remote (eg S3, B2, swift) then you should
probably put the bucket in the remote `s3:bucket`. If you just use
`s3:` then rclone will make encrypted bucket names too (if using file
name encryption) which may or may not be what you want.
Before configuring the crypt remote, check the underlying remote is
working. In this example the underlying remote is called `remote:path`.
Anything inside `remote:path` will be encrypted and anything outside
will not. In the case of an S3 based underlying remote (eg Amazon S3,
B2, Swift) it is generally advisable to define a crypt remote in the
underlying remote `s3:bucket`. If `s3:` alone is specified alongside
file name encryption, rclone will encrypt the bucket name.
Configure `crypt` using `rclone config`. In this example the `crypt`
remote is called `secret`, to differentiate it from the underlying
`remote`.
Now configure `crypt` using `rclone config`. We will call this one
`secret` to differentiate it from the `remote`.
```
No remotes found - make a new one
@@ -99,42 +96,49 @@ d) Delete this remote
y/e/d> y
```
**Important** The crypt password stored in `rclone.conf` is lightly
obscured. That only protects it from cursory inspection. It is not
secure unless encryption of `rclone.conf` is specified.
**Important** The password is stored in the config file is lightly
obscured so it isn't immediately obvious what it is. It is in no way
secure unless you use config file encryption.
A long passphrase is recommended, or `rclone config` can generate a
random one.
A long passphrase is recommended, or you can use a random one.
The obscured password is created using AES-CTR with a static key. The
salt is stored verbatim at the beginning of the obscured password. This
static key is shared between all versions of rclone.
The obscured password is created by using AES-CTR with a static key, with
the salt stored verbatim at the beginning of the obscured password. This
static key is shared by between all versions of rclone.
If you reconfigure rclone with the same passwords/passphrases
elsewhere it will be compatible, but the obscured version will be different
due to the different salt.
Rclone does not encrypt
Note that rclone does not encrypt
* file length - this can be calculated within 16 bytes
* modification time - used for syncing
## Specifying the remote ##
In normal use, ensure the remote has a `:` in. If specified without,
rclone uses a local directory of that name. For example if a remote
`/path/to/secret/files` is specified, rclone encrypts content to that
directory. If a remote `name` is specified, rclone targets a directory
`name` in the current directory.
In normal use, make sure the remote has a `:` in. If you specify the
remote without a `:` then rclone will use a local directory of that
name. So if you use a remote of `/path/to/secret/files` then rclone
will encrypt stuff to that directory. If you use a remote of `name`
then rclone will put files in a directory called `name` in the current
directory.
If remote `remote:path/to/dir` is specified, rclone stores encrypted
files in `path/to/dir` on the remote. With file name encryption, files
saved to `secret:subdir/subfile` are stored in the unencrypted path
`path/to/dir` but the `subdir/subpath` element is encrypted.
If you specify the remote as `remote:path/to/dir` then rclone will
store encrypted files in `path/to/dir` on the remote. If you are using
file name encryption, then when you save files to
`secret:subdir/subfile` this will store them in the unencrypted path
`path/to/dir` but the `subdir/subpath` bit will be encrypted.
Note that unless you want encrypted bucket names (which are difficult
to manage because you won't know what directory they represent in web
interfaces etc), you should probably specify a bucket, eg
`remote:secretbucket` when using bucket based remotes such as S3,
Swift, Hubic, B2, GCS.
## Example ##
Create the following file structure using "standard" file name
To test I made a little directory of files using "standard" file name
encryption.
```
@@ -148,7 +152,7 @@ plaintext/
└── file4.txt
```
Copy these to the remote, and list them
Copy these to the remote and list them back
```
$ rclone -q copy plaintext secret:
@@ -160,7 +164,7 @@ $ rclone -q ls secret:
9 subdir/file3.txt
```
The crypt remote looks like
Now see what that looked like when encrypted
```
$ rclone -q ls remote:path
@@ -171,7 +175,7 @@ $ rclone -q ls remote:path
56 86vhrsv86mpbtd3a0akjuqslj8/8njh1sk437gttmep3p70g81aps
```
The directory structure is preserved
Note that this retains the directory structure which means you can do this
```
$ rclone -q ls secret:subdir
@@ -180,9 +184,9 @@ $ rclone -q ls secret:subdir
10 subsubdir/file4.txt
```
Without file name encryption `.bin` extensions are added to underlying
names. This prevents the cloud provider attempting to interpret file
content.
If don't use file name encryption then the remote will look like this
- note the `.bin` extensions added to prevent the cloud provider
attempting to interpret the data.
```
$ rclone -q ls remote:path
@@ -195,6 +199,8 @@ $ rclone -q ls remote:path
### File name encryption modes ###
Here are some of the features of the file name encryption modes
Off
* doesn't hide file names or directory structure
@@ -213,19 +219,17 @@ Standard
Obfuscation
This is a simple "rotate" of the filename, with each file having a rot
distance based on the filename. Rclone stores the distance at the
beginning of the filename. A file called "hello" may become "53.jgnnq".
distance based on the filename. We store the distance at the beginning
of the filename. So a file called "hello" may become "53.jgnnq".
Obfuscation is not a strong encryption of filenames, but hinders
automated scanning tools picking up on filename patterns. It is an
intermediate between "off" and "standard" which allows for longer path
segment names.
This is not a strong encryption of filenames, but it may stop automated
scanning tools from picking up on filename patterns. As such it's an
intermediate between "off" and "standard". The advantage is that it
allows for longer path segment names.
There is a possibility with some unicode based filenames that the
obfuscation is weak and may map lower case characters to upper case
equivalents.
Obfuscation cannot be relied upon for strong protection.
equivalents. You can not rely on this for strong protection.
* file names very lightly obfuscated
* file names can be longer than standard encryption
@@ -233,14 +237,13 @@ Obfuscation cannot be relied upon for strong protection.
* directory structure visible
* identical files names will have identical uploaded names
Cloud storage systems have limits on file name length and
total path length which rclone is more likely to breach using
"Standard" file name encryption. Where file names are less thn 156
characters in length issues should not be encountered, irrespective of
cloud storage provider.
Cloud storage systems have various limits on file name length and
total path length which you are more likely to hit using "Standard"
file name encryption. If you keep your file names to below 156
characters in length then you should be OK on all providers.
An alternative, future rclone file name encryption mode may tolerate
backend provider path length limits.
There may be an even more secure file name encryption mode in the
future which will address the long file name problem.
### Directory name encryption ###
Crypt offers the option of encrypting dir names or leaving them intact.
@@ -266,10 +269,10 @@ Example:
Crypt stores modification times using the underlying remote so support
depends on that.
Hashes are not stored for crypt. However the data integrity is
Hashes are not stored for crypt. However the data integrity is
protected by an extremely strong crypto authenticator.
Use the `rclone cryptcheck` command to check the
Note that you should use the `rclone cryptcheck` command to check the
integrity of a crypted remote instead of `rclone check` which can't
check the checksums properly.
+2 -10
View File
@@ -757,8 +757,6 @@ This can be useful for tracking down problems with syncs in
combination with the `-v` flag. See the [Logging section](#logging)
for more info.
If FILE exists then rclone will append to it.
Note that if you are using the `logrotate` program to manage rclone's
logs, then you should use the `copytruncate` option as rclone doesn't
have a signal to rotate logs.
@@ -1253,17 +1251,11 @@ or with `--backup-dir`. See `--backup-dir` for more info.
For example
rclone copy -i /path/to/local/file remote:current --suffix .bak
rclone sync -i /path/to/local/file remote:current --suffix .bak
will copy `/path/to/local` to `remote:current`, but for any files
will sync `/path/to/local` to `remote:current`, but for any files
which would have been updated or deleted have .bak added.
If using `rclone sync` with `--suffix` and without `--backup-dir` then
it is recommended to put a filter rule in excluding the suffix
otherwise the `sync` will delete the backup files.
rclone sync -i /path/to/local/file remote:current --suffix .bak --exclude "*.bak"
### --suffix-keep-extension ###
When using `--suffix`, setting this causes rclone put the SUFFIX
+2 -4
View File
@@ -547,10 +547,8 @@ Here are the standard options specific to drive (Google Drive).
#### --drive-client-id
Google Application Client Id
Setting your own is recommended.
See https://rclone.org/drive/#making-your-own-client-id for how to create your own.
If you leave this blank, it will use an internal key which is low performance.
OAuth Client Id
Leave blank normally.
- Config: client_id
- Env Var: RCLONE_DRIVE_CLIENT_ID
+2 -2
View File
@@ -147,7 +147,7 @@ These flags are available for every command.
--use-json-log Use json log format.
--use-mmap Use mmap allocator (see docs).
--use-server-modtime Use server modified time instead of object metadata
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.4")
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.53.0")
-v, --verbose count Print lots more stuff (repeat for more)
```
@@ -246,7 +246,7 @@ and may be set in the config file.
--drive-auth-owner-only Only consider files owned by the authenticated user.
--drive-auth-url string Auth server URL.
--drive-chunk-size SizeSuffix Upload chunk size. Must a power of 2 >= 256k. (default 8M)
--drive-client-id string Google Application Client Id
--drive-client-id string OAuth Client Id
--drive-client-secret string OAuth Client Secret
--drive-disable-http2 Disable drive using http2 (default true)
--drive-encoding MultiEncoder This sets the encoding for the backend. (default InvalidUtf8)
+30 -34
View File
@@ -22,7 +22,7 @@ Here is an overview of the major features of each cloud storage system.
| Backblaze B2 | SHA1 | Yes | No | No | R/W |
| Box | SHA1 | Yes | Yes | No | - |
| Citrix ShareFile | MD5 | Yes | Yes | No | - |
| Dropbox | DBHASH ¹ | Yes | Yes | No | - |
| Dropbox | DBHASH † | Yes | Yes | No | - |
| FTP | - | No | No | No | - |
| Google Cloud Storage | MD5 | Yes | No | No | R/W |
| Google Drive | MD5 | Yes | No | Yes | R/W |
@@ -31,52 +31,25 @@ Here is an overview of the major features of each cloud storage system.
| Hubic | MD5 | Yes | No | No | R/W |
| Jottacloud | MD5 | Yes | Yes | No | R/W |
| Koofr | MD5 | No | Yes | No | - |
| Mail.ru Cloud | Mailru ⁶ | Yes | Yes | No | - |
| Mail.ru Cloud | Mailru ‡‡‡ | Yes | Yes | No | - |
| Mega | - | No | No | Yes | - |
| Memory | MD5 | Yes | No | No | - |
| Microsoft Azure Blob Storage | MD5 | Yes | No | No | R/W |
| Microsoft OneDrive | SHA1 ⁵ | Yes | Yes | No | R |
| OpenDrive | MD5 | Yes | Yes | Partial ⁸ | - |
| Microsoft OneDrive | SHA1 ‡‡ | Yes | Yes | No | R |
| OpenDrive | MD5 | Yes | Yes | Partial \* | - |
| OpenStack Swift | MD5 | Yes | No | No | R/W |
| pCloud | MD5, SHA1 ⁷ | Yes | No | No | W |
| pCloud | MD5, SHA1 | Yes | No | No | W |
| premiumize.me | - | No | Yes | No | R |
| put.io | CRC-32 | Yes | No | Yes | R |
| QingStor | MD5 | No | No | No | R/W |
| Seafile | - | No | No | No | - |
| SFTP | MD5, SHA1 ² | Yes | Depends | No | - |
| SFTP | MD5, SHA1 ‡ | Yes | Depends | No | - |
| SugarSync | - | No | No | No | - |
| Tardigrade | - | Yes | No | No | - |
| WebDAV | MD5, SHA1 ³ | Yes ⁴ | Depends | No | - |
| WebDAV | MD5, SHA1 ††| Yes ††† | Depends | No | - |
| Yandex Disk | MD5 | Yes | No | No | R/W |
| The local filesystem | All | Yes | Depends | No | - |
### Notes
¹ Dropbox supports [its own custom
hash](https://www.dropbox.com/developers/reference/content-hash).
This is an SHA256 sum of all the 4MB block SHA256s.
² SFTP supports checksums if the same login has shell access and
`md5sum` or `sha1sum` as well as `echo` are in the remote's PATH.
³ WebDAV supports hashes when used with Owncloud and Nextcloud only.
⁴ WebDAV supports modtimes when used with Owncloud and Nextcloud only.
⁵ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive
for business and SharePoint server support Microsoft's own
[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash).
⁶ Mail.ru uses its own modified SHA1 hash
⁷ pCloud only supports SHA1 (not MD5) in its EU region
⁸ Opendrive does not support creation of duplicate files using
their web client interface or other stock clients, but the underlying
storage platform has been determined to allow duplicate files, and it
is possible to create them with `rclone`. It may be that this is a
mistake or an unsupported feature.
### Hash ###
The cloud storage system supports various hash types of the objects.
@@ -87,6 +60,23 @@ the `check` command.
To use the verify checksums when transferring between cloud storage
systems they must support a common hash type.
† Note that Dropbox supports [its own custom
hash](https://www.dropbox.com/developers/reference/content-hash).
This is an SHA256 sum of all the 4MB block SHA256s.
‡ SFTP supports checksums if the same login has shell access and `md5sum`
or `sha1sum` as well as `echo` are in the remote's PATH.
†† WebDAV supports hashes when used with Owncloud and Nextcloud only.
††† WebDAV supports modtimes when used with Owncloud and Nextcloud only.
‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive
for business and SharePoint server support Microsoft's own
[QuickXorHash](https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash).
‡‡‡ Mail.ru uses its own modified SHA1 hash
### ModTime ###
The cloud storage system supports setting modification times on
@@ -127,6 +117,12 @@ objects with the same name.
This confuses rclone greatly when syncing - use the `rclone dedupe`
command to rename or remove duplicates.
\* Opendrive does not support creation of duplicate files using
their web client interface or other stock clients, but the underlying
storage platform has been determined to allow duplicate files, and it
is possible to create them with `rclone`. It may be that this is a
mistake or an unsupported feature.
### Restricted filenames ###
Some cloud storage systems might have restrictions on the characters
+2 -5
View File
@@ -90,11 +90,8 @@ second. These will be used to detect whether objects need syncing or
not. In order to set a Modification time pCloud requires the object
be re-uploaded.
pCloud supports MD5 and SHA1 type hashes in the US region but and SHA1
only in the EU region, so you can use the `--checksum` flag.
(Note that pCloud also support SHA256 in the EU region, but rclone
does not have support for that yet.)
pCloud supports MD5 and SHA1 type hashes, so you can use the
`--checksum` flag.
#### Restricted filename characters
+2 -7
View File
@@ -511,22 +511,17 @@ This takes the following parameters
- command - a string with the command name
- arg - a list of arguments for the backend command
- opt - a map of string to string of options
- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR")
- defaults to "COMBINED_OUTPUT" if not set
- the STREAM returnTypes will write the output to the body of the HTTP message
- the COMBINED_OUTPUT will write the output to the "result" parameter
Returns
- result - result from the backend command
- only set when using returnType "COMBINED_OUTPUT"
- error - set if rclone exits with an error code
- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR")
- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT". "STREAM_ONLY_STDERR")
For example
rclone rc core/command command=ls -a mydrive:/ -o max-depth=1
rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
Returns
+13 -261
View File
@@ -18,7 +18,6 @@ The S3 backend can be used with a number of different providers:
{{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
{{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}}
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}}
{{< /provider_list >}}
@@ -139,7 +138,7 @@ Choose a number from below, or type in your own value
/ Asia Pacific (Mumbai)
13 | Needs location constraint ap-south-1.
\ "ap-south-1"
/ Asia Pacific (Hong Kong) Region
/ Asia Patific (Hong Kong) Region
14 | Needs location constraint ap-east-1.
\ "ap-east-1"
/ South America (Sao Paulo) Region
@@ -456,7 +455,7 @@ Vault API, so rclone cannot directly access Glacier Vaults.
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
### Standard Options
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)).
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
#### --s3-provider
@@ -487,8 +486,6 @@ Choose your S3 provider.
- Scaleway Object Storage
- "StackPath"
- StackPath Object Storage
- "TencentCOS"
- Tencent Cloud Object Storage (COS)
- "Wasabi"
- Wasabi Object Storage
- "Other"
@@ -545,12 +542,12 @@ Region to connect to.
- "us-east-2"
- US East (Ohio) Region
- Needs location constraint us-east-2.
- "us-west-1"
- US West (Northern California) Region
- Needs location constraint us-west-1.
- "us-west-2"
- US West (Oregon) Region
- Needs location constraint us-west-2.
- "us-west-1"
- US West (Northern California) Region
- Needs location constraint us-west-1.
- "ca-central-1"
- Canada (Central) Region
- Needs location constraint ca-central-1.
@@ -560,15 +557,9 @@ Region to connect to.
- "eu-west-2"
- EU (London) Region
- Needs location constraint eu-west-2.
- "eu-west-3"
- EU (Paris) Region
- Needs location constraint eu-west-3.
- "eu-north-1"
- EU (Stockholm) Region
- Needs location constraint eu-north-1.
- "eu-south-1"
- EU (Milan) Region
- Needs location constraint eu-south-1.
- "eu-central-1"
- EU (Frankfurt) Region
- Needs location constraint eu-central-1.
@@ -584,36 +575,15 @@ Region to connect to.
- "ap-northeast-2"
- Asia Pacific (Seoul)
- Needs location constraint ap-northeast-2.
- "ap-northeast-3"
- Asia Pacific (Osaka-Local)
- Needs location constraint ap-northeast-3.
- "ap-south-1"
- Asia Pacific (Mumbai)
- Needs location constraint ap-south-1.
- "ap-east-1"
- Asia Pacific (Hong Kong) Region
- Asia Patific (Hong Kong) Region
- Needs location constraint ap-east-1.
- "sa-east-1"
- South America (Sao Paulo) Region
- Needs location constraint sa-east-1.
- "me-south-1"
- Middle East (Bahrain) Region
- Needs location constraint me-south-1.
- "af-south-1"
- Africa (Cape Town) Region
- Needs location constraint af-south-1.
- "cn-north-1"
- China (Beijing) Region
- Needs location constraint cn-north-1.
- "cn-northwest-1"
- China (Ningxia) Region
- Needs location constraint cn-northwest-1.
- "us-gov-east-1"
- AWS GovCloud (US-East) Region
- Needs location constraint us-gov-east-1.
- "us-gov-west-1"
- AWS GovCloud (US) Region
- Needs location constraint us-gov-west-1.
#### --s3-region
@@ -869,54 +839,6 @@ Endpoint for StackPath Object Storage.
#### --s3-endpoint
Endpoint for Tencent COS API.
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Type: string
- Default: ""
- Examples:
- "cos.ap-beijing.myqcloud.com"
- Beijing Region.
- "cos.ap-nanjing.myqcloud.com"
- Nanjing Region.
- "cos.ap-shanghai.myqcloud.com"
- Shanghai Region.
- "cos.ap-guangzhou.myqcloud.com"
- Guangzhou Region.
- "cos.ap-nanjing.myqcloud.com"
- Nanjing Region.
- "cos.ap-chengdu.myqcloud.com"
- Chengdu Region.
- "cos.ap-chongqing.myqcloud.com"
- Chongqing Region.
- "cos.ap-hongkong.myqcloud.com"
- Hong Kong (China) Region.
- "cos.ap-singapore.myqcloud.com"
- Singapore Region.
- "cos.ap-mumbai.myqcloud.com"
- Mumbai Region.
- "cos.ap-seoul.myqcloud.com"
- Seoul Region.
- "cos.ap-bangkok.myqcloud.com"
- Bangkok Region.
- "cos.ap-tokyo.myqcloud.com"
- Tokyo Region.
- "cos.na-siliconvalley.myqcloud.com"
- Silicon Valley Region.
- "cos.na-ashburn.myqcloud.com"
- Virginia Region.
- "cos.na-toronto.myqcloud.com"
- Toronto Region.
- "cos.eu-frankfurt.myqcloud.com"
- Frankfurt Region.
- "cos.eu-moscow.myqcloud.com"
- Moscow Region.
- "cos.accelerate.myqcloud.com"
- Use Tencent COS Accelerate Endpoint.
#### --s3-endpoint
Endpoint for S3 API.
Required when using an S3 clone.
@@ -954,22 +876,18 @@ Used when creating buckets only.
- Empty for US Region, Northern Virginia or Pacific Northwest.
- "us-east-2"
- US East (Ohio) Region.
- "us-west-1"
- US West (Northern California) Region.
- "us-west-2"
- US West (Oregon) Region.
- "us-west-1"
- US West (Northern California) Region.
- "ca-central-1"
- Canada (Central) Region.
- "eu-west-1"
- EU (Ireland) Region.
- "eu-west-2"
- EU (London) Region.
- "eu-west-3"
- EU (Paris) Region.
- "eu-north-1"
- EU (Stockholm) Region.
- "eu-south-1"
- EU (Milan) Region.
- "EU"
- EU Region.
- "ap-southeast-1"
@@ -979,27 +897,13 @@ Used when creating buckets only.
- "ap-northeast-1"
- Asia Pacific (Tokyo) Region.
- "ap-northeast-2"
- Asia Pacific (Seoul) Region.
- "ap-northeast-3"
- Asia Pacific (Osaka-Local) Region.
- Asia Pacific (Seoul)
- "ap-south-1"
- Asia Pacific (Mumbai) Region.
- Asia Pacific (Mumbai)
- "ap-east-1"
- Asia Pacific (Hong Kong) Region.
- Asia Pacific (Hong Kong)
- "sa-east-1"
- South America (Sao Paulo) Region.
- "me-south-1"
- Middle East (Bahrain) Region.
- "af-south-1"
- Africa (Cape Town) Region.
- "cn-north-1"
- China (Beijing) Region
- "cn-northwest-1"
- China (Ningxia) Region.
- "us-gov-east-1"
- AWS GovCloud (US-East) Region.
- "us-gov-west-1"
- AWS GovCloud (US) Region.
#### --s3-location-constraint
@@ -1102,8 +1006,6 @@ doesn't copy the ACL from the source but rather writes a fresh one.
- Type: string
- Default: ""
- Examples:
- "default"
- Owner gets Full_CONTROL. No one else has access rights (default).
- "private"
- Owner gets FULL_CONTROL. No one else has access rights (default).
- "public-read"
@@ -1204,24 +1106,6 @@ The storage class to use when storing new objects in OSS.
#### --s3-storage-class
The storage class to use when storing new objects in Tencent COS.
- Config: storage_class
- Env Var: RCLONE_S3_STORAGE_CLASS
- Type: string
- Default: ""
- Examples:
- ""
- Default
- "STANDARD"
- Standard storage class
- "ARCHIVE"
- Archive storage mode.
- "STANDARD_IA"
- Infrequent access storage mode.
#### --s3-storage-class
The storage class to use when storing new objects in S3.
- Config: storage_class
@@ -1238,7 +1122,7 @@ The storage class to use when storing new objects in S3.
### Advanced Options
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)).
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)).
#### --s3-bucket-acl
@@ -1459,7 +1343,7 @@ if false then rclone will use virtual path style. See [the AWS S3
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
for more info.
Some providers (eg AWS, Aliyun OSS, Netease COS or Tencent COS) require this set to
Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to
false - rclone will do this automatically based on the provider
setting.
@@ -2328,138 +2212,6 @@ d) Delete this remote
y/e/d> y
```
### Tencent COS {#tencent-cos}
[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.
To configure access to Tencent COS, follow the steps below:
1. Run `rclone config` and select `n` for a new remote.
```
rclone config
No remotes found - make a new one
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
```
2. Give the name of the configuration. For example, name it 'cos'.
```
name> cos
```
3. Select `s3` storage.
```
Choose a number from below, or type in your own value
1 / 1Fichier
\ "fichier"
2 / Alias for an existing remote
\ "alias"
3 / Amazon Drive
\ "amazon cloud drive"
4 / Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, Tencent COS, etc)
\ "s3"
[snip]
Storage> s3
```
4. Select `TencentCOS` provider.
```
Choose a number from below, or type in your own value
1 / Amazon Web Services (AWS) S3
\ "AWS"
[snip]
11 / Tencent Cloud Object Storage (COS)
\ "TencentCOS"
[snip]
provider> TencentCOS
```
5. Enter your SecretId and SecretKey of Tencent Cloud.
```
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
Only applies if access_key_id and secret_access_key is blank.
Enter a boolean value (true or false). Press Enter for the default ("false").
Choose a number from below, or type in your own value
1 / Enter AWS credentials in the next step
\ "false"
2 / Get AWS credentials from the environment (env vars or IAM)
\ "true"
env_auth> 1
AWS Access Key ID.
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
access_key_id> AKIDxxxxxxxxxx
AWS Secret Access Key (password)
Leave blank for anonymous access or runtime credentials.
Enter a string value. Press Enter for the default ("").
secret_access_key> xxxxxxxxxxx
```
6. Select endpoint for Tencent COS. This is the standard endpoint for different region.
```
1 / Beijing Region.
\ "cos.ap-beijing.myqcloud.com"
2 / Nanjing Region.
\ "cos.ap-nanjing.myqcloud.com"
3 / Shanghai Region.
\ "cos.ap-shanghai.myqcloud.com"
4 / Guangzhou Region.
\ "cos.ap-guangzhou.myqcloud.com"
[snip]
endpoint> 4
```
7. Choose acl and storage class.
```
Note that this ACL is applied when server side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Owner gets Full_CONTROL. No one else has access rights (default).
\ "default"
[snip]
acl> 1
The storage class to use when storing new objects in Tencent COS.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
1 / Default
\ ""
[snip]
storage_class> 1
Edit advanced config? (y/n)
y) Yes
n) No (default)
y/n> n
Remote config
--------------------
[cos]
type = s3
provider = TencentCOS
env_auth = false
access_key_id = xxx
secret_access_key = xxx
endpoint = cos.ap-guangzhou.myqcloud.com
acl = default
--------------------
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
Current remotes:
Name Type
==== ====
cos s3
```
### Netease NOS ###
For Netease NOS configure as per the configurator `rclone config`
+1 -1
View File
@@ -1 +1 @@
v1.53.4
v1.54.0
-2
View File
@@ -337,8 +337,6 @@ func (s *StatsInfo) String() string {
// Transferred returns list of all completed transfers including checked and
// failed ones.
func (s *StatsInfo) Transferred() []TransferSnapshot {
s.mu.RLock()
defer s.mu.RUnlock()
ts := make([]TransferSnapshot, 0, len(s.startedTransfers))
for _, tr := range s.startedTransfers {
-2
View File
@@ -366,8 +366,6 @@ func (sg *statsGroups) sum() *StatsInfo {
sum.lastError = stats.lastError
}
sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...)
sum.oldDuration += stats.oldDuration
sum.oldTimeRanges = append(sum.oldTimeRanges, stats.oldTimeRanges...)
}
stats.mu.RUnlock()
}
+6 -17
View File
@@ -4,10 +4,8 @@ import (
"fmt"
"runtime"
"testing"
"time"
"github.com/rclone/rclone/fstest/testy"
"github.com/stretchr/testify/assert"
)
func TestStatsGroupOperations(t *testing.T) {
@@ -45,26 +43,17 @@ func TestStatsGroupOperations(t *testing.T) {
t.Parallel()
stats1 := NewStats()
stats1.bytes = 5
stats1.errors = 6
stats1.oldDuration = time.Second
stats1.oldTimeRanges = []timeRange{{time.Now(), time.Now().Add(time.Second)}}
stats1.errors = 5
stats2 := NewStats()
stats2.bytes = 10
stats2.errors = 12
stats2.oldDuration = 2 * time.Second
stats2.oldTimeRanges = []timeRange{{time.Now(), time.Now().Add(2 * time.Second)}}
sg := newStatsGroups()
sg.set("test1", stats1)
sg.set("test2", stats2)
sum := sg.sum()
assert.Equal(t, stats1.bytes+stats2.bytes, sum.bytes)
assert.Equal(t, stats1.errors+stats2.errors, sum.errors)
assert.Equal(t, stats1.oldDuration+stats2.oldDuration, sum.oldDuration)
// dict can iterate in either order
a := timeRanges{stats1.oldTimeRanges[0], stats2.oldTimeRanges[0]}
b := timeRanges{stats2.oldTimeRanges[0], stats1.oldTimeRanges[0]}
if !assert.ObjectsAreEqual(a, sum.oldTimeRanges) {
assert.Equal(t, b, sum.oldTimeRanges)
if sum.bytes != stats1.bytes+stats2.bytes {
t.Fatalf("sum() => bytes %d, expected %d", sum.bytes, stats1.bytes+stats2.bytes)
}
if sum.errors != stats1.errors+stats2.errors {
t.Fatalf("sum() => errors %d, expected %d", sum.errors, stats1.errors+stats2.errors)
}
})
+1 -9
View File
@@ -72,16 +72,8 @@ func (tm *transferMap) _sortedSlice() []*Transfer {
for _, tr := range tm.items {
s = append(s, tr)
}
// sort by time first and if equal by name. Note that the relatively
// low time resolution on Windows can cause equal times.
sort.Slice(s, func(i, j int) bool {
a, b := s[i], s[j]
if a.startedAt.Before(b.startedAt) {
return true
} else if !a.startedAt.Equal(b.startedAt) {
return false
}
return a.remote < b.remote
return s[i].startedAt.Before(s[j].startedAt)
})
return s
}
-5
View File
@@ -31,7 +31,6 @@ var (
noTransport = new(sync.Once)
tpsBucket *rate.Limiter // for limiting number of http transactions per second
cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
logMutex sync.Mutex
)
// StartHTTPTokenBucket starts the token bucket if necessary
@@ -329,18 +328,15 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
if t.dump&fs.DumpAuth == 0 {
buf = cleanAuths(buf)
}
logMutex.Lock()
fs.Debugf(nil, "%s", separatorReq)
fs.Debugf(nil, "%s (req %p)", "HTTP REQUEST", req)
fs.Debugf(nil, "%s", string(buf))
fs.Debugf(nil, "%s", separatorReq)
logMutex.Unlock()
}
// Do round trip
resp, err = t.Transport.RoundTrip(req)
// Logf response
if t.dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
logMutex.Lock()
fs.Debugf(nil, "%s", separatorResp)
fs.Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req)
if err != nil {
@@ -350,7 +346,6 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
fs.Debugf(nil, "%s", string(buf))
}
fs.Debugf(nil, "%s", separatorResp)
logMutex.Unlock()
}
if err == nil {
checkServerTime(req, resp)
+7 -9
View File
@@ -172,12 +172,9 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
return false
}
if ht == hash.None {
common := src.Fs().Hashes().Overlap(dst.Fs().Hashes())
if common.Count() == 0 {
checksumWarning.Do(func() {
fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only")
})
}
checksumWarning.Do(func() {
fs.Logf(dst.Fs(), "--checksum is in use but the source and destination have no hashes in common; falling back to --size-only")
})
fs.Debugf(src, "Size of src and dst objects identical")
} else {
fs.Debugf(src, "Size and %v of src and dst objects identical", ht)
@@ -1525,11 +1522,12 @@ func BackupDir(fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err
}
}
}
} else if fs.Config.Suffix != "" {
} else {
if srcFileName == "" {
return nil, fserrors.FatalError(errors.New("--suffix must be used with a file or with --backup-dir"))
}
// --backup-dir is not set but --suffix is - use the destination as the backupDir
backupDir = fdst
} else {
return nil, fserrors.FatalError(errors.New("internal error: BackupDir called when --backup-dir and --suffix both empty"))
}
if !CanServerSideMove(backupDir) {
return nil, fserrors.FatalError(errors.New("can't use --backup-dir on a remote which doesn't support server side move or copy"))
+3 -3
View File
@@ -37,9 +37,9 @@ func TestParseDuration(t *testing.T) {
{"1x", 0, true},
{"off", time.Duration(DurationOff), false},
{"1h2m3s", time.Hour + 2*time.Minute + 3*time.Second, false},
{"2001-02-03", time.Since(time.Date(2001, 2, 3, 0, 0, 0, 0, time.UTC)), false},
{"2001-02-03 10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.UTC)), false},
{"2001-02-03T10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.UTC)), false},
{"2001-02-03", time.Since(time.Date(2001, 2, 3, 0, 0, 0, 0, time.Local)), false},
{"2001-02-03 10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.Local)), false},
{"2001-02-03T10:11:12", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 0, time.Local)), false},
{"2001-02-03T10:11:12.123Z", time.Since(time.Date(2001, 2, 3, 10, 11, 12, 123, time.UTC)), false},
} {
duration, err := ParseDuration(test.in)
+8 -14
View File
@@ -353,22 +353,17 @@ func init() {
- command - a string with the command name
- arg - a list of arguments for the backend command
- opt - a map of string to string of options
- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR")
- defaults to "COMBINED_OUTPUT" if not set
- the STREAM returnTypes will write the output to the body of the HTTP message
- the COMBINED_OUTPUT will write the output to the "result" parameter
Returns
- result - result from the backend command
- only set when using returnType "COMBINED_OUTPUT"
- error - set if rclone exits with an error code
- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT", "STREAM_ONLY_STDERR")
- returnType - one of ("COMBINED_OUTPUT", "STREAM", "STREAM_ONLY_STDOUT". "STREAM_ONLY_STDERR")
For example
rclone rc core/command command=ls -a mydrive:/ -o max-depth=1
rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
rclone rc core/command -a ls -a mydrive:/ -o max-depth=1
Returns
@@ -391,6 +386,7 @@ OR
// rcRunCommand runs an rclone command with the given args and flags
func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
command, err := in.GetString("command")
if err != nil {
command = ""
@@ -413,7 +409,7 @@ func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
returnType = "COMBINED_OUTPUT"
}
var httpResponse http.ResponseWriter
var httpResponse *http.ResponseWriter
httpResponse, err = in.GetHTTPResponseWriter()
if err != nil {
return nil, errors.Errorf("response object is required\n" + err.Error())
@@ -464,14 +460,12 @@ func rcRunCommand(ctx context.Context, in Params) (out Params, err error) {
"error": false,
}, nil
} else if returnType == "STREAM_ONLY_STDOUT" {
cmd.Stdout = httpResponse
cmd.Stdout = *httpResponse
} else if returnType == "STREAM_ONLY_STDERR" {
cmd.Stderr = httpResponse
cmd.Stderr = *httpResponse
} else if returnType == "STREAM" {
cmd.Stdout = httpResponse
cmd.Stderr = httpResponse
} else {
return nil, errors.Errorf("Unknown returnType %q", returnType)
cmd.Stdout = *httpResponse
cmd.Stderr = *httpResponse
}
err = cmd.Run()
+10 -56
View File
@@ -7,7 +7,6 @@ import (
"net/http/httptest"
"os"
"runtime"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -23,12 +22,6 @@ func TestMain(m *testing.M) {
fmt.Printf("rclone %s\n", fs.Version)
os.Exit(0)
}
// Pretend to error if we have an unknown command
if os.Args[len(os.Args)-1] == "unknown_command" {
fmt.Printf("rclone %s\n", fs.Version)
fmt.Fprintf(os.Stderr, "Unknown command\n")
os.Exit(1)
}
os.Exit(m.Run())
}
@@ -143,56 +136,17 @@ func TestCoreQuit(t *testing.T) {
func TestCoreCommand(t *testing.T) {
call := Calls.Get("core/command")
test := func(command string, returnType string, wantOutput string, fail bool) {
var rec = httptest.NewRecorder()
var w http.ResponseWriter = rec
var httpResponse http.ResponseWriter = httptest.NewRecorder()
in := Params{
"command": command,
"opt": map[string]string{},
"arg": []string{},
"_response": w,
}
if returnType != "" {
in["returnType"] = returnType
} else {
returnType = "COMBINED_OUTPUT"
}
stream := strings.HasPrefix(returnType, "STREAM")
got, err := call.Fn(context.Background(), in)
if stream && fail {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
if !stream {
assert.Equal(t, wantOutput, got["result"])
assert.Equal(t, fail, got["error"])
} else {
assert.Equal(t, wantOutput, rec.Body.String())
}
assert.Equal(t, http.StatusOK, rec.Result().StatusCode)
in := Params{
"command": "version",
"opt": map[string]string{},
"arg": []string{},
"_response": &httpResponse,
}
got, err := call.Fn(context.Background(), in)
require.NoError(t, err)
version := fmt.Sprintf("rclone %s\n", fs.Version)
errorString := "Unknown command\n"
t.Run("OK", func(t *testing.T) {
test("version", "", version, false)
})
t.Run("Fail", func(t *testing.T) {
test("unknown_command", "", version+errorString, true)
})
t.Run("Combined", func(t *testing.T) {
test("unknown_command", "COMBINED_OUTPUT", version+errorString, true)
})
t.Run("Stderr", func(t *testing.T) {
test("unknown_command", "STREAM_ONLY_STDERR", errorString, true)
})
t.Run("Stdout", func(t *testing.T) {
test("unknown_command", "STREAM_ONLY_STDOUT", version, true)
})
t.Run("Stream", func(t *testing.T) {
test("unknown_command", "STREAM", version+errorString, true)
})
assert.Equal(t, fmt.Sprintf("rclone %s\n", fs.Version), got["result"])
assert.Equal(t, false, got["error"])
}
+3 -12
View File
@@ -79,15 +79,6 @@ func Reshape(out interface{}, in interface{}) error {
return nil
}
// Copy shallow copies the Params
func (p Params) Copy() (out Params) {
out = make(Params, len(p))
for k, v := range p {
out[k] = v
}
return out
}
// Get gets a parameter from the input
//
// If the parameter isn't found then error will be of type
@@ -121,15 +112,15 @@ func (p Params) GetHTTPRequest() (*http.Request, error) {
//
// If the parameter isn't found then error will be of type
// ErrParamNotFound and the returned value will be nil.
func (p Params) GetHTTPResponseWriter() (http.ResponseWriter, error) {
func (p Params) GetHTTPResponseWriter() (*http.ResponseWriter, error) {
key := "_response"
value, err := p.Get(key)
if err != nil {
return nil, err
}
request, ok := value.(http.ResponseWriter)
request, ok := value.(*http.ResponseWriter)
if !ok {
return nil, ErrParamInvalid{errors.Errorf("expecting http.ResponseWriter value for key %q (was %T)", key, value)}
return nil, ErrParamInvalid{errors.Errorf("expecting *http.ResponseWriter value for key %q (was %T)", key, value)}
}
return request, nil
}
-65
View File
@@ -2,8 +2,6 @@ package rc
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
@@ -63,19 +61,6 @@ func TestReshape(t *testing.T) {
}
func TestParamsCopy(t *testing.T) {
in := Params{
"ok": 1,
"x": "seventeen",
"nil": nil,
}
out := in.Copy()
assert.Equal(t, in, out)
if &in == &out {
t.Error("didn't copy")
}
}
func TestParamsGet(t *testing.T) {
in := Params{
"ok": 1,
@@ -361,53 +346,3 @@ func TestParamsGetStructMissingOK(t *testing.T) {
assert.Equal(t, 4.2, out.Float)
assert.Equal(t, true, IsErrParamInvalid(e3), e3.Error())
}
func TestParamsGetHTTPRequest(t *testing.T) {
in := Params{}
req, err := in.GetHTTPRequest()
assert.Nil(t, req)
assert.Error(t, err)
assert.Equal(t, true, IsErrParamNotFound(err), err.Error())
in = Params{
"_request": 42,
}
req, err = in.GetHTTPRequest()
assert.Nil(t, req)
assert.Error(t, err)
assert.Equal(t, true, IsErrParamInvalid(err), err.Error())
r := new(http.Request)
in = Params{
"_request": r,
}
req, err = in.GetHTTPRequest()
assert.NotNil(t, req)
assert.NoError(t, err)
assert.Equal(t, r, req)
}
func TestParamsGetHTTPResponseWriter(t *testing.T) {
in := Params{}
wr, err := in.GetHTTPResponseWriter()
assert.Nil(t, wr)
assert.Error(t, err)
assert.Equal(t, true, IsErrParamNotFound(err), err.Error())
in = Params{
"_response": 42,
}
wr, err = in.GetHTTPResponseWriter()
assert.Nil(t, wr)
assert.Error(t, err)
assert.Equal(t, true, IsErrParamInvalid(err), err.Error())
var w http.ResponseWriter = httptest.NewRecorder()
in = Params{
"_response": w,
}
wr, err = in.GetHTTPResponseWriter()
assert.NotNil(t, wr)
assert.NoError(t, err)
assert.Equal(t, w, wr)
}
+14 -19
View File
@@ -183,7 +183,7 @@ func writeError(path string, in rc.Params, w http.ResponseWriter, err error, sta
})
if err != nil {
// can't return the error at this point
fs.Errorf(nil, "rc: writeError: failed to write JSON output from %#v: %v", in, err)
fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
}
}
@@ -267,9 +267,6 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
writeError(path, in, w, errors.Errorf("authentication must be set up on the rc server to use %q or the --rc-no-auth flag must be in use", path), http.StatusForbidden)
return
}
inOrig := in.Copy()
if call.NeedsRequest {
// Add the request to RC
in["_request"] = r
@@ -282,7 +279,7 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
// Check to see if it is async or not
isAsync, err := in.GetBool("_async")
if rc.NotErrParamNotFound(err) {
writeError(path, inOrig, w, err, http.StatusBadRequest)
writeError(path, in, w, err, http.StatusBadRequest)
return
}
delete(in, "_async") // remove the async parameter after parsing so vfs operations don't get confused
@@ -297,7 +294,7 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
w.Header().Add("x-rclone-jobid", fmt.Sprintf("%d", jobID))
}
if err != nil {
writeError(path, inOrig, w, err, http.StatusInternalServerError)
writeError(path, in, w, err, http.StatusInternalServerError)
return
}
if out == nil {
@@ -308,8 +305,8 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
err = rc.WriteJSON(w, out)
if err != nil {
// can't return the error at this point - but have a go anyway
writeError(path, inOrig, w, err, http.StatusInternalServerError)
fs.Errorf(nil, "rc: handlePost: failed to write JSON output: %v", err)
writeError(path, in, w, err, http.StatusInternalServerError)
fs.Errorf(nil, "rc: failed to write JSON output: %v", err)
}
}
@@ -390,20 +387,18 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
s.serveRoot(w, r)
return
case s.files != nil:
if s.opt.WebUI {
pluginsMatchResult := webgui.PluginsMatch.FindStringSubmatch(path)
pluginsMatchResult := webgui.PluginsMatch.FindStringSubmatch(path)
if pluginsMatchResult != nil && len(pluginsMatchResult) > 2 {
ok := webgui.ServePluginOK(w, r, pluginsMatchResult)
if !ok {
r.URL.Path = fmt.Sprintf("/%s/%s/app/build/%s", pluginsMatchResult[1], pluginsMatchResult[2], pluginsMatchResult[3])
s.pluginsHandler.ServeHTTP(w, r)
return
}
return
} else if webgui.ServePluginWithReferrerOK(w, r, path) {
if s.opt.WebUI && pluginsMatchResult != nil && len(pluginsMatchResult) > 2 {
ok := webgui.ServePluginOK(w, r, pluginsMatchResult)
if !ok {
r.URL.Path = fmt.Sprintf("/%s/%s/app/build/%s", pluginsMatchResult[1], pluginsMatchResult[2], pluginsMatchResult[3])
s.pluginsHandler.ServeHTTP(w, r)
return
}
return
} else if s.opt.WebUI && webgui.ServePluginWithReferrerOK(w, r, path) {
return
}
// Serve the files
r.URL.Path = "/" + path
-84
View File
@@ -7,7 +7,6 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"regexp"
"strings"
"testing"
@@ -17,7 +16,6 @@ import (
"github.com/stretchr/testify/require"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/rc"
)
@@ -29,21 +27,6 @@ const (
remoteURL = "[" + testFs + "]/" // initial URL path to fetch from that remote
)
func TestMain(m *testing.M) {
// Pretend to be rclone version if we have a version string parameter
if os.Args[len(os.Args)-1] == "version" {
fmt.Printf("rclone %s\n", fs.Version)
os.Exit(0)
}
// Pretend to error if we have an unknown command
if os.Args[len(os.Args)-1] == "unknown_command" {
fmt.Printf("rclone %s\n", fs.Version)
fmt.Fprintf(os.Stderr, "Unknown command\n")
os.Exit(1)
}
os.Exit(m.Run())
}
// Test the RC server runs and we can do HTTP fetches from it.
// We'll do the majority of the testing with the httptest framework
func TestRcServer(t *testing.T) {
@@ -472,73 +455,6 @@ func TestRC(t *testing.T) {
testServer(t, tests, &opt)
}
func TestRCWithAuth(t *testing.T) {
tests := []testRun{{
Name: "core-command",
URL: "core/command",
Method: "POST",
Body: `command=version`,
ContentType: "application/x-www-form-urlencoded",
Status: http.StatusOK,
Expected: fmt.Sprintf(`{
"error": false,
"result": "rclone %s\n"
}
`, fs.Version),
}, {
Name: "core-command-bad-returnType",
URL: "core/command",
Method: "POST",
Body: `command=version&returnType=POTATO`,
ContentType: "application/x-www-form-urlencoded",
Status: http.StatusInternalServerError,
Expected: `{
"error": "Unknown returnType \"POTATO\"",
"input": {
"command": "version",
"returnType": "POTATO"
},
"path": "core/command",
"status": 500
}
`,
}, {
Name: "core-command-stream",
URL: "core/command",
Method: "POST",
Body: `command=version&returnType=STREAM`,
ContentType: "application/x-www-form-urlencoded",
Status: http.StatusOK,
Expected: fmt.Sprintf(`rclone %s
{}
`, fs.Version),
}, {
Name: "core-command-stream-error",
URL: "core/command",
Method: "POST",
Body: `command=unknown_command&returnType=STREAM`,
ContentType: "application/x-www-form-urlencoded",
Status: http.StatusOK,
Expected: fmt.Sprintf(`rclone %s
Unknown command
{
"error": "exit status 1",
"input": {
"command": "unknown_command",
"returnType": "STREAM"
},
"path": "core/command",
"status": 500
}
`, fs.Version),
}}
opt := newTestOpt()
opt.Serve = true
opt.Files = testFs
opt.NoAuth = true
testServer(t, tests, &opt)
}
func TestMethods(t *testing.T) {
tests := []testRun{{
Name: "options",
+9 -26
View File
@@ -15,8 +15,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/rc/rcflags"
"github.com/rclone/rclone/lib/errors"
)
// PackageJSON is the structure of package.json of a plugin
@@ -64,8 +62,6 @@ var (
PluginsPath string
pluginsConfigPath string
availablePluginsJSONPath = "availablePlugins.json"
initSuccess = false
initMutex = &sync.Mutex{}
)
func init() {
@@ -73,6 +69,11 @@ func init() {
PluginsPath = filepath.Join(cachePath, "plugins")
pluginsConfigPath = filepath.Join(PluginsPath, "config")
loadedPlugins = newPlugins(availablePluginsJSONPath)
err := loadedPlugins.readFromFile()
if err != nil {
fs.Errorf(nil, "error reading available plugins: %v", err)
}
}
// Plugins represents the structure how plugins are saved onto disk
@@ -89,25 +90,9 @@ func newPlugins(fileName string) *Plugins {
return &p
}
func initPluginsOrError() error {
if !rcflags.Opt.WebUI {
return errors.New("WebUI needs to be enabled for plugins to work")
}
initMutex.Lock()
defer initMutex.Unlock()
if !initSuccess {
loadedPlugins = newPlugins(availablePluginsJSONPath)
err := loadedPlugins.readFromFile()
if err != nil {
fs.Errorf(nil, "error reading available plugins: %v", err)
}
initSuccess = true
}
return nil
}
func (p *Plugins) readFromFile() (err error) {
//p.mutex.Lock()
//defer p.mutex.Unlock()
err = CreatePathIfNotExist(pluginsConfigPath)
if err != nil {
return err
@@ -184,6 +169,8 @@ func (p *Plugins) addTestPlugin(pluginName string, testURL string, handlesType [
}
func (p *Plugins) writeToFile() (err error) {
//p.mutex.Lock()
//defer p.mutex.Unlock()
availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName)
file, err := json.MarshalIndent(p, "", " ")
@@ -303,10 +290,6 @@ var referrerPathReg = regexp.MustCompile("^(https?):\\/\\/(.+):([0-9]+)?\\/(.*)\
// sends a redirect to actual url. This function is useful for plugins to refer to absolute paths when
// the referrer in http.Request is set
func ServePluginWithReferrerOK(w http.ResponseWriter, r *http.Request, path string) (ok bool) {
err := initPluginsOrError()
if err != nil {
return false
}
referrer := r.Referer()
referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer)
-24
View File
@@ -30,10 +30,6 @@ Eg
}
func rcListTestPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
err = initPluginsOrError()
if err != nil {
return nil, err
}
return rc.Params{
"loadedTestPlugins": filterPlugins(loadedPlugins, func(json *PackageJSON) bool { return json.isTesting() }),
}, nil
@@ -58,10 +54,6 @@ Eg
})
}
func rcRemoveTestPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
err = initPluginsOrError()
if err != nil {
return nil, err
}
name, err := in.GetString("name")
if err != nil {
return nil, err
@@ -93,10 +85,6 @@ Eg
}
func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
err = initPluginsOrError()
if err != nil {
return nil, err
}
pluginURL, err := in.GetString("url")
if err != nil {
return nil, err
@@ -204,10 +192,6 @@ Eg
}
func rcGetPlugins(_ context.Context, _ rc.Params) (out rc.Params, err error) {
err = initPluginsOrError()
if err != nil {
return nil, err
}
err = loadedPlugins.readFromFile()
if err != nil {
return nil, err
@@ -238,10 +222,6 @@ Eg
}
func rcRemovePlugin(_ context.Context, in rc.Params) (out rc.Params, err error) {
err = initPluginsOrError()
if err != nil {
return nil, err
}
name, err := in.GetString("name")
if err != nil {
return nil, err
@@ -280,10 +260,6 @@ Eg
}
func rcGetPluginsForType(_ context.Context, in rc.Params) (out rc.Params, err error) {
err = initPluginsOrError()
if err != nil {
return nil, err
}
handlesType, err := in.GetString("type")
if err != nil {
handlesType = ""
-5
View File
@@ -9,7 +9,6 @@ import (
"testing"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/rc/rcflags"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -19,10 +18,6 @@ const testPluginAuthor = "rclone"
const testPluginKey = testPluginAuthor + "/" + testPluginName
const testPluginURL = "https://github.com/" + testPluginAuthor + "/" + testPluginName + "/"
func init() {
rcflags.Opt.WebUI = true
}
func setCacheDir(t *testing.T) string {
cacheDir, err := ioutil.TempDir("", "rclone-cache-dir")
assert.Nil(t, err)
+11 -34
View File
@@ -1590,7 +1590,7 @@ func TestSyncCopyDest(t *testing.T) {
}
// Test with BackupDir set
func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeepExtension bool) {
func testSyncBackupDir(t *testing.T, suffix string, suffixKeepExtension bool) {
r := fstest.NewRun(t)
defer r.Finalise()
@@ -1599,23 +1599,7 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep
}
r.Mkdir(context.Background(), r.Fremote)
if backupDir != "" {
fs.Config.BackupDir = r.FremoteName + "/" + backupDir
backupDir += "/"
} else {
fs.Config.BackupDir = ""
backupDir = "dst/"
// Exclude the suffix from the sync otherwise the sync
// deletes the old backup files
flt, err := filter.NewFilter(nil)
require.NoError(t, err)
require.NoError(t, flt.AddRule("- *"+suffix))
oldFlt := filter.Active
filter.Active = flt
defer func() {
filter.Active = oldFlt
}()
}
fs.Config.BackupDir = r.FremoteName + "/backup"
fs.Config.Suffix = suffix
fs.Config.SuffixKeepExtension = suffixKeepExtension
defer func() {
@@ -1643,14 +1627,14 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep
require.NoError(t, err)
// one should be moved to the backup dir and the new one installed
file1.Path = backupDir + "one" + suffix
file1.Path = "backup/one" + suffix
file1a.Path = "dst/one"
// two should be unchanged
// three should be moved to the backup dir
if suffixKeepExtension {
file3.Path = backupDir + "three" + suffix + ".txt"
file3.Path = "backup/three" + suffix + ".txt"
} else {
file3.Path = backupDir + "three.txt" + suffix
file3.Path = "backup/three.txt" + suffix
}
fstest.CheckItems(t, r.Fremote, file1, file2, file3, file1a)
@@ -1668,29 +1652,22 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep
require.NoError(t, err)
// one should be moved to the backup dir and the new one installed
file1a.Path = backupDir + "one" + suffix
file1a.Path = "backup/one" + suffix
file1b.Path = "dst/one"
// two should be unchanged
// three should be moved to the backup dir
if suffixKeepExtension {
file3a.Path = backupDir + "three" + suffix + ".txt"
file3a.Path = "backup/three" + suffix + ".txt"
} else {
file3a.Path = backupDir + "three.txt" + suffix
file3a.Path = "backup/three.txt" + suffix
}
fstest.CheckItems(t, r.Fremote, file1b, file2, file3a, file1a)
}
func TestSyncBackupDir(t *testing.T) {
testSyncBackupDir(t, "backup", "", false)
}
func TestSyncBackupDirWithSuffix(t *testing.T) {
testSyncBackupDir(t, "backup", ".bak", false)
}
func TestSyncBackupDir(t *testing.T) { testSyncBackupDir(t, "", false) }
func TestSyncBackupDirWithSuffix(t *testing.T) { testSyncBackupDir(t, ".bak", false) }
func TestSyncBackupDirWithSuffixKeepExtension(t *testing.T) {
testSyncBackupDir(t, "backup", "-2019-01-01", true)
}
func TestSyncBackupDirSuffixOnly(t *testing.T) {
testSyncBackupDir(t, "", ".bak", false)
testSyncBackupDir(t, "-2019-01-01", true)
}
// Test with Suffix set
+1 -1
View File
@@ -1,4 +1,4 @@
package fs
// Version of rclone
var Version = "v1.53.4-DEV"
var Version = "v1.54.0-DEV"
+19 -21
View File
@@ -27,16 +27,15 @@ type Test struct {
//
// FIXME make bucket based remotes set sub-dir automatically???
type Backend struct {
Backend string // name of the backend directory
Remote string // name of the test remote
FastList bool // set to test with -fast-list
Short bool // set to test with -short
OneOnly bool // set to run only one backend test at once
MaxFile string // file size limit
CleanUp bool // when running clean, run cleanup first
Ignore []string // test names to ignore the failure of
Tests []string // paths of tests to run, blank for all
ListRetries int // -list-retries if > 0
Backend string // name of the backend directory
Remote string // name of the test remote
FastList bool // set to test with -fast-list
Short bool // set to test with -short
OneOnly bool // set to run only one backend test at once
MaxFile string // file size limit
CleanUp bool // when running clean, run cleanup first
Ignore []string // test names to ignore the failure of
Tests []string // paths of tests to run, blank for all
}
// includeTest returns true if this backend should be included in this
@@ -80,17 +79,16 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
continue
}
run := &Run{
Remote: b.Remote,
Backend: b.Backend,
Path: t.Path,
FastList: fastlist,
Short: (b.Short && t.Short),
NoRetries: t.NoRetries,
OneOnly: b.OneOnly,
NoBinary: t.NoBinary,
SizeLimit: int64(maxSize),
Ignore: ignore,
ListRetries: b.ListRetries,
Remote: b.Remote,
Backend: b.Backend,
Path: t.Path,
FastList: fastlist,
Short: (b.Short && t.Short),
NoRetries: t.NoRetries,
OneOnly: b.OneOnly,
NoBinary: t.NoBinary,
SizeLimit: int64(maxSize),
Ignore: ignore,
}
if t.AddBackend {
run.Path = path.Join(run.Path, b.Backend)
+20 -23
View File
@@ -20,7 +20,6 @@ backends:
- backend: "b2"
remote: "TestB2:"
fastlist: true
listretries: 5
- backend: "crypt"
remote: "TestCryptDrive:"
fastlist: true
@@ -42,13 +41,15 @@ backends:
remote: "TestChunkerChunk3bNometaLocal:"
fastlist: true
maxfile: 6k
- backend: "chunker"
remote: "TestChunkerMailru:"
fastlist: true
- backend: "chunker"
remote: "TestChunkerChunk50bMailru:"
fastlist: true
maxfile: 10k
# Disable chunker with mailru tests until Mailru is fixed - see
# https://github.com/rclone/rclone/issues/4376
# - backend: "chunker"
# remote: "TestChunkerMailru:"
# fastlist: true
# - backend: "chunker"
# remote: "TestChunkerChunk50bMailru:"
# fastlist: true
# maxfile: 10k
- backend: "chunker"
remote: "TestChunkerChunk50bYandex:"
fastlist: true
@@ -72,10 +73,6 @@ backends:
remote: "TestChunkerChunk50bSHA1HashS3:"
fastlist: true
maxfile: 1k
- backend: "chunker"
remote: "TestChunkerOverCrypt:"
fastlist: true
maxfile: 6k
- backend: "chunker"
remote: "TestChunkerChunk50bMD5QuickS3:"
fastlist: true
@@ -147,12 +144,12 @@ backends:
# ignore:
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
# - TestIntegration/FsMkdir/FsPutFiles/SetTier
# - backend: "s3"
# remote: "TestS3Ceph:"
# fastlist: true
# ignore:
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
# - TestIntegration/FsMkdir/FsPutFiles/SetTier
- backend: "s3"
remote: "TestS3Ceph:"
fastlist: true
ignore:
- TestIntegration/FsMkdir/FsPutFiles/FsCopy
- TestIntegration/FsMkdir/FsPutFiles/SetTier
- backend: "s3"
remote: "TestS3Alibaba:"
fastlist: true
@@ -173,11 +170,11 @@ backends:
- backend: "swift"
remote: "TestSwift:"
fastlist: true
# - backend: "swift"
# remote: "TestSwiftCeph:"
# fastlist: true
# ignore:
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
- backend: "swift"
remote: "TestSwiftCeph:"
fastlist: true
ignore:
- TestIntegration/FsMkdir/FsPutFiles/FsCopy
- backend: "yandex"
remote: "TestYandex:"
fastlist: false
+12 -17
View File
@@ -35,17 +35,16 @@ var (
// if retries are needed.
type Run struct {
// Config
Remote string // name of the test remote
Backend string // name of the backend
Path string // path to the source directory
FastList bool // add -fast-list to tests
Short bool // add -short
NoRetries bool // don't retry if set
OneOnly bool // only run test for this backend at once
NoBinary bool // set to not build a binary
SizeLimit int64 // maximum test file size
Ignore map[string]struct{}
ListRetries int // -list-retries if > 0
Remote string // name of the test remote
Backend string // name of the backend
Path string // path to the source directory
FastList bool // add -fast-list to tests
Short bool // add -short
NoRetries bool // don't retry if set
OneOnly bool // only run test for this backend at once
NoBinary bool // set to not build a binary
SizeLimit int64 // maximum test file size
Ignore map[string]struct{}
// Internals
CmdLine []string
CmdString string
@@ -337,12 +336,8 @@ func (r *Run) Init() {
r.CmdLine = []string{"./" + r.BinaryName()}
}
r.CmdLine = append(r.CmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote)
listRetries := *listRetries
if r.ListRetries > 0 {
listRetries = r.ListRetries
}
if listRetries > 0 {
r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(listRetries))
if *listRetries > 0 {
r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(*listRetries))
}
r.Try = 1
if *verbose {
+3 -21
View File
@@ -2,10 +2,8 @@
package random
import (
cryptorand "crypto/rand"
"encoding/base64"
"encoding/binary"
mathrand "math/rand"
"math/rand"
"github.com/pkg/errors"
)
@@ -25,7 +23,7 @@ func String(n int) string {
for i := range out {
source := pattern[p]
p = (p + 1) % len(pattern)
out[i] = source[mathrand.Intn(len(source))]
out[i] = source[rand.Intn(len(source))]
}
return string(out)
}
@@ -43,7 +41,7 @@ func Password(bits int) (password string, err error) {
bytes++
}
var pw = make([]byte, bytes)
n, err := cryptorand.Read(pw)
n, err := rand.Read(pw)
if err != nil {
return "", errors.Wrap(err, "password read failed")
}
@@ -53,19 +51,3 @@ func Password(bits int) (password string, err error) {
password = base64.RawURLEncoding.EncodeToString(pw)
return password, nil
}
// Seed the global math/rand with crypto strong data
//
// This doesn't make it OK to use math/rand in crypto sensitive
// environments - don't do that! However it does help to mitigate the
// problem if that happens accidentally. This would have helped with
// CVE-2020-28924 - #4783
func Seed() error {
var seed int64
err := binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
if err != nil {
return errors.Wrap(err, "failed to read random seed")
}
mathrand.Seed(seed)
return nil
}
-14
View File
@@ -1,7 +1,6 @@
package random
import (
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
@@ -49,16 +48,3 @@ func TestPasswordDuplicates(t *testing.T) {
seen[s] = true
}
}
func TestSeed(t *testing.T) {
// seed 100 times and check the first random number doesn't repeat
// This test could fail with a probability of ~ 10**-15
const n = 100
var seen = map[int64]bool{}
for i := 0; i < n; i++ {
assert.NoError(t, Seed())
first := rand.Int63()
assert.False(t, seen[first])
seen[first] = true
}
}
Generated
+8936 -14827
View File
File diff suppressed because it is too large. Load diff
-5
View File
@@ -342,14 +342,9 @@ func (f *File) Size() int64 {
}
// SetModTime sets the modtime for the file
//
// if NoModTime is set then it does nothing
func (f *File) SetModTime(modTime time.Time) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.d.vfs.Opt.NoModTime {
return nil
}
if f.d.vfs.Opt.ReadOnly {
return EROFS
}
-5
View File
@@ -166,11 +166,6 @@ whereas the --vfs-read-ahead is buffered on disk.
When using this mode it is recommended that --buffer-size is not set
too big and --vfs-read-ahead is set large if required.
**IMPORTANT** not all file systems support sparse files. In particular
FAT/exFAT do not. Rclone will perform very badly if the cache
directory is on a filesystem which doesn't support sparse files and it
will log an ERROR message if one is detected.
### VFS Performance
These flags may be used to enable/disable features of the VFS for
-3
View File
@@ -23,8 +23,6 @@ must be supplied.`
//
// If "fs" is not set and there is one and only one VFS in the active
// cache then it returns it. This is for backwards compatibility.
//
// This deletes the "fs" parameter from in if it is valid
func getVFS(in rc.Params) (vfs *VFS, err error) {
fsString, err := in.GetString("fs")
if rc.IsErrParamNotFound(err) {
@@ -48,7 +46,6 @@ func getVFS(in rc.Params) (vfs *VFS, err error) {
} else if len(activeVFS) > 1 {
return nil, errors.Errorf("more than one VFS active with name %q", fsString)
}
delete(in, "fs") // delete the fs parameter
return activeVFS[0], nil
}
+2 -5
View File
@@ -57,7 +57,6 @@ func TestRcGetVFS(t *testing.T) {
assert.Contains(t, err.Error(), "more than one VFS active - need")
assert.Nil(t, vfs)
inPresent = rc.Params{"fs": fs.ConfigString(r.Fremote)}
vfs, err = getVFS(inPresent)
require.Error(t, err)
assert.Contains(t, err.Error(), "more than one VFS active with name")
@@ -68,8 +67,7 @@ func TestRcForget(t *testing.T) {
r, vfs, cleanup, call := rcNewRun(t, "vfs/forget")
defer cleanup()
_, _ = r, vfs
in := rc.Params{"fs": fs.ConfigString(r.Fremote)}
out, err := call.Fn(context.Background(), in)
out, err := call.Fn(context.Background(), nil)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"forgotten": []string{},
@@ -81,8 +79,7 @@ func TestRcRefresh(t *testing.T) {
r, vfs, cleanup, call := rcNewRun(t, "vfs/refresh")
defer cleanup()
_, _ = r, vfs
in := rc.Params{"fs": fs.ConfigString(r.Fremote)}
out, err := call.Fn(context.Background(), in)
out, err := call.Fn(context.Background(), nil)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"result": map[string]string{
Loaded 100 of 103 files, more files were not shown because too many files have changed in this diff. Show more