Compare commits

..
Author SHA1 Message Date
Nick Craig-Wood b0c54538b0 drive: make backend config -o config add a combined AllDrives remote
This adjusts

    rclone backend drives -o config drive:

So that it also emits a config section called `AllDrives` which uses
the combine backend to make a backend which combines all the shared
drives into one.

It also makes sure that all the shared drive names are valid rclone
config names, deduplicating if necessary.

Fixes #4506
2022-04-21 09:58:59 +01:00
Nick Craig-Wood e62b9d017d Add combine backend to combine multiple remotes in one directory tree - FIXME WIP
Needs
- docs
- integration tests

Fixes #5600
2022-04-21 09:12:50 +01:00
Nick Craig-Wood 632f626f08 fstests: check for wrapped errors in ListR test 2022-04-20 17:56:15 +01:00
131 changed files with 29956 additions and 34388 deletions

No files matched your search

+16 -10
View File
@@ -25,7 +25,7 @@ jobs:
strategy:
fail-fast: false
matrix:
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17']
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.16', 'go1.17']
include:
- job_name: linux
@@ -55,12 +55,23 @@ jobs:
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
deploy: true
- job_name: windows
- job_name: windows_amd64
os: windows-latest
go: '1.18.x'
gotags: cmount
cgo: '0'
build_flags: '-include "^windows/"'
build_flags: '-include "^windows/amd64" -cgo'
build_args: '-buildmode exe'
quicktest: true
racequicktest: true
deploy: true
- job_name: windows_386
os: windows-latest
go: '1.18.x'
gotags: cmount
goarch: '386'
cgo: '1'
build_flags: '-include "^windows/386" -cgo'
build_args: '-buildmode exe'
quicktest: true
deploy: true
@@ -68,7 +79,7 @@ jobs:
- job_name: other_os
os: ubuntu-latest
go: '1.18.x'
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
build_flags: '-exclude "^(windows/(386|amd64)|darwin/|linux/)"'
compile_all: true
deploy: true
@@ -167,11 +178,6 @@ jobs:
run: |
make
- name: Rclone version
shell: bash
run: |
rclone version
- name: Run tests
shell: bash
run: |
@@ -20,7 +20,7 @@ jobs:
with:
tag: beta
imageName: rclone/rclone
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
publish: true
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
@@ -28,7 +28,7 @@ jobs:
with:
tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
imageName: rclone/rclone
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
publish: true
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
@@ -50,7 +50,7 @@ jobs:
PLUGIN_USER=rclone
docker login --username ${{ secrets.DOCKER_HUB_USER }} \
--password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
for PLUGIN_ARCH in amd64 arm64 arm/v7 ;do
export PLUGIN_USER PLUGIN_ARCH
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+5 -5
View File
@@ -97,7 +97,7 @@ release_dep_linux:
# Get the release dependencies we only install on Windows
release_dep_windows:
GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
# Update dependencies
showupdates:
@@ -245,18 +245,18 @@ retag:
startdev:
@echo "Version is $(VERSION)"
@echo "Next version is $(NEXT_VERSION)"
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_VERSION)" > VERSION
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
startstable:
@echo "Version is $(VERSION)"
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_PATCH_VERSION)" > VERSION
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
winzip:
zip -9 rclone-$(TAG).zip rclone.exe
-3
View File
@@ -29,8 +29,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* Box [:page_facing_up:](https://rclone.org/box/)
* Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
* China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
* Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
* Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
* Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
* DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
* Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
@@ -44,7 +42,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
* HTTP [:page_facing_up:](https://rclone.org/http/)
* Hubic [:page_facing_up:](https://rclone.org/hubic/)
* Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
* Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
* IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
* Koofr [:page_facing_up:](https://rclone.org/koofr/)
+1 -1
View File
@@ -9,6 +9,7 @@ import (
_ "github.com/rclone/rclone/backend/box"
_ "github.com/rclone/rclone/backend/cache"
_ "github.com/rclone/rclone/backend/chunker"
_ "github.com/rclone/rclone/backend/combine"
_ "github.com/rclone/rclone/backend/compress"
_ "github.com/rclone/rclone/backend/crypt"
_ "github.com/rclone/rclone/backend/drive"
@@ -22,7 +23,6 @@ import (
_ "github.com/rclone/rclone/backend/hdfs"
_ "github.com/rclone/rclone/backend/http"
_ "github.com/rclone/rclone/backend/hubic"
_ "github.com/rclone/rclone/backend/internetarchive"
_ "github.com/rclone/rclone/backend/jottacloud"
_ "github.com/rclone/rclone/backend/koofr"
_ "github.com/rclone/rclone/backend/local"
+12 -17
View File
@@ -26,7 +26,6 @@ import (
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
@@ -44,9 +43,8 @@ import (
const (
minSleep = 10 * time.Millisecond
maxSleep = 10 * time.Second
decayConstant = 1 // bigger for slower decay, exponential
maxListChunkSize = 5000 // number of items to read at once
maxUploadParts = 50000 // maximum allowed number of parts/blocks in a multi-part upload
decayConstant = 1 // bigger for slower decay, exponential
maxListChunkSize = 5000 // number of items to read at once
modTimeKey = "mtime"
timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
@@ -373,9 +371,15 @@ func (o *Object) split() (container, containerPath string) {
// validateAccessTier checks if azureblob supports user supplied tier
func validateAccessTier(tier string) bool {
return strings.EqualFold(tier, string(azblob.AccessTierHot)) ||
strings.EqualFold(tier, string(azblob.AccessTierCool)) ||
strings.EqualFold(tier, string(azblob.AccessTierArchive))
switch tier {
case string(azblob.AccessTierHot),
string(azblob.AccessTierCool),
string(azblob.AccessTierArchive):
// valid cases
return true
default:
return false
}
}
// validatePublicAccess checks if azureblob supports use supplied public access level
@@ -1685,17 +1689,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
}
uploadParts := int64(maxUploadParts)
if uploadParts < 1 {
uploadParts = 1
} else if uploadParts > maxUploadParts {
uploadParts = maxUploadParts
}
// calculate size of parts/blocks
partSize := chunksize.Calculator(o, int(uploadParts), o.fs.opt.ChunkSize)
putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
BufferSize: int(partSize),
BufferSize: int(o.fs.opt.ChunkSize),
MaxBuffers: o.fs.opt.UploadConcurrency,
Metadata: o.meta,
BlobHTTPHeaders: httpHeaders,
-22
View File
@@ -61,25 +61,3 @@ func TestServicePrincipalFileFailure(t *testing.T) {
assert.Error(t, err)
assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
}
func TestValidateAccessTier(t *testing.T) {
tests := map[string]struct {
accessTier string
want bool
}{
"hot": {"hot", true},
"HOT": {"HOT", true},
"Hot": {"Hot", true},
"cool": {"cool", true},
"archive": {"archive", true},
"empty": {"", false},
"unknown": {"unknown", false},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
got := validateAccessTier(test.accessTier)
assert.Equal(t, test.want, got)
})
}
}
+2 -32
View File
@@ -64,8 +64,7 @@ const (
// Globals
var (
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
)
// Register with Fs
@@ -107,11 +106,6 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
Help: "Include old versions in directory listings.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
Default: false,
Advanced: true,
}, {
Name: "version_at",
Help: "Show file versions as they were at the specified time.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
Default: fs.Time{},
Advanced: true,
}, {
Name: "hard_delete",
Help: "Permanently delete files on remote removal, otherwise hide files.",
@@ -217,7 +211,6 @@ type Options struct {
Endpoint string `config:"endpoint"`
TestMode string `config:"test_mode"`
Versions bool `config:"versions"`
VersionAt fs.Time `config:"version_at"`
HardDelete bool `config:"hard_delete"`
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
@@ -703,12 +696,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
Method: "POST",
Path: "/b2_list_file_names",
}
if hidden || f.opt.VersionAt.IsSet() {
if hidden {
opts.Path = "/b2_list_file_versions"
}
lastFileName := ""
for {
var response api.ListFileNamesResponse
err := f.pacer.Call(func() (bool, error) {
@@ -738,21 +728,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
if addBucket {
remote = path.Join(bucket, remote)
}
if f.opt.VersionAt.IsSet() {
if time.Time(file.UploadTimestamp).After(time.Time(f.opt.VersionAt)) {
// Ignore versions that were created after the specified time
continue
}
if file.Name == lastFileName {
// Ignore versions before the already returned version
continue
}
}
// Send object
lastFileName = file.Name
err = fn(remote, file, isDirectory)
if err != nil {
if err == errEndList {
@@ -1852,9 +1828,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if o.fs.opt.Versions {
return errNotWithVersions
}
if o.fs.opt.VersionAt.IsSet() {
return errNotWithVersionAt
}
size := src.Size()
bucket, bucketPath := o.split()
@@ -2010,9 +1983,6 @@ func (o *Object) Remove(ctx context.Context) error {
if o.fs.opt.Versions {
return errNotWithVersions
}
if o.fs.opt.VersionAt.IsSet() {
return errNotWithVersionAt
}
if o.fs.opt.HardDelete {
return o.fs.deleteByID(ctx, o.id, bucketPath)
}
+5 -4
View File
@@ -18,7 +18,6 @@ import (
"github.com/rclone/rclone/backend/b2/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/rest"
@@ -89,19 +88,21 @@ type largeUpload struct {
// newLargeUpload starts an upload of object o from in with metadata in src
//
// If newInfo is set then metadata from that will be used instead of reading it from src
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
remote := o.remote
size := src.Size()
parts := int64(0)
sha1SliceSize := int64(maxParts)
chunkSize := defaultChunkSize
if size == -1 {
fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
} else {
chunkSize = chunksize.Calculator(src, maxParts, defaultChunkSize)
parts = size / int64(chunkSize)
if size%int64(chunkSize) != 0 {
parts++
}
if parts > maxParts {
return nil, fmt.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
}
sha1SliceSize = parts
}
+1 -1
View File
@@ -1747,7 +1747,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil, errors.New("not supported by underlying remote")
return nil, errors.New("About not supported")
}
return do(ctx)
}
+4 -4
View File
@@ -515,7 +515,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
strRegex := regexp.QuoteMeta(pattern)
strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl)
strRegex = strings.ReplaceAll(strRegex, "\\*", mainNameRegStr)
strRegex = strings.Replace(strRegex, "\\*", mainNameRegStr, -1)
strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld)
f.nameRegexp = regexp.MustCompile(strRegex)
@@ -524,7 +524,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
if numDigits > 1 {
fmtDigits = fmt.Sprintf("%%0%dd", numDigits)
}
strFmt := strings.ReplaceAll(pattern, "%", "%%")
strFmt := strings.Replace(pattern, "%", "%%", -1)
strFmt = strings.Replace(strFmt, "*", "%s", 1)
f.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits)
f.ctrlNameFmt = reHashes.ReplaceAllLiteralString(strFmt, "_%s")
@@ -1895,7 +1895,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
func (f *Fs) CleanUp(ctx context.Context) error {
do := f.base.Features().CleanUp
if do == nil {
return errors.New("not supported by underlying remote")
return errors.New("can't CleanUp")
}
return do(ctx)
}
@@ -1904,7 +1904,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.base.Features().About
if do == nil {
return nil, errors.New("not supported by underlying remote")
return nil, errors.New("About not supported")
}
return do(ctx)
}
+877
View File
@@ -0,0 +1,877 @@
// Package combine implents a backend to combine multipe remotes in a directory tree
package combine
/*
Have API to add/remove branches in the combine
*/
import (
"context"
"errors"
"fmt"
"io"
"path"
"strings"
"sync"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/walk"
"golang.org/x/sync/errgroup"
)
// Register with Fs
func init() {
fsi := &fs.RegInfo{
Name: "combine",
Description: "Combine several remotes into one",
NewFs: NewFs,
Options: []fs.Option{{
Name: "upstreams",
Help: `Upstreams for combining
These should be in the form
dir=remote:path dir2=remote2:path
Where before the = is specified the root directory and after is the remote to
put there.
Embedded spaces can be added using quotes
"dir=remote:path with space" "dir2=remote2:path with space"
`,
Required: true,
Default: fs.SpaceSepList(nil),
}},
}
fs.Register(fsi)
}
// Options defines the configuration for this backend
type Options struct {
Upstreams fs.SpaceSepList `config:"upstreams"`
}
// Fs represents a combine of upstreams
type Fs struct {
name string // name of this remote
features *fs.Features // optional features
opt Options // options for this Fs
root string // the path we are working on
hashSet hash.Set // common hashes
when time.Time // directory times
upstreams map[string]*upstream // map of upstreams
}
// adjustment stores the info to add a prefix to a path or chop characters off it
type adjustment struct {
prefix string
chop int
}
// do makes the adjustment on s
func (a *adjustment) do(s string) string {
if a.prefix != "" {
return join(a.prefix, s)
}
return s[a.chop:]
}
// upstream represents an upstream Fs
type upstream struct {
f fs.Fs
parent *Fs
dir string // directory the upstream is mounted
pathAdjustment adjustment // how to fiddle with the path
}
// Create an upstream from the directory it is mounted on and the remote
func (f *Fs) newUpstream(ctx context.Context, dir, remote string) (*upstream, error) {
uFs, err := cache.Get(ctx, remote)
if err == fs.ErrorIsFile {
return nil, fmt.Errorf("can't combine files yet, only directories %q: %w", remote, err)
}
if err != nil {
return nil, fmt.Errorf("failed to create upstream %q: %w", remote, err)
}
u := &upstream{
f: uFs,
parent: f,
dir: dir,
}
if len(f.root) < len(dir) {
u.pathAdjustment.prefix = dir[:len(dir)-len(f.root)]
} else {
u.pathAdjustment.chop = len(f.root) - len(dir)
}
return u, nil
}
// NewFs constructs an Fs from the path.
//
// The returned Fs is the actual Fs, referenced by remote in the config
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) {
// defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err)
// Parse config into Options struct
opt := new(Options)
err = configstruct.Set(m, opt)
if err != nil {
return nil, err
}
// Backward compatible to old config
if len(opt.Upstreams) == 0 {
return nil, errors.New("combine can't point to an empty upstream - check the value of the upstreams setting")
}
for _, u := range opt.Upstreams {
if strings.HasPrefix(u, name+":") {
return nil, errors.New("can't point combine remote at itself - check the value of the upstreams setting")
}
}
f := &Fs{
name: name,
root: root,
opt: *opt,
upstreams: make(map[string]*upstream, len(opt.Upstreams)),
when: time.Now(),
}
g, ctx := errgroup.WithContext(ctx)
var mu sync.Mutex
for _, upstream := range opt.Upstreams {
upstream := upstream
g.Go(func() (err error) {
equal := strings.IndexRune(upstream, '=')
if equal < 0 {
return fmt.Errorf("no \"=\" in upstream definition %q", upstream)
}
dir, remote := upstream[:equal], upstream[equal+1:]
if dir == "" {
return fmt.Errorf("empty dir in upstream definition %q", upstream)
}
if remote == "" {
return fmt.Errorf("empty remote in upstream definition %q", upstream)
}
u, err := f.newUpstream(ctx, dir, remote)
if err != nil {
return err
}
mu.Lock()
f.upstreams[dir] = u
mu.Unlock()
return nil
})
}
err = g.Wait()
if err != nil {
return nil, err
}
// check features
var features = (&fs.Features{
CaseInsensitive: true,
DuplicateFiles: false,
ReadMimeType: true,
WriteMimeType: true,
CanHaveEmptyDirectories: true,
BucketBased: true,
SetTier: true,
GetTier: true,
}).Fill(ctx, f)
canMove := true
for _, u := range f.upstreams {
features = features.Mask(ctx, u.f) // Mask all upstream fs
if !operations.CanServerSideMove(u.f) {
canMove = false
}
}
// We can move if all remotes support Move or Copy
if canMove {
features.Move = f.Move
}
// Enable ListR when upstreams either support ListR or is local
// But not when all upstreams are local
if features.ListR == nil {
for _, u := range f.upstreams {
if u.f.Features().ListR != nil {
features.ListR = f.ListR
} else if !u.f.Features().IsLocal {
features.ListR = nil
break
}
}
}
// Enable Purge when any upstreams support it
if features.Purge == nil {
for _, u := range f.upstreams {
if u.f.Features().Purge != nil {
features.Purge = f.Purge
break
}
}
}
// Enable Shutdown when any upstreams support it
if features.Shutdown == nil {
for _, u := range f.upstreams {
if u.f.Features().Shutdown != nil {
features.Shutdown = f.Shutdown
break
}
}
}
// Enable DirCacheFlush when any upstreams support it
if features.DirCacheFlush == nil {
for _, u := range f.upstreams {
if u.f.Features().DirCacheFlush != nil {
features.DirCacheFlush = f.DirCacheFlush
break
}
}
}
f.features = features
// Get common intersection of hashes
var hashSet hash.Set
var first = true
for _, u := range f.upstreams {
if first {
hashSet = u.f.Hashes()
first = false
} else {
hashSet = hashSet.Overlap(u.f.Hashes())
}
}
f.hashSet = hashSet
// Check to see if the root is actually a file
if f.root != "" {
_, err := f.NewObject(ctx, "")
if err != nil {
if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile || err == fs.ErrorIsDir {
// File doesn't exist or is a directory so return old f
return f, nil
}
return nil, err
}
// Check to see if the root path is actually an existing file
oldRoot := f.root
newRoot, leaf := path.Split(oldRoot)
f.root = newRoot
// Adjust path adjustment to remove leaf
for _, u := range f.upstreams {
u.pathAdjustment.chop -= len(leaf) + 1
}
return f, fs.ErrorIsFile
}
return f, nil
}
// Run a function over all the upstreams in parallel
func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream) error) error {
g, gCtx := errgroup.WithContext(ctx)
for _, u := range f.upstreams {
u := u
g.Go(func() (err error) {
return fn(gCtx, u)
})
}
return g.Wait()
}
// join the elements together but unline path.Join return empty string
func join(elem ...string) string {
result := path.Join(elem...)
if result == "." {
return ""
}
return result
}
// find the upstream for the remote passed in, returning the upstream and the adjusted path
func (f *Fs) findUpstream(remote string) (u *upstream, uRemote string, err error) {
// defer log.Trace(remote, "")("f=%v, uRemote=%q, err=%v", &u, &uRemote, &err)
absolute := join(f.root, remote)
for dir, u := range f.upstreams {
dirSlash := dir + "/"
foundStart := -1
foundEnd := -1
if absolute == dir {
foundEnd = len(dir)
foundStart = foundEnd
} else if strings.HasPrefix(absolute, dirSlash) {
foundEnd = len(dirSlash)
foundStart = foundEnd - 1
}
if foundStart > 0 {
uRemote = absolute[foundEnd:]
return u, uRemote, nil
}
}
return nil, "", fmt.Errorf("combine for remote %q: %w", remote, fs.ErrorDirNotFound)
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
// String converts this Fs to a string
func (f *Fs) String() string {
return fmt.Sprintf("combine root '%s'", f.root)
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// Rmdir removes the root directory of the Fs object
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
// The root always exists
if f.root == "" && dir == "" {
return nil
}
u, uRemote, err := f.findUpstream(dir)
if err != nil {
return err
}
return u.f.Rmdir(ctx, uRemote)
}
// Hashes returns hash.HashNone to indicate remote hashing is unavailable
func (f *Fs) Hashes() hash.Set {
return f.hashSet
}
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// The root always exists
if f.root == "" && dir == "" {
return nil
}
u, uRemote, err := f.findUpstream(dir)
if err != nil {
return err
}
return u.f.Mkdir(ctx, uRemote)
}
// purge the upstream or fallback to a slow way
func (u *upstream) purge(ctx context.Context, dir string) (err error) {
if do := u.f.Features().Purge; do != nil {
err = do(ctx, dir)
} else {
err = operations.Purge(ctx, u.f, dir)
}
return err
}
// Purge all files in the directory
//
// Implement this if you have a way of deleting all the files
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
func (f *Fs) Purge(ctx context.Context, dir string) error {
if f.root == "" && dir == "" {
return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
return u.purge(ctx, "")
})
}
u, uRemote, err := f.findUpstream(dir)
if err != nil {
return err
}
return u.purge(ctx, uRemote)
}
// Copy src to this remote using server-side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
dstU, dstRemote, err := f.findUpstream(remote)
if err != nil {
return nil, err
}
do := dstU.f.Features().Copy
if do == nil {
return nil, fs.ErrorCantCopy
}
o, err := do(ctx, srcObj.Object, dstRemote)
if err != nil {
return nil, err
}
return dstU.newObject(o), nil
}
// Move src to this remote using server-side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
dstU, dstRemote, err := f.findUpstream(remote)
if err != nil {
return nil, err
}
do := dstU.f.Features().Move
useCopy := false
if do == nil {
do = dstU.f.Features().Copy
if do == nil {
return nil, fs.ErrorCantMove
}
useCopy = true
}
o, err := do(ctx, srcObj.Object, dstRemote)
if err != nil {
return nil, err
}
// If did Copy then remove the source object
if useCopy {
err = srcObj.Remove(ctx)
if err != nil {
return nil, err
}
}
return dstU.newObject(o), nil
}
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
// defer log.Trace(f, "src=%v, srcRemote=%q, dstRemote=%q", src, srcRemote, dstRemote)("err=%v", &err)
srcFs, ok := src.(*Fs)
if !ok {
fs.Debugf(src, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
dstU, dstURemote, err := f.findUpstream(dstRemote)
if err != nil {
return err
}
srcU, srcURemote, err := srcFs.findUpstream(srcRemote)
if err != nil {
return err
}
do := dstU.f.Features().DirMove
if do == nil {
return fs.ErrorCantDirMove
}
fs.Logf(dstU.f, "srcU.f=%v, srcURemote=%q, dstURemote=%q", srcU.f, srcURemote, dstURemote)
return do(ctx, srcU.f, srcURemote, dstURemote)
}
// ChangeNotify calls the passed function with a path
// that has had changes. If the implementation
// uses polling, it should adhere to the given interval.
// At least one value will be written to the channel,
// specifying the initial value and updated values might
// follow. A 0 Duration should pause the polling.
// The ChangeNotify implementation must empty the channel
// regularly. When the channel gets closed, the implementation
// should stop polling and release resources.
func (f *Fs) ChangeNotify(ctx context.Context, fn func(string, fs.EntryType), ch <-chan time.Duration) {
var uChans []chan time.Duration
for _, u := range f.upstreams {
if do := u.f.Features().ChangeNotify; do != nil {
ch := make(chan time.Duration)
uChans = append(uChans, ch)
do(ctx, fn, ch)
}
}
go func() {
for i := range ch {
for _, c := range uChans {
c <- i
}
}
for _, c := range uChans {
close(c)
}
}()
}
// DirCacheFlush resets the directory cache - used in testing
// as an optional interface
func (f *Fs) DirCacheFlush() {
ctx := context.Background()
_ = f.multithread(ctx, func(ctx context.Context, u *upstream) error {
if do := u.f.Features().DirCacheFlush; do != nil {
do()
}
return nil
})
}
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
srcPath := src.Remote()
u, uRemote, err := f.findUpstream(srcPath)
if err != nil {
return nil, err
}
uSrc := operations.NewOverrideRemote(src, uRemote)
var o fs.Object
if stream {
o, err = u.f.Features().PutStream(ctx, in, uSrc, options...)
} else {
o, err = u.f.Put(ctx, in, uSrc, options...)
}
if err != nil {
return nil, err
}
return u.newObject(o), nil
}
// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
o, err := f.NewObject(ctx, src.Remote())
switch err {
case nil:
return o, o.Update(ctx, in, src, options...)
case fs.ErrorObjectNotFound:
return f.put(ctx, in, src, false, options...)
default:
return nil, err
}
}
// PutStream uploads to the remote path with the modTime given of indeterminate size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
o, err := f.NewObject(ctx, src.Remote())
switch err {
case nil:
return o, o.Update(ctx, in, src, options...)
case fs.ErrorObjectNotFound:
return f.put(ctx, in, src, true, options...)
default:
return nil, err
}
}
// About gets quota information from the Fs
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
usage := &fs.Usage{
Total: new(int64),
Used: new(int64),
Trashed: new(int64),
Other: new(int64),
Free: new(int64),
Objects: new(int64),
}
for _, u := range f.upstreams {
doAbout := u.f.Features().About
if doAbout == nil {
continue
}
usg, err := doAbout(ctx)
if errors.Is(err, fs.ErrorDirNotFound) {
continue
}
if err != nil {
return nil, err
}
if usg.Total != nil && usage.Total != nil {
*usage.Total += *usg.Total
} else {
usage.Total = nil
}
if usg.Used != nil && usage.Used != nil {
*usage.Used += *usg.Used
} else {
usage.Used = nil
}
if usg.Trashed != nil && usage.Trashed != nil {
*usage.Trashed += *usg.Trashed
} else {
usage.Trashed = nil
}
if usg.Other != nil && usage.Other != nil {
*usage.Other += *usg.Other
} else {
usage.Other = nil
}
if usg.Free != nil && usage.Free != nil {
*usage.Free += *usg.Free
} else {
usage.Free = nil
}
if usg.Objects != nil && usage.Objects != nil {
*usage.Objects += *usg.Objects
} else {
usage.Objects = nil
}
}
return usage, nil
}
// Wraps entries for this upstream
func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.DirEntries, error) {
for i, entry := range entries {
switch x := entry.(type) {
case fs.Object:
entries[i] = u.newObject(x)
case fs.Directory:
newDir := fs.NewDirCopy(ctx, x)
newDir.SetRemote(u.pathAdjustment.do(newDir.Remote()))
entries[i] = newDir
default:
return nil, fmt.Errorf("unknown entry type %T", entry)
}
}
return entries, nil
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
if f.root == "" && dir == "" {
entries = make(fs.DirEntries, 0, len(f.upstreams))
for combineDir := range f.upstreams {
d := fs.NewDir(combineDir, f.when)
entries = append(entries, d)
}
return entries, nil
}
u, uRemote, err := f.findUpstream(dir)
if err != nil {
return nil, err
}
entries, err = u.f.List(ctx, uRemote)
if err != nil {
return nil, err
}
return u.wrapEntries(ctx, entries)
}
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
//
// dir should be "" to start from the root, and should not
// have trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
//
// It should call callback for each tranche of entries read.
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
// defer log.Trace(f, "dir=%q, callback=%v", dir, callback)("err=%v", &err)
if f.root == "" && dir == "" {
rootEntries, err := f.List(ctx, "")
if err != nil {
return err
}
err = callback(rootEntries)
if err != nil {
return err
}
var mu sync.Mutex
syncCallback := func(entries fs.DirEntries) error {
mu.Lock()
defer mu.Unlock()
return callback(entries)
}
err = f.multithread(ctx, func(ctx context.Context, u *upstream) error {
return f.ListR(ctx, u.dir, syncCallback)
})
if err != nil {
return err
}
return nil
}
u, uRemote, err := f.findUpstream(dir)
if err != nil {
return err
}
wrapCallback := func(entries fs.DirEntries) error {
entries, err := u.wrapEntries(ctx, entries)
if err != nil {
return err
}
return callback(entries)
}
if do := u.f.Features().ListR; do != nil {
err = do(ctx, uRemote, wrapCallback)
} else {
err = walk.ListR(ctx, u.f, uRemote, true, -1, walk.ListAll, wrapCallback)
}
if err == fs.ErrorDirNotFound {
err = nil
}
return err
}
// NewObject creates a new remote combine file object
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
u, uRemote, err := f.findUpstream(remote)
if err != nil {
return nil, err
}
o, err := u.f.NewObject(ctx, uRemote)
if err != nil {
return nil, err
}
return u.newObject(o), nil
}
// Precision is the greatest Precision of all upstreams
func (f *Fs) Precision() time.Duration {
var greatestPrecision time.Duration
for _, u := range f.upstreams {
uPrecision := u.f.Precision()
if uPrecision > greatestPrecision {
greatestPrecision = uPrecision
}
}
return greatestPrecision
}
// Shutdown the backend, closing any background tasks and any
// cached connections.
func (f *Fs) Shutdown(ctx context.Context) error {
return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
if do := u.f.Features().Shutdown; do != nil {
return do(ctx)
}
return nil
})
}
// Object describes a wrapped Object
//
// This is a wrapped Object which knows its path prefix
type Object struct {
fs.Object
u *upstream
}
func (u *upstream) newObject(o fs.Object) *Object {
return &Object{
Object: o,
u: u,
}
}
// Fs returns read only access to the Fs that this object is part of
func (o *Object) Fs() fs.Info {
return o.u.parent
}
// String returns the remote path
func (o *Object) String() string {
return o.Remote()
}
// Remote returns the remote path
func (o *Object) Remote() string {
return o.u.pathAdjustment.do(o.Object.String())
}
// MimeType returns the content type of the Object if known
func (o *Object) MimeType(ctx context.Context) (mimeType string) {
if do, ok := o.Object.(fs.MimeTyper); ok {
mimeType = do.MimeType(ctx)
}
return mimeType
}
// UnWrap returns the Object that this Object is wrapping or
// nil if it isn't wrapping anything
func (o *Object) UnWrap() fs.Object {
return o.Object
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.PutStreamer = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ fs.ChangeNotifier = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.Shutdowner = (*Fs)(nil)
)
+79
View File
@@ -0,0 +1,79 @@
// Test Combine filesystem interface
package combine_test
import (
"testing"
_ "github.com/rclone/rclone/backend/local"
_ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
if *fstest.RemoteName == "" {
t.Skip("Skipping as -remote not set")
}
fstests.Run(t, &fstests.Opt{
RemoteName: *fstest.RemoteName,
UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"},
UnimplementableObjectMethods: []string{"MimeType"},
})
}
func TestLocal(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skipping as -remote set")
}
dirs := MakeTestDirs(t, 3)
upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=" + dirs[2]
name := "TestCombineLocal"
fstests.Run(t, &fstests.Opt{
RemoteName: name + ":dir1",
ExtraConfig: []fstests.ExtraConfigItem{
{Name: name, Key: "type", Value: "combine"},
{Name: name, Key: "upstreams", Value: upstreams},
},
})
}
func TestMemory(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skipping as -remote set")
}
upstreams := "dir1=:memory:dir1 dir2=:memory:dir2 dir3=:memory:dir3"
name := "TestCombineMemory"
fstests.Run(t, &fstests.Opt{
RemoteName: name + ":dir1",
ExtraConfig: []fstests.ExtraConfigItem{
{Name: name, Key: "type", Value: "combine"},
{Name: name, Key: "upstreams", Value: upstreams},
},
})
}
func TestMixed(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skipping as -remote set")
}
dirs := MakeTestDirs(t, 2)
upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=:memory:dir3"
name := "TestCombineMixed"
fstests.Run(t, &fstests.Opt{
RemoteName: name + ":dir1",
ExtraConfig: []fstests.ExtraConfigItem{
{Name: name, Key: "type", Value: "combine"},
{Name: name, Key: "upstreams", Value: upstreams},
},
})
}
// MakeTestDirs makes directories in /tmp for testing
func MakeTestDirs(t *testing.T, n int) (dirs []string) {
for i := 1; i <= n; i++ {
dir := t.TempDir()
dirs = append(dirs, dir)
}
return dirs
}
+2 -2
View File
@@ -906,7 +906,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
func (f *Fs) CleanUp(ctx context.Context) error {
do := f.Fs.Features().CleanUp
if do == nil {
return errors.New("not supported by underlying remote")
return errors.New("can't CleanUp: not supported by underlying remote")
}
return do(ctx)
}
@@ -915,7 +915,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil, errors.New("not supported by underlying remote")
return nil, errors.New("can't About: not supported by underlying remote")
}
return do(ctx)
}
+2 -2
View File
@@ -597,7 +597,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
func (f *Fs) CleanUp(ctx context.Context) error {
do := f.Fs.Features().CleanUp
if do == nil {
return errors.New("not supported by underlying remote")
return errors.New("can't CleanUp")
}
return do(ctx)
}
@@ -606,7 +606,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil, errors.New("not supported by underlying remote")
return nil, errors.New("About not supported")
}
return do(ctx)
}
+35 -28
View File
@@ -18,6 +18,7 @@ import (
"mime"
"net/http"
"path"
"regexp"
"sort"
"strconv"
"strings"
@@ -70,7 +71,7 @@ const (
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
defaultChunkSize = 8 * fs.Mebi
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
listRGrouping = 50 // number of IDs to search at once when using ListR
listRInputBuffer = 1000 // size of input buffer when using ListR
defaultXDGIcon = "text-html"
@@ -660,7 +661,6 @@ type baseObject struct {
mimeType string // The object MIME type
bytes int64 // size of the object
parents []string // IDs of the parent directories
resourceKey *string // resourceKey is needed for link shared objects
}
type documentObject struct {
baseObject
@@ -830,8 +830,8 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
if title != "" {
searchTitle := f.opt.Enc.FromStandardName(title)
// Escaping the backslash isn't documented but seems to work
searchTitle = strings.ReplaceAll(searchTitle, `\`, `\\`)
searchTitle = strings.ReplaceAll(searchTitle, `'`, `\'`)
searchTitle = strings.Replace(searchTitle, `\`, `\\`, -1)
searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
var titleQuery bytes.Buffer
_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
@@ -1320,16 +1320,12 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
}
}
}
o := &Object{
return &Object{
baseObject: f.newBaseObject(remote, info),
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
md5sum: strings.ToLower(info.Md5Checksum),
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
}
if info.ResourceKey != "" {
o.resourceKey = &info.ResourceKey
}
return o
}
// newDocumentObject creates an fs.Object for a google docs drive.File
@@ -2434,12 +2430,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var info *drive.File
err = f.pacer.Call(func() (bool, error) {
copy := f.svc.Files.Copy(id, createInfo).
info, err = f.svc.Files.Copy(id, createInfo).
Fields(partialFields).
SupportsAllDrives(true).
KeepRevisionForever(f.opt.KeepRevisionForever)
srcObj.addResourceKey(copy.Header())
info, err = copy.Context(ctx).Do()
KeepRevisionForever(f.opt.KeepRevisionForever).
Context(ctx).Do()
return f.shouldRetry(ctx, err)
})
if err != nil {
@@ -3241,7 +3236,7 @@ This will return a JSON list of objects like this
With the -o config parameter it will output the list in a format
suitable for adding to a config file to make aliases for all the
drives found.
drives found and a combined drive.
[My Drive]
type = alias
@@ -3251,10 +3246,15 @@ drives found.
type = alias
remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
Adding this to the rclone config file will cause those team drives to
be accessible with the aliases shown. This may require manual editing
of the names.
[AllDrives]
type = combine
remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
Adding this to the rclone config file will cause those team drives to
be accessible with the aliases shown. Any illegal charactes will be
substituted with "_" and duplicate names will have numbers suffixed.
It will also add a remote called AllDrives which shows all the shared
drives combined into one directory tree.
`,
}, {
Name: "untrash",
@@ -3370,14 +3370,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
if err != nil {
return nil, err
}
re := regexp.MustCompile(`[^\w_. -]+`)
if _, ok := opt["config"]; ok {
lines := []string{}
for _, drive := range drives {
upstreams := []string{}
names := make(map[string]struct{}, len(drives))
for i, drive := range drives {
name := re.ReplaceAllString(drive.Name, "_")
for {
if _, found := names[name]; !found {
break
}
name += fmt.Sprintf("-%d", i)
}
names[name] = struct{}{}
lines = append(lines, "")
lines = append(lines, fmt.Sprintf("[%s]", drive.Name))
lines = append(lines, fmt.Sprintf("[%s]", name))
lines = append(lines, fmt.Sprintf("type = alias"))
lines = append(lines, fmt.Sprintf("remote = %s,team_drive=%s,root_folder_id=:", f.name, drive.Id))
upstreams = append(upstreams, fmt.Sprintf(`"%s=%s:"`, name, name))
}
lines = append(lines, "")
lines = append(lines, fmt.Sprintf("[AllDrives]"))
lines = append(lines, fmt.Sprintf("type = combine"))
lines = append(lines, fmt.Sprintf("upstreams = %s", strings.Join(upstreams, " ")))
return lines, nil
}
return drives, nil
@@ -3536,14 +3552,6 @@ func (o *baseObject) Storable() bool {
return true
}
// addResourceKey adds a X-Goog-Drive-Resource-Keys header for this
// object if required.
func (o *baseObject) addResourceKey(header http.Header) {
if o.resourceKey != nil {
header.Add("X-Goog-Drive-Resource-Keys", fmt.Sprintf("%s/%s", o.id, *o.resourceKey))
}
}
// httpResponse gets an http.Response object for the object
// using the url and method passed in
func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
@@ -3559,7 +3567,6 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
// Don't supply range requests for 0 length objects as they always fail
delete(req.Header, "Range")
}
o.addResourceKey(req.Header)
err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.client.Do(req)
if err == nil {
+19 -4
View File
@@ -118,12 +118,12 @@ func (b *batcher) Batching() bool {
}
// finishBatch commits the batch, returning a batch status to poll or maybe complete
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (batchStatus *files.UploadSessionFinishBatchLaunch, err error) {
var arg = &files.UploadSessionFinishBatchArg{
Entries: items,
}
err = b.f.pacer.Call(func() (bool, error) {
complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
batchStatus, err = b.f.srv.UploadSessionFinishBatch(arg)
// If error is insufficient space then don't retry
if e, ok := err.(files.UploadSessionFinishAPIError); ok {
if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
@@ -137,7 +137,7 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
if err != nil {
return nil, fmt.Errorf("batch commit failed: %w", err)
}
return complete, nil
return batchStatus, nil
}
// finishBatchJobStatus waits for the batch to complete returning completed entries
@@ -199,11 +199,26 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
fs.Debugf(b.f, "Committing %s", desc)
// finalise the batch getting either a result or a job id to poll
complete, err := b.finishBatch(ctx, items)
batchStatus, err := b.finishBatch(ctx, items)
if err != nil {
return err
}
// check whether batch is complete
var complete *files.UploadSessionFinishBatchResult
switch batchStatus.Tag {
case "async_job_id":
// wait for batch to complete
complete, err = b.finishBatchJobStatus(ctx, batchStatus)
if err != nil {
return err
}
case "complete":
complete = batchStatus.Complete
default:
return fmt.Errorf("batch returned unknown status %q", batchStatus.Tag)
}
// Check we got the right number of entries
entries := complete.Entries
if len(entries) != len(results) {
+1 -3
View File
@@ -1269,7 +1269,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
return nil, fmt.Errorf("about failed: %w", err)
}
var total uint64
if q.Allocation != nil {
@@ -1370,12 +1370,10 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
if timeout < 30 {
timeout = 30
fs.Debugf(f, "Increasing poll interval to minimum 30s")
}
if timeout > 480 {
timeout = 480
fs.Debugf(f, "Decreasing poll interval to maximum 480s")
}
err = f.pacer.Call(func() (bool, error) {
+1 -10
View File
@@ -15,7 +15,7 @@ import (
"sync"
"time"
"github.com/jlaffaye/ftp"
"github.com/rclone/ftp"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config"
@@ -100,11 +100,6 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
Help: "Disable using MLSD even if server advertises support.",
Default: false,
Advanced: true,
}, {
Name: "disable_utf8",
Help: "Disable using UTF-8 even if server advertises support.",
Default: false,
Advanced: true,
}, {
Name: "writing_mdtm",
Help: "Use MDTM to set modification time (VsFtpd quirk)",
@@ -189,7 +184,6 @@ type Options struct {
SkipVerifyTLSCert bool `config:"no_check_certificate"`
DisableEPSV bool `config:"disable_epsv"`
DisableMLSD bool `config:"disable_mlsd"`
DisableUTF8 bool `config:"disable_utf8"`
WritingMDTM bool `config:"writing_mdtm"`
IdleTimeout fs.Duration `config:"idle_timeout"`
CloseTimeout fs.Duration `config:"close_timeout"`
@@ -344,9 +338,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
if f.opt.DisableMLSD {
ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
}
if f.opt.DisableUTF8 {
ftpConfig = append(ftpConfig, ftp.DialWithDisabledUTF8(true))
}
if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
}
@@ -24,7 +24,6 @@ import (
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/rclone/rclone/fs"
@@ -305,21 +304,6 @@ rclone does if you know the bucket exists already.
`,
Default: false,
Advanced: true,
}, {
Name: "download_compressed",
Help: `If set this will download compressed objects as-is.
It is possible to upload objects to GCS with "Content-Encoding: gzip"
set. Normally rclone will transparently decompress these files on
download. This means that rclone can't check the hash or the size of
the file as both of these refer to the compressed object.
If this flag is set then rclone will download files with
"Content-Encoding: gzip" as they are received. This means that rclone
can check the size and hash but the file contents will be compressed.
`,
Advanced: true,
Default: false,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
@@ -343,23 +327,21 @@ type Options struct {
Location string `config:"location"`
StorageClass string `config:"storage_class"`
NoCheckBucket bool `config:"no_check_bucket"`
DownloadCompressed bool `config:"download_compressed"`
Enc encoder.MultiEncoder `config:"encoding"`
}
// Fs represents a remote storage server
type Fs struct {
name string // name of this remote
root string // the path we are working on if any
opt Options // parsed options
features *fs.Features // optional features
svc *storage.Service // the connection to the storage server
client *http.Client // authorized client
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
cache *bucket.Cache // cache of bucket status
pacer *fs.Pacer // To pace the API calls
warnCompressed sync.Once // warn once about compressed files
name string // name of this remote
root string // the path we are working on if any
opt Options // parsed options
features *fs.Features // optional features
svc *storage.Service // the connection to the storage server
client *http.Client // authorized client
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
cache *bucket.Cache // cache of bucket status
pacer *fs.Pacer // To pace the API calls
}
// Object describes a storage object
@@ -373,7 +355,6 @@ type Object struct {
bytes int64 // Bytes in the object
modTime time.Time // Modified time of the object
mimeType string
gzipped bool // set if object has Content-Encoding: gzip
}
// ------------------------------------------------------------
@@ -994,7 +975,6 @@ func (o *Object) setMetaData(info *storage.Object) {
o.url = info.MediaLink
o.bytes = int64(info.Size)
o.mimeType = info.ContentType
o.gzipped = info.ContentEncoding == "gzip"
// Read md5sum
md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
@@ -1033,15 +1013,6 @@ func (o *Object) setMetaData(info *storage.Object) {
} else {
o.modTime = modTime
}
// If gunzipping then size and md5sum are unknown
if o.gzipped && !o.fs.opt.DownloadCompressed {
o.bytes = -1
o.md5sum = ""
o.fs.warnCompressed.Do(func() {
fs.Logf(o.fs, "Decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-download-compressed to override")
})
}
}
// readObjectInfo reads the definition for an object
@@ -1142,15 +1113,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return nil, err
}
fs.FixRangeOption(options, o.bytes)
if o.gzipped && o.fs.opt.DownloadCompressed {
// Allow files which are stored on the cloud storage system
// compressed to be downloaded without being decompressed. Note
// that setting this here overrides the automatic decompression
// in the Transport.
//
// See: https://cloud.google.com/storage/docs/transcoding
req.Header.Set("Accept-Encoding", "gzip")
}
fs.OpenOptionAddHTTPHeaders(req.Header, options)
var res *http.Response
err = o.fs.pacer.Call(func() (bool, error) {
+1 -1
View File
@@ -562,7 +562,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
for i := range items {
item := &result.MediaItems[i]
remote := item.Filename
remote = strings.ReplaceAll(remote, "/", "/")
remote = strings.Replace(remote, "/", "/", -1)
err = fn(remote, item, false)
if err != nil {
return err
+2 -2
View File
@@ -282,7 +282,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
if do := f.Fs.Features().CleanUp; do != nil {
return do(ctx)
}
return errors.New("not supported by underlying remote")
return errors.New("CleanUp not supported")
}
// About gets quota information from the Fs
@@ -290,7 +290,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
if do := f.Fs.Features().About; do != nil {
return do(ctx)
}
return nil, errors.New("not supported by underlying remote")
return nil, errors.New("About not supported")
}
// ChangeNotify calls the passed function with a path that has had changes.
+1 -1
View File
@@ -119,7 +119,7 @@ func (f *Fs) getCredentials(ctx context.Context) (err error) {
defer fs.CheckClose(resp.Body, &err)
if resp.StatusCode < 200 || resp.StatusCode > 299 {
body, _ := ioutil.ReadAll(resp.Body)
bodyStr := strings.TrimSpace(strings.ReplaceAll(string(body), "\n", " "))
bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
}
decoder := json.NewDecoder(resp.Body)
File diff suppressed because it is too large. Load diff
@@ -1,17 +0,0 @@
// Test internetarchive filesystem interface
package internetarchive_test
import (
"testing"
"github.com/rclone/rclone/backend/internetarchive"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestIA:lesmi-rclone-test/",
NilObject: (*internetarchive.Object)(nil),
})
}
+43 -65
View File
@@ -8,69 +8,42 @@ import (
)
const (
// default time format historically used for all request and responses.
// Similar to time.RFC3339, but with an extra '-' in front of 'T',
// and no ':' separator in timezone offset. Some newer endpoints have
// moved to proper time.RFC3339 conformant format instead.
jottaTimeFormat = "2006-01-02-T15:04:05Z0700"
// default time format for almost all request and responses
timeFormat = "2006-01-02-T15:04:05Z0700"
// the API server seems to use a different format
apiTimeFormat = "2006-01-02T15:04:05Z07:00"
)
// unmarshalXML turns XML into a Time
func unmarshalXMLTime(d *xml.Decoder, start xml.StartElement, timeFormat string) (time.Time, error) {
// Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
type Time time.Time
// UnmarshalXML turns XML into a Time
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var v string
if err := d.DecodeElement(&v, &start); err != nil {
return time.Time{}, err
return err
}
if v == "" {
return time.Time{}, nil
*t = Time(time.Time{})
return nil
}
newTime, err := time.Parse(timeFormat, v)
if err == nil {
return newTime, nil
*t = Time(newTime)
}
return time.Time{}, err
}
// JottaTime represents time values in the classic API using a custom RFC3339 like format
type JottaTime time.Time
// String returns JottaTime string in Jottacloud classic format
func (t JottaTime) String() string { return time.Time(t).Format(jottaTimeFormat) }
// UnmarshalXML turns XML into a JottaTime
func (t *JottaTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
tm, err := unmarshalXMLTime(d, start, jottaTimeFormat)
*t = JottaTime(tm)
return err
}
// MarshalXML turns a JottaTime into XML
func (t *JottaTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
// MarshalXML turns a Time into XML
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
return e.EncodeElement(t.String(), start)
}
// Rfc3339Time represents time values in the newer APIs using standard RFC3339 format
type Rfc3339Time time.Time
// Return Time string in Jottacloud format
func (t Time) String() string { return time.Time(t).Format(timeFormat) }
// String returns Rfc3339Time string in Jottacloud RFC3339 format
func (t Rfc3339Time) String() string { return time.Time(t).Format(time.RFC3339) }
// UnmarshalXML turns XML into a Rfc3339Time
func (t *Rfc3339Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
tm, err := unmarshalXMLTime(d, start, time.RFC3339)
*t = Rfc3339Time(tm)
return err
}
// MarshalXML turns a Rfc3339Time into XML
func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
return e.EncodeElement(t.String(), start)
}
// MarshalJSON turns a Rfc3339Time into JSON
func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
}
// APIString returns Time string in Jottacloud API format
func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
// LoginToken is struct representing the login token generated in the WebUI
type LoginToken struct {
@@ -149,11 +122,16 @@ type AllocateFileResponse struct {
// UploadResponse after an upload
type UploadResponse struct {
Path string `json:"path"`
ContentID string `json:"content_id"`
Bytes int64 `json:"bytes"`
Md5 string `json:"md5"`
Modified int64 `json:"modified"`
Name string `json:"name"`
Path string `json:"path"`
Kind string `json:"kind"`
ContentID string `json:"content_id"`
Bytes int64 `json:"bytes"`
Md5 string `json:"md5"`
Created int64 `json:"created"`
Modified int64 `json:"modified"`
Deleted interface{} `json:"deleted"`
Mime string `json:"mime"`
}
// DeviceRegistrationResponse is the response to registering a device
@@ -360,9 +338,9 @@ type JottaFolder struct {
Name string `xml:"name,attr"`
Deleted Flag `xml:"deleted,attr"`
Path string `xml:"path"`
CreatedAt JottaTime `xml:"created"`
ModifiedAt JottaTime `xml:"modified"`
Updated JottaTime `xml:"updated"`
CreatedAt Time `xml:"created"`
ModifiedAt Time `xml:"modified"`
Updated Time `xml:"updated"`
Folders []JottaFolder `xml:"folders>folder"`
Files []JottaFile `xml:"files>file"`
}
@@ -387,17 +365,17 @@ GET http://www.jottacloud.com/JFS/<account>/<device>/<mountpoint>/.../<file>
// JottaFile represents a Jottacloud file
type JottaFile struct {
XMLName xml.Name
Name string `xml:"name,attr"`
Deleted Flag `xml:"deleted,attr"`
PublicURI string `xml:"publicURI"`
PublicSharePath string `xml:"publicSharePath"`
State string `xml:"currentRevision>state"`
CreatedAt JottaTime `xml:"currentRevision>created"`
ModifiedAt JottaTime `xml:"currentRevision>modified"`
Updated JottaTime `xml:"currentRevision>updated"`
Size int64 `xml:"currentRevision>size"`
MimeType string `xml:"currentRevision>mime"`
MD5 string `xml:"currentRevision>md5"`
Name string `xml:"name,attr"`
Deleted Flag `xml:"deleted,attr"`
PublicURI string `xml:"publicURI"`
PublicSharePath string `xml:"publicSharePath"`
State string `xml:"currentRevision>state"`
CreatedAt Time `xml:"currentRevision>created"`
ModifiedAt Time `xml:"currentRevision>modified"`
Updated Time `xml:"currentRevision>updated"`
Size int64 `xml:"currentRevision>size"`
MimeType string `xml:"currentRevision>mime"`
MD5 string `xml:"currentRevision>md5"`
}
// Error is a custom Error for wrapping Jottacloud error responses
+66 -71
View File
@@ -191,7 +191,7 @@ machines.`)
m.Set("auth_code", "")
return fs.ConfigGoto("legacy_do_auth")
case "legacy_auth_code":
authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
m.Set("auth_code", authCode)
return fs.ConfigGoto("legacy_do_auth")
case "legacy_do_auth":
@@ -519,7 +519,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
values.Set("client_id", defaultClientID)
values.Set("grant_type", "password")
values.Set("password", loginToken.AuthToken)
values.Set("scope", "openid offline_access")
values.Set("scope", "offline_access+openid")
values.Set("username", loginToken.Username)
values.Encode()
opts = rest.Opts{
@@ -649,7 +649,7 @@ func errorHandler(resp *http.Response) error {
// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
func urlPathEscape(in string) string {
return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
}
// filePathRaw returns an unescaped file path (f.root, file)
@@ -932,7 +932,26 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return entries, nil
}
func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
type listStreamTime time.Time
func (c *listStreamTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var v string
if err := d.DecodeElement(&v, &start); err != nil {
return err
}
t, err := time.Parse(time.RFC3339, v)
if err != nil {
return err
}
*c = listStreamTime(t)
return nil
}
func (c listStreamTime) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf("\"%s\"", time.Time(c).Format(time.RFC3339))), nil
}
func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, filesystem *Fs, callback func(fs.DirEntry) error) error {
type stats struct {
Folders int `xml:"folders"`
@@ -941,12 +960,12 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback
var expected, actual stats
type xmlFile struct {
Path string `xml:"path"`
Name string `xml:"filename"`
Checksum string `xml:"md5"`
Size int64 `xml:"size"`
Modified api.Rfc3339Time `xml:"modified"` // Note: Liststream response includes 3 decimal milliseconds, but we ignore them since there is second precision everywhere else
Created api.Rfc3339Time `xml:"created"`
Path string `xml:"path"`
Name string `xml:"filename"`
Checksum string `xml:"md5"`
Size int64 `xml:"size"`
Modified listStreamTime `xml:"modified"`
Created listStreamTime `xml:"created"`
}
type xmlFolder struct {
@@ -968,12 +987,8 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback
})
}
// liststream paths are /mountpoint/root/path
// so the returned paths should have /mountpoint/root/ trimmed
// as the caller is expecting path.
pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
trimPathPrefix := func(p string) string {
p = strings.TrimPrefix(p, pathPrefix)
p = strings.TrimPrefix(p, trimPrefix)
p = strings.TrimPrefix(p, "/")
return p
}
@@ -1056,7 +1071,11 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return shouldRetry(ctx, resp, err)
}
err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
// liststream paths are /mountpoint/root/path
// so the returned paths should have /mountpoint/root/ trimmed
// as the caller is expecting path.
trimPrefix := path.Join("/", f.opt.Mountpoint, f.root)
err = parseListRStream(ctx, resp.Body, trimPrefix, f, func(d fs.DirEntry) error {
if d.Remote() == dir {
return nil
}
@@ -1191,45 +1210,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
return f.purgeCheck(ctx, dir, false)
}
// createOrUpdate tries to make remote file match without uploading.
// If the remote file exists, and has matching size and md5, only
// timestamps are updated. If the file does not exist or does does
// not match size and md5, but matching content can be constructed
// from deduplication, the file will be updated/created. If the file
// is currently in trash, but can be made to match, it will be
// restored. Returns ErrorObjectNotFound if upload will be necessary
// to get a matching remote file.
func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
opts := rest.Opts{
Method: "POST",
Path: f.filePath(file),
Parameters: url.Values{},
ExtraHeaders: make(map[string]string),
}
opts.Parameters.Set("cphash", "true")
fileDate := api.JottaTime(modTime).String()
opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
opts.ExtraHeaders["JMd5"] = md5
opts.ExtraHeaders["JCreated"] = fileDate
opts.ExtraHeaders["JModified"] = fileDate
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
return shouldRetry(ctx, resp, err)
})
if apiErr, ok := err.(*api.Error); ok {
// does not exist, i.e. not matching size and md5, and not possible to make it by deduplication
if apiErr.StatusCode == http.StatusNotFound {
return nil, fs.ErrorObjectNotFound
}
}
return info, nil
}
// copyOrMoves copies or moves directories or files depending on the method parameter
func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
opts := rest.Opts{
@@ -1273,12 +1253,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
}
info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
// if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?)
if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
}
if err != nil {
return nil, fmt.Errorf("couldn't copy file: %w", err)
}
@@ -1580,19 +1554,40 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
return err
}
// request check/update with existing metadata and new modtime
// (note that if size/md5 does not match, the file content will
// also be modified if deduplication is possible, i.e. it is
// important to use correct/latest values)
_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
// prepare allocate request with existing metadata but changed timestamps
var resp *http.Response
var options []fs.OpenOption
opts := rest.Opts{
Method: "POST",
Path: "files/v1/allocate",
Options: options,
ExtraHeaders: make(map[string]string),
}
fileDate := api.Time(modTime).APIString()
var request = api.AllocateFileRequest{
Bytes: o.size,
Created: fileDate,
Modified: fileDate,
Md5: o.md5,
Path: path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
}
// send it
var response api.AllocateFileResponse
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
return shouldRetry(ctx, resp, err)
})
if err != nil {
if err == fs.ErrorObjectNotFound {
// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
return errors.New("metadata did not match")
}
return err
}
// check response
if response.State != "COMPLETED" {
// could be the file was modified (size/md5 changed) between readMetaData and the allocate request
return errors.New("metadata did not match")
}
// update local metadata
o.modTime = modTime
return nil
@@ -1730,7 +1725,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Options: options,
ExtraHeaders: make(map[string]string),
}
fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
fileDate := api.Time(src.ModTime(ctx)).APIString()
// the allocate request
var request = api.AllocateFileRequest{
+1 -141
View File
@@ -301,10 +301,6 @@ type siteResource struct {
type siteResponse struct {
Sites []siteResource `json:"value"`
}
type deltaResponse struct {
DeltaLink string `json:"@odata.deltaLink"`
Value []api.Item `json:"value"`
}
// Get the region and graphURL from the config
func getRegionURL(m configmap.Mapper) (region, graphURL string) {
@@ -1506,7 +1502,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
return nil, fmt.Errorf("about failed: %w", err)
}
q := drive.Quota
// On (some?) Onedrive sharepoints these are all 0 so return unknown in that case
@@ -2306,142 +2302,6 @@ func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
return canonicalDriveID
}
// ChangeNotify calls the passed function with a path that has had changes.
// If the implementation uses polling, it should adhere to the given interval.
//
// Automatically restarts itself in case of unexpected behavior of the remote.
//
// Close the returned channel to stop being notified.
//
// The Onedrive implementation gives the whole hierarchy up to the top when
// an object is changed. For instance, if a/b/c is changed, this function
// will call notifyFunc with a, a/b and a/b/c.
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
go func() {
// get the StartPageToken early so all changes from now on get processed
nextDeltaToken, err := f.changeNotifyStartPageToken(ctx)
if err != nil {
fs.Errorf(f, "Could not get first deltaLink: %s", err)
return
}
fs.Debugf(f, "Next delta token is: %s", nextDeltaToken)
var ticker *time.Ticker
var tickerC <-chan time.Time
for {
select {
case pollInterval, ok := <-pollIntervalChan:
if !ok {
if ticker != nil {
ticker.Stop()
}
return
}
if ticker != nil {
ticker.Stop()
ticker, tickerC = nil, nil
}
if pollInterval != 0 {
ticker = time.NewTicker(pollInterval)
tickerC = ticker.C
}
case <-tickerC:
fs.Debugf(f, "Checking for changes on remote")
nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
if err != nil {
fs.Infof(f, "Change notify listener failure: %s", err)
}
}
}
}()
}
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
delta, err := f.changeNotifyNextChange(ctx, "latest")
parsedURL, err := url.Parse(delta.DeltaLink)
if err != nil {
return
}
nextDeltaToken = parsedURL.Query().Get("token")
return
}
func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta deltaResponse, err error) {
opts := f.buildDriveDeltaOpts(token)
_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
return
}
func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
rootURL := graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
return rest.Opts{
Method: "GET",
RootURL: rootURL,
Path: "/" + f.driveID + "/root/delta",
Parameters: map[string][]string{"token": {token}},
}
}
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
delta, err := f.changeNotifyNextChange(ctx, deltaToken)
parsedURL, err := url.Parse(delta.DeltaLink)
if err != nil {
return
}
nextDeltaToken = parsedURL.Query().Get("token")
for _, item := range delta.Value {
isDriveRootFolder := item.GetParentReference().ID == ""
if isDriveRootFolder {
continue
}
fullPath, err := getItemFullPath(&item)
if err != nil {
fs.Errorf(f, "Could not get item full path: %s", err)
continue
}
if fullPath == f.root {
continue
}
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
if !insideRoot {
continue
}
if item.GetFile() != nil {
notifyFunc(relName, fs.EntryObject)
} else if item.GetFolder() != nil {
notifyFunc(relName, fs.EntryDirectory)
}
}
return
}
func getItemFullPath(item *api.Item) (fullPath string, err error) {
err = nil
fullPath = item.GetName()
if parent := item.GetParentReference(); parent != nil && parent.Path != "" {
pathParts := strings.SplitN(parent.Path, ":", 2)
if len(pathParts) != 2 {
err = fmt.Errorf("invalid parent path: %s", parent.Path)
return
}
if pathParts[1] != "" {
fullPath = strings.TrimPrefix(pathParts[1], "/") + "/" + fullPath
}
}
return
}
// getRelativePathInsideBase checks if `target` is inside `base`. If so, it
// returns a relative path for `target` based on `base` and a boolean `true`.
// Otherwise returns "", false.
+1 -1
View File
@@ -906,7 +906,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
return nil, fmt.Errorf("about failed: %w", err)
}
usage = &fs.Usage{
Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used
+2 -2
View File
@@ -783,10 +783,10 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
return nil, fmt.Errorf("CreateDir http: %w", err)
}
if err = info.AsErr(); err != nil {
return nil, err
return nil, fmt.Errorf("CreateDir: %w", err)
}
usage = &fs.Usage{
Used: fs.NewUsageValue(int64(info.SpaceUsed)),
+1 -1
View File
@@ -647,7 +647,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
return nil, fmt.Errorf("about failed: %w", err)
}
return &fs.Usage{
Total: fs.NewUsageValue(ai.Disk.Size), // quota of bytes that can be used
+91 -258
View File
@@ -32,11 +32,9 @@ import (
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/ncw/swift/v2"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
@@ -60,7 +58,7 @@ import (
func init() {
fs.Register(&fs.RegInfo{
Name: "s3",
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
NewFs: NewFs,
CommandHelp: commandHelp,
Options: []fs.Option{{
@@ -80,12 +78,6 @@ func init() {
}, {
Value: "ChinaMobile",
Help: "China Mobile Ecloud Elastic Object Storage (EOS)",
}, {
Value: "Cloudflare",
Help: "Cloudflare R2 Storage",
}, {
Value: "ArvanCloud",
Help: "Arvan Cloud Object Storage (AOS)",
}, {
Value: "DigitalOcean",
Help: "Digital Ocean Spaces",
@@ -301,22 +293,11 @@ func init() {
}, {
Value: "fr-par",
Help: "Paris, France",
}, {
Value: "pl-waw",
Help: "Warsaw, Poland",
}},
}, {
Name: "region",
Help: "Region to connect to.",
Provider: "Cloudflare",
Examples: []fs.OptionExample{{
Value: "auto",
Help: "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
}},
}, {
Name: "region",
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
Provider: "!AWS,Alibaba,ChinaMobile,RackCorp,Scaleway,Storj,TencentCOS",
Examples: []fs.OptionExample{{
Value: "",
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -424,18 +405,6 @@ func init() {
Value: "eos-anhui-1.cmecloud.cn",
Help: "Anhui China (Huainan)",
}},
}, {
// ArvanCloud endpoints: https://www.arvancloud.com/en/products/cloud-storage
Name: "endpoint",
Help: "Endpoint for Arvan Cloud Object Storage (AOS) API.",
Provider: "ArvanCloud",
Examples: []fs.OptionExample{{
Value: "s3.ir-thr-at1.arvanstorage.com",
Help: "The default endpoint - a good choice if you are unsure.\nTehran Iran (Asiatech)",
}, {
Value: "s3.ir-tbz-sh1.arvanstorage.com",
Help: "Tabriz Iran (Shahriar)",
}},
}, {
Name: "endpoint",
Help: "Endpoint for IBM COS S3 API.\n\nSpecify if using an IBM COS On Premise.",
@@ -718,9 +687,6 @@ func init() {
}, {
Value: "s3.fr-par.scw.cloud",
Help: "Paris Endpoint",
}, {
Value: "s3.pl-waw.scw.cloud",
Help: "Warsaw Endpoint",
}},
}, {
Name: "endpoint",
@@ -879,7 +845,7 @@ func init() {
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp",
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,Scaleway,StackPath,Storj,RackCorp",
Examples: []fs.OptionExample{{
Value: "objects-us-east-1.dream.io",
Help: "Dream Objects endpoint",
@@ -932,10 +898,6 @@ func init() {
Value: "s3.ap-northeast-2.wasabisys.com",
Help: "Wasabi AP Northeast 2 (Osaka) endpoint",
Provider: "Wasabi",
}, {
Value: "s3.ir-thr-at1.arvanstorage.com",
Help: "ArvanCloud Tehran Iran (Asiatech) endpoint",
Provider: "ArvanCloud",
}},
}, {
Name: "location_constraint",
@@ -1112,17 +1074,6 @@ func init() {
Value: "anhui1",
Help: "Anhui China (Huainan)",
}},
}, {
Name: "location_constraint",
Help: "Location constraint - must match endpoint.\n\nUsed when creating buckets only.",
Provider: "ArvanCloud",
Examples: []fs.OptionExample{{
Value: "ir-thr-at1",
Help: "Tehran Iran (Asiatech)",
}, {
Value: "ir-tbz-sh1",
Help: "Tabriz Iran (Shahriar)",
}},
}, {
Name: "location_constraint",
Help: "Location constraint - must match endpoint when using IBM Cloud Public.\n\nFor on-prem COS, do not make a selection from this list, hit enter.",
@@ -1289,7 +1240,7 @@ func init() {
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
}, {
Name: "acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1300,7 +1251,7 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
Note that this ACL is applied when server-side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.`,
Provider: "!Storj,Cloudflare",
Provider: "!Storj",
Examples: []fs.OptionExample{{
Value: "default",
Help: "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@@ -1501,15 +1452,6 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
Value: "STANDARD_IA",
Help: "Infrequent access storage mode",
}},
}, {
// Mapping from here: https://www.arvancloud.com/en/products/cloud-storage
Name: "storage_class",
Help: "The storage class to use when storing new objects in ArvanCloud.",
Provider: "ArvanCloud",
Examples: []fs.OptionExample{{
Value: "STANDARD",
Help: "Standard storage class",
}},
}, {
// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
Name: "storage_class",
@@ -1842,20 +1784,6 @@ This should be true, false or left unset to use the default for the provider.
`,
Default: fs.Tristate{},
Advanced: true,
}, {
Name: "use_presigned_request",
Help: `Whether to use a presigned request or PutObject for single part uploads
If this is false rclone will use PutObject from the AWS SDK to upload
an object.
Versions of rclone < 1.59 use presigned requests to upload a single
part object and setting this flag to true will re-enable that
functionality. This shouldn't be necessary except in exceptional
circumstances or for testing.
`,
Default: false,
Advanced: true,
},
}})
}
@@ -1921,7 +1849,6 @@ type Options struct {
DisableHTTP2 bool `config:"disable_http2"`
DownloadURL string `config:"download_url"`
UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`
UsePresignedRequest bool `config:"use_presigned_request"`
}
// Fs represents a remote s3 server
@@ -1933,7 +1860,6 @@ type Fs struct {
ctx context.Context // global context for reading config
features *fs.Features // optional features
c *s3.S3 // the connection to the s3 server
cu *s3.S3 // unsigned connection to the s3 server for PutObject
ses *session.Session // the s3 session
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
@@ -2066,11 +1992,7 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
}
// s3Connection makes a connection to s3
//
// If unsignedBody is set then the connection is configured for
// unsigned bodies which is necessary for PutObject if we don't want
// it to seek
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) {
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
ci := fs.GetConfig(ctx)
// Make the auth
v := credentials.Value{
@@ -2087,7 +2009,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
// start a new AWS session
awsSession, err := session.NewSession()
if err != nil {
return nil, nil, nil, fmt.Errorf("NewSession: %w", err)
return nil, nil, fmt.Errorf("NewSession: %w", err)
}
// first provider to supply a credential set "wins"
@@ -2127,9 +2049,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
cred = credentials.AnonymousCredentials
case v.AccessKeyID == "":
return nil, nil, nil, errors.New("access_key_id not found")
return nil, nil, errors.New("access_key_id not found")
case v.SecretAccessKey == "":
return nil, nil, nil, errors.New("secret_access_key not found")
return nil, nil, errors.New("secret_access_key not found")
}
if opt.Region == "" {
@@ -2168,36 +2090,25 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
awsSessionOpts.Config.Credentials = nil
}
// Setting this stops PutObject reading the body twice and seeking
// We add our own Content-MD5 for data protection
awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true)
ses, err := session.NewSessionWithOptions(awsSessionOpts)
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
newC := func(unsignedBody bool) *s3.S3 {
c := s3.New(ses)
if opt.V2Auth || opt.Region == "other-v2-signature" {
fs.Debugf(nil, "Using v2 auth")
signer := func(req *request.Request) {
// Ignore AnonymousCredentials object
if req.Config.Credentials == credentials.AnonymousCredentials {
return
}
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
c := s3.New(ses)
if opt.V2Auth || opt.Region == "other-v2-signature" {
fs.Debugf(nil, "Using v2 auth")
signer := func(req *request.Request) {
// Ignore AnonymousCredentials object
if req.Config.Credentials == credentials.AnonymousCredentials {
return
}
c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
c.Handlers.Sign.PushBack(signer)
} else if unsignedBody {
// If the body is unsigned then tell the signer that we aren't signing the payload
c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload))
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
}
return c
c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
c.Handlers.Sign.PushBack(signer)
}
return newC(false), newC(true), ses, nil
return c, ses, nil
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
@@ -2256,13 +2167,6 @@ func setQuirks(opt *Options) {
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
case "Cloudflare":
virtualHostStyle = false
useMultipartEtag = false // currently multipart Etags are random
case "ArvanCloud":
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
case "DigitalOcean":
urlEncodeListings = false
case "Dreamhost":
@@ -2382,7 +2286,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
}
srv := getClient(ctx, opt)
c, cu, ses, err := s3Connection(ctx, opt, srv)
c, ses, err := s3Connection(ctx, opt, srv)
if err != nil {
return nil, err
}
@@ -2400,7 +2304,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci: ci,
ctx: ctx,
c: c,
cu: cu,
ses: ses,
pacer: pc,
cache: bucket.NewCache(),
@@ -2524,12 +2427,11 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
// Make a new session with the new region
oldRegion := f.opt.Region
f.opt.Region = region
c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
if err != nil {
return fmt.Errorf("creating new session failed: %w", err)
}
f.c = c
f.cu = cu
f.ses = ses
fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
@@ -2972,7 +2874,7 @@ func (f *Fs) Precision() time.Duration {
// pathEscape escapes s as for a URL path. It uses rest.URLPathEscape
// but also escapes '+' for S3 and Digital Ocean spaces compatibility
func pathEscape(s string) string {
return strings.ReplaceAll(rest.URLPathEscape(s), "+", "%2B")
return strings.Replace(rest.URLPathEscape(s), "+", "%2B", -1)
}
// copy does a server-side copy
@@ -3345,10 +3247,6 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
st.Status = "Not an S3 object"
return
}
if o.storageClass != "GLACIER" && o.storageClass != "DEEP_ARCHIVE" {
st.Status = "Not GLACIER or DEEP_ARCHIVE storage class"
return
}
bucket, bucketPath := o.split()
reqCopy := req
reqCopy.Bucket = &bucket
@@ -3845,7 +3743,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
}
// calculate size of parts
partSize := f.opt.ChunkSize
partSize := int(f.opt.ChunkSize)
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
@@ -3856,7 +3754,11 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
})
} else {
partSize = chunksize.Calculator(o, int(uploadParts), f.opt.ChunkSize)
// Adjust partSize until the number of parts is small enough.
if size/int64(partSize) >= uploadParts {
// Calculate partition size rounded up to the nearest MiB
partSize = int((((size / uploadParts) >> 20) + 1) << 20)
}
}
memPool := f.getMemoryPool(int64(partSize))
@@ -4026,117 +3928,6 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
return etag, nil
}
// unWrapAwsError unwraps AWS errors, looking for a non AWS error
//
// It returns true if one was found and the error, or false and the
// error passed in.
func unWrapAwsError(err error) (found bool, outErr error) {
if awsErr, ok := err.(awserr.Error); ok {
var origErrs []error
if batchErr, ok := awsErr.(awserr.BatchError); ok {
origErrs = batchErr.OrigErrs()
} else {
origErrs = []error{awsErr.OrigErr()}
}
for _, origErr := range origErrs {
found, newErr := unWrapAwsError(origErr)
if found {
return found, newErr
}
}
return false, err
}
return true, err
}
// Upload a single part using PutObject
func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
req.Body = readers.NewFakeSeeker(in, size)
var resp *s3.PutObjectOutput
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.cu.PutObject(req)
return o.fs.shouldRetry(ctx, err)
})
if err != nil {
// Return the underlying error if we have a Serialization error if possible
//
// Serialization errors are synthesized locally in the SDK (not returned from the
// server). We'll get one if the SDK attempts a retry, however the FakeSeeker will
// remember the previous error from Read and return that.
if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization {
if found, newErr := unWrapAwsError(err); found {
err = newErr
}
}
return etag, lastModified, err
}
lastModified = time.Now()
etag = aws.StringValue(resp.ETag)
return etag, lastModified, nil
}
// Upload a single part using a presigned request
func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
// Create the request
putObj, _ := o.fs.c.PutObjectRequest(req)
// Sign it so we can upload using a presigned request.
//
// Note the SDK didn't used to support streaming to
// PutObject so we used this work-around.
url, headers, err := putObj.PresignRequest(15 * time.Minute)
if err != nil {
return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err)
}
if o.fs.opt.V2Auth && headers == nil {
headers = putObj.HTTPRequest.Header
}
// Set request to nil if empty so as not to make chunked encoding
if size == 0 {
in = nil
}
// create the vanilla http request
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
if err != nil {
return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err)
}
// set the headers we signed and the length
httpReq.Header = headers
httpReq.ContentLength = size
var resp *http.Response
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
var err error
resp, err = o.fs.srv.Do(httpReq)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
body, err := rest.ReadBody(resp)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
return false, nil
}
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
})
if err != nil {
return etag, lastModified, err
}
if resp != nil {
if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil {
lastModified = date
}
etag = resp.Header.Get("Etag")
}
return etag, lastModified, nil
}
// Update the Object from in with modTime and size
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
bucket, bucketPath := o.split()
@@ -4188,9 +3979,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
ContentType: &mimeType,
Metadata: metadata,
}
if size >= 0 {
req.ContentLength = &size
}
if md5sumBase64 != "" {
req.ContentMD5 = &md5sumBase64
}
@@ -4245,20 +4033,65 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
}
var wantETag string // Multipart upload Etag to check
var gotEtag string // Etag we got from the upload
var lastModified time.Time // Time we got from the upload
var resp *http.Response // response from PUT
var wantETag string // Multipart upload Etag to check
if multipart {
wantETag, err = o.uploadMultipart(ctx, &req, size, in)
if err != nil {
return err
}
} else {
if o.fs.opt.UsePresignedRequest {
gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
} else {
gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
// Create the request
putObj, _ := o.fs.c.PutObjectRequest(&req)
// Sign it so we can upload using a presigned request.
//
// Note the SDK doesn't currently support streaming to
// PutObject so we'll use this work-around.
url, headers, err := putObj.PresignRequest(15 * time.Minute)
if err != nil {
return fmt.Errorf("s3 upload: sign request: %w", err)
}
if o.fs.opt.V2Auth && headers == nil {
headers = putObj.HTTPRequest.Header
}
// Set request to nil if empty so as not to make chunked encoding
if size == 0 {
in = nil
}
// create the vanilla http request
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
if err != nil {
return fmt.Errorf("s3 upload: new request: %w", err)
}
// set the headers we signed and the length
httpReq.Header = headers
httpReq.ContentLength = size
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
var err error
resp, err = o.fs.srv.Do(httpReq)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
body, err := rest.ReadBody(resp)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
return false, nil
}
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
})
if err != nil {
return err
}
}
if err != nil {
return err
}
// User requested we don't HEAD the object after uploading it
@@ -4272,11 +4105,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
o.mimeType = aws.StringValue(req.ContentType)
o.storageClass = aws.StringValue(req.StorageClass)
// If we have done a single part PUT request then we can read these
if gotEtag != "" {
o.setMD5FromEtag(gotEtag)
}
if !o.lastModified.IsZero() {
o.lastModified = lastModified
if resp != nil {
if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
o.lastModified = date
}
o.setMD5FromEtag(resp.Header.Get("Etag"))
}
return nil
}
+3 -3
View File
@@ -109,7 +109,7 @@ when the ssh-agent contains many keys.`,
Default: false,
}, {
Name: "use_insecure_cipher",
Help: `Enable the use of insecure ciphers and key exchange methods.
Help: `Enable the use of insecure ciphers and key exchange methods.
This enables the use of the following insecure ciphers and key exchange methods:
@@ -1220,7 +1220,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
}
stdout, err := f.run(ctx, "df -k "+escapedPath)
if err != nil {
return nil, fmt.Errorf("your remote may not have the required df utility: %w", err)
return nil, fmt.Errorf("your remote may not support About: %w", err)
}
usageTotal, usageUsed, usageAvail := parseUsage(stdout)
@@ -1311,7 +1311,7 @@ var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\
// when sending it to a shell.
func shellEscape(str string) string {
safe := shellEscapeRegex.ReplaceAllString(str, `\$0`)
return strings.ReplaceAll(safe, "\n", "'\n'")
return strings.Replace(safe, "\n", "'\n'", -1)
}
// Converts a byte array from the SSH session returned by
+1 -1
View File
@@ -579,7 +579,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
return nil, err
}
return newObjectFromUplink(f, src.Remote(), upload.Info()), nil
return newObjectFromUplink(f, "", upload.Info()), nil
}
// PutStream uploads to the remote path with the modTime given of indeterminate
+13 -18
View File
@@ -141,20 +141,22 @@ func (f *Fs) Hashes() hash.Set {
return f.hashSet
}
// mkdir makes the directory passed in and returns the upstreams used
func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
upstreams, err := f.create(ctx, dir)
if err == fs.ErrorObjectNotFound {
parent := parentDir(dir)
if dir != parent {
upstreams, err = f.mkdir(ctx, parent)
if dir != parentDir(dir) {
if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
return err
}
upstreams, err = f.create(ctx, dir)
} else if dir == "" {
// If root dirs not created then create them
upstreams, err = f.upstreams, nil
}
}
if err != nil {
return nil, err
return err
}
errs := Errors(make([]error, len(upstreams)))
multithread(len(upstreams), func(i int) {
@@ -163,17 +165,7 @@ func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
}
})
err = errs.Err()
if err != nil {
return nil, err
}
return upstreams, nil
}
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
_, err := f.mkdir(ctx, dir)
return err
return errs.Err()
}
// Purge all files in the directory
@@ -457,7 +449,10 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
srcPath := src.Remote()
upstreams, err := f.create(ctx, srcPath)
if err == fs.ErrorObjectNotFound {
upstreams, err = f.mkdir(ctx, parentDir(srcPath))
if err := f.Mkdir(ctx, parentDir(srcPath)); err != nil {
return nil, err
}
upstreams, err = f.create(ctx, srcPath)
}
if err != nil {
return nil, err
+5 -39
View File
@@ -124,22 +124,6 @@ You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'
`,
Default: fs.CommaSepList{},
Advanced: true,
}, {
Name: "base_path",
Help: `Base path of expected replies
Normally WebDAV servers return the files they are listing under the
url path as specified above. However some WebDAV servers return files
with URLs that are not under the endpoint URL. This causes rclone to
get confused and return errors like
Item with unknown path received: "/remote.php/webdav/folder1/", "/elsewhere/remote.php/webdav/folder1/"
errors. If that is the case, then set "base_path" to the path
specified in the error message up to the first item, in the above
example "/elsewhere/remote.php/webdav/".
`,
Advanced: true,
}},
})
}
@@ -154,7 +138,6 @@ type Options struct {
BearerTokenCommand string `config:"bearer_token_command"`
Enc encoder.MultiEncoder `config:"encoding"`
Headers fs.CommaSepList `config:"headers"`
BasePath string `config:"base_path"`
}
// Fs represents a remote webdav
@@ -471,9 +454,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, err
}
if !f.findHeader(opt.Headers, "Referer") {
f.srv.SetHeader("Referer", u.String())
}
f.srv.SetHeader("Referer", u.String())
if root != "" && !rootIsDir {
// Check to see if the root actually an existing file
@@ -536,17 +517,6 @@ func (f *Fs) addHeaders(headers fs.CommaSepList) {
}
}
// Returns true if the header was configured
func (f *Fs) findHeader(headers fs.CommaSepList, find string) bool {
for i := 0; i < len(headers); i += 2 {
key := f.opt.Headers[i]
if strings.EqualFold(key, find) {
return true
}
}
return false
}
// fetch the bearer token and set it if successful
func (f *Fs) fetchAndSetBearerToken() error {
if f.opt.BearerTokenCommand == "" {
@@ -710,10 +680,6 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
if err != nil {
return false, fmt.Errorf("couldn't join URL: %w", err)
}
basePath := baseURL.Path
if f.opt.BasePath != "" {
basePath = f.opt.BasePath
}
for i := range result.Responses {
item := &result.Responses[i]
isDir := itemIsDir(item)
@@ -728,11 +694,11 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
if isDir {
u.Path = addSlash(u.Path)
}
if !strings.HasPrefix(u.Path, basePath) {
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, basePath)
if !strings.HasPrefix(u.Path, baseURL.Path) {
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path)
continue
}
subPath := u.Path[len(basePath):]
subPath := u.Path[len(baseURL.Path):]
if f.opt.Enc != encoder.EncodeZero {
subPath = f.opt.Enc.ToStandardPath(subPath)
}
@@ -1182,7 +1148,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
return f.shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, err
return nil, fmt.Errorf("about call failed: %w", err)
}
usage := &fs.Usage{}
if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 {
+3 -6
View File
@@ -173,8 +173,8 @@ func buildZip(dir string) string {
func buildDebAndRpm(dir, version, goarch string) []string {
// Make internal version number acceptable to .deb and .rpm
pkgVersion := version[1:]
pkgVersion = strings.ReplaceAll(pkgVersion, "β", "-beta")
pkgVersion = strings.ReplaceAll(pkgVersion, "-", ".")
pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1)
pkgVersion = strings.Replace(pkgVersion, "-", ".", -1)
nfpmArch, ok := goarchToNfpm[goarch]
if !ok {
nfpmArch = goarch
@@ -265,12 +265,9 @@ func buildWindowsResourceSyso(goarch string, versionTag string) string {
"-o",
sysoPath,
}
if strings.Contains(goarch, "64") {
if goarch == "amd64" {
args = append(args, "-64") // Make the syso a 64-bit coff file
}
if strings.Contains(goarch, "arm") {
args = append(args, "-arm") // Make the syso an arm binary
}
args = append(args, jsonPath)
err = runEnv(args, nil)
if err != nil {
-1
View File
@@ -48,7 +48,6 @@ docs = [
"hdfs.md",
"http.md",
"hubic.md",
"internetarchive.md",
"jottacloud.md",
"koofr.md",
"mailru.md",
+1 -1
View File
@@ -102,7 +102,7 @@ see complete list in [documentation](https://rclone.org/overview/#optional-featu
}
u, err := doAbout(context.Background())
if err != nil {
return fmt.Errorf("about call failed: %w", err)
return fmt.Errorf("About call failed: %w", err)
}
if u == nil {
return errors.New("nil usage returned")
+1 -1
View File
@@ -489,7 +489,7 @@ func resolveExitCode(err error) {
os.Exit(exitcode.TransferExceeded)
case fserrors.ShouldRetry(err):
os.Exit(exitcode.RetryError)
case fserrors.IsNoRetryError(err), fserrors.IsNoLowLevelRetryError(err):
case fserrors.IsNoRetryError(err):
os.Exit(exitcode.NoRetryError)
case fserrors.IsFatalError(err):
os.Exit(exitcode.FatalError)
+4 -3
View File
@@ -1,6 +1,7 @@
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
// +build cmount
// +build linux,cgo darwin,cgo freebsd,cgo windows
// +build cgo
// +build linux darwin freebsd windows
package cmount
@@ -12,12 +13,12 @@ import (
"sync/atomic"
"time"
"github.com/winfsp/cgofuse/fuse"
"github.com/rclone/rclone/cmd/mountlib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/vfs"
"github.com/winfsp/cgofuse/fuse"
)
const fhUnset = ^uint64(0)
+4 -3
View File
@@ -2,9 +2,10 @@
//
// This uses the cgo based cgofuse library
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
// +build cmount
// +build linux,cgo darwin,cgo freebsd,cgo windows
// +build cgo
// +build linux darwin freebsd windows
package cmount
@@ -17,12 +18,12 @@ import (
"sync/atomic"
"time"
"github.com/winfsp/cgofuse/fuse"
"github.com/rclone/rclone/cmd/mountlib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/buildinfo"
"github.com/rclone/rclone/vfs"
"github.com/winfsp/cgofuse/fuse"
)
func init() {
+3 -2
View File
@@ -1,6 +1,7 @@
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows) && (!race || !windows)
//go:build cmount && cgo && (linux || darwin || freebsd || windows) && (!race || !windows)
// +build cmount
// +build linux,cgo darwin,cgo freebsd,cgo windows
// +build cgo
// +build linux darwin freebsd windows
// +build !race !windows
// FIXME this doesn't work with the race detector under Windows either
+2 -5
View File
@@ -1,10 +1,7 @@
// Build for cmount for unsupported platforms to stop go complaining
// about "no buildable Go source files "
//go:build !((linux && cgo && cmount) || (darwin && cgo && cmount) || (freebsd && cgo && cmount) || (windows && cmount))
// +build !linux !cgo !cmount
// +build !darwin !cgo !cmount
// +build !freebsd !cgo !cmount
// +build !windows !cmount
//go:build (!linux && !darwin && !freebsd && !windows) || !brew || !cgo || !cmount
// +build !linux,!darwin,!freebsd,!windows !brew !cgo !cmount
package cmount
+2 -2
View File
@@ -1,5 +1,5 @@
//go:build cmount && windows
// +build cmount,windows
//go:build cmount && cgo && windows
// +build cmount,cgo,windows
package cmount
+3 -3
View File
@@ -79,7 +79,7 @@ rclone.org website.`,
var description = map[string]string{}
var addDescription func(root *cobra.Command)
addDescription = func(root *cobra.Command) {
name := strings.ReplaceAll(root.CommandPath(), " ", "_") + ".md"
name := strings.Replace(root.CommandPath(), " ", "_", -1) + ".md"
description[name] = root.Short
for _, c := range root.Commands() {
addDescription(c)
@@ -93,11 +93,11 @@ rclone.org website.`,
base := strings.TrimSuffix(name, path.Ext(name))
data := frontmatter{
Date: now,
Title: strings.ReplaceAll(base, "_", " "),
Title: strings.Replace(base, "_", " ", -1),
Description: description[name],
Slug: base,
URL: "/commands/" + strings.ToLower(base) + "/",
Source: strings.ReplaceAll(strings.ReplaceAll(base, "rclone", "cmd"), "_", "/") + "/",
Source: strings.Replace(strings.Replace(base, "rclone", "cmd", -1), "_", "/", -1) + "/",
}
var buf bytes.Buffer
err := frontmatterTemplate.Execute(&buf, data)
+8 -13
View File
@@ -78,17 +78,6 @@ type MountPoint struct {
ErrChan <-chan error
}
// NewMountPoint makes a new mounting structure
func NewMountPoint(mount MountFn, mountPoint string, f fs.Fs, mountOpt *Options, vfsOpt *vfscommon.Options) *MountPoint {
return &MountPoint{
MountFn: mount,
MountPoint: mountPoint,
Fs: f,
MountOpt: *mountOpt,
VFSOpt: *vfsOpt,
}
}
// Global constants
const (
MaxLeafSize = 1024 // don't pass file names longer than this
@@ -178,7 +167,14 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm
defer cmd.StartStats()()
}
mnt := NewMountPoint(mount, args[1], cmd.NewFsDir(args), &Opt, &vfsflags.Opt)
mnt := &MountPoint{
MountFn: mount,
MountPoint: args[1],
Fs: cmd.NewFsDir(args),
MountOpt: Opt,
VFSOpt: vfsflags.Opt,
}
daemon, err := mnt.Mount()
// Wait for foreground mount, if any...
@@ -257,7 +253,6 @@ func (m *MountPoint) Mount() (daemon *os.Process, err error) {
if err != nil {
return nil, fmt.Errorf("failed to mount FUSE fs: %w", err)
}
m.MountedOn = time.Now()
return nil, nil
}
+12 -3
View File
@@ -10,6 +10,7 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
)
@@ -116,15 +117,23 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
return nil, err
}
mnt := NewMountPoint(mountFn, mountPoint, fdst, &mountOpt, &vfsOpt)
_, err = mnt.Mount()
VFS := vfs.New(fdst, &vfsOpt)
_, unmountFn, err := mountFn(VFS, mountPoint, &mountOpt)
if err != nil {
log.Printf("mount FAILED: %v", err)
return nil, err
}
// Add mount to list if mount point was successfully created
liveMounts[mountPoint] = mnt
liveMounts[mountPoint] = &MountPoint{
MountPoint: mountPoint,
MountedOn: time.Now(),
MountFn: mountFn,
UnmountFn: unmountFn,
MountOpt: mountOpt,
VFSOpt: vfsOpt,
Fs: fdst,
}
fs.Debugf(nil, "Mount for %s created at %s using %s", fdst.String(), mountPoint, mountType)
return nil, nil
+39 -56
View File
@@ -42,31 +42,15 @@ builds an in memory representation. rclone ncdu can be used during
this scanning phase and you will see it building up the directory
structure as it goes along.
You can interact with the user interface using key presses,
press '?' to toggle the help on and off. The supported keys are:
Here are the keys - press '?' to toggle the help on and off
` + strings.Join(helpText()[1:], "\n ") + `
Listed files/directories may be prefixed by a one-character flag,
some of them combined with a description in brackes at end of line.
These flags have the following meaning:
e means this is an empty directory, i.e. contains no files (but
may contain empty subdirectories)
~ means this is a directory where some of the files (possibly in
subdirectories) have unknown size, and therefore the directory
size may be underestimated (and average size inaccurate, as it
is average of the files with known sizes).
. means an error occurred while reading a subdirectory, and
therefore the directory size may be underestimated (and average
size inaccurate)
! means an error occurred while reading this directory
This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
rclone remotes. It is missing lots of features at the moment
but is useful as it stands.
Note that it might take some time to delete big files/directories. The
Note that it might take some time to delete big files/folders. The
UI won't respond in the meantime since the deletion is done synchronously.
`,
Run: func(command *cobra.Command, args []string) {
@@ -299,9 +283,9 @@ func (u *UI) biggestEntry() (biggest int64) {
return
}
for i := range u.entries {
attrs, _ := u.d.AttrI(u.sortPerm[i])
if attrs.Size > biggest {
biggest = attrs.Size
size, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
if size > biggest {
biggest = size
}
}
return
@@ -313,8 +297,8 @@ func (u *UI) hasEmptyDir() bool {
return false
}
for i := range u.entries {
attrs, _ := u.d.AttrI(u.sortPerm[i])
if attrs.IsDir && attrs.Count == 0 {
_, count, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
if isDir && count == 0 {
return true
}
}
@@ -359,9 +343,9 @@ func (u *UI) Draw() error {
if y >= h-1 {
break
}
attrs, err := u.d.AttrI(u.sortPerm[n])
size, count, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
fg := termbox.ColorWhite
if attrs.EntriesHaveErrors {
if entriesHaveErrors {
fg = termbox.ColorYellow
}
if err != nil {
@@ -372,19 +356,15 @@ func (u *UI) Draw() error {
fg, bg = bg, fg
}
mark := ' '
if attrs.IsDir {
if isDir {
mark = '/'
}
fileFlag := ' '
message := ""
if !attrs.Readable {
if !readable {
message = " [not read yet]"
}
if attrs.CountUnknownSize > 0 {
message = fmt.Sprintf(" [%d of %d files have unknown size, size may be underestimated]", attrs.CountUnknownSize, attrs.Count)
fileFlag = '~'
}
if attrs.EntriesHaveErrors {
if entriesHaveErrors {
message = " [some subdirectories could not be read, size may be underestimated]"
fileFlag = '.'
}
@@ -394,29 +374,32 @@ func (u *UI) Draw() error {
}
extras := ""
if u.showCounts {
ss := operations.CountStringField(attrs.Count, u.humanReadable, 9) + " "
if attrs.Count > 0 {
ss := operations.CountStringField(count, u.humanReadable, 9) + " "
if count > 0 {
extras += ss
} else {
extras += strings.Repeat(" ", len(ss))
}
}
var averageSize float64
if count > 0 {
averageSize = float64(size) / float64(count)
}
if u.showDirAverageSize {
avg := attrs.AverageSize()
ss := operations.SizeStringField(int64(avg), u.humanReadable, 9) + " "
if avg > 0 {
ss := operations.SizeStringField(int64(averageSize), u.humanReadable, 9) + " "
if averageSize > 0 {
extras += ss
} else {
extras += strings.Repeat(" ", len(ss))
}
}
if showEmptyDir {
if attrs.IsDir && attrs.Count == 0 && fileFlag == ' ' {
if isDir && count == 0 && fileFlag == ' ' {
fileFlag = 'e'
}
}
if u.showGraph {
bars := (attrs.Size + perBar/2 - 1) / perBar
bars := (size + perBar/2 - 1) / perBar
// clip if necessary - only happens during startup
if bars > 10 {
bars = 10
@@ -425,7 +408,7 @@ func (u *UI) Draw() error {
}
extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
}
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(attrs.Size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
y++
}
}
@@ -576,14 +559,14 @@ type ncduSort struct {
// Less is part of sort.Interface.
func (ds *ncduSort) Less(i, j int) bool {
var iAvgSize, jAvgSize float64
iattrs, _ := ds.d.AttrI(ds.sortPerm[i])
jattrs, _ := ds.d.AttrI(ds.sortPerm[j])
isize, icount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
jsize, jcount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
if iattrs.Count > 0 {
iAvgSize = iattrs.AverageSize()
if icount > 0 {
iAvgSize = float64(isize / icount)
}
if jattrs.Count > 0 {
jAvgSize = jattrs.AverageSize()
if jcount > 0 {
jAvgSize = float64(jsize / jcount)
}
switch {
@@ -592,33 +575,33 @@ func (ds *ncduSort) Less(i, j int) bool {
case ds.u.sortByName > 0:
break
case ds.u.sortBySize < 0:
if iattrs.Size != jattrs.Size {
return iattrs.Size < jattrs.Size
if isize != jsize {
return isize < jsize
}
case ds.u.sortBySize > 0:
if iattrs.Size != jattrs.Size {
return iattrs.Size > jattrs.Size
if isize != jsize {
return isize > jsize
}
case ds.u.sortByCount < 0:
if iattrs.Count != jattrs.Count {
return iattrs.Count < jattrs.Count
if icount != jcount {
return icount < jcount
}
case ds.u.sortByCount > 0:
if iattrs.Count != jattrs.Count {
return iattrs.Count > jattrs.Count
if icount != jcount {
return icount > jcount
}
case ds.u.sortByAverageSize < 0:
if iAvgSize != jAvgSize {
return iAvgSize < jAvgSize
}
// if avgSize is equal, sort by size
return iattrs.Size < jattrs.Size
return isize < jsize
case ds.u.sortByAverageSize > 0:
if iAvgSize != jAvgSize {
return iAvgSize > jAvgSize
}
// if avgSize is equal, sort by size
return iattrs.Size > jattrs.Size
return isize > jsize
}
// if everything equal, sort by name
return iname < jname
+8 -51
View File
@@ -16,42 +16,14 @@ type Dir struct {
parent *Dir
path string
mu sync.Mutex
size int64
count int64
countUnknownSize int64
size int64
entries fs.DirEntries
dirs map[string]*Dir
readError error
entriesHaveErrors bool
}
// Attrs contains accumulated properties for a directory entry
//
// Files with unknown size are counted separately but also included
// in the total count. They are not included in the size, i.e. treated
// as empty files, which means the size may be underestimated.
type Attrs struct {
Size int64
Count int64
CountUnknownSize int64
IsDir bool
Readable bool
EntriesHaveErrors bool
}
// AverageSize calculates average size of files in directory
//
// If there are files with unknown size, this returns the average over
// files with known sizes, which means it may be under- or
// overestimated.
func (a *Attrs) AverageSize() float64 {
countKnownSize := a.Count - a.CountUnknownSize
if countKnownSize > 0 {
return float64(a.Size) / float64(countKnownSize)
}
return 0
}
// Parent returns the directory above this one
func (d *Dir) Parent() *Dir {
// no locking needed since these are write once in newDir()
@@ -77,13 +49,7 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
for _, entry := range entries {
if o, ok := entry.(fs.Object); ok {
d.count++
size := o.Size()
if size < 0 {
// Some backends may return -1 because size of object is not known
d.countUnknownSize++
} else {
d.size += size
}
d.size += o.Size()
}
}
// Set my directory entry in parent
@@ -96,9 +62,8 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
// Accumulate counts in parents
for ; parent != nil; parent = parent.parent {
parent.mu.Lock()
parent.size += d.size
parent.count += d.count
parent.countUnknownSize += d.countUnknownSize
parent.size += d.size
if d.readError != nil {
parent.entriesHaveErrors = true
}
@@ -126,24 +91,17 @@ func (d *Dir) Remove(i int) {
// Call with d.mu held
func (d *Dir) remove(i int) {
size := d.entries[i].Size()
countUnknownSize := int64(0)
if size < 0 {
size = 0
countUnknownSize = 1
}
count := int64(1)
subDir, ok := d.getDir(i)
if ok {
size = subDir.size
count = subDir.count
countUnknownSize = subDir.countUnknownSize
delete(d.dirs, path.Base(subDir.path))
}
d.size -= size
d.count -= count
d.countUnknownSize -= countUnknownSize
d.entries = append(d.entries[:i], d.entries[i+1:]...)
dir := d
@@ -153,7 +111,6 @@ func (d *Dir) remove(i int) {
parent.dirs[path.Base(dir.path)] = dir
parent.size -= size
parent.count -= count
parent.countUnknownSize -= countUnknownSize
dir = parent
parent.mu.Unlock()
}
@@ -194,19 +151,19 @@ func (d *Dir) Attr() (size int64, count int64) {
}
// AttrI returns the size, count and flags for the i-th directory entry
func (d *Dir) AttrI(i int) (attrs Attrs, err error) {
func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
d.mu.Lock()
defer d.mu.Unlock()
subDir, isDir := d.getDir(i)
if !isDir {
return Attrs{d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors}, d.readError
return d.entries[i].Size(), 0, false, true, d.entriesHaveErrors, d.readError
}
if subDir == nil {
return Attrs{0, 0, 0, true, false, false}, nil
return 0, 0, true, false, false, nil
}
size, count := subDir.Attr()
return Attrs{size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors}, subDir.readError
size, count = subDir.Attr()
return size, count, true, true, subDir.entriesHaveErrors, subDir.readError
}
// Scan the Fs passed in, returning a root directory channel and an
+1 -1
View File
@@ -290,7 +290,7 @@ func list(ctx context.Context) error {
if !ok {
return errors.New("bad JSON")
}
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.ReplaceAll(info["Path"].(string), "/", "-"))
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.Replace(info["Path"].(string), "/", "-", -1))
fmt.Printf("%s\n\n", info["Help"])
if authRequired := info["AuthRequired"]; authRequired != nil {
if authRequired.(bool) {
+1
View File
@@ -274,6 +274,7 @@ func (vol *Volume) mount(id string) error {
if _, err := vol.mnt.Mount(); err != nil {
return err
}
vol.mnt.MountedOn = time.Now()
vol.mountReqs[id] = nil
vol.drv.monChan <- false // ask monitor to refresh channels
return nil
+1 -1
View File
@@ -43,7 +43,7 @@ var shellUnEscapeRegex = regexp.MustCompile(`\\(.)`)
// Unescape a string that was escaped by rclone
func shellUnEscape(str string) string {
str = strings.ReplaceAll(str, "'\n'", "\n")
str = strings.Replace(str, "'\n'", "\n", -1)
str = shellUnEscapeRegex.ReplaceAllString(str, `$1`)
return str
}
+4 -29
View File
@@ -24,51 +24,26 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "size remote:path",
Short: `Prints the total size and number of objects in remote:path.`,
Long: `
Counts objects in the path and calculates the total size. Prints the
result to standard output.
By default the output is in human-readable format, but shows values in
both human-readable format as well as the raw numbers (global option
` + "`--human-readable`" + ` is not considered). Use option ` + "`--json`" + `
to format output as JSON instead.
Recurses by default, use ` + "`--max-depth 1`" + ` to stop the
recursion.
Some backends do not always provide file sizes, see for example
[Google Photos](/googlephotos/#size) and
[Google Drive](/drive/#limitations-of-google-docs).
Rclone will then show a notice in the log indicating how many such
files were encountered, and count them in as empty files in the output
of the size command.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
var err error
var results struct {
Count int64 `json:"count"`
Bytes int64 `json:"bytes"`
Sizeless int64 `json:"sizeless"`
Count int64 `json:"count"`
Bytes int64 `json:"bytes"`
}
results.Count, results.Bytes, results.Sizeless, err = operations.Count(context.Background(), fsrc)
results.Count, results.Bytes, err = operations.Count(context.Background(), fsrc)
if err != nil {
return err
}
if results.Sizeless > 0 {
fs.Logf(fsrc, "Size may be underestimated due to %d objects with unknown size", results.Sizeless)
}
if jsonOutput {
return json.NewEncoder(os.Stdout).Encode(results)
}
fmt.Printf("Total objects: %s (%d)\n", fs.CountSuffix(results.Count), results.Count)
fmt.Printf("Total size: %s (%d Byte)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
if results.Sizeless > 0 {
fmt.Printf("Total objects with unknown size: %s (%d)\n", fs.CountSuffix(results.Sizeless), results.Sizeless)
}
return nil
})
},
+1 -1
View File
@@ -25,7 +25,7 @@ var commandDefinition = &cobra.Command{
cmd.Run(false, false, command, func() error {
ctx := context.Background()
ci := fs.GetConfig(context.Background())
objects, _, _, err := operations.Count(ctx, fsrc)
objects, _, err := operations.Count(ctx, fsrc)
if err != nil {
return err
}
+1 -4
View File
@@ -113,10 +113,8 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Box" home="https://www.box.com/" config="/box/" >}}
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
{{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
{{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}}
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@@ -129,7 +127,6 @@ WebDAV or S3, that work out of the box.)
{{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}}
{{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
{{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}}
{{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}
{{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
{{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
@@ -170,7 +167,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}}
{{< /provider_list >}}
## Links
Links
* {{< icon "fa fa-home" >}} [Home page](https://rclone.org/)
* {{< icon "fab fa-github" >}} [GitHub project page for source and bug tracker](https://github.com/rclone/rclone)
+2 -1
View File
@@ -334,5 +334,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
-22
View File
@@ -583,25 +583,3 @@ put them back in again.` >}}
* rafma0 <int.main@gmail.com>
* Adrien Rey-Jarthon <jobs@adrienjarthon.com>
* Nick Gooding <73336146+nickgooding@users.noreply.github.com>
* Leroy van Logchem <lr.vanlogchem@gmail.com>
* Zsolt Ero <zsolt.ero@gmail.com>
* Lesmiscore <nao20010128@gmail.com>
* ehsantdy <ehsan.tadayon@arvancloud.com>
* SwazRGB <65694696+swazrgb@users.noreply.github.com>
* Mateusz Puczyński <mati6095@gmail.com>
* Michael C Tiernan - MIT-Research Computing Project <mtiernan@mit.edu>
* Kaspian <34658474+KaspianDev@users.noreply.github.com>
* Werner <EvilOlaf@users.noreply.github.com>
* Hugal31 <hugo.laloge@gmail.com>
* Christian Galo <36752715+cgalo5758@users.noreply.github.com>
* Erik van Velzen <erik@evanv.nl>
* Derek Battams <derek@battams.ca>
* SimonLiu <simonliu009@users.noreply.github.com>
* Hugo Laloge <hla@lescompanions.com>
* Mr-Kanister <68117355+Mr-Kanister@users.noreply.github.com>
* Rob Pickerill <r.pickerill@gmail.com>
* Andrey <to.merge@gmail.com>
* Eric Wolf <19wolf@gmail.com>
* Nick <nick.naumann@mailbox.tu-dresden.de>
* Jason Zheng <jszheng17@gmail.com>
* Matthew Vernon <mvernon@wikimedia.org>
+2 -1
View File
@@ -522,7 +522,8 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
## Azure Storage Emulator Support
+2 -6
View File
@@ -173,11 +173,6 @@ the file instead of hiding it.
Old versions of files, where available, are visible using the
`--b2-versions` flag.
It is also possible to view a bucket as it was at a certain point in time,
using the `--b2-version-at` flag. This will show the file versions as they
were at that time, showing files that have been deleted afterwards, and
hiding files that were created since.
If you wish to remove all the old versions then you can use the
`rclone cleanup remote:bucket` command which will delete all the old
versions of files, leaving the current ones intact. You can also
@@ -566,6 +561,7 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
+2 -3
View File
@@ -106,7 +106,7 @@ Optional Flags:
Arbitrary rclone flags may be specified on the
[bisync command line](/commands/rclone_bisync/), for example
`rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
`rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
Note that interactions of various rclone flags with bisync process flow
has not been fully tested yet.
@@ -363,7 +363,6 @@ Bisync is considered _BETA_ and has been tested with the following backends:
- OneDrive
- S3
- SFTP
- Yandex Disk
It has not been fully tested with other services yet.
If it works, or sorta works, please let us know and we'll update the list.
@@ -701,7 +700,7 @@ consider using the flag
Google docs exist as virtual files on Google Drive and cannot be transferred
to other filesystems natively. While it is possible to export a Google doc to
a normal file (with `.xlsx` extension, for example), it is not possible
a normal file (with `.xlsx` extension, for example), it's not possible
to import a normal file back into a Google document.
Bisync's handling of Google Doc files is to flag them in the run log output
+2 -1
View File
@@ -466,5 +466,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
-42
View File
@@ -5,48 +5,6 @@ description: "Rclone Changelog"
# Changelog
## v1.58.1 - 2022-04-29
[See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.58.1)
* Bug Fixes
* build: Update github.com/billziss-gh to github.com/winfsp (Nick Craig-Wood)
* filter: Fix timezone of `--min-age`/`-max-age` from UTC to local as documented (Nick Craig-Wood)
* rc/js: Correct RC method names (Sơn Trần-Nguyễn)
* docs
* Fix some links to command pages (albertony)
* Add `--multi-thread-streams` note to `--transfers`. (Zsolt Ero)
* Mount
* Fix `--devname` and fusermount: unknown option 'fsname' when mounting via rc (Nick Craig-Wood)
* VFS
* Remove wording which suggests VFS is only for mounting (Nick Craig-Wood)
* Dropbox
* Fix retries of multipart uploads with incorrect_offset error (Nick Craig-Wood)
* Google Cloud Storage
* Use the s3 pacer to speed up transactions (Nick Craig-Wood)
* pacer: Default the Google pacer to a burst of 100 to fix gcs pacing (Nick Craig-Wood)
* Jottacloud
* Fix scope in token request (albertony)
* Netstorage
* Fix unescaped HTML in documentation (Nick Craig-Wood)
* Make levels of headings consistent (Nick Craig-Wood)
* Add support contacts to netstorage doc (Nil Alexandrov)
* Onedrive
* Note that sharepoint also changes web files (.html, .aspx) (GH)
* Putio
* Handle rate limit errors (Berkan Teber)
* Fix multithread download and other ranged requests (rafma0)
* S3
* Add ChinaMobile EOS to provider list (GuoXingbin)
* Sync providers in config description with providers (Nick Craig-Wood)
* SFTP
* Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr)
* Note that Scaleway C14 is deprecating SFTP in favor of S3 (Adrien Rey-Jarthon)
* Storj
* Fix bucket creation on Move (Nick Craig-Wood)
* WebDAV
* Don't override Referer if user sets it (Nick Craig-Wood)
## v1.58.0 - 2022-03-18
[See commits](https://github.com/rclone/rclone/compare/v1.57.0...v1.58.0)
+2 -19
View File
@@ -50,7 +50,6 @@ See the following for detailed instructions for
* [HDFS](/hdfs/)
* [HTTP](/http/)
* [Hubic](/hubic/)
* [Internet Archive](/internetarchive/)
* [Jottacloud](/jottacloud/)
* [Koofr](/koofr/)
* [Mail.ru Cloud](/mailru/)
@@ -788,9 +787,7 @@ See `--compare-dest` and `--backup-dir`.
### --dedupe-mode MODE ###
Mode to run dedupe command in. One of `interactive`, `skip`, `first`,
`newest`, `oldest`, `rename`. The default is `interactive`.
See the dedupe command for more information as to what these options mean.
Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean.
### --disable FEATURE,FEATURE,... ###
@@ -1088,12 +1085,7 @@ have a signal to rotate logs.
### --log-format LIST ###
Comma separated list of log format options. Accepted options are `date`,
`time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other
keywords will be silently ignored. `pid` will tag log messages with process
identifier which useful with `rclone mount --daemon`. Other accepted
options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants).
The default log format is "`date`,`time`".
Comma separated list of log format options. Accepted options are `date`, `time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other keywords will be silently ignored. `pid` will tag log messages with process identifier which useful with `rclone mount --daemon`. Other accepted options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants). The default log format is "`date`,`time`".
### --log-level LEVEL ###
@@ -1801,8 +1793,6 @@ of timeouts or bigger if you have lots of bandwidth and a fast remote.
The default is to run 4 file transfers in parallel.
Look at --multi-thread-streams if you would like to control single file transfers.
### -u, --update ###
This forces rclone to skip any files which exist on the destination
@@ -1874,9 +1864,6 @@ With `-vv` rclone will become very verbose telling you about every
file it considers and transfers. Please send bug reports with a log
with this setting.
When setting verbosity as an environment variable, use
`RCLONE_VERBOSE=1` or `RCLONE_VERBOSE=2` for `-v` and `-vv` respectively.
### -V, --version ###
Prints the version number
@@ -2242,10 +2229,6 @@ override the environment variable setting.
Or to always use the trash in drive `--drive-use-trash`, set
`RCLONE_DRIVE_USE_TRASH=true`.
Verbosity is slightly different, the environment variable
equivalent of `--verbose` or `-v` is `RCLONE_VERBOSE=1`,
or for `-vv`, `RCLONE_VERBOSE=2`.
The same parser is used for the options and the environment variables
so they take exactly the same form.
+2 -5
View File
@@ -1375,11 +1375,8 @@ and upload the files if you prefer.
### Limitations of Google Docs
Google docs will appear as size -1 in `rclone ls`, `rclone ncdu` etc,
and as size 0 in anything which uses the VFS layer, e.g. `rclone mount`
and `rclone serve`. When calculating directory totals, e.g. in
`rclone size` and `rclone ncdu`, they will be counted in as empty
files.
Google docs will appear as size -1 in `rclone ls` and as size 0 in
anything which uses the VFS layer, e.g. `rclone mount`, `rclone serve`.
This is because rclone can't find out the size of the Google docs
without downloading them.
+1 -1
View File
@@ -31,7 +31,7 @@ the node running rclone would need to have lots of bandwidth.
The syncs would be incremental (on a file by file basis).
e.g.
Eg
rclone sync -i drive:Folder s3:bucket
+2 -1
View File
@@ -192,5 +192,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
+2 -10
View File
@@ -131,11 +131,7 @@ The regular expressions used are as defined in the [Go regular
expression reference](https://golang.org/pkg/regexp/syntax/). Regular
expressions should be enclosed in `{{` `}}`. They will match only the
last path segment if the glob doesn't start with `/` or the whole path
name if it does. Note that rclone does not attempt to parse the
supplied regular expression, meaning that using any regular expression
filter will prevent rclone from using [directory filter rules](#directory_filter),
as it will instead check every path against
the supplied regular expression(s).
name if it does.
Here is how the `{{regexp}}` is transformed into an full regular
expression to match the entire path:
@@ -251,7 +247,7 @@ currently a means provided to pass regular expression filter options into
rclone directly though character class filter rules contain character
classes. [Go regular expression reference](https://golang.org/pkg/regexp/syntax/)
### How filter rules are applied to directories {#directory_filter}
### How filter rules are applied to directories
Rclone commands are applied to path/file names not
directories. The entire contents of a directory can be matched
@@ -267,10 +263,6 @@ recurse into subdirectories. This potentially optimises access to a remote
by avoiding listing unnecessary directories. Whether optimisation is
desirable depends on the specific filter rules and source remote content.
If any [regular expression filters](#regexp) are in use, then no
directory recursion optimisation is possible, as rclone must check
every path against the supplied regular expression(s).
Directory recursion optimisation occurs if either:
* A source remote does not support the rclone `ListR` primitive. local,
-1
View File
@@ -337,7 +337,6 @@ and may be set in the config file.
--ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited
--ftp-disable-epsv Disable using EPSV even if server advertises support
--ftp-disable-mlsd Disable using MLSD even if server advertises support
--ftp-disable-utf8 Disable using UTF-8 even if server advertises support
--ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
--ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
+2 -12
View File
@@ -267,17 +267,6 @@ Properties:
- Type: bool
- Default: false
#### --ftp-disable-utf8
Disable using UTF-8 even if server advertises support.
Properties:
- Config: disable_utf8
- Env Var: RCLONE_FTP_DISABLE_UTF8
- Type: bool
- Default: false
#### --ftp-writing-mdtm
Use MDTM to set modification time (VsFtpd quirk)
@@ -406,7 +395,8 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
The implementation of : `--dump headers`,
`--dump bodies`, `--dump auth` for debugging isn't the same as
+2 -1
View File
@@ -609,5 +609,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
+2 -1
View File
@@ -220,5 +220,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
+38 -92
View File
@@ -3,11 +3,11 @@ title: "Install"
description: "Rclone Installation"
---
# Install
# Install #
Rclone is a Go program and comes as a single binary file.
## Quickstart
## Quickstart ##
* [Download](/downloads/) the relevant binary.
* Extract the `rclone` executable, `rclone.exe` on Windows, from the archive.
@@ -22,20 +22,20 @@ run `rclone -h`.
Already installed rclone can be easily updated to the latest version
using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
## Script installation
## Script installation ##
To install rclone on Linux/macOS/BSD systems, run:
sudo -v ; curl https://rclone.org/install.sh | sudo bash
curl https://rclone.org/install.sh | sudo bash
For beta installation, run:
sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
curl https://rclone.org/install.sh | sudo bash -s beta
Note that this script checks the version of rclone installed first and
won't re-download if not needed.
## Linux installation from precompiled binary
## Linux installation from precompiled binary ##
Fetch and unpack
@@ -59,7 +59,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
rclone config
## macOS installation with brew
## macOS installation with brew ##
brew install rclone
@@ -68,7 +68,7 @@ NOTE: This version of rclone will not support `mount` any more (see
on macOS, either install a precompiled binary or enable the relevant option
when [installing from source](#install-from-source).
## macOS installation from precompiled binary, using curl
## macOS installation from precompiled binary, using curl ##
To avoid problems with macOS gatekeeper enforcing the binary to be signed and
notarized it is enough to download with `curl`.
@@ -96,20 +96,20 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
rclone config
## macOS installation from precompiled binary, using a web browser
## macOS installation from precompiled binary, using a web browser ##
When downloading a binary with a web browser, the browser will set the macOS
gatekeeper quarantine attribute. Starting from Catalina, when attempting to run
`rclone`, a pop-up will appear saying:
"rclone" cannot be opened because the developer cannot be verified.
“rclone” cannot be opened because the developer cannot be verified.
macOS cannot verify that this app is free from malware.
The simplest fix is to run
xattr -d com.apple.quarantine rclone
## Install with docker
## Install with docker ##
The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
These images are autobuilt by docker hub from the rclone source based
@@ -188,93 +188,39 @@ ls ~/data/mount
kill %1
```
## Install from source
## Install from source ##
Make sure you have git and [Go](https://golang.org/) installed.
Go version 1.16 or newer is required, latest release is recommended.
You can get it from your package manager, or download it from
[golang.org/dl](https://golang.org/dl/). Then you can run the following:
Make sure you have at least [Go](https://golang.org/) go1.16
installed. [Download go](https://golang.org/dl/) if necessary. The
latest release is recommended. Then
```
```sh
git clone https://github.com/rclone/rclone.git
cd rclone
go build
# If on macOS and mount is wanted, instead run: make GOTAGS=cmount
./rclone version
```
This will check out the rclone source in subfolder rclone, which you can later
modify and send pull requests with. Then it will build the rclone executable
in the same folder. As an initial check you can now run `./rclone version`
(`.\rclone version` on Windows).
This will leave you a checked out version of rclone you can modify and
send pull requests with. If you use `make` instead of `go build` then
the rclone build will have the correct version information in it.
Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/)
command will not be available unless you specify additional build tag `cmount`.
You can also build the latest stable rclone with:
```
go build -tags cmount
```
go get github.com/rclone/rclone
This assumes you have a GCC compatible C compiler (GCC or Clang) in your PATH,
as it uses [cgo](https://pkg.go.dev/cmd/cgo). But on Windows, the
[cgofuse](https://github.com/winfsp/cgofuse) library that the cmount
implementation is based on, also supports building
[without cgo](https://github.com/golang/go/wiki/WindowsDLLs), i.e. by setting
environment variable CGO_ENABLED to value 0 (static linking). This is how the
official Windows release of rclone is being built, starting with version 1.59.
It is still possible to build with cgo on Windows as well, by using the MinGW
port of GCC, e.g. by installing it in a [MSYS2](https://www.msys2.org)
distribution (make sure you install it in the classic mingw64 subsystem, the
ucrt64 version is not compatible).
or the latest version (equivalent to the beta) with
Additionally, on Windows, you must install the third party utility
[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected.
If building with cgo, you must also set environment variable CPATH pointing to
the fuse include directory within the WinFsp installation
(normally `C:\Program Files (x86)\WinFsp\inc\fuse`).
go get github.com/rclone/rclone@master
You may also add arguments `-ldflags -s` (with or without `-tags cmount`),
to omit symbol table and debug information, making the executable file smaller,
and `-trimpath` to remove references to local file system paths. This is how
the official rclone releases are built.
These will build the binary in `$(go env GOPATH)/bin`
(`~/go/bin/rclone` by default) after downloading the source to the go
module cache. Note - do **not** use the `-u` flag here. This causes go
to try to update the dependencies that rclone uses and sometimes these
don't work with the current version of rclone.
```
go build -trimpath -ldflags -s -tags cmount
```
Instead of executing the `go build` command directly, you can run it via the
Makefile, which also sets version information and copies the resulting rclone
executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
corresponds to `~/go/bin/rclone` by default).
```
make
```
To include mount command on macOS and Windows with Makefile build:
```
make GOTAGS=cmount
```
As an alternative you can download the source, build and install rclone in one
operation, as a regular Go package. The source will be stored it in the Go
module cache, and the resulting executable will be in your GOPATH bin folder
(`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).
With Go version 1.17 or newer:
```
go install github.com/rclone/rclone@latest
```
With Go versions older than 1.17 (do **not** use the `-u` flag, it causes Go to
try to update the dependencies that rclone uses and sometimes these don't work
with the current version):
```
go get github.com/rclone/rclone
```
## Installation with Ansible
## Installation with Ansible ##
This can be done with [Stefan Weichinger's ansible
role](https://github.com/stefangweichinger/ansible-rclone).
@@ -290,7 +236,7 @@ Instructions
- rclone
```
## Portable installation
## Portable installation ##
As mentioned [above](https://rclone.org/install/#quickstart), rclone is single
executable (`rclone`, or `rclone.exe` on Windows) that you can download as a
@@ -368,7 +314,7 @@ the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
utility from Microsoft's Sysinternals suite, which takes option `-s` to
execute commands as the `SYSTEM` user.
#### Start from Startup folder
#### Start from Startup folder ###
To quickly execute an rclone command you can simply create a standard
Windows Explorer shortcut for the complete rclone command you want to run. If you
@@ -383,7 +329,7 @@ functionality to set it to run as different user, or to set conditions or
actions on certain events. Setting up a scheduled task as described below
will often give you better results.
#### Start from Task Scheduler
#### Start from Task Scheduler ###
Task Scheduler is an administrative tool built into Windows, and it can be used to
configure rclone to be started automatically in a highly configurable way, e.g.
@@ -393,14 +339,14 @@ be available to all users it can run as the `SYSTEM` user.
For technical information, see
https://docs.microsoft.com/windows/win32/taskschd/task-scheduler-start-page.
#### Run as service
#### Run as service ###
For running rclone at system startup, you can create a Windows service that executes
your rclone command, as an alternative to scheduled task configured to run at startup.
##### Mount command built-in service integration
##### Mount command built-in service integration ####
For mount commands, rclone has a built-in Windows service integration via the third-party
For mount commands, Rclone has a built-in Windows service integration via the third-party
WinFsp library it uses. Registering as a regular Windows service easy, as you just have to
execute the built-in PowerShell command `New-Service` (requires administrative privileges).
@@ -420,7 +366,7 @@ Windows standard methods for managing network drives. This is currently not
officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later
it should be possible through path rewriting as described [here](https://github.com/rclone/rclone/issues/3340).
##### Third-party service integration
##### Third-party service integration #####
To Windows service running any rclone command, the excellent third-party utility
[NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used.
+6 -10
View File
@@ -12,7 +12,7 @@ set -e
#when adding a tool to the list make sure to also add its corresponding command further in the script
unzip_tools_list=('unzip' '7z' 'busybox')
usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
#check for beta flag
if [ -n "$1" ] && [ "$1" != "beta" ]; then
@@ -81,8 +81,6 @@ case $OS in
;;
Darwin)
OS='osx'
binTgtDir=/usr/local/bin
man1TgtDir=/usr/local/share/man/man1
;;
SunOS)
OS='solaris'
@@ -173,14 +171,12 @@ case "$OS" in
;;
'osx')
#binary
mkdir -m 0555 -p ${binTgtDir}
cp rclone ${binTgtDir}/rclone.new
mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone
chmod a=x ${binTgtDir}/rclone
mkdir -p /usr/local/bin
cp rclone /usr/local/bin/rclone.new
mv /usr/local/bin/rclone.new /usr/local/bin/rclone
#manual
mkdir -m 0555 -p ${man1TgtDir}
cp rclone.1 ${man1TgtDir}
chmod a=r ${man1TgtDir}/rclone.1
mkdir -p /usr/local/share/man/man1
cp rclone.1 /usr/local/share/man/man1/
;;
*)
echo 'OS not supported'
-222
View File
@@ -1,222 +0,0 @@
---
title: "Internet Archive"
description: "Rclone docs for Internet Archive"
---
# {{< icon "fa fa-archive" >}} Internet Archive
The Internet Archive backend utilizes Items on [archive.org](https://archive.org/)
Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.html) for the API this backend uses.
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
Once you have made a remote (see the provider specific section above)
you can use it like this:
Unlike S3, listing up all items uploaded by you isn't supported.
Make a new item
rclone mkdir remote:item
List the contents of a item
rclone ls remote:item
Sync `/home/local/directory` to the remote item, deleting any excess
files in the item.
rclone sync -i /home/local/directory remote:item
## Notes
Because of Internet Archive's architecture, it enqueues write operations (and extra post-processings) in a per-item queue. You can check item's queue at https://catalogd.archive.org/history/item-name-here . Because of that, all uploads/deletes will not show up immediately and takes some time to be available.
The per-item queue is enqueued to an another queue, Item Deriver Queue. [You can check the status of Item Deriver Queue here.](https://catalogd.archive.org/catalog.php?whereami=1) This queue has a limit, and it may block you from uploading, or even deleting. You should avoid uploading a lot of small files for better behavior.
You can optionally wait for the server's processing to finish, by setting non-zero value to `wait_archive` key.
By making it wait, rclone can do normal file comparison.
Make sure to set a large enough value (e.g. `30m0s` for smaller files) as it can take a long time depending on server's queue.
## Configuration
Here is an example of making an internetarchive configuration.
Most applies to the other providers as well, any differences are described [below](#providers).
First run
rclone config
This will guide you through an interactive setup process.
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> remote
Option Storage.
Type of storage to configure.
Choose a number from below, or type in your own value.
XX / InternetArchive Items
\ (internetarchive)
Storage> internetarchive
Option access_key_id.
IAS3 Access Key.
Leave blank for anonymous access.
You can find one here: https://archive.org/account/s3.php
Enter a value. Press Enter to leave empty.
access_key_id> XXXX
Option secret_access_key.
IAS3 Secret Key (password).
Leave blank for anonymous access.
Enter a value. Press Enter to leave empty.
secret_access_key> XXXX
Edit advanced config?
y) Yes
n) No (default)
y/n> y
Option endpoint.
IAS3 Endpoint.
Leave blank for default value.
Enter a string value. Press Enter for the default (https://s3.us.archive.org).
endpoint>
Option front_endpoint.
Host of InternetArchive Frontend.
Leave blank for default value.
Enter a string value. Press Enter for the default (https://archive.org).
front_endpoint>
Option disable_checksum.
Don't store MD5 checksum with object metadata.
Normally rclone will calculate the MD5 checksum of the input before
uploading it so it can ask the server to check the object against checksum.
This is great for data integrity checking but can cause long delays for
large files to start uploading.
Enter a boolean value (true or false). Press Enter for the default (true).
disable_checksum> true
Option encoding.
The encoding for the backend.
See the [encoding section in the overview](/overview/#encoding) for more info.
Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot).
encoding>
Edit advanced config?
y) Yes
n) No (default)
y/n> n
--------------------
[remote]
type = internetarchive
access_key_id = XXXX
secret_access_key = XXXX
--------------------
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
```
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/internetarchive/internetarchive.go then run make backenddocs" >}}
### Standard options
Here are the standard options specific to internetarchive (Internet Archive).
#### --internetarchive-access-key-id
IAS3 Access Key.
Leave blank for anonymous access.
You can find one here: https://archive.org/account/s3.php
Properties:
- Config: access_key_id
- Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID
- Type: string
- Required: false
#### --internetarchive-secret-access-key
IAS3 Secret Key (password).
Leave blank for anonymous access.
Properties:
- Config: secret_access_key
- Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY
- Type: string
- Required: false
### Advanced options
Here are the advanced options specific to internetarchive (Internet Archive).
#### --internetarchive-endpoint
IAS3 Endpoint.
Leave blank for default value.
Properties:
- Config: endpoint
- Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT
- Type: string
- Default: "https://s3.us.archive.org"
#### --internetarchive-front-endpoint
Host of InternetArchive Frontend.
Leave blank for default value.
Properties:
- Config: front_endpoint
- Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT
- Type: string
- Default: "https://archive.org"
#### --internetarchive-disable-checksum
Don't ask the server to test against MD5 checksum calculated by rclone.
Normally rclone will calculate the MD5 checksum of the input before
uploading it so it can ask the server to check the object against checksum.
This is great for data integrity checking but can cause long delays for
large files to start uploading.
Properties:
- Config: disable_checksum
- Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM
- Type: bool
- Default: true
#### --internetarchive-wait-archive
Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish.
Only enable if you need to be guaranteed to be reflected after write operations.
0 to disable waiting. No errors to be thrown in case of timeout.
Properties:
- Config: wait_archive
- Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE
- Type: Duration
- Default: 0s
#### --internetarchive-encoding
The encoding for the backend.
See the [encoding section in the overview](/overview/#encoding) for more info.
Properties:
- Config: encoding
- Env Var: RCLONE_INTERNETARCHIVE_ENCODING
- Type: MultiEncoder
- Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot
{{< rem autogenerated options stop >}}
+2 -1
View File
@@ -177,6 +177,7 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
+29 -32
View File
@@ -32,7 +32,6 @@ Here is an overview of the major features of each cloud storage system.
| HDFS | - | Yes | No | No | - |
| HTTP | - | No | No | No | R |
| Hubic | MD5 | Yes | No | No | R/W |
| Internet Archive | MD5, SHA1, CRC32 | Yes | No | No | - |
| Jottacloud | MD5 | Yes | Yes | No | R |
| Koofr | MD5 | No | Yes | No | - |
| Mail.ru Cloud | Mailru ⁶ | Yes | Yes | No | - |
@@ -302,36 +301,35 @@ list of all possible values by passing an invalid value to this
flag, e.g. `--local-encoding "help"`. The command `rclone help flags encoding`
will show you the defaults for the backends.
| Encoding | Characters | Encoded as |
| --------- | ---------- | ---------- |
| Asterisk | `*` | `*` |
| BackQuote | `` ` `` | ``` |
| BackSlash | `\` | `\` |
| Colon | `:` | `:` |
| CrLf | CR 0x0D, LF 0x0A | `␍`, `␊` |
| Ctl | All control characters 0x00-0x1F | `␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟` |
| Del | DEL 0x7F | `␡` |
| Dollar | `$` | `$` |
| Dot | `.` or `..` as entire string | `.`, `..` |
| DoubleQuote | `"` | `"` |
| Hash | `#` | `#` |
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | `�` |
| LeftCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string | `␍`, `␊`, `␉`, `␋` |
| LeftPeriod | `.` on the left of a string | `.` |
| LeftSpace | SPACE on the left of a string | `␠` |
| LeftTilde | `~` on the left of a string | `~` |
| LtGt | `<`, `>` | `<`, `>` |
| None | No characters are encoded | |
| Percent | `%` | `%` |
| Pipe | \| | `|` |
| Question | `?` | `?` |
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string | `␍`, `␊`, `␉`, `␋` |
| RightPeriod | `.` on the right of a string | `.` |
| RightSpace | SPACE on the right of a string | `␠` |
| Semicolon | `;` | `;` |
| SingleQuote | `'` | `'` |
| Slash | `/` | `/` |
| SquareBracket | `[`, `]` | `[`, `]` |
| Encoding | Characters |
| --------- | ---------- |
| Asterisk | `*` |
| BackQuote | `` ` `` |
| BackSlash | `\` |
| Colon | `:` |
| CrLf | CR 0x0D, LF 0x0A |
| Ctl | All control characters 0x00-0x1F |
| Del | DEL 0x7F |
| Dollar | `$` |
| Dot | `.` or `..` as entire string |
| DoubleQuote | `"` |
| Hash | `#` |
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) |
| LeftCrLfHtVt | CR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a string |
| LeftPeriod | `.` on the left of a string |
| LeftSpace | SPACE on the left of a string |
| LeftTilde | `~` on the left of a string |
| LtGt | `<`, `>` |
| None | No characters are encoded |
| Percent | `%` |
| Pipe | \| |
| Question | `?` |
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string |
| RightPeriod | `.` on the right of a string |
| RightSpace | SPACE on the right of a string |
| SingleQuote | `'` |
| Slash | `/` |
| SquareBracket | `[`, `]` |
##### Encoding example: FTP
@@ -428,7 +426,6 @@ upon backend-specific capabilities.
| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes |
| HTTP | No | No | No | No | No | No | No | No | No | Yes |
| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No |
| Internet Archive | No | Yes | No | No | Yes | Yes | No | Yes | Yes | No |
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
| Mega | Yes | No | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
+2 -1
View File
@@ -318,5 +318,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
-11
View File
@@ -150,16 +150,6 @@ use these methods. The alternative is to use `--rc-user` and
Default Off.
### --rc-baseurl
Prefix for URLs.
Default is root
### --rc-template
User-specified template.
## Accessing the remote control via the rclone rc command {#api-rc}
Rclone itself implements the remote control protocol in its `rclone
@@ -1295,7 +1285,6 @@ Returns:
- count - number of files
- bytes - number of bytes in those files
- sizeless - number of files with unknown size, included in count but not accounted for in bytes
See the [size command](/commands/rclone_size/) command for more information on the above.
-21
View File
@@ -92,24 +92,3 @@ Configuration file is stored at:
Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and
place it in the correct place (use `rclone config file` on the remote
box to find out where).
## Configuring using SSH Tunnel ##
Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 53682 to local machine by using the following command:
```
ssh -L localhost:53682:localhost:53682 username@remote_server
```
Then on the headless box run `rclone` config and answer `Y` to the `Use
auto config?` question.
```
...
Remote config
Use auto config?
* Say Y if not sure
* Say N if you are working on a remote or headless machine
y) Yes (default)
n) No
y/n> y
```
Then copy and paste the auth url `http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx` to the browser on your local machine, complete the auth and it is done.
+26 -278
View File
@@ -12,8 +12,6 @@ The S3 backend can be used with a number of different providers:
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
@@ -72,7 +70,7 @@ name> remote
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, Dreamhost, IBM COS, Minio, and Tencent COS
\ "s3"
[snip]
Storage> s3
@@ -569,7 +567,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
### Standard options
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
#### --s3-provider
@@ -590,8 +588,6 @@ Properties:
- Ceph Object Storage
- "ChinaMobile"
- China Mobile Ecloud Elastic Object Storage (EOS)
- "ArvanCloud"
- Arvan Cloud Object Storage (AOS)
- "DigitalOcean"
- Digital Ocean Spaces
- "Dreamhost"
@@ -833,7 +829,7 @@ Properties:
- Config: region
- Env Var: RCLONE_S3_REGION
- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS
- Provider: !AWS,Alibaba,ChinaMobile,RackCorp,Scaleway,Storj,TencentCOS
- Type: string
- Required: false
- Examples:
@@ -930,24 +926,6 @@ Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.
- "eos-anhui-1.cmecloud.cn"
- Anhui China (Huainan)
#### --s3-endpoint
Endpoint for Arvan Cloud Object Storage (AOS) API.
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Provider: ArvanCloud
- Type: string
- Required: false
- Examples:
- "s3.ir-thr-at1.arvanstorage.com"
- The default endpoint - a good choice if you are unsure.
- Tehran Iran (Asiatech)
- "s3.ir-tbz-sh1.arvanstorage.com"
- Tabriz Iran (Shahriar)
#### --s3-endpoint
Endpoint for IBM COS S3 API.
@@ -1317,7 +1295,7 @@ Properties:
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,Scaleway,StackPath,Storj,RackCorp
- Type: string
- Required: false
- Examples:
@@ -1347,8 +1325,6 @@ Properties:
- Wasabi AP Northeast 1 (Tokyo) endpoint
- "s3.ap-northeast-2.wasabisys.com"
- Wasabi AP Northeast 2 (Osaka) endpoint
- "s3.ir-thr-at1.arvanstorage.com"
- ArvanCloud Tehran Iran (Asiatech) endpoint
#### --s3-location-constraint
@@ -1556,7 +1532,7 @@ Properties:
- Config: location_constraint
- Env Var: RCLONE_S3_LOCATION_CONSTRAINT
- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,RackCorp,Scaleway,StackPath,Storj,TencentCOS
- Type: string
- Required: false
@@ -1628,7 +1604,7 @@ Properties:
- Config: server_side_encryption
- Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Provider: AWS,Ceph,ChinaMobile,Minio
- Type: string
- Required: false
- Examples:
@@ -1729,19 +1705,6 @@ The storage class to use when storing new objects in ChinaMobile.
#### --s3-storage-class
The storage class to use when storing new objects in ArvanCloud.
- Config: storage_class
- Env Var: RCLONE_S3_STORAGE_CLASS
- Provider: ArvanCloud
- Type: string
- Required: false
- Examples:
- "STANDARD"
- Standard storage class
#### --s3-storage-class
The storage class to use when storing new objects in Tencent COS.
Properties:
@@ -1784,7 +1747,7 @@ Properties:
### Advanced options
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
#### --s3-bucket-acl
@@ -1836,7 +1799,7 @@ Properties:
- Config: sse_customer_algorithm
- Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Provider: AWS,Ceph,ChinaMobile,Minio
- Type: string
- Required: false
- Examples:
@@ -1853,7 +1816,7 @@ Properties:
- Config: sse_customer_key
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Provider: AWS,Ceph,ChinaMobile,Minio
- Type: string
- Required: false
- Examples:
@@ -1871,7 +1834,7 @@ Properties:
- Config: sse_customer_key_md5
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Provider: AWS,Ceph,ChinaMobile,Minio
- Type: string
- Required: false
- Examples:
@@ -2480,19 +2443,12 @@ You will be able to list and copy data but not upload it.
This is the provider used as main example and described in the [configuration](#configuration) section above.
### AWS Snowball Edge
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring
bulk data back to AWS. Its main software interface is S3 object storage.
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware
appliance used for transferring bulk data back to AWS. Its main
software interface is S3 object storage.
To use rclone with AWS Snowball Edge devices, configure as standard
for an 'S3 Compatible Service'.
If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise
you will run into authentication header issues as the snowball device
does not support query parameter based authentication.
With rclone v1.59 or later setting `upload_cutoff` should not be necessary.
To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'
be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as
the snowball device does not support query parameter based authentication.
eg.
```
@@ -2531,11 +2487,10 @@ server_side_encryption =
storage_class =
```
If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a
version of rclone before v1.59 then you may need to supply the
parameter `--s3-upload-cutoff 0` or put this in the config file as
`upload_cutoff 0` to work around a bug which causes uploading of small
files to fail.
If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you
may need to supply the parameter `--s3-upload-cutoff 0` or put this in
the config file as `upload_cutoff 0` to work around a bug which causes
uploading of small files to fail.
Note also that Ceph sometimes puts `/` in the passwords it gives
users. If you read the secret access key using the command line tools
@@ -2562,106 +2517,6 @@ removed).
Because this is a json dump, it is encoding the `/` as `\/`, so if you
use the secret key as `xxxxxx/xxxx` it will work fine.
### Cloudflare R2 {#cloudflare-r2}
[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage
allows developers to store large amounts of unstructured data without
the costly egress bandwidth fees associated with typical cloud storage
services.
Here is an example of making a Cloudflare R2 configuration. First run:
rclone config
This will guide you through an interactive setup process.
Note that all buckets are private, and all are stored in the same
"auto" region. It is necessary to use Cloudflare workers to share the
content of a bucket publicly.
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> r2
Option Storage.
Type of storage to configure.
Choose a number from below, or type in your own value.
...
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
\ (s3)
...
Storage> s3
Option provider.
Choose your S3 provider.
Choose a number from below, or type in your own value.
Press Enter to leave empty.
...
XX / Cloudflare R2 Storage
\ (Cloudflare)
...
provider> Cloudflare
Option env_auth.
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
Only applies if access_key_id and secret_access_key is blank.
Choose a number from below, or type in your own boolean value (true or false).
Press Enter for the default (false).
1 / Enter AWS credentials in the next step.
\ (false)
2 / Get AWS credentials from the environment (env vars or IAM).
\ (true)
env_auth> 1
Option access_key_id.
AWS Access Key ID.
Leave blank for anonymous access or runtime credentials.
Enter a value. Press Enter to leave empty.
access_key_id> ACCESS_KEY
Option secret_access_key.
AWS Secret Access Key (password).
Leave blank for anonymous access or runtime credentials.
Enter a value. Press Enter to leave empty.
secret_access_key> SECRET_ACCESS_KEY
Option region.
Region to connect to.
Choose a number from below, or type in your own value.
Press Enter to leave empty.
1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
\ (auto)
region> 1
Option endpoint.
Endpoint for S3 API.
Required when using an S3 clone.
Enter a value. Press Enter to leave empty.
endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
Edit advanced config?
y) Yes
n) No (default)
y/n> n
--------------------
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
```
This will leave your config looking something like:
```
[r2]
type = s3
provider = Cloudflare
access_key_id = ACCESS_KEY
secret_access_key = SECRET_ACCESS_KEY
region = auto
endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
acl = private
```
Now run `rclone lsf r2:` to see your buckets and `rclone lsf
r2:bucket` to look within a bucket.
### Dreamhost
Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is
@@ -2760,7 +2615,7 @@ Choose a number from below, or type in your own value
\ "alias"
2 / Amazon Drive
\ "amazon cloud drive"
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS)
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Minio, IBM COS)
\ "s3"
4 / Backblaze B2
\ "b2"
@@ -3045,7 +2900,7 @@ Choose `s3` backend
Type of storage to configure.
Choose a number from below, or type in your own value.
[snip]
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
\ (s3)
[snip]
Storage> s3
@@ -3232,7 +3087,7 @@ name> wasabi
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, Minio)
\ "s3"
[snip]
Storage> s3
@@ -3346,7 +3201,7 @@ Type of storage to configure.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
[snip]
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
\ "s3"
[snip]
Storage> s3
@@ -3686,114 +3541,6 @@ d) Delete this remote
y/e/d> y
```
### ArvanCloud
[ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage.
It gives you access to backup and archived files and allows sharing.
Files like profile image in the app, images sent by users or scanned documents can be stored securely and easily in our Object Storage service.
ArvanCloud provides an S3 interface which can be configured for use with
rclone like this.
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
n/s> n
name> ArvanCloud
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
\ "s3"
[snip]
Storage> s3
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
Choose a number from below, or type in your own value
1 / Enter AWS credentials in the next step
\ "false"
2 / Get AWS credentials from the environment (env vars or IAM)
\ "true"
env_auth> 1
AWS Access Key ID - leave blank for anonymous access or runtime credentials.
access_key_id> YOURACCESSKEY
AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
secret_access_key> YOURSECRETACCESSKEY
Region to connect to.
Choose a number from below, or type in your own value
/ The default endpoint - a good choice if you are unsure.
1 | US Region, Northern Virginia, or Pacific Northwest.
| Leave location constraint empty.
\ "us-east-1"
[snip]
region>
Endpoint for S3 API.
Leave blank if using ArvanCloud to use the default endpoint for the region.
Specify if using an S3 clone such as Ceph.
endpoint> s3.arvanstorage.com
Location constraint - must be set to match the Region. Used when creating buckets only.
Choose a number from below, or type in your own value
1 / Empty for Iran-Tehran Region.
\ ""
[snip]
location_constraint>
Canned ACL used when creating buckets and/or storing objects in S3.
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Choose a number from below, or type in your own value
1 / Owner gets FULL_CONTROL. No one else has access rights (default).
\ "private"
[snip]
acl>
The server-side encryption algorithm used when storing this object in S3.
Choose a number from below, or type in your own value
1 / None
\ ""
2 / AES256
\ "AES256"
server_side_encryption>
The storage class to use when storing objects in S3.
Choose a number from below, or type in your own value
1 / Default
\ ""
2 / Standard storage class
\ "STANDARD"
storage_class>
Remote config
--------------------
[ArvanCloud]
env_auth = false
access_key_id = YOURACCESSKEY
secret_access_key = YOURSECRETACCESSKEY
region = ir-thr-at1
endpoint = s3.arvanstorage.com
location_constraint =
acl =
server_side_encryption =
storage_class =
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
This will leave the config file looking like this.
```
[ArvanCloud]
type = s3
provider = ArvanCloud
env_auth = false
access_key_id = YOURACCESSKEY
secret_access_key = YOURSECRETACCESSKEY
region =
endpoint = s3.arvanstorage.com
location_constraint =
acl =
server_side_encryption =
storage_class =
```
### Tencent COS {#tencent-cos}
[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.
@@ -3827,7 +3574,7 @@ Choose a number from below, or type in your own value
\ "alias"
3 / Amazon Drive
\ "amazon cloud drive"
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
\ "s3"
[snip]
Storage> s3
@@ -4040,4 +3787,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
+2 -1
View File
@@ -251,5 +251,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
+2 -1
View File
@@ -415,7 +415,8 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
## Known issues
+2 -1
View File
@@ -280,5 +280,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
-1
View File
@@ -73,7 +73,6 @@
<a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe"></i> HDFS (Hadoop Distributed Filesystem)</a>
<a class="dropdown-item" href="/http/"><i class="fa fa-globe"></i> HTTP</a>
<a class="dropdown-item" href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a>
<a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive"></i> Internet Archive</a>
<a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud"></i> Jottacloud</a>
<a class="dropdown-item" href="/koofr/"><i class="fa fa-suitcase"></i> Koofr</a>
<a class="dropdown-item" href="/mailru/"><i class="fa fa-at"></i> Mail.ru Cloud</a>
+2 -2
View File
@@ -230,7 +230,7 @@ func ConfigChoose(state string, name string, help string, n int, getItem func(i
// StatePush pushes a new values onto the front of the config string
func StatePush(state string, values ...string) string {
for i := range values {
values[i] = strings.ReplaceAll(values[i], ",", ",") // replace comma with unicode wide version
values[i] = strings.Replace(values[i], ",", ",", -1) // replace comma with unicode wide version
}
if state != "" {
values = append(values[:len(values):len(values)], state)
@@ -262,7 +262,7 @@ func StatePop(state string) (newState string, value string) {
return "", state
}
value, newState = state[:comma], state[comma+1:]
value = strings.ReplaceAll(value, ",", ",") // replace unicode wide comma with comma
value = strings.Replace(value, ",", ",", -1) // replace unicode wide comma with comma
return newState, value
}
-36
View File
@@ -1,36 +0,0 @@
// Package chunksize calculates a suitable chunk size for large uploads
package chunksize
import (
"github.com/rclone/rclone/fs"
)
/*
Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi
For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
that some backends may only allow integer type parameters when specifying the chunk size.
Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
smallest chunk size necessary to allow the upload to proceed.
*/
func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
fileSize := fs.SizeSuffix(objInfo.Size())
requiredChunks := fileSize / defaultChunkSize
if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
return defaultChunkSize
}
minChunk := fileSize / fs.SizeSuffix(maxParts)
remainder := minChunk % fs.Mebi
if remainder != 0 {
minChunk += fs.Mebi - remainder
}
if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
minChunk += fs.Mebi
}
fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
return minChunk
}
-40
View File
@@ -1,40 +0,0 @@
package chunksize
import (
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/object"
)
func TestComputeChunkSize(t *testing.T) {
tests := map[string]struct {
fileSize fs.SizeSuffix
maxParts int
defaultChunkSize fs.SizeSuffix
expected fs.SizeSuffix
}{
"default size returned when file size is small enough": {fileSize: 1000, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
"default size returned when file size is just 1 byte small enough": {fileSize: toSizeSuffixMiB(100000) - 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
"no rounding up when everything divides evenly": {fileSize: toSizeSuffixMiB(1000000), maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(100)},
"rounding up to nearest MiB when not quite enough parts": {fileSize: toSizeSuffixMiB(1000000), maxParts: 9999, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
"rounding up to nearest MiB when one extra byte": {fileSize: toSizeSuffixMiB(1000000) + 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
"expected MiB value when rounding sets to absolute minimum": {fileSize: toSizeSuffixMiB(1) - 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(1)},
"expected MiB value when rounding to absolute min with extra": {fileSize: toSizeSuffixMiB(1) + 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(2)},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
src := object.NewStaticObjectInfo("mock", time.Now(), int64(tc.fileSize), true, nil, nil)
result := Calculator(src, tc.maxParts, tc.defaultChunkSize)
if result != tc.expected {
t.Fatalf("expected: %v, got: %v", tc.expected, result)
}
})
}
}
func toSizeSuffixMiB(size int64) fs.SizeSuffix {
return fs.SizeSuffix(size * int64(fs.Mebi))
}
+2 -2
View File
@@ -248,11 +248,11 @@ func AddConfig(ctx context.Context) (context.Context, *ConfigInfo) {
// "ignore-size") into an environment name
// "RCLONE_CONFIG_MY-REMOTE_IGNORE_SIZE"
func ConfigToEnv(section, name string) string {
return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.ReplaceAll(name, "-", "_"))
return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.Replace(name, "-", "_", -1))
}
// OptionToEnv converts an option name, e.g. "ignore-size" into an
// environment name "RCLONE_IGNORE_SIZE"
func OptionToEnv(name string) string {
return "RCLONE_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
}
+1 -1
View File
@@ -50,7 +50,7 @@ func setConfigFile(t *testing.T, data string) func() {
// toUnix converts \r\n to \n in buf
func toUnix(buf string) string {
if runtime.GOOS == "windows" {
return strings.ReplaceAll(buf, "\r\n", "\n")
return strings.Replace(buf, "\r\n", "\n", -1)
}
return buf
}
+1 -1
View File
@@ -417,7 +417,7 @@ func ChooseOption(o *fs.Option, name string) string {
fmt.Printf("Option %s.\n", o.Name)
if o.Help != "" {
// Show help string without empty lines.
help := strings.ReplaceAll(strings.TrimSpace(o.Help), "\n\n", "\n")
help := strings.Replace(strings.TrimSpace(o.Help), "\n\n", "\n", -1)
fmt.Println(help)
}
+1 -1
View File
@@ -208,7 +208,7 @@ loop:
value := path[prev : i-1]
// replace any doubled quotes if there were any
if doubled {
value = strings.ReplaceAll(value, string(quote)+string(quote), string(quote))
value = strings.Replace(value, string(quote)+string(quote), string(quote), -1)
}
prev = i + 1
parsed.Config[param] = value
+1 -1
View File
@@ -208,7 +208,7 @@ func parseHelperOptionString(optString string) (opts []string, err error) {
if len(param) == 0 {
return nil, errHelperEmptyOption
}
if param[0] == '-' {
if param[0] == '-' || param[0] == '_' {
return nil, errHelperOptionName
}
prev = i + 1
+1 -1
View File
@@ -20,7 +20,7 @@ func TestMountHelperArgs(t *testing.T) {
src: []string{},
dst: []string{"mount", "--daemon"},
}, {
src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env,_netdev`},
src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env`},
dst: []string{"mount", "--read-only", "--verbose=3", "--daemon"},
env: "HTTPS_PROXY=a b;c,d?EF",
}}
+5 -14
View File
@@ -138,14 +138,8 @@ func (o *RangeOption) Decode(size int64) (offset, limit int64) {
// absolute fetch using the size passed in and makes sure the range does
// not exceed filesize. Some remotes (e.g. Onedrive, Box) don't support
// range requests which index from the end.
//
// It also adjusts any SeekOption~s, turning them into absolute
// RangeOption~s instead.
func FixRangeOption(options []OpenOption, size int64) {
if size < 0 {
// Can't do anything for unknown length objects
return
} else if size == 0 {
if size == 0 {
// if size 0 then remove RangeOption~s
// replacing with a NullOptions~s which won't be rendered
for i := range options {
@@ -156,21 +150,18 @@ func FixRangeOption(options []OpenOption, size int64) {
}
return
}
for i, option := range options {
switch x := option.(type) {
case *RangeOption:
for i := range options {
option := options[i]
if x, ok := option.(*RangeOption); ok {
// If start is < 0 then fetch from the end
if x.Start < 0 {
x = &RangeOption{Start: size - x.End, End: -1}
options[i] = x
}
// If end is too big or undefined, fetch to the end
if x.End > size || x.End < 0 {
if x.End > size {
x = &RangeOption{Start: x.Start, End: size - 1}
options[i] = x
}
case *SeekOption:
options[i] = &RangeOption{Start: x.Offset, End: size - 1}
}
}
}
+2 -26
View File
@@ -149,16 +149,6 @@ func TestFixRangeOptions(t *testing.T) {
in: []OpenOption{},
want: []OpenOption{},
},
{
name: "Unknown size -1",
in: []OpenOption{
&RangeOption{Start: 1, End: -1},
},
want: []OpenOption{
&RangeOption{Start: 1, End: -1},
},
size: -1,
},
{
name: "Fetch a range with size=0",
in: []OpenOption{
@@ -193,7 +183,7 @@ func TestFixRangeOptions(t *testing.T) {
&RangeOption{Start: 1, End: -1},
},
want: []OpenOption{
&RangeOption{Start: 1, End: 99},
&RangeOption{Start: 1, End: -1},
},
size: 100,
},
@@ -203,7 +193,7 @@ func TestFixRangeOptions(t *testing.T) {
&RangeOption{Start: -1, End: 10},
},
want: []OpenOption{
&RangeOption{Start: 90, End: 99},
&RangeOption{Start: 90, End: -1},
},
size: 100,
},
@@ -217,20 +207,6 @@ func TestFixRangeOptions(t *testing.T) {
},
size: 100,
},
{
name: "SeekOption",
in: []OpenOption{
&HTTPOption{Key: "a", Value: "1"},
&SeekOption{Offset: 10},
&HTTPOption{Key: "b", Value: "2"},
},
want: []OpenOption{
&HTTPOption{Key: "a", Value: "1"},
&RangeOption{Start: 10, End: 99},
&HTTPOption{Key: "b", Value: "2"},
},
size: 100,
},
} {
FixRangeOption(test.in, test.size)
assert.Equal(t, test.want, test.in, test.name)
+1 -1
View File
@@ -121,7 +121,7 @@ func TestDeduplicateFirst(t *testing.T) {
// list until we get one object
var objects, size int64
for try := 1; try <= *fstest.ListRetries; try++ {
objects, size, _, err = operations.Count(context.Background(), r.Fremote)
objects, size, err = operations.Count(context.Background(), r.Fremote)
require.NoError(t, err)
if objects == 1 {
break
+2 -45
View File
@@ -796,47 +796,6 @@ func Overlapping(fdst, fsrc fs.Info) bool {
return strings.HasPrefix(fdstRoot, fsrcRoot) || strings.HasPrefix(fsrcRoot, fdstRoot)
}
// OverlappingFilterCheck returns true if fdst and fsrc point to the same
// underlying Fs and they overlap without fdst being excluded by any filter rule.
func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool {
if !SameConfig(fdst, fsrc) {
return false
}
fdstRoot := fixRoot(fdst)
fsrcRoot := fixRoot(fsrc)
if strings.HasPrefix(fdstRoot, fsrcRoot) {
fdstRelative := fdstRoot[len(fsrcRoot):]
return filterCheckR(ctx, fdstRelative, 0, fsrc)
}
return strings.HasPrefix(fsrcRoot, fdstRoot)
}
// filterCheckR checks if fdst would be included in the sync
func filterCheckR(ctx context.Context, fdstRelative string, pos int, fsrc fs.Fs) bool {
include := true
fi := filter.GetConfig(ctx)
includeDirectory := fi.IncludeDirectory(ctx, fsrc)
dirs := strings.SplitAfterN(fdstRelative, "/", pos+2)
newPath := ""
for i := 0; i <= pos; i++ {
newPath += dirs[i]
}
if !strings.HasSuffix(newPath, "/") {
newPath += "/"
}
if strings.HasPrefix(fdstRelative, newPath) {
include, _ = includeDirectory(newPath)
if include {
if newPath == fdstRelative {
return true
}
pos++
include = filterCheckR(ctx, fdstRelative, pos, fsrc)
}
}
return include
}
// SameDir returns true if fdst and fsrc point to the same
// underlying Fs and they are the same directory.
func SameDir(fdst, fsrc fs.Info) bool {
@@ -1111,13 +1070,11 @@ func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Write
// Count counts the objects and their sizes in the Fs
//
// Obeys includes and excludes
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) {
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error) {
err = ListFn(ctx, f, func(o fs.Object) {
atomic.AddInt64(&objects, 1)
objectSize := o.Size()
if objectSize < 0 {
atomic.AddInt64(&sizelessObjects, 1)
} else if objectSize > 0 {
if objectSize > 0 {
atomic.AddInt64(&size, objectSize)
}
})
+3 -91
View File
@@ -401,11 +401,10 @@ func TestCount(t *testing.T) {
// Check the MaxDepth too
ci.MaxDepth = 1
objects, size, sizeless, err := operations.Count(ctx, r.Fremote)
objects, size, err := operations.Count(ctx, r.Fremote)
require.NoError(t, err)
assert.Equal(t, int64(2), objects)
assert.Equal(t, int64(61), size)
assert.Equal(t, int64(0), sizeless)
}
func TestDelete(t *testing.T) {
@@ -1253,93 +1252,6 @@ func TestOverlapping(t *testing.T) {
}
}
// testFs is for unit testing fs.Fs
type testFs struct {
testFsInfo
}
func (i *testFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
return nil, nil
}
func (i *testFs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return nil, nil }
func (i *testFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return nil, nil
}
func (i *testFs) Mkdir(ctx context.Context, dir string) error { return nil }
func (i *testFs) Rmdir(ctx context.Context, dir string) error { return nil }
// copied from TestOverlapping because the behavior of OverlappingFilterCheck should be identical to Overlapping
// when no filters are set
func TestOverlappingFilterCheckWithoutFilter(t *testing.T) {
ctx := context.Background()
src := &testFs{testFsInfo{name: "name", root: "root"}}
slash := string(os.PathSeparator) // native path separator
for _, test := range []struct {
name string
root string
expected bool
}{
{"name", "root", true},
{"namey", "root", false},
{"name", "rooty", false},
{"namey", "rooty", false},
{"name", "roo", false},
{"name", "root/toot", true},
{"name", "root/toot/", true},
{"name", "root" + slash + "toot", true},
{"name", "root" + slash + "toot" + slash, true},
{"name", "", true},
{"name", "/", true},
} {
dst := &testFs{testFsInfo{name: test.name, root: test.root}}
what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
actual := operations.OverlappingFilterCheck(ctx, src, dst)
assert.Equal(t, test.expected, actual, what)
actual = operations.OverlappingFilterCheck(ctx, dst, src)
assert.Equal(t, test.expected, actual, what)
}
}
func TestOverlappingFilterCheckWithFilter(t *testing.T) {
ctx := context.Background()
fi, err := filter.NewFilter(nil)
require.NoError(t, err)
require.NoError(t, fi.Add(false, "*/exclude/"))
fi.Opt.ExcludeFile = ".ignore"
ctx = filter.ReplaceConfig(ctx, fi)
src := &testFs{testFsInfo{name: "name", root: "root"}}
slash := string(os.PathSeparator) // native path separator
for _, test := range []struct {
name string
root string
expected bool
}{
{"name", "root", true},
{"name", "root/", true},
{"name", "root" + slash, true},
{"name", "root/exclude", false},
{"name", "root/exclude/", false},
{"name", "root" + slash + "exclude", false},
{"name", "root" + slash + "exclude" + slash, false},
{"name", "root/.ignore", false},
{"name", "root" + slash + ".ignore", false},
{"namey", "root/include", false},
{"namey", "root/include/", false},
{"namey", "root" + slash + "include", false},
{"namey", "root" + slash + "include" + slash, false},
} {
dst := &testFs{testFsInfo{name: test.name, root: test.root}}
what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
actual := operations.OverlappingFilterCheck(ctx, dst, src)
assert.Equal(t, test.expected, actual, what)
}
}
func TestListFormat(t *testing.T) {
item0 := &operations.ListJSONItem{
Path: "a",
@@ -1494,7 +1406,7 @@ func TestDirMove(t *testing.T) {
require.NoError(t, operations.DirMove(ctx, r.Fremote, "A1", "A2"))
for i := range files {
files[i].Path = strings.ReplaceAll(files[i].Path, "A1/", "A2/")
files[i].Path = strings.Replace(files[i].Path, "A1/", "A2/", -1)
}
fstest.CheckListingWithPrecision(
@@ -1519,7 +1431,7 @@ func TestDirMove(t *testing.T) {
require.NoError(t, operations.DirMove(ctx, r.Fremote, "A2", "A3"))
for i := range files {
files[i].Path = strings.ReplaceAll(files[i].Path, "A2/", "A3/")
files[i].Path = strings.Replace(files[i].Path, "A2/", "A3/", -1)
}
fstest.CheckListingWithPrecision(
Loaded 100 of 131 files, more files were not shown because too many files have changed in this diff. Show more