Compare commits
103
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dfc5b0460b | ||
|
|
26db80c270 | ||
|
|
9eb3470c9c | ||
|
|
a449dd7d1c | ||
|
|
fc4fe33703 | ||
|
|
e11bfacfcf | ||
|
|
a9c49c50a0 | ||
|
|
8979337313 | ||
|
|
7ffab5d998 | ||
|
|
3ccf222acb | ||
|
|
2781f8e2f1 | ||
|
|
3d55f69338 | ||
|
|
cc9bc2cb80 | ||
|
|
80ac59ee5b | ||
|
|
5d6a6dd6c0 | ||
|
|
c676e2139d | ||
|
|
7361c98b2d | ||
|
|
5cc47de912 | ||
|
|
6d342a3c5b | ||
|
|
336051870e | ||
|
|
38c6d022bd | ||
|
|
c138367df6 | ||
|
|
da404dc0f2 | ||
|
|
28e43fe7af | ||
|
|
3ec25f437b | ||
|
|
a34276e9b3 | ||
|
|
c2baacc0a4 | ||
|
|
fcec4bedbe | ||
|
|
813a5e0931 | ||
|
|
bd4abb15a3 | ||
|
|
7f84283539 | ||
|
|
47b1a0d6fa | ||
|
|
ce168ecac2 | ||
|
|
4f0ddb60e7 | ||
|
|
b929a56f46 | ||
|
|
74af6409d4 | ||
|
|
0e77072dcc | ||
|
|
2437eb3cce | ||
|
|
a12c94caff | ||
|
|
542c1616b8 | ||
|
|
8697f0bd26 | ||
|
|
a9f18f8093 | ||
|
|
8e5e230b81 | ||
|
|
c0985e93b7 | ||
|
|
fb4f7555c7 | ||
|
|
f2e7a2e794 | ||
|
|
9e4854955c | ||
|
|
319ac225e4 | ||
|
|
a9d3283d97 | ||
|
|
edf0412464 | ||
|
|
e6194a4b83 | ||
|
|
7f05990623 | ||
|
|
e16f2a566f | ||
|
|
a36fef8a66 | ||
|
|
6500e1d205 | ||
|
|
9f7484e4e9 | ||
|
|
0ba702ccf4 | ||
|
|
6f91198b57 | ||
|
|
cf0a72aecd | ||
|
|
f6fd6ee777 | ||
|
|
1e66d052fd | ||
|
|
e5974ac4b0 | ||
|
|
50a0c3482d | ||
|
|
389a29b017 | ||
|
|
9dcf9375e8 | ||
|
|
1d6d41fb91 | ||
|
|
a3d4307892 | ||
|
|
a446106041 | ||
|
|
607172b6ec | ||
|
|
94757277bc | ||
|
|
deab86867c | ||
|
|
c0c5b3bc6b | ||
|
|
a947f298e6 | ||
|
|
1b0128ecb2 | ||
|
|
c5395db1f1 | ||
|
|
6e5382fc99 | ||
|
|
134592adaa | ||
|
|
36e614f550 | ||
|
|
7bfed98b48 | ||
|
|
f471096fd0 | ||
|
|
4cebade95d | ||
|
|
a8cd18faf3 | ||
|
|
e34c543660 | ||
|
|
598364ad0f | ||
|
|
211dbe9aee | ||
|
|
4829527dac | ||
|
|
cc8dde402f | ||
|
|
2b67ad17aa | ||
|
|
6da3522499 | ||
|
|
97606bbdef | ||
|
|
a15885dd74 | ||
|
|
87c201c92a | ||
|
|
d77736c21a | ||
|
|
86bd5f6922 | ||
|
|
fe271a4e35 | ||
|
|
75455d4000 | ||
|
|
82e24f521f | ||
|
|
5605e34f7b | ||
|
|
06598531e0 | ||
|
|
b1d43f8d41 | ||
|
|
b53c38c9fd | ||
|
|
03715f6c6b | ||
|
|
07481396e0 |
No files matched your search
+10
-16
@@ -25,7 +25,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.16', 'go1.17']
|
||||
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
@@ -55,23 +55,12 @@ jobs:
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows_amd64
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '1.18.x'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^windows/amd64" -cgo'
|
||||
build_args: '-buildmode exe'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
deploy: true
|
||||
|
||||
- job_name: windows_386
|
||||
os: windows-latest
|
||||
go: '1.18.x'
|
||||
gotags: cmount
|
||||
goarch: '386'
|
||||
cgo: '1'
|
||||
build_flags: '-include "^windows/386" -cgo'
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
build_args: '-buildmode exe'
|
||||
quicktest: true
|
||||
deploy: true
|
||||
@@ -79,7 +68,7 @@ jobs:
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '1.18.x'
|
||||
build_flags: '-exclude "^(windows/(386|amd64)|darwin/|linux/)"'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
@@ -178,6 +167,11 @@ jobs:
|
||||
run: |
|
||||
make
|
||||
|
||||
- name: Rclone version
|
||||
shell: bash
|
||||
run: |
|
||||
rclone version
|
||||
|
||||
- name: Run tests
|
||||
shell: bash
|
||||
run: |
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
with:
|
||||
tag: beta
|
||||
imageName: rclone/rclone
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
|
||||
publish: true
|
||||
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
|
||||
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
with:
|
||||
tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
|
||||
imageName: rclone/rclone
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
|
||||
publish: true
|
||||
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
|
||||
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
PLUGIN_USER=rclone
|
||||
docker login --username ${{ secrets.DOCKER_HUB_USER }} \
|
||||
--password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
|
||||
for PLUGIN_ARCH in amd64 arm64 arm/v7 ;do
|
||||
for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
|
||||
export PLUGIN_USER PLUGIN_ARCH
|
||||
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
|
||||
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
|
||||
|
||||
@@ -97,7 +97,7 @@ release_dep_linux:
|
||||
|
||||
# Get the release dependencies we only install on Windows
|
||||
release_dep_windows:
|
||||
GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
|
||||
GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
|
||||
|
||||
# Update dependencies
|
||||
showupdates:
|
||||
@@ -245,18 +245,18 @@ retag:
|
||||
startdev:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next version is $(NEXT_VERSION)"
|
||||
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
startstable:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
|
||||
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_PATCH_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
winzip:
|
||||
zip -9 rclone-$(TAG).zip rclone.exe
|
||||
|
||||
@@ -29,6 +29,8 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* Box [:page_facing_up:](https://rclone.org/box/)
|
||||
* Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
|
||||
* China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
|
||||
* Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
|
||||
* Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
|
||||
* Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
|
||||
* DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
* Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
@@ -42,6 +44,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
|
||||
* HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
* Hubic [:page_facing_up:](https://rclone.org/hubic/)
|
||||
* Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
|
||||
* Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
* IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
* Koofr [:page_facing_up:](https://rclone.org/koofr/)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/hdfs"
|
||||
_ "github.com/rclone/rclone/backend/http"
|
||||
_ "github.com/rclone/rclone/backend/hubic"
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
_ "github.com/rclone/rclone/backend/jottacloud"
|
||||
_ "github.com/rclone/rclone/backend/koofr"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
"github.com/Azure/go-autorest/autorest/adal"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
@@ -43,8 +44,9 @@ import (
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 10 * time.Second
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
maxListChunkSize = 5000 // number of items to read at once
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
maxListChunkSize = 5000 // number of items to read at once
|
||||
maxUploadParts = 50000 // maximum allowed number of parts/blocks in a multi-part upload
|
||||
modTimeKey = "mtime"
|
||||
timeFormatIn = time.RFC3339
|
||||
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
@@ -371,15 +373,9 @@ func (o *Object) split() (container, containerPath string) {
|
||||
|
||||
// validateAccessTier checks if azureblob supports user supplied tier
|
||||
func validateAccessTier(tier string) bool {
|
||||
switch tier {
|
||||
case string(azblob.AccessTierHot),
|
||||
string(azblob.AccessTierCool),
|
||||
string(azblob.AccessTierArchive):
|
||||
// valid cases
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(tier, string(azblob.AccessTierHot)) ||
|
||||
strings.EqualFold(tier, string(azblob.AccessTierCool)) ||
|
||||
strings.EqualFold(tier, string(azblob.AccessTierArchive))
|
||||
}
|
||||
|
||||
// validatePublicAccess checks if azureblob supports use supplied public access level
|
||||
@@ -1689,8 +1685,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
uploadParts := int64(maxUploadParts)
|
||||
if uploadParts < 1 {
|
||||
uploadParts = 1
|
||||
} else if uploadParts > maxUploadParts {
|
||||
uploadParts = maxUploadParts
|
||||
}
|
||||
// calculate size of parts/blocks
|
||||
partSize := chunksize.Calculator(o, int(uploadParts), o.fs.opt.ChunkSize)
|
||||
|
||||
putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
|
||||
BufferSize: int(o.fs.opt.ChunkSize),
|
||||
BufferSize: int(partSize),
|
||||
MaxBuffers: o.fs.opt.UploadConcurrency,
|
||||
Metadata: o.meta,
|
||||
BlobHTTPHeaders: httpHeaders,
|
||||
|
||||
@@ -61,3 +61,25 @@ func TestServicePrincipalFileFailure(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
|
||||
}
|
||||
|
||||
func TestValidateAccessTier(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
accessTier string
|
||||
want bool
|
||||
}{
|
||||
"hot": {"hot", true},
|
||||
"HOT": {"HOT", true},
|
||||
"Hot": {"Hot", true},
|
||||
"cool": {"cool", true},
|
||||
"archive": {"archive", true},
|
||||
"empty": {"", false},
|
||||
"unknown": {"unknown", false},
|
||||
}
|
||||
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got := validateAccessTier(test.accessTier)
|
||||
assert.Equal(t, test.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
+32
-2
@@ -64,7 +64,8 @@ const (
|
||||
|
||||
// Globals
|
||||
var (
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -106,6 +107,11 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
|
||||
Help: "Include old versions in directory listings.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "version_at",
|
||||
Help: "Show file versions as they were at the specified time.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
|
||||
Default: fs.Time{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "hard_delete",
|
||||
Help: "Permanently delete files on remote removal, otherwise hide files.",
|
||||
@@ -211,6 +217,7 @@ type Options struct {
|
||||
Endpoint string `config:"endpoint"`
|
||||
TestMode string `config:"test_mode"`
|
||||
Versions bool `config:"versions"`
|
||||
VersionAt fs.Time `config:"version_at"`
|
||||
HardDelete bool `config:"hard_delete"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
|
||||
@@ -696,9 +703,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
Method: "POST",
|
||||
Path: "/b2_list_file_names",
|
||||
}
|
||||
if hidden {
|
||||
if hidden || f.opt.VersionAt.IsSet() {
|
||||
opts.Path = "/b2_list_file_versions"
|
||||
}
|
||||
|
||||
lastFileName := ""
|
||||
|
||||
for {
|
||||
var response api.ListFileNamesResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -728,7 +738,21 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
if addBucket {
|
||||
remote = path.Join(bucket, remote)
|
||||
}
|
||||
|
||||
if f.opt.VersionAt.IsSet() {
|
||||
if time.Time(file.UploadTimestamp).After(time.Time(f.opt.VersionAt)) {
|
||||
// Ignore versions that were created after the specified time
|
||||
continue
|
||||
}
|
||||
|
||||
if file.Name == lastFileName {
|
||||
// Ignore versions before the already returned version
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Send object
|
||||
lastFileName = file.Name
|
||||
err = fn(remote, file, isDirectory)
|
||||
if err != nil {
|
||||
if err == errEndList {
|
||||
@@ -1828,6 +1852,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if o.fs.opt.Versions {
|
||||
return errNotWithVersions
|
||||
}
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
return errNotWithVersionAt
|
||||
}
|
||||
size := src.Size()
|
||||
|
||||
bucket, bucketPath := o.split()
|
||||
@@ -1983,6 +2010,9 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
if o.fs.opt.Versions {
|
||||
return errNotWithVersions
|
||||
}
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
return errNotWithVersionAt
|
||||
}
|
||||
if o.fs.opt.HardDelete {
|
||||
return o.fs.deleteByID(ctx, o.id, bucketPath)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/rclone/rclone/backend/b2/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
@@ -88,21 +89,19 @@ type largeUpload struct {
|
||||
// newLargeUpload starts an upload of object o from in with metadata in src
|
||||
//
|
||||
// If newInfo is set then metadata from that will be used instead of reading it from src
|
||||
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
|
||||
remote := o.remote
|
||||
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
|
||||
size := src.Size()
|
||||
parts := int64(0)
|
||||
sha1SliceSize := int64(maxParts)
|
||||
chunkSize := defaultChunkSize
|
||||
if size == -1 {
|
||||
fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
|
||||
} else {
|
||||
chunkSize = chunksize.Calculator(src, maxParts, defaultChunkSize)
|
||||
parts = size / int64(chunkSize)
|
||||
if size%int64(chunkSize) != 0 {
|
||||
parts++
|
||||
}
|
||||
if parts > maxParts {
|
||||
return nil, fmt.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
|
||||
}
|
||||
sha1SliceSize = parts
|
||||
}
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1747,7 +1747,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.Fs.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
@@ -515,7 +515,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
|
||||
|
||||
strRegex := regexp.QuoteMeta(pattern)
|
||||
strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl)
|
||||
strRegex = strings.Replace(strRegex, "\\*", mainNameRegStr, -1)
|
||||
strRegex = strings.ReplaceAll(strRegex, "\\*", mainNameRegStr)
|
||||
strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld)
|
||||
f.nameRegexp = regexp.MustCompile(strRegex)
|
||||
|
||||
@@ -524,7 +524,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
|
||||
if numDigits > 1 {
|
||||
fmtDigits = fmt.Sprintf("%%0%dd", numDigits)
|
||||
}
|
||||
strFmt := strings.Replace(pattern, "%", "%%", -1)
|
||||
strFmt := strings.ReplaceAll(pattern, "%", "%%")
|
||||
strFmt = strings.Replace(strFmt, "*", "%s", 1)
|
||||
f.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits)
|
||||
f.ctrlNameFmt = reHashes.ReplaceAllLiteralString(strFmt, "_%s")
|
||||
@@ -1895,7 +1895,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.base.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("can't CleanUp")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
@@ -1904,7 +1904,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.base.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
@@ -906,7 +906,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.Fs.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("can't CleanUp: not supported by underlying remote")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
@@ -915,7 +915,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.Fs.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("can't About: not supported by underlying remote")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
@@ -597,7 +597,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.Fs.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("can't CleanUp")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
@@ -606,7 +606,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.Fs.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
+22
-7
@@ -70,7 +70,7 @@ const (
|
||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
|
||||
defaultChunkSize = 8 * fs.Mebi
|
||||
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
|
||||
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
|
||||
listRGrouping = 50 // number of IDs to search at once when using ListR
|
||||
listRInputBuffer = 1000 // size of input buffer when using ListR
|
||||
defaultXDGIcon = "text-html"
|
||||
@@ -660,6 +660,7 @@ type baseObject struct {
|
||||
mimeType string // The object MIME type
|
||||
bytes int64 // size of the object
|
||||
parents []string // IDs of the parent directories
|
||||
resourceKey *string // resourceKey is needed for link shared objects
|
||||
}
|
||||
type documentObject struct {
|
||||
baseObject
|
||||
@@ -829,8 +830,8 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
|
||||
if title != "" {
|
||||
searchTitle := f.opt.Enc.FromStandardName(title)
|
||||
// Escaping the backslash isn't documented but seems to work
|
||||
searchTitle = strings.Replace(searchTitle, `\`, `\\`, -1)
|
||||
searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
|
||||
searchTitle = strings.ReplaceAll(searchTitle, `\`, `\\`)
|
||||
searchTitle = strings.ReplaceAll(searchTitle, `'`, `\'`)
|
||||
|
||||
var titleQuery bytes.Buffer
|
||||
_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
|
||||
@@ -1319,12 +1320,16 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||
}
|
||||
}
|
||||
}
|
||||
return &Object{
|
||||
o := &Object{
|
||||
baseObject: f.newBaseObject(remote, info),
|
||||
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
||||
md5sum: strings.ToLower(info.Md5Checksum),
|
||||
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
||||
}
|
||||
if info.ResourceKey != "" {
|
||||
o.resourceKey = &info.ResourceKey
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// newDocumentObject creates an fs.Object for a google docs drive.File
|
||||
@@ -2429,11 +2434,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
|
||||
var info *drive.File
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
info, err = f.svc.Files.Copy(id, createInfo).
|
||||
copy := f.svc.Files.Copy(id, createInfo).
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
KeepRevisionForever(f.opt.KeepRevisionForever).
|
||||
Context(ctx).Do()
|
||||
KeepRevisionForever(f.opt.KeepRevisionForever)
|
||||
srcObj.addResourceKey(copy.Header())
|
||||
info, err = copy.Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -3530,6 +3536,14 @@ func (o *baseObject) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// addResourceKey adds a X-Goog-Drive-Resource-Keys header for this
|
||||
// object if required.
|
||||
func (o *baseObject) addResourceKey(header http.Header) {
|
||||
if o.resourceKey != nil {
|
||||
header.Add("X-Goog-Drive-Resource-Keys", fmt.Sprintf("%s/%s", o.id, *o.resourceKey))
|
||||
}
|
||||
}
|
||||
|
||||
// httpResponse gets an http.Response object for the object
|
||||
// using the url and method passed in
|
||||
func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
|
||||
@@ -3545,6 +3559,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
|
||||
// Don't supply range requests for 0 length objects as they always fail
|
||||
delete(req.Header, "Range")
|
||||
}
|
||||
o.addResourceKey(req.Header)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
res, err = o.fs.client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
@@ -118,12 +118,12 @@ func (b *batcher) Batching() bool {
|
||||
}
|
||||
|
||||
// finishBatch commits the batch, returning a batch status to poll or maybe complete
|
||||
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (batchStatus *files.UploadSessionFinishBatchLaunch, err error) {
|
||||
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
|
||||
var arg = &files.UploadSessionFinishBatchArg{
|
||||
Entries: items,
|
||||
}
|
||||
err = b.f.pacer.Call(func() (bool, error) {
|
||||
batchStatus, err = b.f.srv.UploadSessionFinishBatch(arg)
|
||||
complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
|
||||
// If error is insufficient space then don't retry
|
||||
if e, ok := err.(files.UploadSessionFinishAPIError); ok {
|
||||
if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
|
||||
@@ -137,7 +137,7 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch commit failed: %w", err)
|
||||
}
|
||||
return batchStatus, nil
|
||||
return complete, nil
|
||||
}
|
||||
|
||||
// finishBatchJobStatus waits for the batch to complete returning completed entries
|
||||
@@ -199,26 +199,11 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
|
||||
fs.Debugf(b.f, "Committing %s", desc)
|
||||
|
||||
// finalise the batch getting either a result or a job id to poll
|
||||
batchStatus, err := b.finishBatch(ctx, items)
|
||||
complete, err := b.finishBatch(ctx, items)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check whether batch is complete
|
||||
var complete *files.UploadSessionFinishBatchResult
|
||||
switch batchStatus.Tag {
|
||||
case "async_job_id":
|
||||
// wait for batch to complete
|
||||
complete, err = b.finishBatchJobStatus(ctx, batchStatus)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case "complete":
|
||||
complete = batchStatus.Complete
|
||||
default:
|
||||
return fmt.Errorf("batch returned unknown status %q", batchStatus.Tag)
|
||||
}
|
||||
|
||||
// Check we got the right number of entries
|
||||
entries := complete.Entries
|
||||
if len(entries) != len(results) {
|
||||
|
||||
@@ -1269,7 +1269,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
var total uint64
|
||||
if q.Allocation != nil {
|
||||
@@ -1370,10 +1370,12 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
|
||||
if timeout < 30 {
|
||||
timeout = 30
|
||||
fs.Debugf(f, "Increasing poll interval to minimum 30s")
|
||||
}
|
||||
|
||||
if timeout > 480 {
|
||||
timeout = 480
|
||||
fs.Debugf(f, "Decreasing poll interval to maximum 480s")
|
||||
}
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
|
||||
+10
-1
@@ -15,7 +15,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/ftp"
|
||||
"github.com/jlaffaye/ftp"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
@@ -100,6 +100,11 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
|
||||
Help: "Disable using MLSD even if server advertises support.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "disable_utf8",
|
||||
Help: "Disable using UTF-8 even if server advertises support.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "writing_mdtm",
|
||||
Help: "Use MDTM to set modification time (VsFtpd quirk)",
|
||||
@@ -184,6 +189,7 @@ type Options struct {
|
||||
SkipVerifyTLSCert bool `config:"no_check_certificate"`
|
||||
DisableEPSV bool `config:"disable_epsv"`
|
||||
DisableMLSD bool `config:"disable_mlsd"`
|
||||
DisableUTF8 bool `config:"disable_utf8"`
|
||||
WritingMDTM bool `config:"writing_mdtm"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
CloseTimeout fs.Duration `config:"close_timeout"`
|
||||
@@ -338,6 +344,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
|
||||
if f.opt.DisableMLSD {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
|
||||
}
|
||||
if f.opt.DisableUTF8 {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithDisabledUTF8(true))
|
||||
}
|
||||
if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -304,6 +305,21 @@ rclone does if you know the bucket exists already.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "download_compressed",
|
||||
Help: `If set this will download compressed objects as-is.
|
||||
|
||||
It is possible to upload objects to GCS with "Content-Encoding: gzip"
|
||||
set. Normally rclone will transparently decompress these files on
|
||||
download. This means that rclone can't check the hash or the size of
|
||||
the file as both of these refer to the compressed object.
|
||||
|
||||
If this flag is set then rclone will download files with
|
||||
"Content-Encoding: gzip" as they are received. This means that rclone
|
||||
can check the size and hash but the file contents will be compressed.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -327,21 +343,23 @@ type Options struct {
|
||||
Location string `config:"location"`
|
||||
StorageClass string `config:"storage_class"`
|
||||
NoCheckBucket bool `config:"no_check_bucket"`
|
||||
DownloadCompressed bool `config:"download_compressed"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote storage server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
svc *storage.Service // the connection to the storage server
|
||||
client *http.Client // authorized client
|
||||
rootBucket string // bucket part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
cache *bucket.Cache // cache of bucket status
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
svc *storage.Service // the connection to the storage server
|
||||
client *http.Client // authorized client
|
||||
rootBucket string // bucket part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
cache *bucket.Cache // cache of bucket status
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
warnCompressed sync.Once // warn once about compressed files
|
||||
}
|
||||
|
||||
// Object describes a storage object
|
||||
@@ -355,6 +373,7 @@ type Object struct {
|
||||
bytes int64 // Bytes in the object
|
||||
modTime time.Time // Modified time of the object
|
||||
mimeType string
|
||||
gzipped bool // set if object has Content-Encoding: gzip
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -975,6 +994,7 @@ func (o *Object) setMetaData(info *storage.Object) {
|
||||
o.url = info.MediaLink
|
||||
o.bytes = int64(info.Size)
|
||||
o.mimeType = info.ContentType
|
||||
o.gzipped = info.ContentEncoding == "gzip"
|
||||
|
||||
// Read md5sum
|
||||
md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
|
||||
@@ -1013,6 +1033,15 @@ func (o *Object) setMetaData(info *storage.Object) {
|
||||
} else {
|
||||
o.modTime = modTime
|
||||
}
|
||||
|
||||
// If gunzipping then size and md5sum are unknown
|
||||
if o.gzipped && !o.fs.opt.DownloadCompressed {
|
||||
o.bytes = -1
|
||||
o.md5sum = ""
|
||||
o.fs.warnCompressed.Do(func() {
|
||||
fs.Logf(o.fs, "Decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-download-compressed to override")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// readObjectInfo reads the definition for an object
|
||||
@@ -1113,6 +1142,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return nil, err
|
||||
}
|
||||
fs.FixRangeOption(options, o.bytes)
|
||||
if o.gzipped && o.fs.opt.DownloadCompressed {
|
||||
// Allow files which are stored on the cloud storage system
|
||||
// compressed to be downloaded without being decompressed. Note
|
||||
// that setting this here overrides the automatic decompression
|
||||
// in the Transport.
|
||||
//
|
||||
// See: https://cloud.google.com/storage/docs/transcoding
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
}
|
||||
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
||||
var res *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
|
||||
@@ -562,7 +562,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
|
||||
for i := range items {
|
||||
item := &result.MediaItems[i]
|
||||
remote := item.Filename
|
||||
remote = strings.Replace(remote, "/", "/", -1)
|
||||
remote = strings.ReplaceAll(remote, "/", "/")
|
||||
err = fn(remote, item, false)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -282,7 +282,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
if do := f.Fs.Features().CleanUp; do != nil {
|
||||
return do(ctx)
|
||||
}
|
||||
return errors.New("CleanUp not supported")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
@@ -290,7 +290,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
if do := f.Fs.Features().About; do != nil {
|
||||
return do(ctx)
|
||||
}
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
|
||||
// ChangeNotify calls the passed function with a path that has had changes.
|
||||
|
||||
@@ -119,7 +119,7 @@ func (f *Fs) getCredentials(ctx context.Context) (err error) {
|
||||
defer fs.CheckClose(resp.Body, &err)
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
body, _ := ioutil.ReadAll(resp.Body)
|
||||
bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
|
||||
bodyStr := strings.TrimSpace(strings.ReplaceAll(string(body), "\n", " "))
|
||||
return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
|
||||
}
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,17 @@
|
||||
// Test internetarchive filesystem interface
|
||||
package internetarchive_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/internetarchive"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestIA:lesmi-rclone-test/",
|
||||
NilObject: (*internetarchive.Object)(nil),
|
||||
})
|
||||
}
|
||||
@@ -8,42 +8,69 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// default time format for almost all request and responses
|
||||
timeFormat = "2006-01-02-T15:04:05Z0700"
|
||||
// the API server seems to use a different format
|
||||
apiTimeFormat = "2006-01-02T15:04:05Z07:00"
|
||||
// default time format historically used for all request and responses.
|
||||
// Similar to time.RFC3339, but with an extra '-' in front of 'T',
|
||||
// and no ':' separator in timezone offset. Some newer endpoints have
|
||||
// moved to proper time.RFC3339 conformant format instead.
|
||||
jottaTimeFormat = "2006-01-02-T15:04:05Z0700"
|
||||
)
|
||||
|
||||
// Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
|
||||
type Time time.Time
|
||||
|
||||
// UnmarshalXML turns XML into a Time
|
||||
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
// unmarshalXML turns XML into a Time
|
||||
func unmarshalXMLTime(d *xml.Decoder, start xml.StartElement, timeFormat string) (time.Time, error) {
|
||||
var v string
|
||||
if err := d.DecodeElement(&v, &start); err != nil {
|
||||
return err
|
||||
return time.Time{}, err
|
||||
}
|
||||
if v == "" {
|
||||
*t = Time(time.Time{})
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
}
|
||||
newTime, err := time.Parse(timeFormat, v)
|
||||
if err == nil {
|
||||
*t = Time(newTime)
|
||||
return newTime, nil
|
||||
}
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
// JottaTime represents time values in the classic API using a custom RFC3339 like format
|
||||
type JottaTime time.Time
|
||||
|
||||
// String returns JottaTime string in Jottacloud classic format
|
||||
func (t JottaTime) String() string { return time.Time(t).Format(jottaTimeFormat) }
|
||||
|
||||
// UnmarshalXML turns XML into a JottaTime
|
||||
func (t *JottaTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
tm, err := unmarshalXMLTime(d, start, jottaTimeFormat)
|
||||
*t = JottaTime(tm)
|
||||
return err
|
||||
}
|
||||
|
||||
// MarshalXML turns a Time into XML
|
||||
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
// MarshalXML turns a JottaTime into XML
|
||||
func (t *JottaTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return e.EncodeElement(t.String(), start)
|
||||
}
|
||||
|
||||
// Return Time string in Jottacloud format
|
||||
func (t Time) String() string { return time.Time(t).Format(timeFormat) }
|
||||
// Rfc3339Time represents time values in the newer APIs using standard RFC3339 format
|
||||
type Rfc3339Time time.Time
|
||||
|
||||
// APIString returns Time string in Jottacloud API format
|
||||
func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
|
||||
// String returns Rfc3339Time string in Jottacloud RFC3339 format
|
||||
func (t Rfc3339Time) String() string { return time.Time(t).Format(time.RFC3339) }
|
||||
|
||||
// UnmarshalXML turns XML into a Rfc3339Time
|
||||
func (t *Rfc3339Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
tm, err := unmarshalXMLTime(d, start, time.RFC3339)
|
||||
*t = Rfc3339Time(tm)
|
||||
return err
|
||||
}
|
||||
|
||||
// MarshalXML turns a Rfc3339Time into XML
|
||||
func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return e.EncodeElement(t.String(), start)
|
||||
}
|
||||
|
||||
// MarshalJSON turns a Rfc3339Time into JSON
|
||||
func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
|
||||
return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
|
||||
}
|
||||
|
||||
// LoginToken is struct representing the login token generated in the WebUI
|
||||
type LoginToken struct {
|
||||
@@ -122,16 +149,11 @@ type AllocateFileResponse struct {
|
||||
|
||||
// UploadResponse after an upload
|
||||
type UploadResponse struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
ContentID string `json:"content_id"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Md5 string `json:"md5"`
|
||||
Created int64 `json:"created"`
|
||||
Modified int64 `json:"modified"`
|
||||
Deleted interface{} `json:"deleted"`
|
||||
Mime string `json:"mime"`
|
||||
Path string `json:"path"`
|
||||
ContentID string `json:"content_id"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Md5 string `json:"md5"`
|
||||
Modified int64 `json:"modified"`
|
||||
}
|
||||
|
||||
// DeviceRegistrationResponse is the response to registering a device
|
||||
@@ -338,9 +360,9 @@ type JottaFolder struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Deleted Flag `xml:"deleted,attr"`
|
||||
Path string `xml:"path"`
|
||||
CreatedAt Time `xml:"created"`
|
||||
ModifiedAt Time `xml:"modified"`
|
||||
Updated Time `xml:"updated"`
|
||||
CreatedAt JottaTime `xml:"created"`
|
||||
ModifiedAt JottaTime `xml:"modified"`
|
||||
Updated JottaTime `xml:"updated"`
|
||||
Folders []JottaFolder `xml:"folders>folder"`
|
||||
Files []JottaFile `xml:"files>file"`
|
||||
}
|
||||
@@ -365,17 +387,17 @@ GET http://www.jottacloud.com/JFS/<account>/<device>/<mountpoint>/.../<file>
|
||||
// JottaFile represents a Jottacloud file
|
||||
type JottaFile struct {
|
||||
XMLName xml.Name
|
||||
Name string `xml:"name,attr"`
|
||||
Deleted Flag `xml:"deleted,attr"`
|
||||
PublicURI string `xml:"publicURI"`
|
||||
PublicSharePath string `xml:"publicSharePath"`
|
||||
State string `xml:"currentRevision>state"`
|
||||
CreatedAt Time `xml:"currentRevision>created"`
|
||||
ModifiedAt Time `xml:"currentRevision>modified"`
|
||||
Updated Time `xml:"currentRevision>updated"`
|
||||
Size int64 `xml:"currentRevision>size"`
|
||||
MimeType string `xml:"currentRevision>mime"`
|
||||
MD5 string `xml:"currentRevision>md5"`
|
||||
Name string `xml:"name,attr"`
|
||||
Deleted Flag `xml:"deleted,attr"`
|
||||
PublicURI string `xml:"publicURI"`
|
||||
PublicSharePath string `xml:"publicSharePath"`
|
||||
State string `xml:"currentRevision>state"`
|
||||
CreatedAt JottaTime `xml:"currentRevision>created"`
|
||||
ModifiedAt JottaTime `xml:"currentRevision>modified"`
|
||||
Updated JottaTime `xml:"currentRevision>updated"`
|
||||
Size int64 `xml:"currentRevision>size"`
|
||||
MimeType string `xml:"currentRevision>mime"`
|
||||
MD5 string `xml:"currentRevision>md5"`
|
||||
}
|
||||
|
||||
// Error is a custom Error for wrapping Jottacloud error responses
|
||||
|
||||
@@ -191,7 +191,7 @@ machines.`)
|
||||
m.Set("auth_code", "")
|
||||
return fs.ConfigGoto("legacy_do_auth")
|
||||
case "legacy_auth_code":
|
||||
authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
|
||||
authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
|
||||
m.Set("auth_code", authCode)
|
||||
return fs.ConfigGoto("legacy_do_auth")
|
||||
case "legacy_do_auth":
|
||||
@@ -519,7 +519,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
|
||||
values.Set("client_id", defaultClientID)
|
||||
values.Set("grant_type", "password")
|
||||
values.Set("password", loginToken.AuthToken)
|
||||
values.Set("scope", "offline_access+openid")
|
||||
values.Set("scope", "openid offline_access")
|
||||
values.Set("username", loginToken.Username)
|
||||
values.Encode()
|
||||
opts = rest.Opts{
|
||||
@@ -649,7 +649,7 @@ func errorHandler(resp *http.Response) error {
|
||||
|
||||
// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
|
||||
func urlPathEscape(in string) string {
|
||||
return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
|
||||
return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
|
||||
}
|
||||
|
||||
// filePathRaw returns an unescaped file path (f.root, file)
|
||||
@@ -932,26 +932,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
type listStreamTime time.Time
|
||||
|
||||
func (c *listStreamTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
var v string
|
||||
if err := d.DecodeElement(&v, &start); err != nil {
|
||||
return err
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*c = listStreamTime(t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c listStreamTime) MarshalJSON() ([]byte, error) {
|
||||
return []byte(fmt.Sprintf("\"%s\"", time.Time(c).Format(time.RFC3339))), nil
|
||||
}
|
||||
|
||||
func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, filesystem *Fs, callback func(fs.DirEntry) error) error {
|
||||
func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
|
||||
|
||||
type stats struct {
|
||||
Folders int `xml:"folders"`
|
||||
@@ -960,12 +941,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
|
||||
var expected, actual stats
|
||||
|
||||
type xmlFile struct {
|
||||
Path string `xml:"path"`
|
||||
Name string `xml:"filename"`
|
||||
Checksum string `xml:"md5"`
|
||||
Size int64 `xml:"size"`
|
||||
Modified listStreamTime `xml:"modified"`
|
||||
Created listStreamTime `xml:"created"`
|
||||
Path string `xml:"path"`
|
||||
Name string `xml:"filename"`
|
||||
Checksum string `xml:"md5"`
|
||||
Size int64 `xml:"size"`
|
||||
Modified api.Rfc3339Time `xml:"modified"` // Note: Liststream response includes 3 decimal milliseconds, but we ignore them since there is second precision everywhere else
|
||||
Created api.Rfc3339Time `xml:"created"`
|
||||
}
|
||||
|
||||
type xmlFolder struct {
|
||||
@@ -987,8 +968,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
|
||||
})
|
||||
}
|
||||
|
||||
// liststream paths are /mountpoint/root/path
|
||||
// so the returned paths should have /mountpoint/root/ trimmed
|
||||
// as the caller is expecting path.
|
||||
pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
|
||||
trimPathPrefix := func(p string) string {
|
||||
p = strings.TrimPrefix(p, trimPrefix)
|
||||
p = strings.TrimPrefix(p, pathPrefix)
|
||||
p = strings.TrimPrefix(p, "/")
|
||||
return p
|
||||
}
|
||||
@@ -1071,11 +1056,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
return shouldRetry(ctx, resp, err)
|
||||
}
|
||||
|
||||
// liststream paths are /mountpoint/root/path
|
||||
// so the returned paths should have /mountpoint/root/ trimmed
|
||||
// as the caller is expecting path.
|
||||
trimPrefix := path.Join("/", f.opt.Mountpoint, f.root)
|
||||
err = parseListRStream(ctx, resp.Body, trimPrefix, f, func(d fs.DirEntry) error {
|
||||
err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
|
||||
if d.Remote() == dir {
|
||||
return nil
|
||||
}
|
||||
@@ -1210,6 +1191,45 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
return f.purgeCheck(ctx, dir, false)
|
||||
}
|
||||
|
||||
// createOrUpdate tries to make remote file match without uploading.
|
||||
// If the remote file exists, and has matching size and md5, only
|
||||
// timestamps are updated. If the file does not exist or does does
|
||||
// not match size and md5, but matching content can be constructed
|
||||
// from deduplication, the file will be updated/created. If the file
|
||||
// is currently in trash, but can be made to match, it will be
|
||||
// restored. Returns ErrorObjectNotFound if upload will be necessary
|
||||
// to get a matching remote file.
|
||||
func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: f.filePath(file),
|
||||
Parameters: url.Values{},
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
|
||||
opts.Parameters.Set("cphash", "true")
|
||||
|
||||
fileDate := api.JottaTime(modTime).String()
|
||||
opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
|
||||
opts.ExtraHeaders["JMd5"] = md5
|
||||
opts.ExtraHeaders["JCreated"] = fileDate
|
||||
opts.ExtraHeaders["JModified"] = fileDate
|
||||
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
// does not exist, i.e. not matching size and md5, and not possible to make it by deduplication
|
||||
if apiErr.StatusCode == http.StatusNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// copyOrMoves copies or moves directories or files depending on the method parameter
|
||||
func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
|
||||
opts := rest.Opts{
|
||||
@@ -1253,6 +1273,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
|
||||
|
||||
// if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?)
|
||||
if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
|
||||
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
|
||||
info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
@@ -1554,40 +1580,19 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// prepare allocate request with existing metadata but changed timestamps
|
||||
var resp *http.Response
|
||||
var options []fs.OpenOption
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "files/v1/allocate",
|
||||
Options: options,
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
fileDate := api.Time(modTime).APIString()
|
||||
var request = api.AllocateFileRequest{
|
||||
Bytes: o.size,
|
||||
Created: fileDate,
|
||||
Modified: fileDate,
|
||||
Md5: o.md5,
|
||||
Path: path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
|
||||
}
|
||||
|
||||
// send it
|
||||
var response api.AllocateFileResponse
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
// request check/update with existing metadata and new modtime
|
||||
// (note that if size/md5 does not match, the file content will
|
||||
// also be modified if deduplication is possible, i.e. it is
|
||||
// important to use correct/latest values)
|
||||
_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
|
||||
return errors.New("metadata did not match")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// check response
|
||||
if response.State != "COMPLETED" {
|
||||
// could be the file was modified (size/md5 changed) between readMetaData and the allocate request
|
||||
return errors.New("metadata did not match")
|
||||
}
|
||||
|
||||
// update local metadata
|
||||
o.modTime = modTime
|
||||
return nil
|
||||
@@ -1725,7 +1730,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
fileDate := api.Time(src.ModTime(ctx)).APIString()
|
||||
fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
|
||||
|
||||
// the allocate request
|
||||
var request = api.AllocateFileRequest{
|
||||
|
||||
@@ -301,6 +301,10 @@ type siteResource struct {
|
||||
type siteResponse struct {
|
||||
Sites []siteResource `json:"value"`
|
||||
}
|
||||
type deltaResponse struct {
|
||||
DeltaLink string `json:"@odata.deltaLink"`
|
||||
Value []api.Item `json:"value"`
|
||||
}
|
||||
|
||||
// Get the region and graphURL from the config
|
||||
func getRegionURL(m configmap.Mapper) (region, graphURL string) {
|
||||
@@ -1502,7 +1506,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
q := drive.Quota
|
||||
// On (some?) Onedrive sharepoints these are all 0 so return unknown in that case
|
||||
@@ -2302,6 +2306,142 @@ func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
|
||||
return canonicalDriveID
|
||||
}
|
||||
|
||||
// ChangeNotify calls the passed function with a path that has had changes.
|
||||
// If the implementation uses polling, it should adhere to the given interval.
|
||||
//
|
||||
// Automatically restarts itself in case of unexpected behavior of the remote.
|
||||
//
|
||||
// Close the returned channel to stop being notified.
|
||||
//
|
||||
// The Onedrive implementation gives the whole hierarchy up to the top when
|
||||
// an object is changed. For instance, if a/b/c is changed, this function
|
||||
// will call notifyFunc with a, a/b and a/b/c.
|
||||
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
|
||||
go func() {
|
||||
// get the StartPageToken early so all changes from now on get processed
|
||||
nextDeltaToken, err := f.changeNotifyStartPageToken(ctx)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get first deltaLink: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
fs.Debugf(f, "Next delta token is: %s", nextDeltaToken)
|
||||
|
||||
var ticker *time.Ticker
|
||||
var tickerC <-chan time.Time
|
||||
for {
|
||||
select {
|
||||
case pollInterval, ok := <-pollIntervalChan:
|
||||
if !ok {
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
}
|
||||
return
|
||||
}
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
ticker, tickerC = nil, nil
|
||||
}
|
||||
if pollInterval != 0 {
|
||||
ticker = time.NewTicker(pollInterval)
|
||||
tickerC = ticker.C
|
||||
}
|
||||
case <-tickerC:
|
||||
fs.Debugf(f, "Checking for changes on remote")
|
||||
nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
|
||||
if err != nil {
|
||||
fs.Infof(f, "Change notify listener failure: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, "latest")
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
nextDeltaToken = parsedURL.Query().Get("token")
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta deltaResponse, err error) {
|
||||
opts := f.buildDriveDeltaOpts(token)
|
||||
|
||||
_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
|
||||
rootURL := graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
|
||||
|
||||
return rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: rootURL,
|
||||
Path: "/" + f.driveID + "/root/delta",
|
||||
Parameters: map[string][]string{"token": {token}},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, deltaToken)
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
nextDeltaToken = parsedURL.Query().Get("token")
|
||||
|
||||
for _, item := range delta.Value {
|
||||
isDriveRootFolder := item.GetParentReference().ID == ""
|
||||
if isDriveRootFolder {
|
||||
continue
|
||||
}
|
||||
|
||||
fullPath, err := getItemFullPath(&item)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get item full path: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if fullPath == f.root {
|
||||
continue
|
||||
}
|
||||
|
||||
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
|
||||
if !insideRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
if item.GetFile() != nil {
|
||||
notifyFunc(relName, fs.EntryObject)
|
||||
} else if item.GetFolder() != nil {
|
||||
notifyFunc(relName, fs.EntryDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func getItemFullPath(item *api.Item) (fullPath string, err error) {
|
||||
err = nil
|
||||
fullPath = item.GetName()
|
||||
if parent := item.GetParentReference(); parent != nil && parent.Path != "" {
|
||||
pathParts := strings.SplitN(parent.Path, ":", 2)
|
||||
if len(pathParts) != 2 {
|
||||
err = fmt.Errorf("invalid parent path: %s", parent.Path)
|
||||
return
|
||||
}
|
||||
|
||||
if pathParts[1] != "" {
|
||||
fullPath = strings.TrimPrefix(pathParts[1], "/") + "/" + fullPath
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// getRelativePathInsideBase checks if `target` is inside `base`. If so, it
|
||||
// returns a relative path for `target` based on `base` and a boolean `true`.
|
||||
// Otherwise returns "", false.
|
||||
|
||||
@@ -906,7 +906,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
usage = &fs.Usage{
|
||||
Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used
|
||||
|
||||
@@ -783,10 +783,10 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("CreateDir http: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
if err = info.AsErr(); err != nil {
|
||||
return nil, fmt.Errorf("CreateDir: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
usage = &fs.Usage{
|
||||
Used: fs.NewUsageValue(int64(info.SpaceUsed)),
|
||||
|
||||
+1
-1
@@ -647,7 +647,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return &fs.Usage{
|
||||
Total: fs.NewUsageValue(ai.Disk.Size), // quota of bytes that can be used
|
||||
|
||||
+258
-91
@@ -32,9 +32,11 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/endpoints"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/ncw/swift/v2"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
@@ -58,7 +60,7 @@ import (
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "s3",
|
||||
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
|
||||
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Options: []fs.Option{{
|
||||
@@ -78,6 +80,12 @@ func init() {
|
||||
}, {
|
||||
Value: "ChinaMobile",
|
||||
Help: "China Mobile Ecloud Elastic Object Storage (EOS)",
|
||||
}, {
|
||||
Value: "Cloudflare",
|
||||
Help: "Cloudflare R2 Storage",
|
||||
}, {
|
||||
Value: "ArvanCloud",
|
||||
Help: "Arvan Cloud Object Storage (AOS)",
|
||||
}, {
|
||||
Value: "DigitalOcean",
|
||||
Help: "Digital Ocean Spaces",
|
||||
@@ -293,11 +301,22 @@ func init() {
|
||||
}, {
|
||||
Value: "fr-par",
|
||||
Help: "Paris, France",
|
||||
}, {
|
||||
Value: "pl-waw",
|
||||
Help: "Warsaw, Poland",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.",
|
||||
Provider: "Cloudflare",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "auto",
|
||||
Help: "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
|
||||
Provider: "!AWS,Alibaba,ChinaMobile,RackCorp,Scaleway,Storj,TencentCOS",
|
||||
Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
|
||||
@@ -405,6 +424,18 @@ func init() {
|
||||
Value: "eos-anhui-1.cmecloud.cn",
|
||||
Help: "Anhui China (Huainan)",
|
||||
}},
|
||||
}, {
|
||||
// ArvanCloud endpoints: https://www.arvancloud.com/en/products/cloud-storage
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for Arvan Cloud Object Storage (AOS) API.",
|
||||
Provider: "ArvanCloud",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "s3.ir-thr-at1.arvanstorage.com",
|
||||
Help: "The default endpoint - a good choice if you are unsure.\nTehran Iran (Asiatech)",
|
||||
}, {
|
||||
Value: "s3.ir-tbz-sh1.arvanstorage.com",
|
||||
Help: "Tabriz Iran (Shahriar)",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for IBM COS S3 API.\n\nSpecify if using an IBM COS On Premise.",
|
||||
@@ -687,6 +718,9 @@ func init() {
|
||||
}, {
|
||||
Value: "s3.fr-par.scw.cloud",
|
||||
Help: "Paris Endpoint",
|
||||
}, {
|
||||
Value: "s3.pl-waw.scw.cloud",
|
||||
Help: "Warsaw Endpoint",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
@@ -845,7 +879,7 @@ func init() {
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
|
||||
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,Scaleway,StackPath,Storj,RackCorp",
|
||||
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "objects-us-east-1.dream.io",
|
||||
Help: "Dream Objects endpoint",
|
||||
@@ -898,6 +932,10 @@ func init() {
|
||||
Value: "s3.ap-northeast-2.wasabisys.com",
|
||||
Help: "Wasabi AP Northeast 2 (Osaka) endpoint",
|
||||
Provider: "Wasabi",
|
||||
}, {
|
||||
Value: "s3.ir-thr-at1.arvanstorage.com",
|
||||
Help: "ArvanCloud Tehran Iran (Asiatech) endpoint",
|
||||
Provider: "ArvanCloud",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
@@ -1074,6 +1112,17 @@ func init() {
|
||||
Value: "anhui1",
|
||||
Help: "Anhui China (Huainan)",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must match endpoint.\n\nUsed when creating buckets only.",
|
||||
Provider: "ArvanCloud",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "ir-thr-at1",
|
||||
Help: "Tehran Iran (Asiatech)",
|
||||
}, {
|
||||
Value: "ir-tbz-sh1",
|
||||
Help: "Tabriz Iran (Shahriar)",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must match endpoint when using IBM Cloud Public.\n\nFor on-prem COS, do not make a selection from this list, hit enter.",
|
||||
@@ -1240,7 +1289,7 @@ func init() {
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
|
||||
}, {
|
||||
Name: "acl",
|
||||
Help: `Canned ACL used when creating buckets and storing or copying objects.
|
||||
@@ -1251,7 +1300,7 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
|
||||
|
||||
Note that this ACL is applied when server-side copying objects as S3
|
||||
doesn't copy the ACL from the source but rather writes a fresh one.`,
|
||||
Provider: "!Storj",
|
||||
Provider: "!Storj,Cloudflare",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "default",
|
||||
Help: "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
|
||||
@@ -1452,6 +1501,15 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
|
||||
Value: "STANDARD_IA",
|
||||
Help: "Infrequent access storage mode",
|
||||
}},
|
||||
}, {
|
||||
// Mapping from here: https://www.arvancloud.com/en/products/cloud-storage
|
||||
Name: "storage_class",
|
||||
Help: "The storage class to use when storing new objects in ArvanCloud.",
|
||||
Provider: "ArvanCloud",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "STANDARD",
|
||||
Help: "Standard storage class",
|
||||
}},
|
||||
}, {
|
||||
// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
|
||||
Name: "storage_class",
|
||||
@@ -1784,6 +1842,20 @@ This should be true, false or left unset to use the default for the provider.
|
||||
`,
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_presigned_request",
|
||||
Help: `Whether to use a presigned request or PutObject for single part uploads
|
||||
|
||||
If this is false rclone will use PutObject from the AWS SDK to upload
|
||||
an object.
|
||||
|
||||
Versions of rclone < 1.59 use presigned requests to upload a single
|
||||
part object and setting this flag to true will re-enable that
|
||||
functionality. This shouldn't be necessary except in exceptional
|
||||
circumstances or for testing.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
}})
|
||||
}
|
||||
@@ -1849,6 +1921,7 @@ type Options struct {
|
||||
DisableHTTP2 bool `config:"disable_http2"`
|
||||
DownloadURL string `config:"download_url"`
|
||||
UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`
|
||||
UsePresignedRequest bool `config:"use_presigned_request"`
|
||||
}
|
||||
|
||||
// Fs represents a remote s3 server
|
||||
@@ -1860,6 +1933,7 @@ type Fs struct {
|
||||
ctx context.Context // global context for reading config
|
||||
features *fs.Features // optional features
|
||||
c *s3.S3 // the connection to the s3 server
|
||||
cu *s3.S3 // unsigned connection to the s3 server for PutObject
|
||||
ses *session.Session // the s3 session
|
||||
rootBucket string // bucket part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
@@ -1992,7 +2066,11 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
|
||||
}
|
||||
|
||||
// s3Connection makes a connection to s3
|
||||
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
|
||||
//
|
||||
// If unsignedBody is set then the connection is configured for
|
||||
// unsigned bodies which is necessary for PutObject if we don't want
|
||||
// it to seek
|
||||
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
// Make the auth
|
||||
v := credentials.Value{
|
||||
@@ -2009,7 +2087,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
// start a new AWS session
|
||||
awsSession, err := session.NewSession()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("NewSession: %w", err)
|
||||
return nil, nil, nil, fmt.Errorf("NewSession: %w", err)
|
||||
}
|
||||
|
||||
// first provider to supply a credential set "wins"
|
||||
@@ -2049,9 +2127,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
|
||||
cred = credentials.AnonymousCredentials
|
||||
case v.AccessKeyID == "":
|
||||
return nil, nil, errors.New("access_key_id not found")
|
||||
return nil, nil, nil, errors.New("access_key_id not found")
|
||||
case v.SecretAccessKey == "":
|
||||
return nil, nil, errors.New("secret_access_key not found")
|
||||
return nil, nil, nil, errors.New("secret_access_key not found")
|
||||
}
|
||||
|
||||
if opt.Region == "" {
|
||||
@@ -2090,25 +2168,36 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
|
||||
awsSessionOpts.Config.Credentials = nil
|
||||
}
|
||||
// Setting this stops PutObject reading the body twice and seeking
|
||||
// We add our own Content-MD5 for data protection
|
||||
awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true)
|
||||
ses, err := session.NewSessionWithOptions(awsSessionOpts)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
c := s3.New(ses)
|
||||
if opt.V2Auth || opt.Region == "other-v2-signature" {
|
||||
fs.Debugf(nil, "Using v2 auth")
|
||||
signer := func(req *request.Request) {
|
||||
// Ignore AnonymousCredentials object
|
||||
if req.Config.Credentials == credentials.AnonymousCredentials {
|
||||
return
|
||||
newC := func(unsignedBody bool) *s3.S3 {
|
||||
c := s3.New(ses)
|
||||
if opt.V2Auth || opt.Region == "other-v2-signature" {
|
||||
fs.Debugf(nil, "Using v2 auth")
|
||||
signer := func(req *request.Request) {
|
||||
// Ignore AnonymousCredentials object
|
||||
if req.Config.Credentials == credentials.AnonymousCredentials {
|
||||
return
|
||||
}
|
||||
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
|
||||
}
|
||||
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
|
||||
c.Handlers.Sign.Clear()
|
||||
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
|
||||
c.Handlers.Sign.PushBack(signer)
|
||||
} else if unsignedBody {
|
||||
// If the body is unsigned then tell the signer that we aren't signing the payload
|
||||
c.Handlers.Sign.Clear()
|
||||
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
|
||||
c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload))
|
||||
}
|
||||
c.Handlers.Sign.Clear()
|
||||
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
|
||||
c.Handlers.Sign.PushBack(signer)
|
||||
return c
|
||||
}
|
||||
return c, ses, nil
|
||||
return newC(false), newC(true), ses, nil
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
@@ -2167,6 +2256,13 @@ func setQuirks(opt *Options) {
|
||||
listObjectsV2 = false
|
||||
virtualHostStyle = false
|
||||
urlEncodeListings = false
|
||||
case "Cloudflare":
|
||||
virtualHostStyle = false
|
||||
useMultipartEtag = false // currently multipart Etags are random
|
||||
case "ArvanCloud":
|
||||
listObjectsV2 = false
|
||||
virtualHostStyle = false
|
||||
urlEncodeListings = false
|
||||
case "DigitalOcean":
|
||||
urlEncodeListings = false
|
||||
case "Dreamhost":
|
||||
@@ -2286,7 +2382,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
|
||||
}
|
||||
srv := getClient(ctx, opt)
|
||||
c, ses, err := s3Connection(ctx, opt, srv)
|
||||
c, cu, ses, err := s3Connection(ctx, opt, srv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -2304,6 +2400,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ci: ci,
|
||||
ctx: ctx,
|
||||
c: c,
|
||||
cu: cu,
|
||||
ses: ses,
|
||||
pacer: pc,
|
||||
cache: bucket.NewCache(),
|
||||
@@ -2427,11 +2524,12 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
|
||||
// Make a new session with the new region
|
||||
oldRegion := f.opt.Region
|
||||
f.opt.Region = region
|
||||
c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
|
||||
c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating new session failed: %w", err)
|
||||
}
|
||||
f.c = c
|
||||
f.cu = cu
|
||||
f.ses = ses
|
||||
|
||||
fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
|
||||
@@ -2874,7 +2972,7 @@ func (f *Fs) Precision() time.Duration {
|
||||
// pathEscape escapes s as for a URL path. It uses rest.URLPathEscape
|
||||
// but also escapes '+' for S3 and Digital Ocean spaces compatibility
|
||||
func pathEscape(s string) string {
|
||||
return strings.Replace(rest.URLPathEscape(s), "+", "%2B", -1)
|
||||
return strings.ReplaceAll(rest.URLPathEscape(s), "+", "%2B")
|
||||
}
|
||||
|
||||
// copy does a server-side copy
|
||||
@@ -3247,6 +3345,10 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
st.Status = "Not an S3 object"
|
||||
return
|
||||
}
|
||||
if o.storageClass != "GLACIER" && o.storageClass != "DEEP_ARCHIVE" {
|
||||
st.Status = "Not GLACIER or DEEP_ARCHIVE storage class"
|
||||
return
|
||||
}
|
||||
bucket, bucketPath := o.split()
|
||||
reqCopy := req
|
||||
reqCopy.Bucket = &bucket
|
||||
@@ -3743,7 +3845,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
}
|
||||
|
||||
// calculate size of parts
|
||||
partSize := int(f.opt.ChunkSize)
|
||||
partSize := f.opt.ChunkSize
|
||||
|
||||
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
|
||||
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
|
||||
@@ -3754,11 +3856,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
|
||||
})
|
||||
} else {
|
||||
// Adjust partSize until the number of parts is small enough.
|
||||
if size/int64(partSize) >= uploadParts {
|
||||
// Calculate partition size rounded up to the nearest MiB
|
||||
partSize = int((((size / uploadParts) >> 20) + 1) << 20)
|
||||
}
|
||||
partSize = chunksize.Calculator(o, int(uploadParts), f.opt.ChunkSize)
|
||||
}
|
||||
|
||||
memPool := f.getMemoryPool(int64(partSize))
|
||||
@@ -3928,6 +4026,117 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
return etag, nil
|
||||
}
|
||||
|
||||
// unWrapAwsError unwraps AWS errors, looking for a non AWS error
|
||||
//
|
||||
// It returns true if one was found and the error, or false and the
|
||||
// error passed in.
|
||||
func unWrapAwsError(err error) (found bool, outErr error) {
|
||||
if awsErr, ok := err.(awserr.Error); ok {
|
||||
var origErrs []error
|
||||
if batchErr, ok := awsErr.(awserr.BatchError); ok {
|
||||
origErrs = batchErr.OrigErrs()
|
||||
} else {
|
||||
origErrs = []error{awsErr.OrigErr()}
|
||||
}
|
||||
for _, origErr := range origErrs {
|
||||
found, newErr := unWrapAwsError(origErr)
|
||||
if found {
|
||||
return found, newErr
|
||||
}
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, err
|
||||
}
|
||||
|
||||
// Upload a single part using PutObject
|
||||
func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
|
||||
req.Body = readers.NewFakeSeeker(in, size)
|
||||
var resp *s3.PutObjectOutput
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.cu.PutObject(req)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
// Return the underlying error if we have a Serialization error if possible
|
||||
//
|
||||
// Serialization errors are synthesized locally in the SDK (not returned from the
|
||||
// server). We'll get one if the SDK attempts a retry, however the FakeSeeker will
|
||||
// remember the previous error from Read and return that.
|
||||
if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization {
|
||||
if found, newErr := unWrapAwsError(err); found {
|
||||
err = newErr
|
||||
}
|
||||
}
|
||||
return etag, lastModified, err
|
||||
}
|
||||
lastModified = time.Now()
|
||||
etag = aws.StringValue(resp.ETag)
|
||||
return etag, lastModified, nil
|
||||
}
|
||||
|
||||
// Upload a single part using a presigned request
|
||||
func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
|
||||
// Create the request
|
||||
putObj, _ := o.fs.c.PutObjectRequest(req)
|
||||
|
||||
// Sign it so we can upload using a presigned request.
|
||||
//
|
||||
// Note the SDK didn't used to support streaming to
|
||||
// PutObject so we used this work-around.
|
||||
url, headers, err := putObj.PresignRequest(15 * time.Minute)
|
||||
if err != nil {
|
||||
return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err)
|
||||
}
|
||||
|
||||
if o.fs.opt.V2Auth && headers == nil {
|
||||
headers = putObj.HTTPRequest.Header
|
||||
}
|
||||
|
||||
// Set request to nil if empty so as not to make chunked encoding
|
||||
if size == 0 {
|
||||
in = nil
|
||||
}
|
||||
|
||||
// create the vanilla http request
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
|
||||
if err != nil {
|
||||
return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err)
|
||||
}
|
||||
|
||||
// set the headers we signed and the length
|
||||
httpReq.Header = headers
|
||||
httpReq.ContentLength = size
|
||||
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = o.fs.srv.Do(httpReq)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
|
||||
return false, nil
|
||||
}
|
||||
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
|
||||
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
})
|
||||
if err != nil {
|
||||
return etag, lastModified, err
|
||||
}
|
||||
if resp != nil {
|
||||
if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil {
|
||||
lastModified = date
|
||||
}
|
||||
etag = resp.Header.Get("Etag")
|
||||
}
|
||||
return etag, lastModified, nil
|
||||
}
|
||||
|
||||
// Update the Object from in with modTime and size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
bucket, bucketPath := o.split()
|
||||
@@ -3979,6 +4188,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
ContentType: &mimeType,
|
||||
Metadata: metadata,
|
||||
}
|
||||
if size >= 0 {
|
||||
req.ContentLength = &size
|
||||
}
|
||||
if md5sumBase64 != "" {
|
||||
req.ContentMD5 = &md5sumBase64
|
||||
}
|
||||
@@ -4033,66 +4245,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
var resp *http.Response // response from PUT
|
||||
var wantETag string // Multipart upload Etag to check
|
||||
var wantETag string // Multipart upload Etag to check
|
||||
var gotEtag string // Etag we got from the upload
|
||||
var lastModified time.Time // Time we got from the upload
|
||||
if multipart {
|
||||
wantETag, err = o.uploadMultipart(ctx, &req, size, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
|
||||
// Create the request
|
||||
putObj, _ := o.fs.c.PutObjectRequest(&req)
|
||||
|
||||
// Sign it so we can upload using a presigned request.
|
||||
//
|
||||
// Note the SDK doesn't currently support streaming to
|
||||
// PutObject so we'll use this work-around.
|
||||
url, headers, err := putObj.PresignRequest(15 * time.Minute)
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3 upload: sign request: %w", err)
|
||||
}
|
||||
|
||||
if o.fs.opt.V2Auth && headers == nil {
|
||||
headers = putObj.HTTPRequest.Header
|
||||
}
|
||||
|
||||
// Set request to nil if empty so as not to make chunked encoding
|
||||
if size == 0 {
|
||||
in = nil
|
||||
}
|
||||
|
||||
// create the vanilla http request
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3 upload: new request: %w", err)
|
||||
}
|
||||
|
||||
// set the headers we signed and the length
|
||||
httpReq.Header = headers
|
||||
httpReq.ContentLength = size
|
||||
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = o.fs.srv.Do(httpReq)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
|
||||
return false, nil
|
||||
}
|
||||
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
|
||||
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
if o.fs.opt.UsePresignedRequest {
|
||||
gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
|
||||
} else {
|
||||
gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// User requested we don't HEAD the object after uploading it
|
||||
// so make up the object as best we can assuming it got
|
||||
@@ -4105,11 +4272,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
o.mimeType = aws.StringValue(req.ContentType)
|
||||
o.storageClass = aws.StringValue(req.StorageClass)
|
||||
// If we have done a single part PUT request then we can read these
|
||||
if resp != nil {
|
||||
if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
|
||||
o.lastModified = date
|
||||
}
|
||||
o.setMD5FromEtag(resp.Header.Get("Etag"))
|
||||
if gotEtag != "" {
|
||||
o.setMD5FromEtag(gotEtag)
|
||||
}
|
||||
if !o.lastModified.IsZero() {
|
||||
o.lastModified = lastModified
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -109,7 +109,7 @@ when the ssh-agent contains many keys.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "use_insecure_cipher",
|
||||
Help: `Enable the use of insecure ciphers and key exchange methods.
|
||||
Help: `Enable the use of insecure ciphers and key exchange methods.
|
||||
|
||||
This enables the use of the following insecure ciphers and key exchange methods:
|
||||
|
||||
@@ -1220,7 +1220,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
}
|
||||
stdout, err := f.run(ctx, "df -k "+escapedPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("your remote may not support About: %w", err)
|
||||
return nil, fmt.Errorf("your remote may not have the required df utility: %w", err)
|
||||
}
|
||||
|
||||
usageTotal, usageUsed, usageAvail := parseUsage(stdout)
|
||||
@@ -1311,7 +1311,7 @@ var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\
|
||||
// when sending it to a shell.
|
||||
func shellEscape(str string) string {
|
||||
safe := shellEscapeRegex.ReplaceAllString(str, `\$0`)
|
||||
return strings.Replace(safe, "\n", "'\n'", -1)
|
||||
return strings.ReplaceAll(safe, "\n", "'\n'")
|
||||
}
|
||||
|
||||
// Converts a byte array from the SSH session returned by
|
||||
|
||||
+1
-1
@@ -579,7 +579,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return newObjectFromUplink(f, "", upload.Info()), nil
|
||||
return newObjectFromUplink(f, src.Remote(), upload.Info()), nil
|
||||
}
|
||||
|
||||
// PutStream uploads to the remote path with the modTime given of indeterminate
|
||||
|
||||
+18
-13
@@ -141,22 +141,20 @@ func (f *Fs) Hashes() hash.Set {
|
||||
return f.hashSet
|
||||
}
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
// mkdir makes the directory passed in and returns the upstreams used
|
||||
func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
|
||||
upstreams, err := f.create(ctx, dir)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
if dir != parentDir(dir) {
|
||||
if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
|
||||
return err
|
||||
}
|
||||
upstreams, err = f.create(ctx, dir)
|
||||
parent := parentDir(dir)
|
||||
if dir != parent {
|
||||
upstreams, err = f.mkdir(ctx, parent)
|
||||
} else if dir == "" {
|
||||
// If root dirs not created then create them
|
||||
upstreams, err = f.upstreams, nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
errs := Errors(make([]error, len(upstreams)))
|
||||
multithread(len(upstreams), func(i int) {
|
||||
@@ -165,7 +163,17 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
|
||||
}
|
||||
})
|
||||
return errs.Err()
|
||||
err = errs.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return upstreams, nil
|
||||
}
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.mkdir(ctx, dir)
|
||||
return err
|
||||
}
|
||||
|
||||
// Purge all files in the directory
|
||||
@@ -449,10 +457,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
|
||||
srcPath := src.Remote()
|
||||
upstreams, err := f.create(ctx, srcPath)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
if err := f.Mkdir(ctx, parentDir(srcPath)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upstreams, err = f.create(ctx, srcPath)
|
||||
upstreams, err = f.mkdir(ctx, parentDir(srcPath))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -124,6 +124,22 @@ You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'
|
||||
`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "base_path",
|
||||
Help: `Base path of expected replies
|
||||
|
||||
Normally WebDAV servers return the files they are listing under the
|
||||
url path as specified above. However some WebDAV servers return files
|
||||
with URLs that are not under the endpoint URL. This causes rclone to
|
||||
get confused and return errors like
|
||||
|
||||
Item with unknown path received: "/remote.php/webdav/folder1/", "/elsewhere/remote.php/webdav/folder1/"
|
||||
|
||||
errors. If that is the case, then set "base_path" to the path
|
||||
specified in the error message up to the first item, in the above
|
||||
example "/elsewhere/remote.php/webdav/".
|
||||
`,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -138,6 +154,7 @@ type Options struct {
|
||||
BearerTokenCommand string `config:"bearer_token_command"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
Headers fs.CommaSepList `config:"headers"`
|
||||
BasePath string `config:"base_path"`
|
||||
}
|
||||
|
||||
// Fs represents a remote webdav
|
||||
@@ -454,7 +471,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.srv.SetHeader("Referer", u.String())
|
||||
if !f.findHeader(opt.Headers, "Referer") {
|
||||
f.srv.SetHeader("Referer", u.String())
|
||||
}
|
||||
|
||||
if root != "" && !rootIsDir {
|
||||
// Check to see if the root actually an existing file
|
||||
@@ -517,6 +536,17 @@ func (f *Fs) addHeaders(headers fs.CommaSepList) {
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if the header was configured
|
||||
func (f *Fs) findHeader(headers fs.CommaSepList, find string) bool {
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
key := f.opt.Headers[i]
|
||||
if strings.EqualFold(key, find) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fetch the bearer token and set it if successful
|
||||
func (f *Fs) fetchAndSetBearerToken() error {
|
||||
if f.opt.BearerTokenCommand == "" {
|
||||
@@ -680,6 +710,10 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("couldn't join URL: %w", err)
|
||||
}
|
||||
basePath := baseURL.Path
|
||||
if f.opt.BasePath != "" {
|
||||
basePath = f.opt.BasePath
|
||||
}
|
||||
for i := range result.Responses {
|
||||
item := &result.Responses[i]
|
||||
isDir := itemIsDir(item)
|
||||
@@ -694,11 +728,11 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
||||
if isDir {
|
||||
u.Path = addSlash(u.Path)
|
||||
}
|
||||
if !strings.HasPrefix(u.Path, baseURL.Path) {
|
||||
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path)
|
||||
if !strings.HasPrefix(u.Path, basePath) {
|
||||
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, basePath)
|
||||
continue
|
||||
}
|
||||
subPath := u.Path[len(baseURL.Path):]
|
||||
subPath := u.Path[len(basePath):]
|
||||
if f.opt.Enc != encoder.EncodeZero {
|
||||
subPath = f.opt.Enc.ToStandardPath(subPath)
|
||||
}
|
||||
@@ -1148,7 +1182,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about call failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
usage := &fs.Usage{}
|
||||
if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 {
|
||||
|
||||
@@ -173,8 +173,8 @@ func buildZip(dir string) string {
|
||||
func buildDebAndRpm(dir, version, goarch string) []string {
|
||||
// Make internal version number acceptable to .deb and .rpm
|
||||
pkgVersion := version[1:]
|
||||
pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1)
|
||||
pkgVersion = strings.Replace(pkgVersion, "-", ".", -1)
|
||||
pkgVersion = strings.ReplaceAll(pkgVersion, "β", "-beta")
|
||||
pkgVersion = strings.ReplaceAll(pkgVersion, "-", ".")
|
||||
nfpmArch, ok := goarchToNfpm[goarch]
|
||||
if !ok {
|
||||
nfpmArch = goarch
|
||||
@@ -265,9 +265,12 @@ func buildWindowsResourceSyso(goarch string, versionTag string) string {
|
||||
"-o",
|
||||
sysoPath,
|
||||
}
|
||||
if goarch == "amd64" {
|
||||
if strings.Contains(goarch, "64") {
|
||||
args = append(args, "-64") // Make the syso a 64-bit coff file
|
||||
}
|
||||
if strings.Contains(goarch, "arm") {
|
||||
args = append(args, "-arm") // Make the syso an arm binary
|
||||
}
|
||||
args = append(args, jsonPath)
|
||||
err = runEnv(args, nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -48,6 +48,7 @@ docs = [
|
||||
"hdfs.md",
|
||||
"http.md",
|
||||
"hubic.md",
|
||||
"internetarchive.md",
|
||||
"jottacloud.md",
|
||||
"koofr.md",
|
||||
"mailru.md",
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@ see complete list in [documentation](https://rclone.org/overview/#optional-featu
|
||||
}
|
||||
u, err := doAbout(context.Background())
|
||||
if err != nil {
|
||||
return fmt.Errorf("About call failed: %w", err)
|
||||
return fmt.Errorf("about call failed: %w", err)
|
||||
}
|
||||
if u == nil {
|
||||
return errors.New("nil usage returned")
|
||||
|
||||
+1
-1
@@ -489,7 +489,7 @@ func resolveExitCode(err error) {
|
||||
os.Exit(exitcode.TransferExceeded)
|
||||
case fserrors.ShouldRetry(err):
|
||||
os.Exit(exitcode.RetryError)
|
||||
case fserrors.IsNoRetryError(err):
|
||||
case fserrors.IsNoRetryError(err), fserrors.IsNoLowLevelRetryError(err):
|
||||
os.Exit(exitcode.NoRetryError)
|
||||
case fserrors.IsFatalError(err):
|
||||
os.Exit(exitcode.FatalError)
|
||||
|
||||
+3
-4
@@ -1,7 +1,6 @@
|
||||
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
|
||||
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
|
||||
// +build cmount
|
||||
// +build cgo
|
||||
// +build linux darwin freebsd windows
|
||||
// +build linux,cgo darwin,cgo freebsd,cgo windows
|
||||
|
||||
package cmount
|
||||
|
||||
@@ -13,12 +12,12 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/winfsp/cgofuse/fuse"
|
||||
"github.com/rclone/rclone/cmd/mountlib"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/winfsp/cgofuse/fuse"
|
||||
)
|
||||
|
||||
const fhUnset = ^uint64(0)
|
||||
|
||||
+3
-4
@@ -2,10 +2,9 @@
|
||||
//
|
||||
// This uses the cgo based cgofuse library
|
||||
|
||||
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
|
||||
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
|
||||
// +build cmount
|
||||
// +build cgo
|
||||
// +build linux darwin freebsd windows
|
||||
// +build linux,cgo darwin,cgo freebsd,cgo windows
|
||||
|
||||
package cmount
|
||||
|
||||
@@ -18,12 +17,12 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/winfsp/cgofuse/fuse"
|
||||
"github.com/rclone/rclone/cmd/mountlib"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/buildinfo"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/winfsp/cgofuse/fuse"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//go:build cmount && cgo && (linux || darwin || freebsd || windows) && (!race || !windows)
|
||||
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows) && (!race || !windows)
|
||||
// +build cmount
|
||||
// +build cgo
|
||||
// +build linux darwin freebsd windows
|
||||
// +build linux,cgo darwin,cgo freebsd,cgo windows
|
||||
// +build !race !windows
|
||||
|
||||
// FIXME this doesn't work with the race detector under Windows either
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
// Build for cmount for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build (!linux && !darwin && !freebsd && !windows) || !brew || !cgo || !cmount
|
||||
// +build !linux,!darwin,!freebsd,!windows !brew !cgo !cmount
|
||||
//go:build !((linux && cgo && cmount) || (darwin && cgo && cmount) || (freebsd && cgo && cmount) || (windows && cmount))
|
||||
// +build !linux !cgo !cmount
|
||||
// +build !darwin !cgo !cmount
|
||||
// +build !freebsd !cgo !cmount
|
||||
// +build !windows !cmount
|
||||
|
||||
package cmount
|
||||
@@ -1,5 +1,5 @@
|
||||
//go:build cmount && cgo && windows
|
||||
// +build cmount,cgo,windows
|
||||
//go:build cmount && windows
|
||||
// +build cmount,windows
|
||||
|
||||
package cmount
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ rclone.org website.`,
|
||||
var description = map[string]string{}
|
||||
var addDescription func(root *cobra.Command)
|
||||
addDescription = func(root *cobra.Command) {
|
||||
name := strings.Replace(root.CommandPath(), " ", "_", -1) + ".md"
|
||||
name := strings.ReplaceAll(root.CommandPath(), " ", "_") + ".md"
|
||||
description[name] = root.Short
|
||||
for _, c := range root.Commands() {
|
||||
addDescription(c)
|
||||
@@ -93,11 +93,11 @@ rclone.org website.`,
|
||||
base := strings.TrimSuffix(name, path.Ext(name))
|
||||
data := frontmatter{
|
||||
Date: now,
|
||||
Title: strings.Replace(base, "_", " ", -1),
|
||||
Title: strings.ReplaceAll(base, "_", " "),
|
||||
Description: description[name],
|
||||
Slug: base,
|
||||
URL: "/commands/" + strings.ToLower(base) + "/",
|
||||
Source: strings.Replace(strings.Replace(base, "rclone", "cmd", -1), "_", "/", -1) + "/",
|
||||
Source: strings.ReplaceAll(strings.ReplaceAll(base, "rclone", "cmd"), "_", "/") + "/",
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err := frontmatterTemplate.Execute(&buf, data)
|
||||
|
||||
+13
-8
@@ -78,6 +78,17 @@ type MountPoint struct {
|
||||
ErrChan <-chan error
|
||||
}
|
||||
|
||||
// NewMountPoint makes a new mounting structure
|
||||
func NewMountPoint(mount MountFn, mountPoint string, f fs.Fs, mountOpt *Options, vfsOpt *vfscommon.Options) *MountPoint {
|
||||
return &MountPoint{
|
||||
MountFn: mount,
|
||||
MountPoint: mountPoint,
|
||||
Fs: f,
|
||||
MountOpt: *mountOpt,
|
||||
VFSOpt: *vfsOpt,
|
||||
}
|
||||
}
|
||||
|
||||
// Global constants
|
||||
const (
|
||||
MaxLeafSize = 1024 // don't pass file names longer than this
|
||||
@@ -167,14 +178,7 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm
|
||||
defer cmd.StartStats()()
|
||||
}
|
||||
|
||||
mnt := &MountPoint{
|
||||
MountFn: mount,
|
||||
MountPoint: args[1],
|
||||
Fs: cmd.NewFsDir(args),
|
||||
MountOpt: Opt,
|
||||
VFSOpt: vfsflags.Opt,
|
||||
}
|
||||
|
||||
mnt := NewMountPoint(mount, args[1], cmd.NewFsDir(args), &Opt, &vfsflags.Opt)
|
||||
daemon, err := mnt.Mount()
|
||||
|
||||
// Wait for foreground mount, if any...
|
||||
@@ -253,6 +257,7 @@ func (m *MountPoint) Mount() (daemon *os.Process, err error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to mount FUSE fs: %w", err)
|
||||
}
|
||||
m.MountedOn = time.Now()
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
|
||||
+3
-12
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfsflags"
|
||||
)
|
||||
|
||||
@@ -117,23 +116,15 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
VFS := vfs.New(fdst, &vfsOpt)
|
||||
_, unmountFn, err := mountFn(VFS, mountPoint, &mountOpt)
|
||||
mnt := NewMountPoint(mountFn, mountPoint, fdst, &mountOpt, &vfsOpt)
|
||||
_, err = mnt.Mount()
|
||||
if err != nil {
|
||||
log.Printf("mount FAILED: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add mount to list if mount point was successfully created
|
||||
liveMounts[mountPoint] = &MountPoint{
|
||||
MountPoint: mountPoint,
|
||||
MountedOn: time.Now(),
|
||||
MountFn: mountFn,
|
||||
UnmountFn: unmountFn,
|
||||
MountOpt: mountOpt,
|
||||
VFSOpt: vfsOpt,
|
||||
Fs: fdst,
|
||||
}
|
||||
liveMounts[mountPoint] = mnt
|
||||
|
||||
fs.Debugf(nil, "Mount for %s created at %s using %s", fdst.String(), mountPoint, mountType)
|
||||
return nil, nil
|
||||
|
||||
+56
-39
@@ -42,15 +42,31 @@ builds an in memory representation. rclone ncdu can be used during
|
||||
this scanning phase and you will see it building up the directory
|
||||
structure as it goes along.
|
||||
|
||||
Here are the keys - press '?' to toggle the help on and off
|
||||
You can interact with the user interface using key presses,
|
||||
press '?' to toggle the help on and off. The supported keys are:
|
||||
|
||||
` + strings.Join(helpText()[1:], "\n ") + `
|
||||
|
||||
Listed files/directories may be prefixed by a one-character flag,
|
||||
some of them combined with a description in brackes at end of line.
|
||||
These flags have the following meaning:
|
||||
|
||||
e means this is an empty directory, i.e. contains no files (but
|
||||
may contain empty subdirectories)
|
||||
~ means this is a directory where some of the files (possibly in
|
||||
subdirectories) have unknown size, and therefore the directory
|
||||
size may be underestimated (and average size inaccurate, as it
|
||||
is average of the files with known sizes).
|
||||
. means an error occurred while reading a subdirectory, and
|
||||
therefore the directory size may be underestimated (and average
|
||||
size inaccurate)
|
||||
! means an error occurred while reading this directory
|
||||
|
||||
This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
|
||||
rclone remotes. It is missing lots of features at the moment
|
||||
but is useful as it stands.
|
||||
|
||||
Note that it might take some time to delete big files/folders. The
|
||||
Note that it might take some time to delete big files/directories. The
|
||||
UI won't respond in the meantime since the deletion is done synchronously.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
@@ -283,9 +299,9 @@ func (u *UI) biggestEntry() (biggest int64) {
|
||||
return
|
||||
}
|
||||
for i := range u.entries {
|
||||
size, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if size > biggest {
|
||||
biggest = size
|
||||
attrs, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if attrs.Size > biggest {
|
||||
biggest = attrs.Size
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -297,8 +313,8 @@ func (u *UI) hasEmptyDir() bool {
|
||||
return false
|
||||
}
|
||||
for i := range u.entries {
|
||||
_, count, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if isDir && count == 0 {
|
||||
attrs, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if attrs.IsDir && attrs.Count == 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -343,9 +359,9 @@ func (u *UI) Draw() error {
|
||||
if y >= h-1 {
|
||||
break
|
||||
}
|
||||
size, count, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
|
||||
attrs, err := u.d.AttrI(u.sortPerm[n])
|
||||
fg := termbox.ColorWhite
|
||||
if entriesHaveErrors {
|
||||
if attrs.EntriesHaveErrors {
|
||||
fg = termbox.ColorYellow
|
||||
}
|
||||
if err != nil {
|
||||
@@ -356,15 +372,19 @@ func (u *UI) Draw() error {
|
||||
fg, bg = bg, fg
|
||||
}
|
||||
mark := ' '
|
||||
if isDir {
|
||||
if attrs.IsDir {
|
||||
mark = '/'
|
||||
}
|
||||
fileFlag := ' '
|
||||
message := ""
|
||||
if !readable {
|
||||
if !attrs.Readable {
|
||||
message = " [not read yet]"
|
||||
}
|
||||
if entriesHaveErrors {
|
||||
if attrs.CountUnknownSize > 0 {
|
||||
message = fmt.Sprintf(" [%d of %d files have unknown size, size may be underestimated]", attrs.CountUnknownSize, attrs.Count)
|
||||
fileFlag = '~'
|
||||
}
|
||||
if attrs.EntriesHaveErrors {
|
||||
message = " [some subdirectories could not be read, size may be underestimated]"
|
||||
fileFlag = '.'
|
||||
}
|
||||
@@ -374,32 +394,29 @@ func (u *UI) Draw() error {
|
||||
}
|
||||
extras := ""
|
||||
if u.showCounts {
|
||||
ss := operations.CountStringField(count, u.humanReadable, 9) + " "
|
||||
if count > 0 {
|
||||
ss := operations.CountStringField(attrs.Count, u.humanReadable, 9) + " "
|
||||
if attrs.Count > 0 {
|
||||
extras += ss
|
||||
} else {
|
||||
extras += strings.Repeat(" ", len(ss))
|
||||
}
|
||||
}
|
||||
var averageSize float64
|
||||
if count > 0 {
|
||||
averageSize = float64(size) / float64(count)
|
||||
}
|
||||
if u.showDirAverageSize {
|
||||
ss := operations.SizeStringField(int64(averageSize), u.humanReadable, 9) + " "
|
||||
if averageSize > 0 {
|
||||
avg := attrs.AverageSize()
|
||||
ss := operations.SizeStringField(int64(avg), u.humanReadable, 9) + " "
|
||||
if avg > 0 {
|
||||
extras += ss
|
||||
} else {
|
||||
extras += strings.Repeat(" ", len(ss))
|
||||
}
|
||||
}
|
||||
if showEmptyDir {
|
||||
if isDir && count == 0 && fileFlag == ' ' {
|
||||
if attrs.IsDir && attrs.Count == 0 && fileFlag == ' ' {
|
||||
fileFlag = 'e'
|
||||
}
|
||||
}
|
||||
if u.showGraph {
|
||||
bars := (size + perBar/2 - 1) / perBar
|
||||
bars := (attrs.Size + perBar/2 - 1) / perBar
|
||||
// clip if necessary - only happens during startup
|
||||
if bars > 10 {
|
||||
bars = 10
|
||||
@@ -408,7 +425,7 @@ func (u *UI) Draw() error {
|
||||
}
|
||||
extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
|
||||
}
|
||||
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
|
||||
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(attrs.Size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
|
||||
y++
|
||||
}
|
||||
}
|
||||
@@ -559,14 +576,14 @@ type ncduSort struct {
|
||||
// Less is part of sort.Interface.
|
||||
func (ds *ncduSort) Less(i, j int) bool {
|
||||
var iAvgSize, jAvgSize float64
|
||||
isize, icount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
|
||||
jsize, jcount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
|
||||
iattrs, _ := ds.d.AttrI(ds.sortPerm[i])
|
||||
jattrs, _ := ds.d.AttrI(ds.sortPerm[j])
|
||||
iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
|
||||
if icount > 0 {
|
||||
iAvgSize = float64(isize / icount)
|
||||
if iattrs.Count > 0 {
|
||||
iAvgSize = iattrs.AverageSize()
|
||||
}
|
||||
if jcount > 0 {
|
||||
jAvgSize = float64(jsize / jcount)
|
||||
if jattrs.Count > 0 {
|
||||
jAvgSize = jattrs.AverageSize()
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -575,33 +592,33 @@ func (ds *ncduSort) Less(i, j int) bool {
|
||||
case ds.u.sortByName > 0:
|
||||
break
|
||||
case ds.u.sortBySize < 0:
|
||||
if isize != jsize {
|
||||
return isize < jsize
|
||||
if iattrs.Size != jattrs.Size {
|
||||
return iattrs.Size < jattrs.Size
|
||||
}
|
||||
case ds.u.sortBySize > 0:
|
||||
if isize != jsize {
|
||||
return isize > jsize
|
||||
if iattrs.Size != jattrs.Size {
|
||||
return iattrs.Size > jattrs.Size
|
||||
}
|
||||
case ds.u.sortByCount < 0:
|
||||
if icount != jcount {
|
||||
return icount < jcount
|
||||
if iattrs.Count != jattrs.Count {
|
||||
return iattrs.Count < jattrs.Count
|
||||
}
|
||||
case ds.u.sortByCount > 0:
|
||||
if icount != jcount {
|
||||
return icount > jcount
|
||||
if iattrs.Count != jattrs.Count {
|
||||
return iattrs.Count > jattrs.Count
|
||||
}
|
||||
case ds.u.sortByAverageSize < 0:
|
||||
if iAvgSize != jAvgSize {
|
||||
return iAvgSize < jAvgSize
|
||||
}
|
||||
// if avgSize is equal, sort by size
|
||||
return isize < jsize
|
||||
return iattrs.Size < jattrs.Size
|
||||
case ds.u.sortByAverageSize > 0:
|
||||
if iAvgSize != jAvgSize {
|
||||
return iAvgSize > jAvgSize
|
||||
}
|
||||
// if avgSize is equal, sort by size
|
||||
return isize > jsize
|
||||
return iattrs.Size > jattrs.Size
|
||||
}
|
||||
// if everything equal, sort by name
|
||||
return iname < jname
|
||||
|
||||
+51
-8
@@ -16,14 +16,42 @@ type Dir struct {
|
||||
parent *Dir
|
||||
path string
|
||||
mu sync.Mutex
|
||||
count int64
|
||||
size int64
|
||||
count int64
|
||||
countUnknownSize int64
|
||||
entries fs.DirEntries
|
||||
dirs map[string]*Dir
|
||||
readError error
|
||||
entriesHaveErrors bool
|
||||
}
|
||||
|
||||
// Attrs contains accumulated properties for a directory entry
|
||||
//
|
||||
// Files with unknown size are counted separately but also included
|
||||
// in the total count. They are not included in the size, i.e. treated
|
||||
// as empty files, which means the size may be underestimated.
|
||||
type Attrs struct {
|
||||
Size int64
|
||||
Count int64
|
||||
CountUnknownSize int64
|
||||
IsDir bool
|
||||
Readable bool
|
||||
EntriesHaveErrors bool
|
||||
}
|
||||
|
||||
// AverageSize calculates average size of files in directory
|
||||
//
|
||||
// If there are files with unknown size, this returns the average over
|
||||
// files with known sizes, which means it may be under- or
|
||||
// overestimated.
|
||||
func (a *Attrs) AverageSize() float64 {
|
||||
countKnownSize := a.Count - a.CountUnknownSize
|
||||
if countKnownSize > 0 {
|
||||
return float64(a.Size) / float64(countKnownSize)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Parent returns the directory above this one
|
||||
func (d *Dir) Parent() *Dir {
|
||||
// no locking needed since these are write once in newDir()
|
||||
@@ -49,7 +77,13 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
|
||||
for _, entry := range entries {
|
||||
if o, ok := entry.(fs.Object); ok {
|
||||
d.count++
|
||||
d.size += o.Size()
|
||||
size := o.Size()
|
||||
if size < 0 {
|
||||
// Some backends may return -1 because size of object is not known
|
||||
d.countUnknownSize++
|
||||
} else {
|
||||
d.size += size
|
||||
}
|
||||
}
|
||||
}
|
||||
// Set my directory entry in parent
|
||||
@@ -62,8 +96,9 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
|
||||
// Accumulate counts in parents
|
||||
for ; parent != nil; parent = parent.parent {
|
||||
parent.mu.Lock()
|
||||
parent.count += d.count
|
||||
parent.size += d.size
|
||||
parent.count += d.count
|
||||
parent.countUnknownSize += d.countUnknownSize
|
||||
if d.readError != nil {
|
||||
parent.entriesHaveErrors = true
|
||||
}
|
||||
@@ -91,17 +126,24 @@ func (d *Dir) Remove(i int) {
|
||||
// Call with d.mu held
|
||||
func (d *Dir) remove(i int) {
|
||||
size := d.entries[i].Size()
|
||||
countUnknownSize := int64(0)
|
||||
if size < 0 {
|
||||
size = 0
|
||||
countUnknownSize = 1
|
||||
}
|
||||
count := int64(1)
|
||||
|
||||
subDir, ok := d.getDir(i)
|
||||
if ok {
|
||||
size = subDir.size
|
||||
count = subDir.count
|
||||
countUnknownSize = subDir.countUnknownSize
|
||||
delete(d.dirs, path.Base(subDir.path))
|
||||
}
|
||||
|
||||
d.size -= size
|
||||
d.count -= count
|
||||
d.countUnknownSize -= countUnknownSize
|
||||
d.entries = append(d.entries[:i], d.entries[i+1:]...)
|
||||
|
||||
dir := d
|
||||
@@ -111,6 +153,7 @@ func (d *Dir) remove(i int) {
|
||||
parent.dirs[path.Base(dir.path)] = dir
|
||||
parent.size -= size
|
||||
parent.count -= count
|
||||
parent.countUnknownSize -= countUnknownSize
|
||||
dir = parent
|
||||
parent.mu.Unlock()
|
||||
}
|
||||
@@ -151,19 +194,19 @@ func (d *Dir) Attr() (size int64, count int64) {
|
||||
}
|
||||
|
||||
// AttrI returns the size, count and flags for the i-th directory entry
|
||||
func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
|
||||
func (d *Dir) AttrI(i int) (attrs Attrs, err error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
subDir, isDir := d.getDir(i)
|
||||
|
||||
if !isDir {
|
||||
return d.entries[i].Size(), 0, false, true, d.entriesHaveErrors, d.readError
|
||||
return Attrs{d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors}, d.readError
|
||||
}
|
||||
if subDir == nil {
|
||||
return 0, 0, true, false, false, nil
|
||||
return Attrs{0, 0, 0, true, false, false}, nil
|
||||
}
|
||||
size, count = subDir.Attr()
|
||||
return size, count, true, true, subDir.entriesHaveErrors, subDir.readError
|
||||
size, count := subDir.Attr()
|
||||
return Attrs{size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors}, subDir.readError
|
||||
}
|
||||
|
||||
// Scan the Fs passed in, returning a root directory channel and an
|
||||
|
||||
+1
-1
@@ -290,7 +290,7 @@ func list(ctx context.Context) error {
|
||||
if !ok {
|
||||
return errors.New("bad JSON")
|
||||
}
|
||||
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.Replace(info["Path"].(string), "/", "-", -1))
|
||||
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.ReplaceAll(info["Path"].(string), "/", "-"))
|
||||
fmt.Printf("%s\n\n", info["Help"])
|
||||
if authRequired := info["AuthRequired"]; authRequired != nil {
|
||||
if authRequired.(bool) {
|
||||
|
||||
@@ -274,7 +274,6 @@ func (vol *Volume) mount(id string) error {
|
||||
if _, err := vol.mnt.Mount(); err != nil {
|
||||
return err
|
||||
}
|
||||
vol.mnt.MountedOn = time.Now()
|
||||
vol.mountReqs[id] = nil
|
||||
vol.drv.monChan <- false // ask monitor to refresh channels
|
||||
return nil
|
||||
|
||||
@@ -43,7 +43,7 @@ var shellUnEscapeRegex = regexp.MustCompile(`\\(.)`)
|
||||
|
||||
// Unescape a string that was escaped by rclone
|
||||
func shellUnEscape(str string) string {
|
||||
str = strings.Replace(str, "'\n'", "\n", -1)
|
||||
str = strings.ReplaceAll(str, "'\n'", "\n")
|
||||
str = shellUnEscapeRegex.ReplaceAllString(str, `$1`)
|
||||
return str
|
||||
}
|
||||
|
||||
+29
-4
@@ -24,26 +24,51 @@ func init() {
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "size remote:path",
|
||||
Short: `Prints the total size and number of objects in remote:path.`,
|
||||
Long: `
|
||||
Counts objects in the path and calculates the total size. Prints the
|
||||
result to standard output.
|
||||
|
||||
By default the output is in human-readable format, but shows values in
|
||||
both human-readable format as well as the raw numbers (global option
|
||||
` + "`--human-readable`" + ` is not considered). Use option ` + "`--json`" + `
|
||||
to format output as JSON instead.
|
||||
|
||||
Recurses by default, use ` + "`--max-depth 1`" + ` to stop the
|
||||
recursion.
|
||||
|
||||
Some backends do not always provide file sizes, see for example
|
||||
[Google Photos](/googlephotos/#size) and
|
||||
[Google Drive](/drive/#limitations-of-google-docs).
|
||||
Rclone will then show a notice in the log indicating how many such
|
||||
files were encountered, and count them in as empty files in the output
|
||||
of the size command.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
fsrc := cmd.NewFsSrc(args)
|
||||
cmd.Run(false, false, command, func() error {
|
||||
var err error
|
||||
var results struct {
|
||||
Count int64 `json:"count"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Count int64 `json:"count"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Sizeless int64 `json:"sizeless"`
|
||||
}
|
||||
|
||||
results.Count, results.Bytes, err = operations.Count(context.Background(), fsrc)
|
||||
results.Count, results.Bytes, results.Sizeless, err = operations.Count(context.Background(), fsrc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if results.Sizeless > 0 {
|
||||
fs.Logf(fsrc, "Size may be underestimated due to %d objects with unknown size", results.Sizeless)
|
||||
}
|
||||
if jsonOutput {
|
||||
return json.NewEncoder(os.Stdout).Encode(results)
|
||||
}
|
||||
fmt.Printf("Total objects: %s (%d)\n", fs.CountSuffix(results.Count), results.Count)
|
||||
fmt.Printf("Total size: %s (%d Byte)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
|
||||
if results.Sizeless > 0 {
|
||||
fmt.Printf("Total objects with unknown size: %s (%d)\n", fs.CountSuffix(results.Sizeless), results.Sizeless)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
|
||||
@@ -25,7 +25,7 @@ var commandDefinition = &cobra.Command{
|
||||
cmd.Run(false, false, command, func() error {
|
||||
ctx := context.Background()
|
||||
ci := fs.GetConfig(context.Background())
|
||||
objects, _, err := operations.Count(ctx, fsrc)
|
||||
objects, _, _, err := operations.Count(ctx, fsrc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -113,8 +113,10 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="Box" home="https://www.box.com/" config="/box/" >}}
|
||||
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
|
||||
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
|
||||
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
|
||||
{{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
|
||||
{{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}}
|
||||
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
|
||||
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
|
||||
{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
|
||||
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
|
||||
@@ -127,6 +129,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}}
|
||||
{{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
|
||||
{{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}}
|
||||
{{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}
|
||||
{{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
|
||||
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
|
||||
{{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
|
||||
@@ -167,7 +170,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}}
|
||||
{{< /provider_list >}}
|
||||
|
||||
Links
|
||||
## Links
|
||||
|
||||
* {{< icon "fa fa-home" >}} [Home page](https://rclone.org/)
|
||||
* {{< icon "fab fa-github" >}} [GitHub project page for source and bug tracker](https://github.com/rclone/rclone)
|
||||
|
||||
@@ -334,6 +334,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -583,3 +583,25 @@ put them back in again.` >}}
|
||||
* rafma0 <int.main@gmail.com>
|
||||
* Adrien Rey-Jarthon <jobs@adrienjarthon.com>
|
||||
* Nick Gooding <73336146+nickgooding@users.noreply.github.com>
|
||||
* Leroy van Logchem <lr.vanlogchem@gmail.com>
|
||||
* Zsolt Ero <zsolt.ero@gmail.com>
|
||||
* Lesmiscore <nao20010128@gmail.com>
|
||||
* ehsantdy <ehsan.tadayon@arvancloud.com>
|
||||
* SwazRGB <65694696+swazrgb@users.noreply.github.com>
|
||||
* Mateusz Puczyński <mati6095@gmail.com>
|
||||
* Michael C Tiernan - MIT-Research Computing Project <mtiernan@mit.edu>
|
||||
* Kaspian <34658474+KaspianDev@users.noreply.github.com>
|
||||
* Werner <EvilOlaf@users.noreply.github.com>
|
||||
* Hugal31 <hugo.laloge@gmail.com>
|
||||
* Christian Galo <36752715+cgalo5758@users.noreply.github.com>
|
||||
* Erik van Velzen <erik@evanv.nl>
|
||||
* Derek Battams <derek@battams.ca>
|
||||
* SimonLiu <simonliu009@users.noreply.github.com>
|
||||
* Hugo Laloge <hla@lescompanions.com>
|
||||
* Mr-Kanister <68117355+Mr-Kanister@users.noreply.github.com>
|
||||
* Rob Pickerill <r.pickerill@gmail.com>
|
||||
* Andrey <to.merge@gmail.com>
|
||||
* Eric Wolf <19wolf@gmail.com>
|
||||
* Nick <nick.naumann@mailbox.tu-dresden.de>
|
||||
* Jason Zheng <jszheng17@gmail.com>
|
||||
* Matthew Vernon <mvernon@wikimedia.org>
|
||||
@@ -522,8 +522,7 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
## Azure Storage Emulator Support
|
||||
|
||||
|
||||
+6
-2
@@ -173,6 +173,11 @@ the file instead of hiding it.
|
||||
Old versions of files, where available, are visible using the
|
||||
`--b2-versions` flag.
|
||||
|
||||
It is also possible to view a bucket as it was at a certain point in time,
|
||||
using the `--b2-version-at` flag. This will show the file versions as they
|
||||
were at that time, showing files that have been deleted afterwards, and
|
||||
hiding files that were created since.
|
||||
|
||||
If you wish to remove all the old versions then you can use the
|
||||
`rclone cleanup remote:bucket` command which will delete all the old
|
||||
versions of files, leaving the current ones intact. You can also
|
||||
@@ -561,7 +566,6 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ Optional Flags:
|
||||
|
||||
Arbitrary rclone flags may be specified on the
|
||||
[bisync command line](/commands/rclone_bisync/), for example
|
||||
`rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
|
||||
`rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
|
||||
Note that interactions of various rclone flags with bisync process flow
|
||||
has not been fully tested yet.
|
||||
|
||||
@@ -363,6 +363,7 @@ Bisync is considered _BETA_ and has been tested with the following backends:
|
||||
- OneDrive
|
||||
- S3
|
||||
- SFTP
|
||||
- Yandex Disk
|
||||
|
||||
It has not been fully tested with other services yet.
|
||||
If it works, or sorta works, please let us know and we'll update the list.
|
||||
@@ -700,7 +701,7 @@ consider using the flag
|
||||
|
||||
Google docs exist as virtual files on Google Drive and cannot be transferred
|
||||
to other filesystems natively. While it is possible to export a Google doc to
|
||||
a normal file (with `.xlsx` extension, for example), it's not possible
|
||||
a normal file (with `.xlsx` extension, for example), it is not possible
|
||||
to import a normal file back into a Google document.
|
||||
|
||||
Bisync's handling of Google Doc files is to flag them in the run log output
|
||||
|
||||
+1
-2
@@ -466,6 +466,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -5,6 +5,48 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.58.1 - 2022-04-29
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.58.1)
|
||||
|
||||
* Bug Fixes
|
||||
* build: Update github.com/billziss-gh to github.com/winfsp (Nick Craig-Wood)
|
||||
* filter: Fix timezone of `--min-age`/`-max-age` from UTC to local as documented (Nick Craig-Wood)
|
||||
* rc/js: Correct RC method names (Sơn Trần-Nguyễn)
|
||||
* docs
|
||||
* Fix some links to command pages (albertony)
|
||||
* Add `--multi-thread-streams` note to `--transfers`. (Zsolt Ero)
|
||||
* Mount
|
||||
* Fix `--devname` and fusermount: unknown option 'fsname' when mounting via rc (Nick Craig-Wood)
|
||||
* VFS
|
||||
* Remove wording which suggests VFS is only for mounting (Nick Craig-Wood)
|
||||
* Dropbox
|
||||
* Fix retries of multipart uploads with incorrect_offset error (Nick Craig-Wood)
|
||||
* Google Cloud Storage
|
||||
* Use the s3 pacer to speed up transactions (Nick Craig-Wood)
|
||||
* pacer: Default the Google pacer to a burst of 100 to fix gcs pacing (Nick Craig-Wood)
|
||||
* Jottacloud
|
||||
* Fix scope in token request (albertony)
|
||||
* Netstorage
|
||||
* Fix unescaped HTML in documentation (Nick Craig-Wood)
|
||||
* Make levels of headings consistent (Nick Craig-Wood)
|
||||
* Add support contacts to netstorage doc (Nil Alexandrov)
|
||||
* Onedrive
|
||||
* Note that sharepoint also changes web files (.html, .aspx) (GH)
|
||||
* Putio
|
||||
* Handle rate limit errors (Berkan Teber)
|
||||
* Fix multithread download and other ranged requests (rafma0)
|
||||
* S3
|
||||
* Add ChinaMobile EOS to provider list (GuoXingbin)
|
||||
* Sync providers in config description with providers (Nick Craig-Wood)
|
||||
* SFTP
|
||||
* Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr)
|
||||
* Note that Scaleway C14 is deprecating SFTP in favor of S3 (Adrien Rey-Jarthon)
|
||||
* Storj
|
||||
* Fix bucket creation on Move (Nick Craig-Wood)
|
||||
* WebDAV
|
||||
* Don't override Referer if user sets it (Nick Craig-Wood)
|
||||
|
||||
## v1.58.0 - 2022-03-18
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.57.0...v1.58.0)
|
||||
|
||||
+19
-2
@@ -50,6 +50,7 @@ See the following for detailed instructions for
|
||||
* [HDFS](/hdfs/)
|
||||
* [HTTP](/http/)
|
||||
* [Hubic](/hubic/)
|
||||
* [Internet Archive](/internetarchive/)
|
||||
* [Jottacloud](/jottacloud/)
|
||||
* [Koofr](/koofr/)
|
||||
* [Mail.ru Cloud](/mailru/)
|
||||
@@ -787,7 +788,9 @@ See `--compare-dest` and `--backup-dir`.
|
||||
|
||||
### --dedupe-mode MODE ###
|
||||
|
||||
Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean.
|
||||
Mode to run dedupe command in. One of `interactive`, `skip`, `first`,
|
||||
`newest`, `oldest`, `rename`. The default is `interactive`.
|
||||
See the dedupe command for more information as to what these options mean.
|
||||
|
||||
### --disable FEATURE,FEATURE,... ###
|
||||
|
||||
@@ -1085,7 +1088,12 @@ have a signal to rotate logs.
|
||||
|
||||
### --log-format LIST ###
|
||||
|
||||
Comma separated list of log format options. Accepted options are `date`, `time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other keywords will be silently ignored. `pid` will tag log messages with process identifier which useful with `rclone mount --daemon`. Other accepted options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants). The default log format is "`date`,`time`".
|
||||
Comma separated list of log format options. Accepted options are `date`,
|
||||
`time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other
|
||||
keywords will be silently ignored. `pid` will tag log messages with process
|
||||
identifier which useful with `rclone mount --daemon`. Other accepted
|
||||
options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants).
|
||||
The default log format is "`date`,`time`".
|
||||
|
||||
### --log-level LEVEL ###
|
||||
|
||||
@@ -1793,6 +1801,8 @@ of timeouts or bigger if you have lots of bandwidth and a fast remote.
|
||||
|
||||
The default is to run 4 file transfers in parallel.
|
||||
|
||||
Look at --multi-thread-streams if you would like to control single file transfers.
|
||||
|
||||
### -u, --update ###
|
||||
|
||||
This forces rclone to skip any files which exist on the destination
|
||||
@@ -1864,6 +1874,9 @@ With `-vv` rclone will become very verbose telling you about every
|
||||
file it considers and transfers. Please send bug reports with a log
|
||||
with this setting.
|
||||
|
||||
When setting verbosity as an environment variable, use
|
||||
`RCLONE_VERBOSE=1` or `RCLONE_VERBOSE=2` for `-v` and `-vv` respectively.
|
||||
|
||||
### -V, --version ###
|
||||
|
||||
Prints the version number
|
||||
@@ -2229,6 +2242,10 @@ override the environment variable setting.
|
||||
Or to always use the trash in drive `--drive-use-trash`, set
|
||||
`RCLONE_DRIVE_USE_TRASH=true`.
|
||||
|
||||
Verbosity is slightly different, the environment variable
|
||||
equivalent of `--verbose` or `-v` is `RCLONE_VERBOSE=1`,
|
||||
or for `-vv`, `RCLONE_VERBOSE=2`.
|
||||
|
||||
The same parser is used for the options and the environment variables
|
||||
so they take exactly the same form.
|
||||
|
||||
|
||||
@@ -1375,8 +1375,11 @@ and upload the files if you prefer.
|
||||
|
||||
### Limitations of Google Docs
|
||||
|
||||
Google docs will appear as size -1 in `rclone ls` and as size 0 in
|
||||
anything which uses the VFS layer, e.g. `rclone mount`, `rclone serve`.
|
||||
Google docs will appear as size -1 in `rclone ls`, `rclone ncdu` etc,
|
||||
and as size 0 in anything which uses the VFS layer, e.g. `rclone mount`
|
||||
and `rclone serve`. When calculating directory totals, e.g. in
|
||||
`rclone size` and `rclone ncdu`, they will be counted in as empty
|
||||
files.
|
||||
|
||||
This is because rclone can't find out the size of the Google docs
|
||||
without downloading them.
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ the node running rclone would need to have lots of bandwidth.
|
||||
|
||||
The syncs would be incremental (on a file by file basis).
|
||||
|
||||
Eg
|
||||
e.g.
|
||||
|
||||
rclone sync -i drive:Folder s3:bucket
|
||||
|
||||
|
||||
@@ -192,6 +192,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -131,7 +131,11 @@ The regular expressions used are as defined in the [Go regular
|
||||
expression reference](https://golang.org/pkg/regexp/syntax/). Regular
|
||||
expressions should be enclosed in `{{` `}}`. They will match only the
|
||||
last path segment if the glob doesn't start with `/` or the whole path
|
||||
name if it does.
|
||||
name if it does. Note that rclone does not attempt to parse the
|
||||
supplied regular expression, meaning that using any regular expression
|
||||
filter will prevent rclone from using [directory filter rules](#directory_filter),
|
||||
as it will instead check every path against
|
||||
the supplied regular expression(s).
|
||||
|
||||
Here is how the `{{regexp}}` is transformed into an full regular
|
||||
expression to match the entire path:
|
||||
@@ -247,7 +251,7 @@ currently a means provided to pass regular expression filter options into
|
||||
rclone directly though character class filter rules contain character
|
||||
classes. [Go regular expression reference](https://golang.org/pkg/regexp/syntax/)
|
||||
|
||||
### How filter rules are applied to directories
|
||||
### How filter rules are applied to directories {#directory_filter}
|
||||
|
||||
Rclone commands are applied to path/file names not
|
||||
directories. The entire contents of a directory can be matched
|
||||
@@ -263,6 +267,10 @@ recurse into subdirectories. This potentially optimises access to a remote
|
||||
by avoiding listing unnecessary directories. Whether optimisation is
|
||||
desirable depends on the specific filter rules and source remote content.
|
||||
|
||||
If any [regular expression filters](#regexp) are in use, then no
|
||||
directory recursion optimisation is possible, as rclone must check
|
||||
every path against the supplied regular expression(s).
|
||||
|
||||
Directory recursion optimisation occurs if either:
|
||||
|
||||
* A source remote does not support the rclone `ListR` primitive. local,
|
||||
|
||||
@@ -337,6 +337,7 @@ and may be set in the config file.
|
||||
--ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited
|
||||
--ftp-disable-epsv Disable using EPSV even if server advertises support
|
||||
--ftp-disable-mlsd Disable using MLSD even if server advertises support
|
||||
--ftp-disable-utf8 Disable using UTF-8 even if server advertises support
|
||||
--ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
|
||||
--ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
|
||||
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
|
||||
|
||||
+12
-2
@@ -267,6 +267,17 @@ Properties:
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --ftp-disable-utf8
|
||||
|
||||
Disable using UTF-8 even if server advertises support.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: disable_utf8
|
||||
- Env Var: RCLONE_FTP_DISABLE_UTF8
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --ftp-writing-mdtm
|
||||
|
||||
Use MDTM to set modification time (VsFtpd quirk)
|
||||
@@ -395,8 +406,7 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
The implementation of : `--dump headers`,
|
||||
`--dump bodies`, `--dump auth` for debugging isn't the same as
|
||||
|
||||
@@ -609,6 +609,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -220,6 +220,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
+92
-38
@@ -3,11 +3,11 @@ title: "Install"
|
||||
description: "Rclone Installation"
|
||||
---
|
||||
|
||||
# Install #
|
||||
# Install
|
||||
|
||||
Rclone is a Go program and comes as a single binary file.
|
||||
|
||||
## Quickstart ##
|
||||
## Quickstart
|
||||
|
||||
* [Download](/downloads/) the relevant binary.
|
||||
* Extract the `rclone` executable, `rclone.exe` on Windows, from the archive.
|
||||
@@ -22,20 +22,20 @@ run `rclone -h`.
|
||||
Already installed rclone can be easily updated to the latest version
|
||||
using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
|
||||
|
||||
## Script installation ##
|
||||
## Script installation
|
||||
|
||||
To install rclone on Linux/macOS/BSD systems, run:
|
||||
|
||||
curl https://rclone.org/install.sh | sudo bash
|
||||
sudo -v ; curl https://rclone.org/install.sh | sudo bash
|
||||
|
||||
For beta installation, run:
|
||||
|
||||
curl https://rclone.org/install.sh | sudo bash -s beta
|
||||
sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
|
||||
|
||||
Note that this script checks the version of rclone installed first and
|
||||
won't re-download if not needed.
|
||||
|
||||
## Linux installation from precompiled binary ##
|
||||
## Linux installation from precompiled binary
|
||||
|
||||
Fetch and unpack
|
||||
|
||||
@@ -59,7 +59,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
|
||||
|
||||
rclone config
|
||||
|
||||
## macOS installation with brew ##
|
||||
## macOS installation with brew
|
||||
|
||||
brew install rclone
|
||||
|
||||
@@ -68,7 +68,7 @@ NOTE: This version of rclone will not support `mount` any more (see
|
||||
on macOS, either install a precompiled binary or enable the relevant option
|
||||
when [installing from source](#install-from-source).
|
||||
|
||||
## macOS installation from precompiled binary, using curl ##
|
||||
## macOS installation from precompiled binary, using curl
|
||||
|
||||
To avoid problems with macOS gatekeeper enforcing the binary to be signed and
|
||||
notarized it is enough to download with `curl`.
|
||||
@@ -96,20 +96,20 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
|
||||
|
||||
rclone config
|
||||
|
||||
## macOS installation from precompiled binary, using a web browser ##
|
||||
## macOS installation from precompiled binary, using a web browser
|
||||
|
||||
When downloading a binary with a web browser, the browser will set the macOS
|
||||
gatekeeper quarantine attribute. Starting from Catalina, when attempting to run
|
||||
`rclone`, a pop-up will appear saying:
|
||||
|
||||
“rclone” cannot be opened because the developer cannot be verified.
|
||||
"rclone" cannot be opened because the developer cannot be verified.
|
||||
macOS cannot verify that this app is free from malware.
|
||||
|
||||
The simplest fix is to run
|
||||
|
||||
xattr -d com.apple.quarantine rclone
|
||||
|
||||
## Install with docker ##
|
||||
## Install with docker
|
||||
|
||||
The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
|
||||
These images are autobuilt by docker hub from the rclone source based
|
||||
@@ -188,39 +188,93 @@ ls ~/data/mount
|
||||
kill %1
|
||||
```
|
||||
|
||||
## Install from source ##
|
||||
## Install from source
|
||||
|
||||
Make sure you have at least [Go](https://golang.org/) go1.16
|
||||
installed. [Download go](https://golang.org/dl/) if necessary. The
|
||||
latest release is recommended. Then
|
||||
Make sure you have git and [Go](https://golang.org/) installed.
|
||||
Go version 1.16 or newer is required, latest release is recommended.
|
||||
You can get it from your package manager, or download it from
|
||||
[golang.org/dl](https://golang.org/dl/). Then you can run the following:
|
||||
|
||||
```sh
|
||||
```
|
||||
git clone https://github.com/rclone/rclone.git
|
||||
cd rclone
|
||||
go build
|
||||
# If on macOS and mount is wanted, instead run: make GOTAGS=cmount
|
||||
./rclone version
|
||||
```
|
||||
|
||||
This will leave you a checked out version of rclone you can modify and
|
||||
send pull requests with. If you use `make` instead of `go build` then
|
||||
the rclone build will have the correct version information in it.
|
||||
This will check out the rclone source in subfolder rclone, which you can later
|
||||
modify and send pull requests with. Then it will build the rclone executable
|
||||
in the same folder. As an initial check you can now run `./rclone version`
|
||||
(`.\rclone version` on Windows).
|
||||
|
||||
You can also build the latest stable rclone with:
|
||||
Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/)
|
||||
command will not be available unless you specify additional build tag `cmount`.
|
||||
|
||||
go get github.com/rclone/rclone
|
||||
```
|
||||
go build -tags cmount
|
||||
```
|
||||
|
||||
or the latest version (equivalent to the beta) with
|
||||
This assumes you have a GCC compatible C compiler (GCC or Clang) in your PATH,
|
||||
as it uses [cgo](https://pkg.go.dev/cmd/cgo). But on Windows, the
|
||||
[cgofuse](https://github.com/winfsp/cgofuse) library that the cmount
|
||||
implementation is based on, also supports building
|
||||
[without cgo](https://github.com/golang/go/wiki/WindowsDLLs), i.e. by setting
|
||||
environment variable CGO_ENABLED to value 0 (static linking). This is how the
|
||||
official Windows release of rclone is being built, starting with version 1.59.
|
||||
It is still possible to build with cgo on Windows as well, by using the MinGW
|
||||
port of GCC, e.g. by installing it in a [MSYS2](https://www.msys2.org)
|
||||
distribution (make sure you install it in the classic mingw64 subsystem, the
|
||||
ucrt64 version is not compatible).
|
||||
|
||||
go get github.com/rclone/rclone@master
|
||||
Additionally, on Windows, you must install the third party utility
|
||||
[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected.
|
||||
If building with cgo, you must also set environment variable CPATH pointing to
|
||||
the fuse include directory within the WinFsp installation
|
||||
(normally `C:\Program Files (x86)\WinFsp\inc\fuse`).
|
||||
|
||||
These will build the binary in `$(go env GOPATH)/bin`
|
||||
(`~/go/bin/rclone` by default) after downloading the source to the go
|
||||
module cache. Note - do **not** use the `-u` flag here. This causes go
|
||||
to try to update the dependencies that rclone uses and sometimes these
|
||||
don't work with the current version of rclone.
|
||||
You may also add arguments `-ldflags -s` (with or without `-tags cmount`),
|
||||
to omit symbol table and debug information, making the executable file smaller,
|
||||
and `-trimpath` to remove references to local file system paths. This is how
|
||||
the official rclone releases are built.
|
||||
|
||||
## Installation with Ansible ##
|
||||
```
|
||||
go build -trimpath -ldflags -s -tags cmount
|
||||
```
|
||||
|
||||
Instead of executing the `go build` command directly, you can run it via the
|
||||
Makefile, which also sets version information and copies the resulting rclone
|
||||
executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
|
||||
corresponds to `~/go/bin/rclone` by default).
|
||||
|
||||
```
|
||||
make
|
||||
```
|
||||
|
||||
To include mount command on macOS and Windows with Makefile build:
|
||||
|
||||
```
|
||||
make GOTAGS=cmount
|
||||
```
|
||||
|
||||
As an alternative you can download the source, build and install rclone in one
|
||||
operation, as a regular Go package. The source will be stored it in the Go
|
||||
module cache, and the resulting executable will be in your GOPATH bin folder
|
||||
(`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).
|
||||
|
||||
With Go version 1.17 or newer:
|
||||
|
||||
```
|
||||
go install github.com/rclone/rclone@latest
|
||||
```
|
||||
|
||||
With Go versions older than 1.17 (do **not** use the `-u` flag, it causes Go to
|
||||
try to update the dependencies that rclone uses and sometimes these don't work
|
||||
with the current version):
|
||||
|
||||
```
|
||||
go get github.com/rclone/rclone
|
||||
```
|
||||
|
||||
## Installation with Ansible
|
||||
|
||||
This can be done with [Stefan Weichinger's ansible
|
||||
role](https://github.com/stefangweichinger/ansible-rclone).
|
||||
@@ -236,7 +290,7 @@ Instructions
|
||||
- rclone
|
||||
```
|
||||
|
||||
## Portable installation ##
|
||||
## Portable installation
|
||||
|
||||
As mentioned [above](https://rclone.org/install/#quickstart), rclone is single
|
||||
executable (`rclone`, or `rclone.exe` on Windows) that you can download as a
|
||||
@@ -314,7 +368,7 @@ the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
|
||||
utility from Microsoft's Sysinternals suite, which takes option `-s` to
|
||||
execute commands as the `SYSTEM` user.
|
||||
|
||||
#### Start from Startup folder ###
|
||||
#### Start from Startup folder
|
||||
|
||||
To quickly execute an rclone command you can simply create a standard
|
||||
Windows Explorer shortcut for the complete rclone command you want to run. If you
|
||||
@@ -329,7 +383,7 @@ functionality to set it to run as different user, or to set conditions or
|
||||
actions on certain events. Setting up a scheduled task as described below
|
||||
will often give you better results.
|
||||
|
||||
#### Start from Task Scheduler ###
|
||||
#### Start from Task Scheduler
|
||||
|
||||
Task Scheduler is an administrative tool built into Windows, and it can be used to
|
||||
configure rclone to be started automatically in a highly configurable way, e.g.
|
||||
@@ -339,14 +393,14 @@ be available to all users it can run as the `SYSTEM` user.
|
||||
For technical information, see
|
||||
https://docs.microsoft.com/windows/win32/taskschd/task-scheduler-start-page.
|
||||
|
||||
#### Run as service ###
|
||||
#### Run as service
|
||||
|
||||
For running rclone at system startup, you can create a Windows service that executes
|
||||
your rclone command, as an alternative to scheduled task configured to run at startup.
|
||||
|
||||
##### Mount command built-in service integration ####
|
||||
##### Mount command built-in service integration
|
||||
|
||||
For mount commands, Rclone has a built-in Windows service integration via the third-party
|
||||
For mount commands, rclone has a built-in Windows service integration via the third-party
|
||||
WinFsp library it uses. Registering as a regular Windows service easy, as you just have to
|
||||
execute the built-in PowerShell command `New-Service` (requires administrative privileges).
|
||||
|
||||
@@ -366,7 +420,7 @@ Windows standard methods for managing network drives. This is currently not
|
||||
officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later
|
||||
it should be possible through path rewriting as described [here](https://github.com/rclone/rclone/issues/3340).
|
||||
|
||||
##### Third-party service integration #####
|
||||
##### Third-party service integration
|
||||
|
||||
To Windows service running any rclone command, the excellent third-party utility
|
||||
[NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used.
|
||||
|
||||
+10
-6
@@ -12,7 +12,7 @@ set -e
|
||||
#when adding a tool to the list make sure to also add its corresponding command further in the script
|
||||
unzip_tools_list=('unzip' '7z' 'busybox')
|
||||
|
||||
usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
|
||||
usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
|
||||
|
||||
#check for beta flag
|
||||
if [ -n "$1" ] && [ "$1" != "beta" ]; then
|
||||
@@ -81,6 +81,8 @@ case $OS in
|
||||
;;
|
||||
Darwin)
|
||||
OS='osx'
|
||||
binTgtDir=/usr/local/bin
|
||||
man1TgtDir=/usr/local/share/man/man1
|
||||
;;
|
||||
SunOS)
|
||||
OS='solaris'
|
||||
@@ -171,12 +173,14 @@ case "$OS" in
|
||||
;;
|
||||
'osx')
|
||||
#binary
|
||||
mkdir -p /usr/local/bin
|
||||
cp rclone /usr/local/bin/rclone.new
|
||||
mv /usr/local/bin/rclone.new /usr/local/bin/rclone
|
||||
mkdir -m 0555 -p ${binTgtDir}
|
||||
cp rclone ${binTgtDir}/rclone.new
|
||||
mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone
|
||||
chmod a=x ${binTgtDir}/rclone
|
||||
#manual
|
||||
mkdir -p /usr/local/share/man/man1
|
||||
cp rclone.1 /usr/local/share/man/man1/
|
||||
mkdir -m 0555 -p ${man1TgtDir}
|
||||
cp rclone.1 ${man1TgtDir}
|
||||
chmod a=r ${man1TgtDir}/rclone.1
|
||||
;;
|
||||
*)
|
||||
echo 'OS not supported'
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
---
|
||||
title: "Internet Archive"
|
||||
description: "Rclone docs for Internet Archive"
|
||||
---
|
||||
|
||||
# {{< icon "fa fa-archive" >}} Internet Archive
|
||||
|
||||
The Internet Archive backend utilizes Items on [archive.org](https://archive.org/)
|
||||
|
||||
Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.html) for the API this backend uses.
|
||||
|
||||
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
|
||||
command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
|
||||
|
||||
Once you have made a remote (see the provider specific section above)
|
||||
you can use it like this:
|
||||
|
||||
Unlike S3, listing up all items uploaded by you isn't supported.
|
||||
|
||||
Make a new item
|
||||
|
||||
rclone mkdir remote:item
|
||||
|
||||
List the contents of a item
|
||||
|
||||
rclone ls remote:item
|
||||
|
||||
Sync `/home/local/directory` to the remote item, deleting any excess
|
||||
files in the item.
|
||||
|
||||
rclone sync -i /home/local/directory remote:item
|
||||
|
||||
## Notes
|
||||
Because of Internet Archive's architecture, it enqueues write operations (and extra post-processings) in a per-item queue. You can check item's queue at https://catalogd.archive.org/history/item-name-here . Because of that, all uploads/deletes will not show up immediately and takes some time to be available.
|
||||
The per-item queue is enqueued to an another queue, Item Deriver Queue. [You can check the status of Item Deriver Queue here.](https://catalogd.archive.org/catalog.php?whereami=1) This queue has a limit, and it may block you from uploading, or even deleting. You should avoid uploading a lot of small files for better behavior.
|
||||
|
||||
You can optionally wait for the server's processing to finish, by setting non-zero value to `wait_archive` key.
|
||||
By making it wait, rclone can do normal file comparison.
|
||||
Make sure to set a large enough value (e.g. `30m0s` for smaller files) as it can take a long time depending on server's queue.
|
||||
|
||||
## Configuration
|
||||
|
||||
Here is an example of making an internetarchive configuration.
|
||||
Most applies to the other providers as well, any differences are described [below](#providers).
|
||||
|
||||
First run
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> remote
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
XX / InternetArchive Items
|
||||
\ (internetarchive)
|
||||
Storage> internetarchive
|
||||
Option access_key_id.
|
||||
IAS3 Access Key.
|
||||
Leave blank for anonymous access.
|
||||
You can find one here: https://archive.org/account/s3.php
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> XXXX
|
||||
Option secret_access_key.
|
||||
IAS3 Secret Key (password).
|
||||
Leave blank for anonymous access.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> XXXX
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> y
|
||||
Option endpoint.
|
||||
IAS3 Endpoint.
|
||||
Leave blank for default value.
|
||||
Enter a string value. Press Enter for the default (https://s3.us.archive.org).
|
||||
endpoint>
|
||||
Option front_endpoint.
|
||||
Host of InternetArchive Frontend.
|
||||
Leave blank for default value.
|
||||
Enter a string value. Press Enter for the default (https://archive.org).
|
||||
front_endpoint>
|
||||
Option disable_checksum.
|
||||
Don't store MD5 checksum with object metadata.
|
||||
Normally rclone will calculate the MD5 checksum of the input before
|
||||
uploading it so it can ask the server to check the object against checksum.
|
||||
This is great for data integrity checking but can cause long delays for
|
||||
large files to start uploading.
|
||||
Enter a boolean value (true or false). Press Enter for the default (true).
|
||||
disable_checksum> true
|
||||
Option encoding.
|
||||
The encoding for the backend.
|
||||
See the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot).
|
||||
encoding>
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
--------------------
|
||||
[remote]
|
||||
type = internetarchive
|
||||
access_key_id = XXXX
|
||||
secret_access_key = XXXX
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/internetarchive/internetarchive.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the standard options specific to internetarchive (Internet Archive).
|
||||
|
||||
#### --internetarchive-access-key-id
|
||||
|
||||
IAS3 Access Key.
|
||||
|
||||
Leave blank for anonymous access.
|
||||
You can find one here: https://archive.org/account/s3.php
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: access_key_id
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --internetarchive-secret-access-key
|
||||
|
||||
IAS3 Secret Key (password).
|
||||
|
||||
Leave blank for anonymous access.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: secret_access_key
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
### Advanced options
|
||||
|
||||
Here are the advanced options specific to internetarchive (Internet Archive).
|
||||
|
||||
#### --internetarchive-endpoint
|
||||
|
||||
IAS3 Endpoint.
|
||||
|
||||
Leave blank for default value.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT
|
||||
- Type: string
|
||||
- Default: "https://s3.us.archive.org"
|
||||
|
||||
#### --internetarchive-front-endpoint
|
||||
|
||||
Host of InternetArchive Frontend.
|
||||
|
||||
Leave blank for default value.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: front_endpoint
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT
|
||||
- Type: string
|
||||
- Default: "https://archive.org"
|
||||
|
||||
#### --internetarchive-disable-checksum
|
||||
|
||||
Don't ask the server to test against MD5 checksum calculated by rclone.
|
||||
Normally rclone will calculate the MD5 checksum of the input before
|
||||
uploading it so it can ask the server to check the object against checksum.
|
||||
This is great for data integrity checking but can cause long delays for
|
||||
large files to start uploading.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: disable_checksum
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM
|
||||
- Type: bool
|
||||
- Default: true
|
||||
|
||||
#### --internetarchive-wait-archive
|
||||
|
||||
Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish.
|
||||
Only enable if you need to be guaranteed to be reflected after write operations.
|
||||
0 to disable waiting. No errors to be thrown in case of timeout.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: wait_archive
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE
|
||||
- Type: Duration
|
||||
- Default: 0s
|
||||
|
||||
#### --internetarchive-encoding
|
||||
|
||||
The encoding for the backend.
|
||||
|
||||
See the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: encoding
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_ENCODING
|
||||
- Type: MultiEncoder
|
||||
- Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
@@ -177,7 +177,6 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
|
||||
+32
-29
@@ -32,6 +32,7 @@ Here is an overview of the major features of each cloud storage system.
|
||||
| HDFS | - | Yes | No | No | - |
|
||||
| HTTP | - | No | No | No | R |
|
||||
| Hubic | MD5 | Yes | No | No | R/W |
|
||||
| Internet Archive | MD5, SHA1, CRC32 | Yes | No | No | - |
|
||||
| Jottacloud | MD5 | Yes | Yes | No | R |
|
||||
| Koofr | MD5 | No | Yes | No | - |
|
||||
| Mail.ru Cloud | Mailru ⁶ | Yes | Yes | No | - |
|
||||
@@ -301,35 +302,36 @@ list of all possible values by passing an invalid value to this
|
||||
flag, e.g. `--local-encoding "help"`. The command `rclone help flags encoding`
|
||||
will show you the defaults for the backends.
|
||||
|
||||
| Encoding | Characters |
|
||||
| --------- | ---------- |
|
||||
| Asterisk | `*` |
|
||||
| BackQuote | `` ` `` |
|
||||
| BackSlash | `\` |
|
||||
| Colon | `:` |
|
||||
| CrLf | CR 0x0D, LF 0x0A |
|
||||
| Ctl | All control characters 0x00-0x1F |
|
||||
| Del | DEL 0x7F |
|
||||
| Dollar | `$` |
|
||||
| Dot | `.` or `..` as entire string |
|
||||
| DoubleQuote | `"` |
|
||||
| Hash | `#` |
|
||||
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) |
|
||||
| LeftCrLfHtVt | CR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a string |
|
||||
| LeftPeriod | `.` on the left of a string |
|
||||
| LeftSpace | SPACE on the left of a string |
|
||||
| LeftTilde | `~` on the left of a string |
|
||||
| LtGt | `<`, `>` |
|
||||
| None | No characters are encoded |
|
||||
| Percent | `%` |
|
||||
| Pipe | \| |
|
||||
| Question | `?` |
|
||||
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string |
|
||||
| RightPeriod | `.` on the right of a string |
|
||||
| RightSpace | SPACE on the right of a string |
|
||||
| SingleQuote | `'` |
|
||||
| Slash | `/` |
|
||||
| SquareBracket | `[`, `]` |
|
||||
| Encoding | Characters | Encoded as |
|
||||
| --------- | ---------- | ---------- |
|
||||
| Asterisk | `*` | `*` |
|
||||
| BackQuote | `` ` `` | ``` |
|
||||
| BackSlash | `\` | `\` |
|
||||
| Colon | `:` | `:` |
|
||||
| CrLf | CR 0x0D, LF 0x0A | `␍`, `␊` |
|
||||
| Ctl | All control characters 0x00-0x1F | `␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟` |
|
||||
| Del | DEL 0x7F | `␡` |
|
||||
| Dollar | `$` | `$` |
|
||||
| Dot | `.` or `..` as entire string | `.`, `..` |
|
||||
| DoubleQuote | `"` | `"` |
|
||||
| Hash | `#` | `#` |
|
||||
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | `�` |
|
||||
| LeftCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string | `␍`, `␊`, `␉`, `␋` |
|
||||
| LeftPeriod | `.` on the left of a string | `.` |
|
||||
| LeftSpace | SPACE on the left of a string | `␠` |
|
||||
| LeftTilde | `~` on the left of a string | `~` |
|
||||
| LtGt | `<`, `>` | `<`, `>` |
|
||||
| None | No characters are encoded | |
|
||||
| Percent | `%` | `%` |
|
||||
| Pipe | \| | `|` |
|
||||
| Question | `?` | `?` |
|
||||
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string | `␍`, `␊`, `␉`, `␋` |
|
||||
| RightPeriod | `.` on the right of a string | `.` |
|
||||
| RightSpace | SPACE on the right of a string | `␠` |
|
||||
| Semicolon | `;` | `;` |
|
||||
| SingleQuote | `'` | `'` |
|
||||
| Slash | `/` | `/` |
|
||||
| SquareBracket | `[`, `]` | `[`, `]` |
|
||||
|
||||
##### Encoding example: FTP
|
||||
|
||||
@@ -426,6 +428,7 @@ upon backend-specific capabilities.
|
||||
| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes |
|
||||
| HTTP | No | No | No | No | No | No | No | No | No | Yes |
|
||||
| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No |
|
||||
| Internet Archive | No | Yes | No | No | Yes | Yes | No | Yes | Yes | No |
|
||||
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
|
||||
| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| Mega | Yes | No | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
|
||||
@@ -318,6 +318,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -150,6 +150,16 @@ use these methods. The alternative is to use `--rc-user` and
|
||||
|
||||
Default Off.
|
||||
|
||||
### --rc-baseurl
|
||||
|
||||
Prefix for URLs.
|
||||
|
||||
Default is root
|
||||
|
||||
### --rc-template
|
||||
|
||||
User-specified template.
|
||||
|
||||
## Accessing the remote control via the rclone rc command {#api-rc}
|
||||
|
||||
Rclone itself implements the remote control protocol in its `rclone
|
||||
@@ -1285,6 +1295,7 @@ Returns:
|
||||
|
||||
- count - number of files
|
||||
- bytes - number of bytes in those files
|
||||
- sizeless - number of files with unknown size, included in count but not accounted for in bytes
|
||||
|
||||
See the [size command](/commands/rclone_size/) command for more information on the above.
|
||||
|
||||
|
||||
@@ -92,3 +92,24 @@ Configuration file is stored at:
|
||||
Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and
|
||||
place it in the correct place (use `rclone config file` on the remote
|
||||
box to find out where).
|
||||
|
||||
## Configuring using SSH Tunnel ##
|
||||
|
||||
Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 53682 to local machine by using the following command:
|
||||
```
|
||||
ssh -L localhost:53682:localhost:53682 username@remote_server
|
||||
```
|
||||
Then on the headless box run `rclone` config and answer `Y` to the `Use
|
||||
auto config?` question.
|
||||
|
||||
```
|
||||
...
|
||||
Remote config
|
||||
Use auto config?
|
||||
* Say Y if not sure
|
||||
* Say N if you are working on a remote or headless machine
|
||||
y) Yes (default)
|
||||
n) No
|
||||
y/n> y
|
||||
```
|
||||
Then copy and paste the auth url `http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx` to the browser on your local machine, complete the auth and it is done.
|
||||
+278
-26
@@ -12,6 +12,8 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
|
||||
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
|
||||
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
|
||||
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
|
||||
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
|
||||
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
|
||||
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
|
||||
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
|
||||
@@ -70,7 +72,7 @@ name> remote
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -567,7 +569,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
|
||||
#### --s3-provider
|
||||
|
||||
@@ -588,6 +590,8 @@ Properties:
|
||||
- Ceph Object Storage
|
||||
- "ChinaMobile"
|
||||
- China Mobile Ecloud Elastic Object Storage (EOS)
|
||||
- "ArvanCloud"
|
||||
- Arvan Cloud Object Storage (AOS)
|
||||
- "DigitalOcean"
|
||||
- Digital Ocean Spaces
|
||||
- "Dreamhost"
|
||||
@@ -829,7 +833,7 @@ Properties:
|
||||
|
||||
- Config: region
|
||||
- Env Var: RCLONE_S3_REGION
|
||||
- Provider: !AWS,Alibaba,ChinaMobile,RackCorp,Scaleway,Storj,TencentCOS
|
||||
- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -926,6 +930,24 @@ Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.
|
||||
- "eos-anhui-1.cmecloud.cn"
|
||||
- Anhui China (Huainan)
|
||||
|
||||
#### --s3-endpoint
|
||||
|
||||
Endpoint for Arvan Cloud Object Storage (AOS) API.
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_S3_ENDPOINT
|
||||
- Provider: ArvanCloud
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
- "s3.ir-thr-at1.arvanstorage.com"
|
||||
- The default endpoint - a good choice if you are unsure.
|
||||
- Tehran Iran (Asiatech)
|
||||
- "s3.ir-tbz-sh1.arvanstorage.com"
|
||||
- Tabriz Iran (Shahriar)
|
||||
|
||||
|
||||
|
||||
#### --s3-endpoint
|
||||
|
||||
Endpoint for IBM COS S3 API.
|
||||
@@ -1295,7 +1317,7 @@ Properties:
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_S3_ENDPOINT
|
||||
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,Scaleway,StackPath,Storj,RackCorp
|
||||
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1325,6 +1347,8 @@ Properties:
|
||||
- Wasabi AP Northeast 1 (Tokyo) endpoint
|
||||
- "s3.ap-northeast-2.wasabisys.com"
|
||||
- Wasabi AP Northeast 2 (Osaka) endpoint
|
||||
- "s3.ir-thr-at1.arvanstorage.com"
|
||||
- ArvanCloud Tehran Iran (Asiatech) endpoint
|
||||
|
||||
#### --s3-location-constraint
|
||||
|
||||
@@ -1532,7 +1556,7 @@ Properties:
|
||||
|
||||
- Config: location_constraint
|
||||
- Env Var: RCLONE_S3_LOCATION_CONSTRAINT
|
||||
- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,RackCorp,Scaleway,StackPath,Storj,TencentCOS
|
||||
- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
@@ -1604,7 +1628,7 @@ Properties:
|
||||
|
||||
- Config: server_side_encryption
|
||||
- Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION
|
||||
- Provider: AWS,Ceph,ChinaMobile,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1705,6 +1729,19 @@ The storage class to use when storing new objects in ChinaMobile.
|
||||
|
||||
#### --s3-storage-class
|
||||
|
||||
The storage class to use when storing new objects in ArvanCloud.
|
||||
|
||||
- Config: storage_class
|
||||
- Env Var: RCLONE_S3_STORAGE_CLASS
|
||||
- Provider: ArvanCloud
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
- "STANDARD"
|
||||
- Standard storage class
|
||||
|
||||
#### --s3-storage-class
|
||||
|
||||
The storage class to use when storing new objects in Tencent COS.
|
||||
|
||||
Properties:
|
||||
@@ -1747,7 +1784,7 @@ Properties:
|
||||
|
||||
### Advanced options
|
||||
|
||||
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
|
||||
#### --s3-bucket-acl
|
||||
|
||||
@@ -1799,7 +1836,7 @@ Properties:
|
||||
|
||||
- Config: sse_customer_algorithm
|
||||
- Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM
|
||||
- Provider: AWS,Ceph,ChinaMobile,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1816,7 +1853,7 @@ Properties:
|
||||
|
||||
- Config: sse_customer_key
|
||||
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY
|
||||
- Provider: AWS,Ceph,ChinaMobile,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1834,7 +1871,7 @@ Properties:
|
||||
|
||||
- Config: sse_customer_key_md5
|
||||
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5
|
||||
- Provider: AWS,Ceph,ChinaMobile,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -2443,12 +2480,19 @@ You will be able to list and copy data but not upload it.
|
||||
This is the provider used as main example and described in the [configuration](#configuration) section above.
|
||||
|
||||
### AWS Snowball Edge
|
||||
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring
|
||||
bulk data back to AWS. Its main software interface is S3 object storage.
|
||||
|
||||
To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'
|
||||
be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as
|
||||
the snowball device does not support query parameter based authentication.
|
||||
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware
|
||||
appliance used for transferring bulk data back to AWS. Its main
|
||||
software interface is S3 object storage.
|
||||
|
||||
To use rclone with AWS Snowball Edge devices, configure as standard
|
||||
for an 'S3 Compatible Service'.
|
||||
|
||||
If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise
|
||||
you will run into authentication header issues as the snowball device
|
||||
does not support query parameter based authentication.
|
||||
|
||||
With rclone v1.59 or later setting `upload_cutoff` should not be necessary.
|
||||
|
||||
eg.
|
||||
```
|
||||
@@ -2487,10 +2531,11 @@ server_side_encryption =
|
||||
storage_class =
|
||||
```
|
||||
|
||||
If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you
|
||||
may need to supply the parameter `--s3-upload-cutoff 0` or put this in
|
||||
the config file as `upload_cutoff 0` to work around a bug which causes
|
||||
uploading of small files to fail.
|
||||
If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a
|
||||
version of rclone before v1.59 then you may need to supply the
|
||||
parameter `--s3-upload-cutoff 0` or put this in the config file as
|
||||
`upload_cutoff 0` to work around a bug which causes uploading of small
|
||||
files to fail.
|
||||
|
||||
Note also that Ceph sometimes puts `/` in the passwords it gives
|
||||
users. If you read the secret access key using the command line tools
|
||||
@@ -2517,6 +2562,106 @@ removed).
|
||||
Because this is a json dump, it is encoding the `/` as `\/`, so if you
|
||||
use the secret key as `xxxxxx/xxxx` it will work fine.
|
||||
|
||||
### Cloudflare R2 {#cloudflare-r2}
|
||||
|
||||
[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage
|
||||
allows developers to store large amounts of unstructured data without
|
||||
the costly egress bandwidth fees associated with typical cloud storage
|
||||
services.
|
||||
|
||||
Here is an example of making a Cloudflare R2 configuration. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
Note that all buckets are private, and all are stored in the same
|
||||
"auto" region. It is necessary to use Cloudflare workers to share the
|
||||
content of a bucket publicly.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> r2
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
...
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
|
||||
\ (s3)
|
||||
...
|
||||
Storage> s3
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
...
|
||||
XX / Cloudflare R2 Storage
|
||||
\ (Cloudflare)
|
||||
...
|
||||
provider> Cloudflare
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth> 1
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> ACCESS_KEY
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> SECRET_ACCESS_KEY
|
||||
Option region.
|
||||
Region to connect to.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
|
||||
\ (auto)
|
||||
region> 1
|
||||
Option endpoint.
|
||||
Endpoint for S3 API.
|
||||
Required when using an S3 clone.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This will leave your config looking something like:
|
||||
|
||||
```
|
||||
[r2]
|
||||
type = s3
|
||||
provider = Cloudflare
|
||||
access_key_id = ACCESS_KEY
|
||||
secret_access_key = SECRET_ACCESS_KEY
|
||||
region = auto
|
||||
endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
|
||||
acl = private
|
||||
```
|
||||
|
||||
Now run `rclone lsf r2:` to see your buckets and `rclone lsf
|
||||
r2:bucket` to look within a bucket.
|
||||
|
||||
### Dreamhost
|
||||
|
||||
Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is
|
||||
@@ -2615,7 +2760,7 @@ Choose a number from below, or type in your own value
|
||||
\ "alias"
|
||||
2 / Amazon Drive
|
||||
\ "amazon cloud drive"
|
||||
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Minio, IBM COS)
|
||||
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS)
|
||||
\ "s3"
|
||||
4 / Backblaze B2
|
||||
\ "b2"
|
||||
@@ -2900,7 +3045,7 @@ Choose `s3` backend
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -3087,7 +3232,7 @@ name> wasabi
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, Minio)
|
||||
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -3201,7 +3346,7 @@ Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -3541,6 +3686,114 @@ d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
### ArvanCloud
|
||||
|
||||
[ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage.
|
||||
It gives you access to backup and archived files and allows sharing.
|
||||
Files like profile image in the app, images sent by users or scanned documents can be stored securely and easily in our Object Storage service.
|
||||
|
||||
ArvanCloud provides an S3 interface which can be configured for use with
|
||||
rclone like this.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
n/s> n
|
||||
name> ArvanCloud
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Enter AWS credentials in the next step
|
||||
\ "false"
|
||||
2 / Get AWS credentials from the environment (env vars or IAM)
|
||||
\ "true"
|
||||
env_auth> 1
|
||||
AWS Access Key ID - leave blank for anonymous access or runtime credentials.
|
||||
access_key_id> YOURACCESSKEY
|
||||
AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
|
||||
secret_access_key> YOURSECRETACCESSKEY
|
||||
Region to connect to.
|
||||
Choose a number from below, or type in your own value
|
||||
/ The default endpoint - a good choice if you are unsure.
|
||||
1 | US Region, Northern Virginia, or Pacific Northwest.
|
||||
| Leave location constraint empty.
|
||||
\ "us-east-1"
|
||||
[snip]
|
||||
region>
|
||||
Endpoint for S3 API.
|
||||
Leave blank if using ArvanCloud to use the default endpoint for the region.
|
||||
Specify if using an S3 clone such as Ceph.
|
||||
endpoint> s3.arvanstorage.com
|
||||
Location constraint - must be set to match the Region. Used when creating buckets only.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Empty for Iran-Tehran Region.
|
||||
\ ""
|
||||
[snip]
|
||||
location_constraint>
|
||||
Canned ACL used when creating buckets and/or storing objects in S3.
|
||||
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Owner gets FULL_CONTROL. No one else has access rights (default).
|
||||
\ "private"
|
||||
[snip]
|
||||
acl>
|
||||
The server-side encryption algorithm used when storing this object in S3.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / None
|
||||
\ ""
|
||||
2 / AES256
|
||||
\ "AES256"
|
||||
server_side_encryption>
|
||||
The storage class to use when storing objects in S3.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Default
|
||||
\ ""
|
||||
2 / Standard storage class
|
||||
\ "STANDARD"
|
||||
storage_class>
|
||||
Remote config
|
||||
--------------------
|
||||
[ArvanCloud]
|
||||
env_auth = false
|
||||
access_key_id = YOURACCESSKEY
|
||||
secret_access_key = YOURSECRETACCESSKEY
|
||||
region = ir-thr-at1
|
||||
endpoint = s3.arvanstorage.com
|
||||
location_constraint =
|
||||
acl =
|
||||
server_side_encryption =
|
||||
storage_class =
|
||||
--------------------
|
||||
y) Yes this is OK
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This will leave the config file looking like this.
|
||||
|
||||
```
|
||||
[ArvanCloud]
|
||||
type = s3
|
||||
provider = ArvanCloud
|
||||
env_auth = false
|
||||
access_key_id = YOURACCESSKEY
|
||||
secret_access_key = YOURSECRETACCESSKEY
|
||||
region =
|
||||
endpoint = s3.arvanstorage.com
|
||||
location_constraint =
|
||||
acl =
|
||||
server_side_encryption =
|
||||
storage_class =
|
||||
```
|
||||
|
||||
### Tencent COS {#tencent-cos}
|
||||
|
||||
[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.
|
||||
@@ -3574,7 +3827,7 @@ Choose a number from below, or type in your own value
|
||||
\ "alias"
|
||||
3 / Amazon Drive
|
||||
\ "amazon cloud drive"
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -3787,5 +4040,4 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
@@ -251,6 +251,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -415,8 +415,7 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
## Known issues
|
||||
|
||||
|
||||
@@ -280,6 +280,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -73,6 +73,7 @@
|
||||
<a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe"></i> HDFS (Hadoop Distributed Filesystem)</a>
|
||||
<a class="dropdown-item" href="/http/"><i class="fa fa-globe"></i> HTTP</a>
|
||||
<a class="dropdown-item" href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a>
|
||||
<a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive"></i> Internet Archive</a>
|
||||
<a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud"></i> Jottacloud</a>
|
||||
<a class="dropdown-item" href="/koofr/"><i class="fa fa-suitcase"></i> Koofr</a>
|
||||
<a class="dropdown-item" href="/mailru/"><i class="fa fa-at"></i> Mail.ru Cloud</a>
|
||||
|
||||
@@ -230,7 +230,7 @@ func ConfigChoose(state string, name string, help string, n int, getItem func(i
|
||||
// StatePush pushes a new values onto the front of the config string
|
||||
func StatePush(state string, values ...string) string {
|
||||
for i := range values {
|
||||
values[i] = strings.Replace(values[i], ",", ",", -1) // replace comma with unicode wide version
|
||||
values[i] = strings.ReplaceAll(values[i], ",", ",") // replace comma with unicode wide version
|
||||
}
|
||||
if state != "" {
|
||||
values = append(values[:len(values):len(values)], state)
|
||||
@@ -262,7 +262,7 @@ func StatePop(state string) (newState string, value string) {
|
||||
return "", state
|
||||
}
|
||||
value, newState = state[:comma], state[comma+1:]
|
||||
value = strings.Replace(value, ",", ",", -1) // replace unicode wide comma with comma
|
||||
value = strings.ReplaceAll(value, ",", ",") // replace unicode wide comma with comma
|
||||
return newState, value
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
// Package chunksize calculates a suitable chunk size for large uploads
|
||||
package chunksize
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
/*
|
||||
Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi
|
||||
|
||||
For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
|
||||
possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
|
||||
that some backends may only allow integer type parameters when specifying the chunk size.
|
||||
|
||||
Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
|
||||
smallest chunk size necessary to allow the upload to proceed.
|
||||
*/
|
||||
func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
|
||||
fileSize := fs.SizeSuffix(objInfo.Size())
|
||||
requiredChunks := fileSize / defaultChunkSize
|
||||
if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
|
||||
return defaultChunkSize
|
||||
}
|
||||
|
||||
minChunk := fileSize / fs.SizeSuffix(maxParts)
|
||||
remainder := minChunk % fs.Mebi
|
||||
if remainder != 0 {
|
||||
minChunk += fs.Mebi - remainder
|
||||
}
|
||||
if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
|
||||
minChunk += fs.Mebi
|
||||
}
|
||||
|
||||
fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
|
||||
return minChunk
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package chunksize
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
)
|
||||
|
||||
func TestComputeChunkSize(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
fileSize fs.SizeSuffix
|
||||
maxParts int
|
||||
defaultChunkSize fs.SizeSuffix
|
||||
expected fs.SizeSuffix
|
||||
}{
|
||||
"default size returned when file size is small enough": {fileSize: 1000, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
|
||||
"default size returned when file size is just 1 byte small enough": {fileSize: toSizeSuffixMiB(100000) - 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
|
||||
"no rounding up when everything divides evenly": {fileSize: toSizeSuffixMiB(1000000), maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(100)},
|
||||
"rounding up to nearest MiB when not quite enough parts": {fileSize: toSizeSuffixMiB(1000000), maxParts: 9999, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
|
||||
"rounding up to nearest MiB when one extra byte": {fileSize: toSizeSuffixMiB(1000000) + 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
|
||||
"expected MiB value when rounding sets to absolute minimum": {fileSize: toSizeSuffixMiB(1) - 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(1)},
|
||||
"expected MiB value when rounding to absolute min with extra": {fileSize: toSizeSuffixMiB(1) + 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(2)},
|
||||
}
|
||||
|
||||
for name, tc := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
src := object.NewStaticObjectInfo("mock", time.Now(), int64(tc.fileSize), true, nil, nil)
|
||||
result := Calculator(src, tc.maxParts, tc.defaultChunkSize)
|
||||
if result != tc.expected {
|
||||
t.Fatalf("expected: %v, got: %v", tc.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func toSizeSuffixMiB(size int64) fs.SizeSuffix {
|
||||
return fs.SizeSuffix(size * int64(fs.Mebi))
|
||||
}
|
||||
+2
-2
@@ -248,11 +248,11 @@ func AddConfig(ctx context.Context) (context.Context, *ConfigInfo) {
|
||||
// "ignore-size") into an environment name
|
||||
// "RCLONE_CONFIG_MY-REMOTE_IGNORE_SIZE"
|
||||
func ConfigToEnv(section, name string) string {
|
||||
return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.Replace(name, "-", "_", -1))
|
||||
return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.ReplaceAll(name, "-", "_"))
|
||||
}
|
||||
|
||||
// OptionToEnv converts an option name, e.g. "ignore-size" into an
|
||||
// environment name "RCLONE_IGNORE_SIZE"
|
||||
func OptionToEnv(name string) string {
|
||||
return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
|
||||
return "RCLONE_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
|
||||
}
|
||||
@@ -50,7 +50,7 @@ func setConfigFile(t *testing.T, data string) func() {
|
||||
// toUnix converts \r\n to \n in buf
|
||||
func toUnix(buf string) string {
|
||||
if runtime.GOOS == "windows" {
|
||||
return strings.Replace(buf, "\r\n", "\n", -1)
|
||||
return strings.ReplaceAll(buf, "\r\n", "\n")
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
+1
-1
@@ -417,7 +417,7 @@ func ChooseOption(o *fs.Option, name string) string {
|
||||
fmt.Printf("Option %s.\n", o.Name)
|
||||
if o.Help != "" {
|
||||
// Show help string without empty lines.
|
||||
help := strings.Replace(strings.TrimSpace(o.Help), "\n\n", "\n", -1)
|
||||
help := strings.ReplaceAll(strings.TrimSpace(o.Help), "\n\n", "\n")
|
||||
fmt.Println(help)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -208,7 +208,7 @@ loop:
|
||||
value := path[prev : i-1]
|
||||
// replace any doubled quotes if there were any
|
||||
if doubled {
|
||||
value = strings.Replace(value, string(quote)+string(quote), string(quote), -1)
|
||||
value = strings.ReplaceAll(value, string(quote)+string(quote), string(quote))
|
||||
}
|
||||
prev = i + 1
|
||||
parsed.Config[param] = value
|
||||
|
||||
+1
-1
@@ -208,7 +208,7 @@ func parseHelperOptionString(optString string) (opts []string, err error) {
|
||||
if len(param) == 0 {
|
||||
return nil, errHelperEmptyOption
|
||||
}
|
||||
if param[0] == '-' || param[0] == '_' {
|
||||
if param[0] == '-' {
|
||||
return nil, errHelperOptionName
|
||||
}
|
||||
prev = i + 1
|
||||
|
||||
@@ -20,7 +20,7 @@ func TestMountHelperArgs(t *testing.T) {
|
||||
src: []string{},
|
||||
dst: []string{"mount", "--daemon"},
|
||||
}, {
|
||||
src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env`},
|
||||
src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env,_netdev`},
|
||||
dst: []string{"mount", "--read-only", "--verbose=3", "--daemon"},
|
||||
env: "HTTPS_PROXY=a b;c,d?EF",
|
||||
}}
|
||||
|
||||
+14
-5
@@ -138,8 +138,14 @@ func (o *RangeOption) Decode(size int64) (offset, limit int64) {
|
||||
// absolute fetch using the size passed in and makes sure the range does
|
||||
// not exceed filesize. Some remotes (e.g. Onedrive, Box) don't support
|
||||
// range requests which index from the end.
|
||||
//
|
||||
// It also adjusts any SeekOption~s, turning them into absolute
|
||||
// RangeOption~s instead.
|
||||
func FixRangeOption(options []OpenOption, size int64) {
|
||||
if size == 0 {
|
||||
if size < 0 {
|
||||
// Can't do anything for unknown length objects
|
||||
return
|
||||
} else if size == 0 {
|
||||
// if size 0 then remove RangeOption~s
|
||||
// replacing with a NullOptions~s which won't be rendered
|
||||
for i := range options {
|
||||
@@ -150,18 +156,21 @@ func FixRangeOption(options []OpenOption, size int64) {
|
||||
}
|
||||
return
|
||||
}
|
||||
for i := range options {
|
||||
option := options[i]
|
||||
if x, ok := option.(*RangeOption); ok {
|
||||
for i, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *RangeOption:
|
||||
// If start is < 0 then fetch from the end
|
||||
if x.Start < 0 {
|
||||
x = &RangeOption{Start: size - x.End, End: -1}
|
||||
options[i] = x
|
||||
}
|
||||
if x.End > size {
|
||||
// If end is too big or undefined, fetch to the end
|
||||
if x.End > size || x.End < 0 {
|
||||
x = &RangeOption{Start: x.Start, End: size - 1}
|
||||
options[i] = x
|
||||
}
|
||||
case *SeekOption:
|
||||
options[i] = &RangeOption{Start: x.Offset, End: size - 1}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+26
-2
@@ -149,6 +149,16 @@ func TestFixRangeOptions(t *testing.T) {
|
||||
in: []OpenOption{},
|
||||
want: []OpenOption{},
|
||||
},
|
||||
{
|
||||
name: "Unknown size -1",
|
||||
in: []OpenOption{
|
||||
&RangeOption{Start: 1, End: -1},
|
||||
},
|
||||
want: []OpenOption{
|
||||
&RangeOption{Start: 1, End: -1},
|
||||
},
|
||||
size: -1,
|
||||
},
|
||||
{
|
||||
name: "Fetch a range with size=0",
|
||||
in: []OpenOption{
|
||||
@@ -183,7 +193,7 @@ func TestFixRangeOptions(t *testing.T) {
|
||||
&RangeOption{Start: 1, End: -1},
|
||||
},
|
||||
want: []OpenOption{
|
||||
&RangeOption{Start: 1, End: -1},
|
||||
&RangeOption{Start: 1, End: 99},
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
@@ -193,7 +203,7 @@ func TestFixRangeOptions(t *testing.T) {
|
||||
&RangeOption{Start: -1, End: 10},
|
||||
},
|
||||
want: []OpenOption{
|
||||
&RangeOption{Start: 90, End: -1},
|
||||
&RangeOption{Start: 90, End: 99},
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
@@ -207,6 +217,20 @@ func TestFixRangeOptions(t *testing.T) {
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
{
|
||||
name: "SeekOption",
|
||||
in: []OpenOption{
|
||||
&HTTPOption{Key: "a", Value: "1"},
|
||||
&SeekOption{Offset: 10},
|
||||
&HTTPOption{Key: "b", Value: "2"},
|
||||
},
|
||||
want: []OpenOption{
|
||||
&HTTPOption{Key: "a", Value: "1"},
|
||||
&RangeOption{Start: 10, End: 99},
|
||||
&HTTPOption{Key: "b", Value: "2"},
|
||||
},
|
||||
size: 100,
|
||||
},
|
||||
} {
|
||||
FixRangeOption(test.in, test.size)
|
||||
assert.Equal(t, test.want, test.in, test.name)
|
||||
|
||||
@@ -121,7 +121,7 @@ func TestDeduplicateFirst(t *testing.T) {
|
||||
// list until we get one object
|
||||
var objects, size int64
|
||||
for try := 1; try <= *fstest.ListRetries; try++ {
|
||||
objects, size, err = operations.Count(context.Background(), r.Fremote)
|
||||
objects, size, _, err = operations.Count(context.Background(), r.Fremote)
|
||||
require.NoError(t, err)
|
||||
if objects == 1 {
|
||||
break
|
||||
|
||||
@@ -796,6 +796,47 @@ func Overlapping(fdst, fsrc fs.Info) bool {
|
||||
return strings.HasPrefix(fdstRoot, fsrcRoot) || strings.HasPrefix(fsrcRoot, fdstRoot)
|
||||
}
|
||||
|
||||
// OverlappingFilterCheck returns true if fdst and fsrc point to the same
|
||||
// underlying Fs and they overlap without fdst being excluded by any filter rule.
|
||||
func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool {
|
||||
if !SameConfig(fdst, fsrc) {
|
||||
return false
|
||||
}
|
||||
fdstRoot := fixRoot(fdst)
|
||||
fsrcRoot := fixRoot(fsrc)
|
||||
if strings.HasPrefix(fdstRoot, fsrcRoot) {
|
||||
fdstRelative := fdstRoot[len(fsrcRoot):]
|
||||
return filterCheckR(ctx, fdstRelative, 0, fsrc)
|
||||
}
|
||||
return strings.HasPrefix(fsrcRoot, fdstRoot)
|
||||
}
|
||||
|
||||
// filterCheckR checks if fdst would be included in the sync
|
||||
func filterCheckR(ctx context.Context, fdstRelative string, pos int, fsrc fs.Fs) bool {
|
||||
include := true
|
||||
fi := filter.GetConfig(ctx)
|
||||
includeDirectory := fi.IncludeDirectory(ctx, fsrc)
|
||||
dirs := strings.SplitAfterN(fdstRelative, "/", pos+2)
|
||||
newPath := ""
|
||||
for i := 0; i <= pos; i++ {
|
||||
newPath += dirs[i]
|
||||
}
|
||||
if !strings.HasSuffix(newPath, "/") {
|
||||
newPath += "/"
|
||||
}
|
||||
if strings.HasPrefix(fdstRelative, newPath) {
|
||||
include, _ = includeDirectory(newPath)
|
||||
if include {
|
||||
if newPath == fdstRelative {
|
||||
return true
|
||||
}
|
||||
pos++
|
||||
include = filterCheckR(ctx, fdstRelative, pos, fsrc)
|
||||
}
|
||||
}
|
||||
return include
|
||||
}
|
||||
|
||||
// SameDir returns true if fdst and fsrc point to the same
|
||||
// underlying Fs and they are the same directory.
|
||||
func SameDir(fdst, fsrc fs.Info) bool {
|
||||
@@ -1070,11 +1111,13 @@ func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Write
|
||||
// Count counts the objects and their sizes in the Fs
|
||||
//
|
||||
// Obeys includes and excludes
|
||||
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error) {
|
||||
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) {
|
||||
err = ListFn(ctx, f, func(o fs.Object) {
|
||||
atomic.AddInt64(&objects, 1)
|
||||
objectSize := o.Size()
|
||||
if objectSize > 0 {
|
||||
if objectSize < 0 {
|
||||
atomic.AddInt64(&sizelessObjects, 1)
|
||||
} else if objectSize > 0 {
|
||||
atomic.AddInt64(&size, objectSize)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -401,10 +401,11 @@ func TestCount(t *testing.T) {
|
||||
// Check the MaxDepth too
|
||||
ci.MaxDepth = 1
|
||||
|
||||
objects, size, err := operations.Count(ctx, r.Fremote)
|
||||
objects, size, sizeless, err := operations.Count(ctx, r.Fremote)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(2), objects)
|
||||
assert.Equal(t, int64(61), size)
|
||||
assert.Equal(t, int64(0), sizeless)
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
@@ -1252,6 +1253,93 @@ func TestOverlapping(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// testFs is for unit testing fs.Fs
|
||||
type testFs struct {
|
||||
testFsInfo
|
||||
}
|
||||
|
||||
func (i *testFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (i *testFs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return nil, nil }
|
||||
|
||||
func (i *testFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (i *testFs) Mkdir(ctx context.Context, dir string) error { return nil }
|
||||
|
||||
func (i *testFs) Rmdir(ctx context.Context, dir string) error { return nil }
|
||||
|
||||
// copied from TestOverlapping because the behavior of OverlappingFilterCheck should be identical to Overlapping
|
||||
// when no filters are set
|
||||
func TestOverlappingFilterCheckWithoutFilter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
src := &testFs{testFsInfo{name: "name", root: "root"}}
|
||||
slash := string(os.PathSeparator) // native path separator
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
root string
|
||||
expected bool
|
||||
}{
|
||||
{"name", "root", true},
|
||||
{"namey", "root", false},
|
||||
{"name", "rooty", false},
|
||||
{"namey", "rooty", false},
|
||||
{"name", "roo", false},
|
||||
{"name", "root/toot", true},
|
||||
{"name", "root/toot/", true},
|
||||
{"name", "root" + slash + "toot", true},
|
||||
{"name", "root" + slash + "toot" + slash, true},
|
||||
{"name", "", true},
|
||||
{"name", "/", true},
|
||||
} {
|
||||
dst := &testFs{testFsInfo{name: test.name, root: test.root}}
|
||||
what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
|
||||
actual := operations.OverlappingFilterCheck(ctx, src, dst)
|
||||
assert.Equal(t, test.expected, actual, what)
|
||||
actual = operations.OverlappingFilterCheck(ctx, dst, src)
|
||||
assert.Equal(t, test.expected, actual, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverlappingFilterCheckWithFilter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fi, err := filter.NewFilter(nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fi.Add(false, "*/exclude/"))
|
||||
fi.Opt.ExcludeFile = ".ignore"
|
||||
ctx = filter.ReplaceConfig(ctx, fi)
|
||||
|
||||
src := &testFs{testFsInfo{name: "name", root: "root"}}
|
||||
slash := string(os.PathSeparator) // native path separator
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
root string
|
||||
expected bool
|
||||
}{
|
||||
{"name", "root", true},
|
||||
{"name", "root/", true},
|
||||
{"name", "root" + slash, true},
|
||||
{"name", "root/exclude", false},
|
||||
{"name", "root/exclude/", false},
|
||||
{"name", "root" + slash + "exclude", false},
|
||||
{"name", "root" + slash + "exclude" + slash, false},
|
||||
{"name", "root/.ignore", false},
|
||||
{"name", "root" + slash + ".ignore", false},
|
||||
{"namey", "root/include", false},
|
||||
{"namey", "root/include/", false},
|
||||
{"namey", "root" + slash + "include", false},
|
||||
{"namey", "root" + slash + "include" + slash, false},
|
||||
} {
|
||||
dst := &testFs{testFsInfo{name: test.name, root: test.root}}
|
||||
what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
|
||||
actual := operations.OverlappingFilterCheck(ctx, dst, src)
|
||||
assert.Equal(t, test.expected, actual, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFormat(t *testing.T) {
|
||||
item0 := &operations.ListJSONItem{
|
||||
Path: "a",
|
||||
@@ -1406,7 +1494,7 @@ func TestDirMove(t *testing.T) {
|
||||
require.NoError(t, operations.DirMove(ctx, r.Fremote, "A1", "A2"))
|
||||
|
||||
for i := range files {
|
||||
files[i].Path = strings.Replace(files[i].Path, "A1/", "A2/", -1)
|
||||
files[i].Path = strings.ReplaceAll(files[i].Path, "A1/", "A2/")
|
||||
}
|
||||
|
||||
fstest.CheckListingWithPrecision(
|
||||
@@ -1431,7 +1519,7 @@ func TestDirMove(t *testing.T) {
|
||||
require.NoError(t, operations.DirMove(ctx, r.Fremote, "A2", "A3"))
|
||||
|
||||
for i := range files {
|
||||
files[i].Path = strings.Replace(files[i].Path, "A2/", "A3/", -1)
|
||||
files[i].Path = strings.ReplaceAll(files[i].Path, "A2/", "A3/")
|
||||
}
|
||||
|
||||
fstest.CheckListingWithPrecision(
|
||||
|
||||
+2
-1
@@ -344,13 +344,14 @@ func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
count, bytes, err := Count(ctx, f)
|
||||
count, bytes, sizeless, err := Count(ctx, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = make(rc.Params)
|
||||
out["count"] = count
|
||||
out["bytes"] = bytes
|
||||
out["sizeless"] = sizeless
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -451,8 +451,9 @@ func TestRcSize(t *testing.T) {
|
||||
out, err := call.Fn(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rc.Params{
|
||||
"count": int64(3),
|
||||
"bytes": int64(120),
|
||||
"count": int64(3),
|
||||
"bytes": int64(120),
|
||||
"sizeless": int64(0),
|
||||
}, out)
|
||||
}
|
||||
|
||||
|
||||
Loaded 100 of 128 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user