Compare commits

..
160 changed files with 1105 additions and 6757 deletions

No files matched your search

+1 -8
View File
@@ -37,6 +37,7 @@ The Rclone Developers
-->
#### The associated forum post URL from `https://forum.rclone.org`
@@ -64,11 +65,3 @@ The Rclone Developers
#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
<!--- Please keep the note below for others who read your bug report. -->
#### How to use GitHub
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
* Subscribe to receive notifications on status change and new comments.
+1 -8
View File
@@ -26,6 +26,7 @@ The Rclone Developers
-->
#### The associated forum post URL from `https://forum.rclone.org`
@@ -41,11 +42,3 @@ The Rclone Developers
#### How do you think rclone should be changed to solve that?
<!--- Please keep the note below for others who read your feature request. -->
#### How to use GitHub
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
* Subscribe to receive notifications on status change and new comments.
-18
View File
@@ -12,15 +12,9 @@ on:
tags:
- '*'
pull_request:
workflow_dispatch:
inputs:
manual:
required: true
default: true
jobs:
build:
if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
timeout-minutes: 60
strategy:
fail-fast: false
@@ -36,7 +30,6 @@ jobs:
check: true
quicktest: true
racequicktest: true
librclonetest: true
deploy: true
- job_name: mac_amd64
@@ -194,14 +187,6 @@ jobs:
make racequicktest
if: matrix.racequicktest
- name: Run librclone tests
shell: bash
run: |
make -C librclone/ctest test
make -C librclone/ctest clean
librclone/python/test_rclone.py
if: matrix.librclonetest
- name: Code quality test
shell: bash
run: |
@@ -229,7 +214,6 @@ jobs:
if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
android:
if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
timeout-minutes: 30
name: "android-all"
runs-on: ubuntu-latest
@@ -237,8 +221,6 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
# Upgrade together with NDK version
- name: Set up Go 1.14
@@ -7,7 +7,6 @@ on:
jobs:
build:
if: github.repository == 'rclone/rclone'
runs-on: ubuntu-latest
name: Build image job
steps:
@@ -6,7 +6,6 @@ on:
jobs:
build:
if: github.repository == 'rclone/rclone'
runs-on: ubuntu-latest
name: Build image job
steps:
+1 -1
View File
@@ -20,7 +20,7 @@ var (
)
func prepare(t *testing.T, root string) {
configfile.Install()
configfile.LoadConfig(context.Background())
// Configure the remote
config.FileSet(remoteName, "type", "alias")
-1
View File
@@ -41,7 +41,6 @@ import (
_ "github.com/rclone/rclone/backend/swift"
_ "github.com/rclone/rclone/backend/tardigrade"
_ "github.com/rclone/rclone/backend/union"
_ "github.com/rclone/rclone/backend/uptobox"
_ "github.com/rclone/rclone/backend/webdav"
_ "github.com/rclone/rclone/backend/yandex"
_ "github.com/rclone/rclone/backend/zoho"
+9 -9
View File
@@ -16,6 +16,7 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"path"
"strings"
@@ -69,12 +70,11 @@ func init() {
Prefix: "acd",
Description: "Amazon Drive",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
err := oauthutil.Config(ctx, "amazon cloud drive", name, m, acdConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "checkpoint",
@@ -83,16 +83,16 @@ func init() {
Advanced: true,
}, {
Name: "upload_wait_per_gb",
Help: `Additional time per GiB to wait after a failed complete upload to see if it appears.
Help: `Additional time per GB to wait after a failed complete upload to see if it appears.
Sometimes Amazon Drive gives an error when a file has been fully
uploaded but the file appears anyway after a little while. This
happens sometimes for files over 1 GiB in size and nearly every time for
files bigger than 10 GiB. This parameter controls the time rclone waits
happens sometimes for files over 1GB in size and nearly every time for
files bigger than 10GB. This parameter controls the time rclone waits
for the file to appear.
The default value for this parameter is 3 minutes per GiB, so by
default it will wait 3 minutes for every GiB uploaded to see if the
The default value for this parameter is 3 minutes per GB, so by
default it will wait 3 minutes for every GB uploaded to see if the
file appears.
You can disable this feature by setting it to 0. This may cause
@@ -112,7 +112,7 @@ in this situation.`,
Files this size or more will be downloaded via their "tempLink". This
is to work around a problem with Amazon Drive which blocks downloads
of files bigger than about 10 GiB. The default for this is 9 GiB which
of files bigger than about 10GB. The default for this is 9GB which
shouldn't need to be changed.
To download files above this threshold, rclone requests a "tempLink"
+5 -5
View File
@@ -47,8 +47,8 @@ const (
timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
storageDefaultBaseURL = "blob.core.windows.net"
defaultChunkSize = 4 * fs.Mebi
maxChunkSize = 100 * fs.Mebi
defaultChunkSize = 4 * fs.MebiByte
maxChunkSize = 100 * fs.MebiByte
uploadConcurrency = 4
defaultAccessTier = azblob.AccessTierNone
maxTryTimeout = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
@@ -129,11 +129,11 @@ msi_client_id, or msi_mi_res_id parameters.`,
Advanced: true,
}, {
Name: "upload_cutoff",
Help: "Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)",
Help: "Cutoff for switching to chunked upload (<= 256MB). (Deprecated)",
Advanced: true,
}, {
Name: "chunk_size",
Help: `Upload chunk size (<= 100 MiB).
Help: `Upload chunk size (<= 100MB).
Note that this is stored in memory and there may be up to
"--transfers" chunks stored at once in memory.`,
@@ -404,7 +404,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.SizeSuffixBase
const minChunkSize = fs.Byte
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
+27 -12
View File
@@ -2,11 +2,12 @@ package api
import (
"fmt"
"path"
"strconv"
"strings"
"time"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/version"
)
// Error describes a B2 error response
@@ -62,17 +63,16 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
return nil
}
// HasVersion returns true if it looks like the passed filename has a timestamp on it.
//
// Note that the passed filename's timestamp may still be invalid even if this
// function returns true.
func HasVersion(remote string) bool {
return version.Match(remote)
}
const versionFormat = "-v2006-01-02-150405.000"
// AddVersion adds the timestamp as a version string into the filename passed in.
func (t Timestamp) AddVersion(remote string) string {
return version.Add(remote, time.Time(t))
ext := path.Ext(remote)
base := remote[:len(remote)-len(ext)]
s := time.Time(t).Format(versionFormat)
// Replace the '.' with a '-'
s = strings.Replace(s, ".", "-", -1)
return base + s + ext
}
// RemoveVersion removes the timestamp from a filename as a version string.
@@ -80,9 +80,24 @@ func (t Timestamp) AddVersion(remote string) string {
// It returns the new file name and a timestamp, or the old filename
// and a zero timestamp.
func RemoveVersion(remote string) (t Timestamp, newRemote string) {
time, newRemote := version.Remove(remote)
t = Timestamp(time)
return
newRemote = remote
ext := path.Ext(remote)
base := remote[:len(remote)-len(ext)]
if len(base) < len(versionFormat) {
return
}
versionStart := len(base) - len(versionFormat)
// Check it ends in -xxx
if base[len(base)-4] != '-' {
return
}
// Replace with .xxx for parsing
base = base[:len(base)-4] + "." + base[len(base)-3:]
newT, err := time.Parse(versionFormat, base[versionStart:])
if err != nil {
return
}
return Timestamp(newT), base[:versionStart] + ext
}
// IsZero returns true if the timestamp is uninitialized
+35
View File
@@ -13,6 +13,7 @@ import (
var (
emptyT api.Timestamp
t0 = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z"))
t0r = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z"))
t1 = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z"))
)
@@ -35,6 +36,40 @@ func TestTimestampUnmarshalJSON(t *testing.T) {
assert.Equal(t, (time.Time)(t1), (time.Time)(tActual))
}
func TestTimestampAddVersion(t *testing.T) {
for _, test := range []struct {
t api.Timestamp
in string
expected string
}{
{t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"},
{t1, "potato", "potato-v2001-02-03-040506-123"},
{t1, "", "-v2001-02-03-040506-123"},
} {
actual := test.t.AddVersion(test.in)
assert.Equal(t, test.expected, actual, test.in)
}
}
func TestTimestampRemoveVersion(t *testing.T) {
for _, test := range []struct {
in string
expectedT api.Timestamp
expectedRemote string
}{
{"potato.txt", emptyT, "potato.txt"},
{"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"},
{"potato-v2001-02-03-040506-123", t1, "potato"},
{"-v2001-02-03-040506-123", t1, ""},
{"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"},
{"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"},
} {
actualT, actualRemote := api.RemoveVersion(test.in)
assert.Equal(t, test.expectedT, actualT, test.in)
assert.Equal(t, test.expectedRemote, actualRemote, test.in)
}
}
func TestTimestampIsZero(t *testing.T) {
assert.True(t, emptyT.IsZero())
assert.False(t, t0.IsZero())
+7 -7
View File
@@ -54,10 +54,10 @@ const (
decayConstant = 1 // bigger for slower decay, exponential
maxParts = 10000
maxVersions = 100 // maximum number of versions we search in --b2-versions mode
minChunkSize = 5 * fs.Mebi
defaultChunkSize = 96 * fs.Mebi
defaultUploadCutoff = 200 * fs.Mebi
largeFileCopyCutoff = 4 * fs.Gibi // 5E9 is the max
minChunkSize = 5 * fs.MebiByte
defaultChunkSize = 96 * fs.MebiByte
defaultUploadCutoff = 200 * fs.MebiByte
largeFileCopyCutoff = 4 * fs.GibiByte // 5E9 is the max
memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
memoryPoolUseMmap = false
)
@@ -116,7 +116,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
Files above this size will be uploaded in chunks of "--b2-chunk-size".
This value should be set no larger than 4.657 GiB (== 5 GB).`,
This value should be set no larger than 4.657GiB (== 5GB).`,
Default: defaultUploadCutoff,
Advanced: true,
}, {
@@ -126,7 +126,7 @@ This value should be set no larger than 4.657 GiB (== 5 GB).`,
Any files larger than this that need to be server-side copied will be
copied in chunks of this size.
The minimum is 0 and the maximum is 4.6 GiB.`,
The minimum is 0 and the maximum is 4.6GB.`,
Default: largeFileCopyCutoff,
Advanced: true,
}, {
@@ -1353,7 +1353,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
}
var request = api.GetDownloadAuthorizationRequest{
BucketID: bucketID,
FileNamePrefix: f.opt.Enc.FromStandardPath(path.Join(f.rootDirectory, remote)),
FileNamePrefix: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
ValidDurationInSeconds: validDurationInSeconds,
}
var response api.GetDownloadAuthorizationResponse
+3 -3
View File
@@ -230,14 +230,14 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
//
// The number of bytes in the file being uploaded. Note that
// this header is required; you cannot leave it out and just
// use chunked encoding. The minimum size of every part but
// the last one is 100 MB (100,000,000 bytes)
// use chunked encoding. The minimum size of every part but
// the last one is 100MB.
//
// X-Bz-Content-Sha1
//
// The SHA1 checksum of the this part of the file. B2 will
// check this when the part is uploaded, to make sure that the
// data arrived correctly. The same SHA1 checksum must be
// data arrived correctly. The same SHA1 checksum must be
// passed to b2_finish_large_file.
opts := rest.Opts{
Method: "POST",
+7 -39
View File
@@ -36,13 +36,13 @@ func (t *Time) UnmarshalJSON(data []byte) error {
// Error is returned from box when things go wrong
type Error struct {
Type string `json:"type"`
Status int `json:"status"`
Code string `json:"code"`
ContextInfo json.RawMessage `json:"context_info"`
HelpURL string `json:"help_url"`
Message string `json:"message"`
RequestID string `json:"request_id"`
Type string `json:"type"`
Status int `json:"status"`
Code string `json:"code"`
ContextInfo json.RawMessage
HelpURL string `json:"help_url"`
Message string `json:"message"`
RequestID string `json:"request_id"`
}
// Error returns a string for the error and satisfies the error interface
@@ -132,38 +132,6 @@ type UploadFile struct {
ContentModifiedAt Time `json:"content_modified_at"`
}
// PreUploadCheck is the request for upload preflight check
type PreUploadCheck struct {
Name string `json:"name"`
Parent Parent `json:"parent"`
Size *int64 `json:"size,omitempty"`
}
// PreUploadCheckResponse is the response from upload preflight check
// if successful
type PreUploadCheckResponse struct {
UploadToken string `json:"upload_token"`
UploadURL string `json:"upload_url"`
}
// PreUploadCheckConflict is returned in the ContextInfo error field
// from PreUploadCheck when the error code is "item_name_in_use"
type PreUploadCheckConflict struct {
Conflicts struct {
Type string `json:"type"`
ID string `json:"id"`
FileVersion struct {
Type string `json:"type"`
ID string `json:"id"`
Sha1 string `json:"sha1"`
} `json:"file_version"`
SequenceID string `json:"sequence_id"`
Etag string `json:"etag"`
Sha1 string `json:"sha1"`
Name string `json:"name"`
} `json:"conflicts"`
}
// UpdateFileModTime is used in Update File Info
type UpdateFileModTime struct {
ContentModifiedAt Time `json:"content_modified_at"`
+17 -75
View File
@@ -17,6 +17,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"path"
@@ -83,7 +84,7 @@ func init() {
Name: "box",
Description: "Box",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
jsonFile, ok := m.Get("box_config_file")
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
boxAccessToken, boxAccessTokenOk := m.Get("access_token")
@@ -92,16 +93,15 @@ func init() {
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
if err != nil {
return errors.Wrap(err, "failed to configure token with jwt authentication")
log.Fatalf("Failed to configure token with jwt authentication: %v", err)
}
// Else, if not using an access token, use oauth2
} else if boxAccessToken == "" || !boxAccessTokenOk {
err = oauthutil.Config(ctx, "box", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token with oauth authentication")
log.Fatalf("Failed to configure token with oauth authentication: %v", err)
}
}
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "root_folder_id",
@@ -126,7 +126,7 @@ func init() {
}},
}, {
Name: "upload_cutoff",
Help: "Cutoff for switching to multipart upload (>= 50 MiB).",
Help: "Cutoff for switching to multipart upload (>= 50MB).",
Default: fs.SizeSuffix(defaultUploadCutoff),
Advanced: true,
}, {
@@ -157,15 +157,15 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
jsonFile = env.ShellExpand(jsonFile)
boxConfig, err := getBoxConfig(jsonFile)
if err != nil {
return errors.Wrap(err, "get box config")
log.Fatalf("Failed to configure token: %v", err)
}
privateKey, err := getDecryptedPrivateKey(boxConfig)
if err != nil {
return errors.Wrap(err, "get decrypted private key")
log.Fatalf("Failed to configure token: %v", err)
}
claims, err := getClaims(boxConfig, boxSubType)
if err != nil {
return errors.Wrap(err, "get claims")
log.Fatalf("Failed to configure token: %v", err)
}
signingHeaders := getSigningHeaders(boxConfig)
queryParams := getQueryParams(boxConfig)
@@ -686,80 +686,22 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
return o, leaf, directoryID, nil
}
// preUploadCheck checks to see if a file can be uploaded
//
// It returns "", nil if the file is good to go
// It returns "ID", nil if the file must be updated
func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (ID string, err error) {
check := api.PreUploadCheck{
Name: f.opt.Enc.FromStandardName(leaf),
Parent: api.Parent{
ID: directoryID,
},
}
if size >= 0 {
check.Size = &size
}
opts := rest.Opts{
Method: "OPTIONS",
Path: "/files/content/",
}
var result api.PreUploadCheckResponse
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(ctx, &opts, &check, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
if apiErr, ok := err.(*api.Error); ok && apiErr.Code == "item_name_in_use" {
var conflict api.PreUploadCheckConflict
err = json.Unmarshal(apiErr.ContextInfo, &conflict)
if err != nil {
return "", errors.Wrap(err, "pre-upload check: JSON decode failed")
}
if conflict.Conflicts.Type != api.ItemTypeFile {
return "", errors.Wrap(err, "pre-upload check: can't overwrite non file with file")
}
return conflict.Conflicts.ID, nil
}
return "", errors.Wrap(err, "pre-upload check")
}
return "", nil
}
// Put the object
//
// Copy the reader in to the new object which is returned
//
// The new object may have been created if an error is returned
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
// If directory doesn't exist, file doesn't exist so can upload
remote := src.Remote()
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
return f.PutUnchecked(ctx, in, src, options...)
}
existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
switch err {
case nil:
return existingObj, existingObj.Update(ctx, in, src, options...)
case fs.ErrorObjectNotFound:
// Not found so create it
return f.PutUnchecked(ctx, in, src)
default:
return nil, err
}
// Preflight check the upload, which returns the ID if the
// object already exists
ID, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
if err != nil {
return nil, err
}
if ID == "" {
return f.PutUnchecked(ctx, in, src, options...)
}
// If object exists then create a skeleton one with just id
o := &Object{
fs: f,
remote: remote,
id: ID,
}
return o, o.Update(ctx, in, src, options...)
}
// PutStream uploads to the remote path with the modTime given of indeterminate size
@@ -1286,7 +1228,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// upload does a single non-multipart upload
//
// This is recommended for less than 50 MiB of content
// This is recommended for less than 50 MB of content
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
upload := api.UploadFile{
Name: o.fs.opt.Enc.FromStandardName(leaf),
+7 -7
View File
@@ -98,14 +98,14 @@ changed, any downloaded chunks will be invalid and cache-chunk-path
will need to be cleared or unexpected EOF errors will occur.`,
Default: DefCacheChunkSize,
Examples: []fs.OptionExample{{
Value: "1M",
Help: "1 MiB",
Value: "1m",
Help: "1MB",
}, {
Value: "5M",
Help: "5 MiB",
Help: "5 MB",
}, {
Value: "10M",
Help: "10 MiB",
Help: "10 MB",
}},
}, {
Name: "info_age",
@@ -132,13 +132,13 @@ oldest chunks until it goes under this value.`,
Default: DefCacheTotalChunkSize,
Examples: []fs.OptionExample{{
Value: "500M",
Help: "500 MiB",
Help: "500 MB",
}, {
Value: "1G",
Help: "1 GiB",
Help: "1 GB",
}, {
Value: "10G",
Help: "10 GiB",
Help: "10 GB",
}},
}, {
Name: "db_path",
+1 -1
View File
@@ -836,7 +836,7 @@ func newRun() *run {
if uploadDir == "" {
r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
if err != nil {
panic(fmt.Sprintf("Failed to create temp dir: %v", err))
log.Fatalf("Failed to create temp dir: %v", err)
}
} else {
r.tmpUploadDir = uploadDir
+2 -2
View File
@@ -155,7 +155,7 @@ Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
}, {
Name: "chunk_size",
Advanced: false,
Default: fs.SizeSuffix(2147483648), // 2 GiB
Default: fs.SizeSuffix(2147483648), // 2GB
Help: `Files larger than chunk size will be split in chunks.`,
}, {
Name: "name_format",
@@ -1448,7 +1448,7 @@ func (c *chunkingReader) dummyRead(in io.Reader, size int64) error {
c.accountBytes(size)
return nil
}
const bufLen = 1048576 // 1 MiB
const bufLen = 1048576 // 1MB
buf := make([]byte, bufLen)
for size > 0 {
n := size
+1 -1
View File
@@ -33,7 +33,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: fmt.Sprintf("chunker-upload-%dk", kilobytes),
Size: int64(kilobytes) * int64(fs.Kibi),
Size: int64(kilobytes) * int64(fs.KibiByte),
})
})
}
+2 -2
View File
@@ -36,7 +36,7 @@ import (
// Globals
const (
initialChunkSize = 262144 // Initial and max sizes of chunks when reading parts of the file. Currently
maxChunkSize = 8388608 // at 256 KiB and 8 MiB.
maxChunkSize = 8388608 // at 256KB and 8 MB.
bufferSize = 8388608
heuristicBytes = 1048576
@@ -53,7 +53,7 @@ const (
Gzip = 2
)
var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9-_]{11})$")
var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9+_]{11})$")
// Register with Fs
func init() {
+3 -55
View File
@@ -12,14 +12,12 @@ import (
"strconv"
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/pkg/errors"
"github.com/rclone/rclone/backend/crypt/pkcs7"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/lib/version"
"github.com/rfjakob/eme"
"golang.org/x/crypto/nacl/secretbox"
"golang.org/x/crypto/scrypt"
@@ -444,32 +442,11 @@ func (c *Cipher) encryptFileName(in string) string {
if !c.dirNameEncrypt && i != (len(segments)-1) {
continue
}
// Strip version string so that only the non-versioned part
// of the file name gets encrypted/obfuscated
hasVersion := false
var t time.Time
if i == (len(segments)-1) && version.Match(segments[i]) {
var s string
t, s = version.Remove(segments[i])
// version.Remove can fail, in which case it returns segments[i]
if s != segments[i] {
segments[i] = s
hasVersion = true
}
}
if c.mode == NameEncryptionStandard {
segments[i] = c.encryptSegment(segments[i])
} else {
segments[i] = c.obfuscateSegment(segments[i])
}
// Add back a version to the encrypted/obfuscated
// file name, if we stripped it off earlier
if hasVersion {
segments[i] = version.Add(segments[i], t)
}
}
return strings.Join(segments, "/")
}
@@ -500,21 +477,6 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
if !c.dirNameEncrypt && i != (len(segments)-1) {
continue
}
// Strip version string so that only the non-versioned part
// of the file name gets decrypted/deobfuscated
hasVersion := false
var t time.Time
if i == (len(segments)-1) && version.Match(segments[i]) {
var s string
t, s = version.Remove(segments[i])
// version.Remove can fail, in which case it returns segments[i]
if s != segments[i] {
segments[i] = s
hasVersion = true
}
}
if c.mode == NameEncryptionStandard {
segments[i], err = c.decryptSegment(segments[i])
} else {
@@ -524,12 +486,6 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
if err != nil {
return "", err
}
// Add back a version to the decrypted/deobfuscated
// file name, if we stripped it off earlier
if hasVersion {
segments[i] = version.Add(segments[i], t)
}
}
return strings.Join(segments, "/"), nil
}
@@ -538,18 +494,10 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
func (c *Cipher) DecryptFileName(in string) (string, error) {
if c.mode == NameEncryptionOff {
remainingLength := len(in) - len(encryptedSuffix)
if remainingLength == 0 || !strings.HasSuffix(in, encryptedSuffix) {
return "", ErrorNotAnEncryptedFile
if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
return in[:remainingLength], nil
}
decrypted := in[:remainingLength]
if version.Match(decrypted) {
_, unversioned := version.Remove(decrypted)
if unversioned == "" {
return "", ErrorNotAnEncryptedFile
}
}
// Leave the version string on, if it was there
return decrypted, nil
return "", ErrorNotAnEncryptedFile
}
return c.decryptFileName(in)
}
-12
View File
@@ -160,29 +160,22 @@ func TestEncryptFileName(t *testing.T) {
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
// Standard mode with directory name encryption off
c, _ = newCipher(NameEncryptionStandard, "", "", false)
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
// Now off mode
c, _ = newCipher(NameEncryptionOff, "", "", true)
assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
// Obfuscation mode
c, _ = newCipher(NameEncryptionObfuscated, "", "", true)
assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
assert.Equal(t, "49.6/99.23/150.890/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
assert.Equal(t, "49.6/99.23/150.890/162.uryyB-v2001-02-03-040506-123.GKG", c.EncryptFileName("1/12/123/hello-v2001-02-03-040506-123.txt"))
assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
// Obfuscation mode with directory name encryption off
c, _ = newCipher(NameEncryptionObfuscated, "", "", false)
assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
assert.Equal(t, "1/12/123/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
}
@@ -201,19 +194,14 @@ func TestDecryptFileName(t *testing.T) {
{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
{NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", "1-v2001-02-03-040506-123", nil},
{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
} {
c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
actual, actualErr := c.DecryptFileName(test.in)
+12 -11
View File
@@ -14,6 +14,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"mime"
"net/http"
"path"
@@ -67,8 +68,8 @@ const (
defaultScope = "drive"
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
minChunkSize = 256 * fs.Kibi
defaultChunkSize = 8 * fs.Mebi
minChunkSize = 256 * fs.KibiByte
defaultChunkSize = 8 * fs.MebiByte
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
listRGrouping = 50 // number of IDs to search at once when using ListR
listRInputBuffer = 1000 // size of input buffer when using ListR
@@ -182,12 +183,13 @@ func init() {
Description: "Google Drive",
NewFs: NewFs,
CommandHelp: commandHelp,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return errors.Wrap(err, "couldn't parse config into struct")
fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
return
}
// Fill in the scopes
@@ -197,17 +199,16 @@ func init() {
m.Set("root_folder_id", "appDataFolder")
}
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
if opt.ServiceAccountFile == "" {
err = oauthutil.Config(ctx, "drive", name, m, driveConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
}
err = configTeamDrive(ctx, opt, m, name)
if err != nil {
return errors.Wrap(err, "failed to configure Shared Drive")
log.Fatalf("Failed to configure Shared Drive: %v", err)
}
return nil
},
Options: append(driveOAuthOptions(), []fs.Option{{
Name: "scope",
@@ -466,7 +467,7 @@ See: https://github.com/rclone/rclone/issues/3631
Default: false,
Help: `Make upload limit errors be fatal
At the time of writing it is only possible to upload 750 GiB of data to
At the time of writing it is only possible to upload 750GB of data to
Google Drive a day (this is an undocumented limit). When this limit is
reached Google Drive produces a slightly different error message. When
this flag is set it causes these errors to be fatal. These will stop
@@ -483,7 +484,7 @@ See: https://github.com/rclone/rclone/issues/3857
Default: false,
Help: `Make download limit errors be fatal
At the time of writing it is only possible to download 10 TiB of data from
At the time of writing it is only possible to download 10TB of data from
Google Drive a day (this is an undocumented limit). When this limit is
reached Google Drive produces a slightly different error message. When
this flag is set it causes these errors to be fatal. These will stop
@@ -521,7 +522,7 @@ If this flag is set then rclone will ignore shortcut files completely.
} {
for mimeType, extension := range m {
if err := mime.AddExtensionType(extension, mimeType); err != nil {
fs.Errorf("Failed to register MIME type %q: %v", mimeType, err)
log.Fatalf("Failed to register MIME type %q: %v", mimeType, err)
}
}
}
+16 -18
View File
@@ -25,6 +25,7 @@ import (
"context"
"fmt"
"io"
"log"
"path"
"regexp"
"strings"
@@ -64,9 +65,9 @@ const (
// Upload chunk size - setting too small makes uploads slow.
// Chunks are buffered into memory for retries.
//
// Speed vs chunk size uploading a 1 GiB file on 2017-11-22
// Speed vs chunk size uploading a 1 GB file on 2017-11-22
//
// Chunk Size MiB, Speed MiByte/s, % of max
// Chunk Size MB, Speed Mbyte/s, % of max
// 1 1.364 11%
// 2 2.443 19%
// 4 4.288 33%
@@ -81,11 +82,11 @@ const (
// 96 12.302 95%
// 128 12.945 100%
//
// Choose 48 MiB which is 91% of Maximum speed. rclone by
// default does 4 transfers so this should use 4*48 MiB = 192 MiB
// Choose 48MB which is 91% of Maximum speed. rclone by
// default does 4 transfers so this should use 4*48MB = 192MB
// by default.
defaultChunkSize = 48 * fs.Mebi
maxChunkSize = 150 * fs.Mebi
defaultChunkSize = 48 * fs.MebiByte
maxChunkSize = 150 * fs.MebiByte
// Max length of filename parts: https://help.dropbox.com/installs-integrations/sync-uploads/files-not-syncing
maxFileNameLength = 255
)
@@ -98,10 +99,8 @@ var (
"files.content.write",
"files.content.read",
"sharing.write",
"account_info.read", // needed for About
// "file_requests.write",
// "members.read", // needed for impersonate - but causes app to need to be approved by Dropbox Team Admin during the flow
// "team_data.member"
},
// Endpoint: oauth2.Endpoint{
// AuthURL: "https://www.dropbox.com/1/oauth2/authorize",
@@ -131,8 +130,8 @@ func getOauthConfig(m configmap.Mapper) *oauth2.Config {
}
// Make a copy of the config
config := *dropboxConfig
// Make a copy of the scopes with extra scopes requires appended
config.Scopes = append(config.Scopes, "members.read", "team_data.member")
// Make a copy of the scopes with "members.read" appended
config.Scopes = append(config.Scopes, "members.read")
return &config
}
@@ -143,7 +142,7 @@ func init() {
Name: "dropbox",
Description: "Dropbox",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
opt := oauthutil.Options{
NoOffline: true,
OAuth2Opts: []oauth2.AuthCodeOption{
@@ -152,9 +151,8 @@ func init() {
}
err := oauthutil.Config(ctx, "dropbox", name, m, getOauthConfig(m), &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "chunk_size",
@@ -164,7 +162,7 @@ Any files larger than this will be uploaded in chunks of this size.
Note that chunks are buffered in memory (one at a time) so rclone can
deal with retries. Setting this larger will increase the speed
slightly (at most 10%% for 128 MiB in tests) at the cost of using more
slightly (at most 10%% for 128MB in tests) at the cost of using more
memory. It can be set smaller if you are tight on memory.`, maxChunkSize),
Default: defaultChunkSize,
Advanced: true,
@@ -325,7 +323,7 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.SizeSuffixBase
const minChunkSize = fs.Byte
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
@@ -1353,13 +1351,13 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
switch info := entry.(type) {
case *files.FolderMetadata:
entryType = fs.EntryDirectory
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
case *files.FileMetadata:
entryType = fs.EntryObject
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
case *files.DeletedMetadata:
entryType = fs.EntryObject
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
default:
fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
continue
+3 -1
View File
@@ -35,7 +35,9 @@ func init() {
fs.Register(&fs.RegInfo{
Name: "fichier",
Description: "1Fichier",
NewFs: NewFs,
Config: func(ctx context.Context, name string, config configmap.Mapper) {
},
NewFs: NewFs,
Options: []fs.Option{{
Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
Name: "api_key",
+20 -16
View File
@@ -241,6 +241,23 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
return len(p), nil
}
type dialCtx struct {
f *Fs
ctx context.Context
}
// dial a new connection with fshttp dialer
func (d *dialCtx) dial(network, address string) (net.Conn, error) {
conn, err := fshttp.NewDialer(d.ctx).Dial(network, address)
if err != nil {
return nil, err
}
if d.f.tlsConf != nil {
conn = tls.Client(conn, d.f.tlsConf)
}
return conn, err
}
// shouldRetry returns a boolean as to whether this err deserve to be
// retried. It returns the err as a convenience
func shouldRetry(ctx context.Context, err error) (bool, error) {
@@ -260,22 +277,9 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
// Open a new connection to the FTP server.
func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
fs.Debugf(f, "Connecting to FTP server")
// Make ftp library dial with fshttp dialer optionally using TLS
dial := func(network, address string) (conn net.Conn, err error) {
conn, err = fshttp.NewDialer(ctx).Dial(network, address)
if f.tlsConf != nil && err == nil {
conn = tls.Client(conn, f.tlsConf)
}
return
}
ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
if f.opt.TLS {
// Our dialer takes care of TLS but ftp library also needs tlsConf
// as a trigger for sending PSBZ and PROT options to server.
ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
} else if f.opt.ExplicitTLS {
dCtx := dialCtx{f, ctx}
ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dCtx.dial)}
if f.opt.ExplicitTLS {
ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
// Initial connection needs to be cleartext for explicit TLS
conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
@@ -19,6 +19,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"path"
"strings"
@@ -75,18 +76,17 @@ func init() {
Prefix: "gcs",
Description: "Google Cloud Storage (this is not Google Drive)",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
saFile, _ := m.Get("service_account_file")
saCreds, _ := m.Get("service_account_credentials")
anonymous, _ := m.Get("anonymous")
if saFile != "" || saCreds != "" || anonymous == "true" {
return nil
return
}
err := oauthutil.Config(ctx, "google cloud storage", name, m, storageConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "project_number",
+5 -4
View File
@@ -8,6 +8,7 @@ import (
"encoding/json"
"fmt"
"io"
golog "log"
"net/http"
"net/url"
"path"
@@ -77,12 +78,13 @@ func init() {
Prefix: "gphotos",
Description: "Google Photos",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return errors.Wrap(err, "couldn't parse config into struct")
fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
return
}
// Fill in the scopes
@@ -95,7 +97,7 @@ func init() {
// Do the oauth
err = oauthutil.Config(ctx, "google photos", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
golog.Fatalf("Failed to configure token: %v", err)
}
// Warn the user
@@ -106,7 +108,6 @@ func init() {
`)
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "read_only",
+1 -1
View File
@@ -47,7 +47,7 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
ts := httptest.NewServer(handler)
// Configure the remote
configfile.Install()
configfile.LoadConfig(context.Background())
// fs.Config.LogLevel = fs.LogLevelDebug
// fs.Config.DumpHeaders = true
// fs.Config.DumpBodies = true
+3 -3
View File
@@ -11,6 +11,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
@@ -55,12 +56,11 @@ func init() {
Name: "hubic",
Description: "Hubic",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
err := oauthutil.Config(ctx, "hubic", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
})
+23 -26
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"net/url"
@@ -86,12 +87,12 @@ func init() {
Name: "jottacloud",
Description: "Jottacloud",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
refresh := false
if version, ok := m.Get("configVersion"); ok {
ver, err := strconv.Atoi(version)
if err != nil {
return errors.Wrap(err, "failed to parse config version - corrupted config")
log.Fatalf("Failed to parse config version - corrupted config")
}
refresh = (ver != configVersion) && (ver != v1configVersion)
}
@@ -103,7 +104,7 @@ func init() {
if ok && tokenString != "" {
fmt.Printf("Already have a token - refresh?\n")
if !config.Confirm(false) {
return nil
return
}
}
}
@@ -115,13 +116,11 @@ func init() {
switch config.ChooseNumber("Your choice", 1, 3) {
case 1:
return v2config(ctx, name, m)
v2config(ctx, name, m)
case 2:
return v1config(ctx, name, m)
v1config(ctx, name, m)
case 3:
return teliaCloudConfig(ctx, name, m)
default:
return errors.New("unknown config choice")
teliaCloudConfig(ctx, name, m)
}
},
Options: []fs.Option{{
@@ -243,7 +242,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) error {
func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
teliaCloudOauthConfig := &oauth2.Config{
Endpoint: oauth2.Endpoint{
AuthURL: teliaCloudAuthURL,
@@ -256,14 +255,15 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) erro
err := oauthutil.Config(ctx, "jottacloud", name, m, teliaCloudOauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
return
}
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
if config.Confirm(false) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, teliaCloudOauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
log.Fatalf("Failed to load oAuthClient: %s", err)
}
srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
@@ -271,7 +271,7 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) erro
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
if err != nil {
return errors.Wrap(err, "failed to setup mountpoint")
log.Fatalf("Failed to setup mountpoint: %s", err)
}
m.Set(configDevice, device)
m.Set(configMountpoint, mountpoint)
@@ -280,18 +280,17 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) erro
m.Set("configVersion", strconv.Itoa(configVersion))
m.Set(configClientID, teliaCloudClientID)
m.Set(configTokenURL, teliaCloudTokenURL)
return nil
}
// v1config configure a jottacloud backend using legacy authentication
func v1config(ctx context.Context, name string, m configmap.Mapper) error {
func v1config(ctx context.Context, name string, m configmap.Mapper) {
srv := rest.NewClient(fshttp.NewClient(ctx))
fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
if config.Confirm(false) {
deviceRegistration, err := registerDevice(ctx, srv)
if err != nil {
return errors.Wrap(err, "failed to register device")
log.Fatalf("Failed to register device: %v", err)
}
m.Set(configClientID, deviceRegistration.ClientID)
@@ -319,18 +318,18 @@ func v1config(ctx context.Context, name string, m configmap.Mapper) error {
token, err := doAuthV1(ctx, srv, username, password)
if err != nil {
return errors.Wrap(err, "failed to get oauth token")
log.Fatalf("Failed to get oauth token: %s", err)
}
err = oauthutil.PutToken(name, m, &token, true)
if err != nil {
return errors.Wrap(err, "error while saving token")
log.Fatalf("Error while saving token: %s", err)
}
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
if config.Confirm(false) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
log.Fatalf("Failed to load oAuthClient: %s", err)
}
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
@@ -338,14 +337,13 @@ func v1config(ctx context.Context, name string, m configmap.Mapper) error {
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
if err != nil {
return errors.Wrap(err, "failed to setup mountpoint")
log.Fatalf("Failed to setup mountpoint: %s", err)
}
m.Set(configDevice, device)
m.Set(configMountpoint, mountpoint)
}
m.Set("configVersion", strconv.Itoa(v1configVersion))
return nil
}
// registerDevice register a new device for use with the jottacloud API
@@ -420,7 +418,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
}
// v2config configure a jottacloud backend using the modern JottaCli token based authentication
func v2config(ctx context.Context, name string, m configmap.Mapper) error {
func v2config(ctx context.Context, name string, m configmap.Mapper) {
srv := rest.NewClient(fshttp.NewClient(ctx))
fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
@@ -432,32 +430,31 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) error {
token, err := doAuthV2(ctx, srv, loginToken, m)
if err != nil {
return errors.Wrap(err, "failed to get oauth token")
log.Fatalf("Failed to get oauth token: %s", err)
}
err = oauthutil.PutToken(name, m, &token, true)
if err != nil {
return errors.Wrap(err, "error while saving token")
log.Fatalf("Error while saving token: %s", err)
}
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
if config.Confirm(false) {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
log.Fatalf("Failed to load oAuthClient: %s", err)
}
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
if err != nil {
return errors.Wrap(err, "failed to setup mountpoint")
log.Fatalf("Failed to setup mountpoint: %s", err)
}
m.Set(configDevice, device)
m.Set(configMountpoint, mountpoint)
}
m.Set("configVersion", strconv.Itoa(configVersion))
return nil
}
// doAuthV2 runs the actual token request for V2 authentication
+1 -1
View File
@@ -534,7 +534,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return nil
}
// About reports space usage (with a MiB precision)
// About reports space usage (with a MB precision)
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
mount, err := f.client.MountsDetails(f.mountID)
if err != nil {
-2
View File
@@ -1017,7 +1017,6 @@ func (file *localOpenFile) Close() (err error) {
func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
// Read the link and return the destination it as the contents of the object
linkdst, err := os.Readlink(o.path)
fs.Infof(o, "openTranslatedLink link = %q, offset = %d, limit = %d, err = %v", linkdst, offset, limit, err)
if err != nil {
return nil, err
}
@@ -1272,7 +1271,6 @@ func (o *Object) setMetadata(info os.FileInfo) {
// Optionally, users can turn this feature on with the zero_size_links flag
if (runtime.GOOS == "windows" || o.fs.opt.ZeroSizeLinks) && o.translatedLink {
linkdst, err := os.Readlink(o.path)
fs.Infof(o, "setMetadata link = %q, err = %v", linkdst, err)
if err != nil {
fs.Errorf(o, "Failed to read link size: %v", err)
} else {
+5 -5
View File
@@ -6,8 +6,8 @@ import (
"bufio"
"bytes"
"encoding/binary"
"fmt"
"io"
"log"
"time"
"github.com/pkg/errors"
@@ -48,7 +48,7 @@ func (w *BinWriter) Reader() io.Reader {
// WritePu16 writes a short as unsigned varint
func (w *BinWriter) WritePu16(val int) {
if val < 0 || val > 65535 {
panic(fmt.Sprintf("Invalid UInt16 %v", val))
log.Fatalf("Invalid UInt16 %v", val)
}
w.WritePu64(int64(val))
}
@@ -56,7 +56,7 @@ func (w *BinWriter) WritePu16(val int) {
// WritePu32 writes a signed long as unsigned varint
func (w *BinWriter) WritePu32(val int64) {
if val < 0 || val > 4294967295 {
panic(fmt.Sprintf("Invalid UInt32 %v", val))
log.Fatalf("Invalid UInt32 %v", val)
}
w.WritePu64(val)
}
@@ -64,7 +64,7 @@ func (w *BinWriter) WritePu32(val int64) {
// WritePu64 writes an unsigned (actually, signed) long as unsigned varint
func (w *BinWriter) WritePu64(val int64) {
if val < 0 {
panic(fmt.Sprintf("Invalid UInt64 %v", val))
log.Fatalf("Invalid UInt64 %v", val)
}
w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
}
@@ -123,7 +123,7 @@ func (r *BinReader) check(err error) bool {
r.err = err
}
if err != io.EOF {
panic(fmt.Sprintf("Error parsing response: %v", err))
log.Fatalf("Error parsing response: %v", err)
}
return false
}
+32 -32
View File
@@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
@@ -51,8 +52,8 @@ const (
driveTypePersonal = "personal"
driveTypeBusiness = "business"
driveTypeSharepoint = "documentLibrary"
defaultChunkSize = 10 * fs.Mebi
chunkSizeMultiple = 320 * fs.Kibi
defaultChunkSize = 10 * fs.MebiByte
chunkSizeMultiple = 320 * fs.KibiByte
regionGlobal = "global"
regionUS = "us"
@@ -98,7 +99,7 @@ func init() {
Name: "onedrive",
Description: "Microsoft OneDrive",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
region, _ := m.Get("region")
graphURL := graphAPIEndpoint[region] + "/v1.0"
oauthConfig.Endpoint = oauth2.Endpoint{
@@ -108,12 +109,13 @@ func init() {
ci := fs.GetConfig(ctx)
err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
return
}
// Stop if we are running non-interactive config
if ci.AutoConfirm {
return nil
return
}
type driveResource struct {
@@ -136,7 +138,7 @@ func init() {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to configure OneDrive")
log.Fatalf("Failed to configure OneDrive: %v", err)
}
srv := rest.NewClient(oAuthClient)
@@ -201,17 +203,18 @@ func init() {
sites := siteResponse{}
_, err := srv.CallJSON(ctx, &opts, nil, &sites)
if err != nil {
return errors.Wrap(err, "failed to query available sites")
log.Fatalf("Failed to query available sites: %v", err)
}
if len(sites.Sites) == 0 {
return errors.Errorf("search for %q returned no results", searchTerm)
log.Fatalf("Search for '%s' returned no results", searchTerm)
} else {
fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
for index, site := range sites.Sites {
fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
}
siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
}
fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
for index, site := range sites.Sites {
fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
}
siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
}
// if we use server-relative URL for finding the drive
@@ -224,7 +227,7 @@ func init() {
site := siteResource{}
_, err := srv.CallJSON(ctx, &opts, nil, &site)
if err != nil {
return errors.Wrap(err, "failed to query available site by relative path")
log.Fatalf("Failed to query available site by relative path: %v", err)
}
siteID = site.SiteID
}
@@ -244,7 +247,7 @@ func init() {
drives := drivesResponse{}
_, err := srv.CallJSON(ctx, &opts, nil, &drives)
if err != nil {
return errors.Wrap(err, "failed to query available drives")
log.Fatalf("Failed to query available drives: %v", err)
}
// Also call /me/drive as sometimes /me/drives doesn't return it #4068
@@ -253,7 +256,7 @@ func init() {
meDrive := driveResource{}
_, err := srv.CallJSON(ctx, &opts, nil, &meDrive)
if err != nil {
return errors.Wrap(err, "failed to query available drives")
log.Fatalf("Failed to query available drives: %v", err)
}
found := false
for _, drive := range drives.Drives {
@@ -270,13 +273,14 @@ func init() {
}
if len(drives.Drives) == 0 {
return errors.New("no drives found")
log.Fatalf("No drives found")
} else {
fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
for index, drive := range drives.Drives {
fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
}
finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
}
fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
for index, drive := range drives.Drives {
fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
}
finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
}
// Test the driveID and get drive type
@@ -287,18 +291,17 @@ func init() {
var rootItem api.Item
_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
if err != nil {
return errors.Wrapf(err, "failed to query root for drive %s", finalDriveID)
log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err)
}
fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
// This does not work, YET :)
if !config.ConfirmWithConfig(ctx, m, "config_drive_ok", true) {
return errors.New("cancelled by user")
log.Fatalf("Cancelled by user")
}
m.Set(configDriveID, finalDriveID)
m.Set(configDriveType, rootItem.ParentReference.DriveType)
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "region",
@@ -563,9 +566,6 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
if len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
retry = true
fs.Debugf(nil, "Should retry: %v", err)
} else if err != nil && strings.Contains(err.Error(), "Unable to initialize RPS") {
retry = true
fs.Debugf(nil, "HTTP 401: Unable to initialize RPS. Trying again.")
}
case 429: // Too Many Requests.
// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
@@ -693,7 +693,7 @@ func errorHandler(resp *http.Response) error {
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.SizeSuffixBase
const minChunkSize = fs.Byte
if cs%chunkSizeMultiple != 0 {
return errors.Errorf("%s is not a multiple of %s", cs, chunkSizeMultiple)
}
@@ -1857,7 +1857,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
fs.Debugf(o, "Cancelling multipart upload: %v", err)
cancelErr := o.cancelUploadSession(ctx, uploadURL)
if cancelErr != nil {
fs.Logf(o, "Failed to cancel multipart upload: %v (upload failed due to: %v)", cancelErr, err)
fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
}
})()
@@ -1882,11 +1882,11 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
return info, nil
}
// Update the content of a remote file within 4 MiB size in one single request
// Update the content of a remote file within 4MB size in one single request
// This function will set modtime after uploading, which will create a new version for the remote file
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, modTime time.Time, options ...fs.OpenOption) (info *api.Item, err error) {
if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")
return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
}
fs.Debugf(o, "Starting singlepart upload")
+1 -1
View File
@@ -88,7 +88,7 @@ func init() {
Note that these chunks are buffered in memory so increasing them will
increase memory use.`,
Default: 10 * fs.Mebi,
Default: 10 * fs.MebiByte,
Advanced: true,
}},
})
+3 -3
View File
@@ -12,6 +12,7 @@ import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
@@ -71,7 +72,7 @@ func init() {
Name: "pcloud",
Description: "Pcloud",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
optc := new(Options)
err := configstruct.Set(m, optc)
if err != nil {
@@ -99,9 +100,8 @@ func init() {
}
err = oauthutil.Config(ctx, "pcloud", name, m, oauthConfig, &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: config.ConfigEncoding,
+3 -3
View File
@@ -20,6 +20,7 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
@@ -77,12 +78,11 @@ func init() {
Name: "premiumizeme",
Description: "premiumize.me",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
err := oauthutil.Config(ctx, "premiumizeme", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: []fs.Option{{
Name: "api_key",
+4 -5
View File
@@ -2,10 +2,10 @@ package putio
import (
"context"
"log"
"regexp"
"time"
"github.com/pkg/errors"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
@@ -35,7 +35,7 @@ const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
defaultChunkSize = 48 * fs.Mebi
defaultChunkSize = 48 * fs.MebiByte
)
var (
@@ -60,15 +60,14 @@ func init() {
Name: "putio",
Description: "Put.io",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
opt := oauthutil.Options{
NoOffline: true,
}
err := oauthutil.Config(ctx, "putio", name, m, putioConfig, &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: []fs.Option{{
Name: config.ConfigEncoding,
+1 -1
View File
@@ -80,7 +80,7 @@ func init() {
Help: `Cutoff for switching to chunked upload
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5 GiB.`,
The minimum is 0 and the maximum is 5GB.`,
Default: defaultUploadCutoff,
Advanced: true,
}, {
+27 -53
View File
@@ -26,6 +26,7 @@ import (
"github.com/aws/aws-sdk-go/aws/corehandlers"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/endpoints"
@@ -1016,7 +1017,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
Help: `Cutoff for switching to chunked upload
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5 GiB.`,
The minimum is 0 and the maximum is 5GB.`,
Default: defaultUploadCutoff,
Advanced: true,
}, {
@@ -1038,9 +1039,9 @@ Rclone will automatically increase the chunk size when uploading a
large file of known size to stay below the 10,000 chunks limit.
Files of unknown size are uploaded with the configured
chunk_size. Since the default chunk size is 5 MiB and there can be at
chunk_size. Since the default chunk size is 5MB and there can be at
most 10,000 chunks, this means that by default the maximum size of
a file you can stream upload is 48 GiB. If you wish to stream upload
a file you can stream upload is 48GB. If you wish to stream upload
larger files then you will need to increase chunk_size.`,
Default: minChunkSize,
Advanced: true,
@@ -1066,7 +1067,7 @@ large file of a known size to stay below this number of chunks limit.
Any files larger than this that need to be server-side copied will be
copied in chunks of this size.
The minimum is 0 and the maximum is 5 GiB.`,
The minimum is 0 and the maximum is 5GB.`,
Default: fs.SizeSuffix(maxSizeForCopy),
Advanced: true,
}, {
@@ -1220,11 +1221,6 @@ very small even with this flag.
`,
Default: false,
Advanced: true,
}, {
Name: "no_head_object",
Help: `If set, don't HEAD objects`,
Default: false,
Advanced: true,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
@@ -1275,7 +1271,7 @@ See: https://github.com/rclone/rclone/issues/4673, https://github.com/rclone/rcl
const (
metaMtime = "Mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime
metaMD5Hash = "Md5chksum" // the meta key to store md5hash in
// The maximum size of object we can COPY - this should be 5 GiB but is < 5 GB for b2 compatibility
// The maximum size of object we can COPY - this should be 5GiB but is < 5GB for b2 compatibility
// See https://forum.rclone.org/t/copying-files-within-a-b2-bucket/16680/76
maxSizeForCopy = 4768 * 1024 * 1024
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
@@ -1323,7 +1319,6 @@ type Options struct {
ListChunk int64 `config:"list_chunk"`
NoCheckBucket bool `config:"no_check_bucket"`
NoHead bool `config:"no_head"`
NoHeadObject bool `config:"no_head_object"`
Enc encoder.MultiEncoder `config:"encoding"`
MemoryPoolFlushTime fs.Duration `config:"memory_pool_flush_time"`
MemoryPoolUseMmap bool `config:"memory_pool_use_mmap"`
@@ -1516,6 +1511,11 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
}),
ExpiryWindow: 3 * time.Minute,
},
// Pick up IAM role if we are in EKS
&stscreds.WebIdentityRoleProvider{
ExpiryWindow: 3 * time.Minute,
},
}
cred := credentials.NewChainCredentials(providers)
@@ -1693,7 +1693,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
GetTier: true,
SlowModTime: true,
}).Fill(ctx, f)
if f.rootBucket != "" && f.rootDirectory != "" && !opt.NoHeadObject {
if f.rootBucket != "" && f.rootDirectory != "" {
// Check to see if the (bucket,directory) is actually an existing file
oldRoot := f.root
newRoot, leaf := path.Split(oldRoot)
@@ -1730,7 +1730,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
o.setMD5FromEtag(aws.StringValue(info.ETag))
o.bytes = aws.Int64Value(info.Size)
o.storageClass = aws.StringValue(info.StorageClass)
} else if !o.fs.opt.NoHeadObject {
} else {
err := o.readMetaData(ctx) // reads info and meta, returning an error
if err != nil {
return nil, err
@@ -2831,23 +2831,15 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
if err != nil {
return err
}
if resp.LastModified == nil {
fs.Logf(o, "Failed to read last modified from HEAD: %v", err)
}
o.setMetaData(resp.ETag, resp.ContentLength, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass)
return nil
}
func (o *Object) setMetaData(etag *string, contentLength *int64, lastModified *time.Time, meta map[string]*string, mimeType *string, storageClass *string) {
var size int64
// Ignore missing Content-Length assuming it is 0
// Some versions of ceph do this due their apache proxies
if contentLength != nil {
size = *contentLength
if resp.ContentLength != nil {
size = *resp.ContentLength
}
o.setMD5FromEtag(aws.StringValue(etag))
o.setMD5FromEtag(aws.StringValue(resp.ETag))
o.bytes = size
o.meta = meta
o.meta = resp.Metadata
if o.meta == nil {
o.meta = map[string]*string{}
}
@@ -2862,13 +2854,15 @@ func (o *Object) setMetaData(etag *string, contentLength *int64, lastModified *t
o.md5 = hex.EncodeToString(md5sumBytes)
}
}
o.storageClass = aws.StringValue(storageClass)
if lastModified == nil {
o.storageClass = aws.StringValue(resp.StorageClass)
if resp.LastModified == nil {
fs.Logf(o, "Failed to read last modified from HEAD: %v", err)
o.lastModified = time.Now()
} else {
o.lastModified = *lastModified
o.lastModified = *resp.LastModified
}
o.mimeType = aws.StringValue(mimeType)
o.mimeType = aws.StringValue(resp.ContentType)
return nil
}
// ModTime returns the modification time of the object
@@ -2978,26 +2972,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
if err != nil {
return nil, err
}
if resp.LastModified == nil {
fs.Logf(o, "Failed to read last modified: %v", err)
}
// read size from ContentLength or ContentRange
size := resp.ContentLength
if resp.ContentRange != nil {
var contentRange = *resp.ContentRange
slash := strings.IndexRune(contentRange, '/')
if slash >= 0 {
i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
if err == nil {
size = &i
} else {
fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
}
} else {
fs.Debugf(o, "Failed to find length in %q", contentRange)
}
}
o.setMetaData(resp.ETag, size, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass)
return resp.Body, nil
}
@@ -3023,9 +2997,9 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
// calculate size of parts
partSize := int(f.opt.ChunkSize)
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
// 48 GiB which seems like a not too unreasonable limit.
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
// buffers here (default 5MB). With a maximum number of parts (10,000) this will be a file of
// 48GB which seems like a not too unreasonable limit.
if size == -1 {
warnStreamUpload.Do(func() {
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
@@ -3034,7 +3008,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
} else {
// Adjust partSize until the number of parts is small enough.
if size/int64(partSize) >= uploadParts {
// Calculate partition size rounded up to the nearest MiB
// Calculate partition size rounded up to the nearest MB
partSize = int((((size / uploadParts) >> 20) + 1) << 20)
}
}
+10 -7
View File
@@ -296,32 +296,36 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// Config callback for 2FA
func Config(ctx context.Context, name string, m configmap.Mapper) error {
func Config(ctx context.Context, name string, m configmap.Mapper) {
ci := fs.GetConfig(ctx)
serverURL, ok := m.Get(configURL)
if !ok || serverURL == "" {
// If there's no server URL, it means we're trying an operation at the backend level, like a "rclone authorize seafile"
return errors.New("operation not supported on this remote. If you need a 2FA code on your account, use the command: nrclone config reconnect <remote name>: ")
fmt.Print("\nOperation not supported on this remote.\nIf you need a 2FA code on your account, use the command:\n\nrclone config reconnect <remote name>:\n\n")
return
}
// Stop if we are running non-interactive config
if ci.AutoConfirm {
return nil
return
}
u, err := url.Parse(serverURL)
if err != nil {
return errors.Errorf("invalid server URL %s", serverURL)
fs.Errorf(nil, "Invalid server URL %s", serverURL)
return
}
is2faEnabled, _ := m.Get(config2FA)
if is2faEnabled != "true" {
return errors.New("two-factor authentication is not enabled on this account")
fmt.Println("Two-factor authentication is not enabled on this account.")
return
}
username, _ := m.Get(configUser)
if username == "" {
return errors.New("a username is required")
fs.Errorf(nil, "A username is required")
return
}
password, _ := m.Get(configPassword)
@@ -372,7 +376,6 @@ func Config(ctx context.Context, name string, m configmap.Mapper) error {
break
}
}
return nil
}
// sets the AuthorizationToken up
+26 -51
View File
@@ -224,17 +224,6 @@ have a server which returns
Then you may need to enable this flag.
If concurrent reads are disabled, the use_fstat option is ignored.
`,
Advanced: true,
}, {
Name: "disable_concurrent_writes",
Default: false,
Help: `If set don't use concurrent writes
Normally rclone uses concurrent writes to upload files. This improves
the performance greatly, especially for distant servers.
This option disables concurrent writes should that be necessary.
`,
Advanced: true,
}, {
@@ -255,30 +244,29 @@ Set to 0 to keep connections indefinitely.
// Options defines the configuration for this backend
type Options struct {
Host string `config:"host"`
User string `config:"user"`
Port string `config:"port"`
Pass string `config:"pass"`
KeyPem string `config:"key_pem"`
KeyFile string `config:"key_file"`
KeyFilePass string `config:"key_file_pass"`
PubKeyFile string `config:"pubkey_file"`
KnownHostsFile string `config:"known_hosts_file"`
KeyUseAgent bool `config:"key_use_agent"`
UseInsecureCipher bool `config:"use_insecure_cipher"`
DisableHashCheck bool `config:"disable_hashcheck"`
AskPassword bool `config:"ask_password"`
PathOverride string `config:"path_override"`
SetModTime bool `config:"set_modtime"`
Md5sumCommand string `config:"md5sum_command"`
Sha1sumCommand string `config:"sha1sum_command"`
SkipLinks bool `config:"skip_links"`
Subsystem string `config:"subsystem"`
ServerCommand string `config:"server_command"`
UseFstat bool `config:"use_fstat"`
DisableConcurrentReads bool `config:"disable_concurrent_reads"`
DisableConcurrentWrites bool `config:"disable_concurrent_writes"`
IdleTimeout fs.Duration `config:"idle_timeout"`
Host string `config:"host"`
User string `config:"user"`
Port string `config:"port"`
Pass string `config:"pass"`
KeyPem string `config:"key_pem"`
KeyFile string `config:"key_file"`
KeyFilePass string `config:"key_file_pass"`
PubKeyFile string `config:"pubkey_file"`
KnownHostsFile string `config:"known_hosts_file"`
KeyUseAgent bool `config:"key_use_agent"`
UseInsecureCipher bool `config:"use_insecure_cipher"`
DisableHashCheck bool `config:"disable_hashcheck"`
AskPassword bool `config:"ask_password"`
PathOverride string `config:"path_override"`
SetModTime bool `config:"set_modtime"`
Md5sumCommand string `config:"md5sum_command"`
Sha1sumCommand string `config:"sha1sum_command"`
SkipLinks bool `config:"skip_links"`
Subsystem string `config:"subsystem"`
ServerCommand string `config:"server_command"`
UseFstat bool `config:"use_fstat"`
DisableConcurrentReads bool `config:"disable_concurrent_reads"`
IdleTimeout fs.Duration `config:"idle_timeout"`
}
// Fs stores the interface to the remote SFTP files
@@ -426,8 +414,8 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
opts = append(opts,
sftp.UseFstat(f.opt.UseFstat),
sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites),
// FIXME disabled after library reversion
// sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
)
if f.opt.DisableConcurrentReads { // FIXME
fs.Errorf(f, "Ignoring disable_concurrent_reads after library reversion - see #5197")
@@ -1506,19 +1494,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return in, nil
}
type sizeReader struct {
io.Reader
size int64
}
// Size returns the expected size of the stream
//
// It is used in sftpFile.ReadFrom as a hint to work out the
// concurrency needed
func (sr *sizeReader) Size() int64 {
return sr.size
}
// Update a remote sftp file using the data <in> and ModTime from <src>
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
o.fs.addTransfer() // Show transfer in progress
@@ -1550,7 +1525,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
fs.Debugf(src, "Removed after failed upload: %v", err)
}
}
_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
_, err = file.ReadFrom(in)
if err != nil {
remove()
return errors.Wrap(err, "Update ReadFrom failed")
+7 -7
View File
@@ -77,6 +77,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"path"
@@ -109,10 +110,10 @@ const (
decayConstant = 2 // bigger for slower decay, exponential
apiPath = "/sf/v3" // add to endpoint to get API path
tokenPath = "/oauth/token" // add to endpoint to get Token path
minChunkSize = 256 * fs.Kibi
maxChunkSize = 2 * fs.Gibi
defaultChunkSize = 64 * fs.Mebi
defaultUploadCutoff = 128 * fs.Mebi
minChunkSize = 256 * fs.KibiByte
maxChunkSize = 2 * fs.GibiByte
defaultChunkSize = 64 * fs.MebiByte
defaultUploadCutoff = 128 * fs.MebiByte
)
// Generate a new oauth2 config which we will update when we know the TokenURL
@@ -135,7 +136,7 @@ func init() {
Name: "sharefile",
Description: "Citrix Sharefile",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
oauthConfig := newOauthConfig("")
checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
if auth == nil || auth.Form == nil {
@@ -156,9 +157,8 @@ func init() {
}
err := oauthutil.Config(ctx, "sharefile", name, m, oauthConfig, &opt)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
return nil
},
Options: []fs.Option{{
Name: "upload_cutoff",
+5 -5
View File
@@ -16,6 +16,7 @@ import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/url"
"path"
@@ -75,17 +76,17 @@ func init() {
Name: "sugarsync",
Description: "Sugarsync",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return errors.Wrap(err, "failed to read options")
log.Fatalf("Failed to read options: %v", err)
}
if opt.RefreshToken != "" {
fmt.Printf("Already have a token - refresh?\n")
if !config.ConfirmWithConfig(ctx, m, "config_refresh_token", true) {
return nil
return
}
}
fmt.Printf("Username (email address)> ")
@@ -113,11 +114,10 @@ func init() {
// return shouldRetry(ctx, resp, err)
//})
if err != nil {
return errors.Wrap(err, "failed to get token")
log.Fatalf("Failed to get token: %v", err)
}
opt.RefreshToken = resp.Header.Get("Location")
m.Set("refresh_token", opt.RefreshToken)
return nil
},
Options: []fs.Option{{
Name: "app_id",
+4 -4
View File
@@ -36,7 +36,7 @@ import (
const (
directoryMarkerContentType = "application/directory" // content type of directory marker objects
listChunks = 1000 // chunk size to read directory listings
defaultChunkSize = 5 * fs.Gibi
defaultChunkSize = 5 * fs.GibiByte
minSleep = 10 * time.Millisecond // In case of error, start at 10ms sleep.
)
@@ -46,7 +46,7 @@ var SharedOptions = []fs.Option{{
Help: `Above this size files will be chunked into a _segments container.
Above this size files will be chunked into a _segments container. The
default for this is 5 GiB which is its maximum value.`,
default for this is 5GB which is its maximum value.`,
Default: defaultChunkSize,
Advanced: true,
}, {
@@ -56,7 +56,7 @@ default for this is 5 GiB which is its maximum value.`,
When doing streaming uploads (e.g. using rcat or mount) setting this
flag will cause the swift backend to not upload chunked files.
This will limit the maximum upload size to 5 GiB. However non chunked
This will limit the maximum upload size to 5GB. However non chunked
files are easier to deal with and have an MD5SUM.
Rclone will still chunk files bigger than chunk_size when doing normal
@@ -419,7 +419,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.SizeSuffixBase
const minChunkSize = fs.Byte
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
+3 -3
View File
@@ -87,7 +87,7 @@ func (f *Fs) testWithChunk(t *testing.T) {
preConfChunkSize := f.opt.ChunkSize
preConfChunk := f.opt.NoChunk
f.opt.NoChunk = false
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
f.opt.ChunkSize = 1024 * fs.Byte
defer func() {
//restore old config after test
f.opt.ChunkSize = preConfChunkSize
@@ -117,7 +117,7 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
preConfChunkSize := f.opt.ChunkSize
preConfChunk := f.opt.NoChunk
f.opt.NoChunk = false
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
f.opt.ChunkSize = 1024 * fs.Byte
segmentContainer := f.root + "_segments"
defer func() {
//restore config
@@ -159,7 +159,7 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
preConfChunkSize := f.opt.ChunkSize
preConfChunk := f.opt.NoChunk
f.opt.NoChunk = false
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
f.opt.ChunkSize = 1024 * fs.Byte
defer func() {
//restore old config after test
f.opt.ChunkSize = preConfChunkSize
+6 -6
View File
@@ -7,6 +7,7 @@ import (
"context"
"fmt"
"io"
"log"
"path"
"strings"
"time"
@@ -41,7 +42,7 @@ func init() {
Name: "tardigrade",
Description: "Tardigrade Decentralized Cloud Storage",
NewFs: NewFs,
Config: func(ctx context.Context, name string, configMapper configmap.Mapper) error {
Config: func(ctx context.Context, name string, configMapper configmap.Mapper) {
provider, _ := configMapper.Get(fs.ConfigProvider)
config.FileDeleteKey(name, fs.ConfigProvider)
@@ -53,7 +54,7 @@ func init() {
// satelliteString contains always default and passphrase can be empty
if apiKey == "" {
return nil
return
}
satellite, found := satMap[satelliteString]
@@ -63,12 +64,12 @@ func init() {
access, err := uplink.RequestAccessWithPassphrase(context.TODO(), satellite, apiKey, passphrase)
if err != nil {
return errors.Wrap(err, "couldn't create access grant")
log.Fatalf("Couldn't create access grant: %v", err)
}
serializedAccess, err := access.Serialize()
if err != nil {
return errors.Wrap(err, "couldn't serialize access grant")
log.Fatalf("Couldn't serialize access grant: %v", err)
}
configMapper.Set("satellite_address", satellite)
configMapper.Set("access_grant", serializedAccess)
@@ -77,9 +78,8 @@ func init() {
config.FileDeleteKey(name, "api_key")
config.FileDeleteKey(name, "passphrase")
} else {
return errors.Errorf("invalid provider type: %s", provider)
log.Fatalf("Invalid provider type: %s", provider)
}
return nil
},
Options: []fs.Option{
{
+7 -1
View File
@@ -148,7 +148,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (_ io.ReadC
case s && !e:
offset = opt.Start
case !s && e:
offset = -opt.End
object, err := o.fs.project.StatObject(ctx, bucketName, bucketPath)
if err != nil {
return nil, err
}
offset = object.System.ContentLength - opt.End
length = opt.End
}
case *fs.SeekOption:
offset = opt.Offset
-170
View File
@@ -1,170 +0,0 @@
package api
import "fmt"
// Error contains the error code and message returned by the API
type Error struct {
Success bool `json:"success,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Message string `json:"message,omitempty"`
Data string `json:"data,omitempty"`
}
// Error returns a string for the error and satisfies the error interface
func (e Error) Error() string {
out := fmt.Sprintf("api error %d", e.StatusCode)
if e.Message != "" {
out += ": " + e.Message
}
if e.Data != "" {
out += ": " + e.Data
}
return out
}
// FolderEntry represents a Uptobox subfolder when listing folder contents
type FolderEntry struct {
FolderID uint64 `json:"fld_id"`
Description string `json:"fld_descr"`
Password string `json:"fld_password"`
FullPath string `json:"fullPath"`
Path string `json:"fld_name"`
Name string `json:"name"`
Hash string `json:"hash"`
}
// FolderInfo represents the current folder when listing folder contents
type FolderInfo struct {
FolderID uint64 `json:"fld_id"`
Hash string `json:"hash"`
FileCount uint64 `json:"fileCount"`
TotalFileSize int64 `json:"totalFileSize"`
}
// FileInfo represents a file when listing folder contents
type FileInfo struct {
Name string `json:"file_name"`
Description string `json:"file_descr"`
Created string `json:"file_created"`
Size int64 `json:"file_size"`
Downloads uint64 `json:"file_downloads"`
Code string `json:"file_code"`
Password string `json:"file_password"`
Public int `json:"file_public"`
LastDownload string `json:"file_last_download"`
ID uint64 `json:"id"`
}
// ReadMetadataResponse is the response when listing folder contents
type ReadMetadataResponse struct {
StatusCode int `json:"statusCode"`
Message string `json:"message"`
Data struct {
CurrentFolder FolderInfo `json:"currentFolder"`
Folders []FolderEntry `json:"folders"`
Files []FileInfo `json:"files"`
PageCount int `json:"pageCount"`
TotalFileCount int `json:"totalFileCount"`
TotalFileSize int64 `json:"totalFileSize"`
} `json:"data"`
}
// UploadInfo is the response when initiating an upload
type UploadInfo struct {
StatusCode int `json:"statusCode"`
Message string `json:"message"`
Data struct {
UploadLink string `json:"uploadLink"`
MaxUpload string `json:"maxUpload"`
} `json:"data"`
}
// UploadResponse is the respnse to a successful upload
type UploadResponse struct {
Files []struct {
Name string `json:"name"`
Size int64 `json:"size"`
URL string `json:"url"`
DeleteURL string `json:"deleteUrl"`
} `json:"files"`
}
// UpdateResponse is a generic response to various action on files (rename/copy/move)
type UpdateResponse struct {
Message string `json:"message"`
StatusCode int `json:"statusCode"`
}
// Download is the response when requesting a download link
type Download struct {
StatusCode int `json:"statusCode"`
Message string `json:"message"`
Data struct {
DownloadLink string `json:"dlLink"`
} `json:"data"`
}
// MetadataRequestOptions represents all the options when listing folder contents
type MetadataRequestOptions struct {
Limit uint64
Offset uint64
SearchField string
Search string
}
// CreateFolderRequest is used for creating a folder
type CreateFolderRequest struct {
Token string `json:"token"`
Path string `json:"path"`
Name string `json:"name"`
}
// DeleteFolderRequest is used for deleting a folder
type DeleteFolderRequest struct {
Token string `json:"token"`
FolderID uint64 `json:"fld_id"`
}
// CopyMoveFileRequest is used for moving/copying a file
type CopyMoveFileRequest struct {
Token string `json:"token"`
FileCodes string `json:"file_codes"`
DestinationFolderID uint64 `json:"destination_fld_id"`
Action string `json:"action"`
}
// MoveFolderRequest is used for moving a folder
type MoveFolderRequest struct {
Token string `json:"token"`
FolderID uint64 `json:"fld_id"`
DestinationFolderID uint64 `json:"destination_fld_id"`
Action string `json:"action"`
}
// RenameFolderRequest is used for renaming a folder
type RenameFolderRequest struct {
Token string `json:"token"`
FolderID uint64 `json:"fld_id"`
NewName string `json:"new_name"`
}
// UpdateFileInformation is used for renaming a file
type UpdateFileInformation struct {
Token string `json:"token"`
FileCode string `json:"file_code"`
NewName string `json:"new_name,omitempty"`
Description string `json:"description,omitempty"`
Password string `json:"password,omitempty"`
Public string `json:"public,omitempty"`
}
// RemoveFileRequest is used for deleting a file
type RemoveFileRequest struct {
Token string `json:"token"`
FileCodes string `json:"file_codes"`
}
// Token represents the authentication token
type Token struct {
Token string `json:"token"`
}
File diff suppressed because it is too large. Load diff
-21
View File
@@ -1,21 +0,0 @@
// Test Uptobox filesystem interface
package uptobox_test
import (
"testing"
"github.com/rclone/rclone/backend/uptobox"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
if *fstest.RemoteName == "" {
*fstest.RemoteName = "TestUptobox:"
}
fstests.Run(t, &fstests.Opt{
RemoteName: *fstest.RemoteName,
NilObject: (*uptobox.Object)(nil),
})
}
+1 -1
View File
@@ -125,7 +125,7 @@ func (ca *CookieAuth) getSPCookie(conf *SharepointSuccessResponse) (*CookieRespo
return nil, errors.Wrap(err, "Error while constructing endpoint URL")
}
u, err := url.Parse(spRoot.Scheme + "://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
if err != nil {
return nil, errors.Wrap(err, "Error while constructing login URL")
}
+7 -7
View File
@@ -60,12 +60,12 @@ func init() {
Name: "yandex",
Description: "Yandex Disk",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
err := oauthutil.Config(ctx, "yandex", name, m, oauthConfig, nil)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
return
}
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: config.ConfigEncoding,
@@ -251,22 +251,22 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
token, err := oauthutil.GetToken(name, m)
if err != nil {
return nil, errors.Wrap(err, "couldn't read OAuth token")
log.Fatalf("Couldn't read OAuth token (this should never happen).")
}
if token.RefreshToken == "" {
return nil, errors.New("unable to get RefreshToken. If you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
log.Fatalf("Unable to get RefreshToken. If you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend.")
}
if token.TokenType != "OAuth" {
token.TokenType = "OAuth"
err = oauthutil.PutToken(name, m, token, false)
if err != nil {
return nil, errors.Wrap(err, "couldn't save OAuth token")
log.Fatalf("Couldn't save OAuth token (this should never happen).")
}
log.Printf("Automatically upgraded OAuth config.")
}
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return nil, errors.Wrap(err, "failed to configure Yandex")
log.Fatalf("Failed to configure Yandex: %v", err)
}
ci := fs.GetConfig(ctx)
+12 -19
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"path"
@@ -72,41 +73,37 @@ func init() {
Name: "zoho",
Description: "Zoho",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
Config: func(ctx context.Context, name string, m configmap.Mapper) {
// Need to setup region before configuring oauth
err := setupRegion(m)
if err != nil {
return err
}
setupRegion(m)
opt := oauthutil.Options{
// No refresh token unless ApprovalForce is set
OAuth2Opts: []oauth2.AuthCodeOption{oauth2.ApprovalForce},
}
if err := oauthutil.Config(ctx, "zoho", name, m, oauthConfig, &opt); err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
// We need to rewrite the token type to "Zoho-oauthtoken" because Zoho wants
// it's own custom type
token, err := oauthutil.GetToken(name, m)
if err != nil {
return errors.Wrap(err, "failed to read token")
log.Fatalf("Failed to read token: %v", err)
}
if token.TokenType != "Zoho-oauthtoken" {
token.TokenType = "Zoho-oauthtoken"
err = oauthutil.PutToken(name, m, token, false)
if err != nil {
return errors.Wrap(err, "failed to configure token")
log.Fatalf("Failed to configure token: %v", err)
}
}
if fs.GetConfig(ctx).AutoConfirm {
return nil
return
}
if err = setupRoot(ctx, name, m); err != nil {
return errors.Wrap(err, "failed to configure root directory")
log.Fatalf("Failed to configure root directory: %v", err)
}
return nil
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
Name: "region",
@@ -167,16 +164,15 @@ type Object struct {
// ------------------------------------------------------------
func setupRegion(m configmap.Mapper) error {
func setupRegion(m configmap.Mapper) {
region, ok := m.Get("region")
if !ok || region == "" {
return errors.New("no region set")
log.Fatalf("No region set\n")
}
rootURL = fmt.Sprintf("https://workdrive.zoho.%s/api/v1", region)
accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region)
oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
return nil
}
// ------------------------------------------------------------
@@ -212,7 +208,7 @@ func listWorkspaces(ctx context.Context, teamID string, srv *rest.Client) ([]api
func setupRoot(ctx context.Context, name string, m configmap.Mapper) error {
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return errors.Wrap(err, "failed to load oAuthClient")
log.Fatalf("Failed to load oAuthClient: %s", err)
}
authSrv := rest.NewClient(oAuthClient).SetRoot(accountsURL)
opts := rest.Opts{
@@ -381,10 +377,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err := configstruct.Set(m, opt); err != nil {
return nil, err
}
err := setupRegion(m)
if err != nil {
return nil, err
}
setupRegion(m)
root = parsePath(root)
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
-1
View File
@@ -62,7 +62,6 @@ docs = [
"sftp.md",
"sugarsync.md",
"tardigrade.md",
"uptobox.md",
"union.md",
"webdav.md",
"yandex.md",
-1
View File
@@ -54,7 +54,6 @@ import (
_ "github.com/rclone/rclone/cmd/size"
_ "github.com/rclone/rclone/cmd/sync"
_ "github.com/rclone/rclone/cmd/test"
_ "github.com/rclone/rclone/cmd/test/changenotify"
_ "github.com/rclone/rclone/cmd/test/histogram"
_ "github.com/rclone/rclone/cmd/test/info"
_ "github.com/rclone/rclone/cmd/test/makefiles"
+2 -2
View File
@@ -49,7 +49,7 @@ var (
cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file")
memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file")
statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)")
dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second")
dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s")
version bool
retries = flags.IntP("retries", "", 3, "Retry operations this many times if they fail")
retriesInterval = flags.DurationP("retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)")
@@ -400,7 +400,7 @@ func initConfig() {
configflags.SetFlags(ci)
// Load the config
configfile.Install()
configfile.LoadConfig(ctx)
// Start accounting
accounting.Start(ctx)
+4 -13
View File
@@ -22,7 +22,6 @@ func init() {
cmd.Root.AddCommand(configCommand)
configCommand.AddCommand(configEditCommand)
configCommand.AddCommand(configFileCommand)
configCommand.AddCommand(configTouchCommand)
configCommand.AddCommand(configShowCommand)
configCommand.AddCommand(configDumpCommand)
configCommand.AddCommand(configProvidersCommand)
@@ -42,9 +41,9 @@ var configCommand = &cobra.Command{
remotes and manage existing ones. You may also set or remove a
password to protect your configuration.
`,
RunE: func(command *cobra.Command, args []string) error {
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 0, command, args)
return config.EditConfig(context.Background())
config.EditConfig(context.Background())
},
}
@@ -64,15 +63,6 @@ var configFileCommand = &cobra.Command{
},
}
var configTouchCommand = &cobra.Command{
Use: "touch",
Short: `Ensure configuration file exists.`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 0, command, args)
config.SaveConfig()
},
}
var configShowCommand = &cobra.Command{
Use: "show [<remote>]",
Short: `Print (decrypted) config file, or the config for a single remote.`,
@@ -272,7 +262,8 @@ This normally means going through the interactive oauth flow again.
if fsInfo.Config == nil {
return errors.Errorf("%s: doesn't support Reconnect", configName)
}
return fsInfo.Config(ctx, configName, config)
fsInfo.Config(ctx, configName, config)
return nil
},
}
+4 -4
View File
@@ -36,8 +36,8 @@ If you supply the |--rmdirs| flag, it will remove all empty directories along wi
You can also use the separate command |rmdir| or |rmdirs| to
delete empty directories only.
For example, to delete all files bigger than 100 MiB, you may first want to
check what would be deleted (use either):
For example, to delete all files bigger than 100MBytes, you may first want to check what
would be deleted (use either):
rclone --min-size 100M lsl remote:path
rclone --dry-run --min-size 100M delete remote:path
@@ -46,8 +46,8 @@ Then proceed with the actual delete:
rclone --min-size 100M delete remote:path
That reads "delete everything with a minimum size of 100 MiB", hence
delete all files bigger than 100 MiB.
That reads "delete everything with a minimum size of 100 MB", hence
delete all files bigger than 100MBytes.
**Important**: Since this can cause data loss, test first with the
|--dry-run| or the |--interactive|/|-i| flag.
+14 -33
View File
@@ -206,9 +206,9 @@ When that happens, it is the user's responsibility to stop the mount manually.
The size of the mounted file system will be set according to information retrieved
from the remote, the same as returned by the [rclone about](https://rclone.org/commands/rclone_about/)
command. Remotes with unlimited storage may report the used size only,
then an additional 1 PiB of free space is assumed. If the remote does not
then an additional 1PB of free space is assumed. If the remote does not
[support](https://rclone.org/overview/#optional-features) the about feature
at all, then 1 PiB is set as both the total and the free size.
at all, then 1PB is set as both the total and the free size.
**Note**: As of |rclone| 1.52.2, |rclone mount| now requires Go version 1.13
or newer on some platforms depending on the underlying FUSE library in use.
@@ -342,38 +342,19 @@ by specifying |-o FileSecurity="D:P(A;;FA;;;OW)"|, for file all access (FA) to t
#### Windows caveats
Drives created as Administrator are not visible to other accounts,
not even an account that was elevated to Administrator with the
User Account Control (UAC) feature. A result of this is that if you mount
to a drive letter from a Command Prompt run as Administrator, and then try
to access the same drive from Windows Explorer (which does not run as
Administrator), you will not be able to see the mounted drive.
Note that drives created as Administrator are not visible by other
accounts (including the account that was elevated as
Administrator). So if you start a Windows drive from an Administrative
Command Prompt and then try to access the same drive from Explorer
(which does not run as Administrator), you will not be able to see the
new drive.
If you don't need to access the drive from applications running with
administrative privileges, the easiest way around this is to always
create the mount from a non-elevated command prompt.
To make mapped drives available to the user account that created them
regardless if elevated or not, there is a special Windows setting called
[linked connections](https://docs.microsoft.com/en-us/troubleshoot/windows-client/networking/mapped-drives-not-available-from-elevated-command#detail-to-configure-the-enablelinkedconnections-registry-entry)
that can be enabled.
It is also possible to make a drive mount available to everyone on the system,
by running the process creating it as the built-in SYSTEM account.
There are several ways to do this: One is to use the command-line
utility [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec),
from Microsoft's Sysinternals suite, which has option |-s| to start
processes as the SYSTEM account. Another alternative is to run the mount
command from a Windows Scheduled Task, or a Windows Service, configured
to run as the SYSTEM account. A third alternative is to use the
[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)).
Note that when running rclone as another user, it will not use
the configuration file from your profile unless you tell it to
with the [|--config|](https://rclone.org/docs/#config-config-file) option.
Read more in the [install documentation](https://rclone.org/install/).
Note that mapping to a directory path, instead of a drive letter,
does not suffer from the same limitations.
The easiest way around this is to start the drive from a normal
command prompt. It is also possible to start a drive from the SYSTEM
account (using [the WinFsp.Launcher
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
which creates drives accessible for everyone on the system or
alternatively using [the nssm service manager](https://nssm.cc/usage).
### Limitations
+1 -1
View File
@@ -21,7 +21,7 @@ import (
func TestRc(t *testing.T) {
ctx := context.Background()
configfile.Install()
configfile.LoadConfig(ctx)
mount := rc.Calls.Get("mount/mount")
assert.NotNil(t, mount)
unmount := rc.Calls.Get("mount/unmount")
+10 -20
View File
@@ -373,7 +373,7 @@ func (u *UI) Draw() error {
extras := ""
if u.showCounts {
if count > 0 {
extras += fmt.Sprintf("%8v ", fs.CountSuffix(count))
extras += fmt.Sprintf("%8v ", fs.SizeSuffix(count))
} else {
extras += " "
}
@@ -385,9 +385,9 @@ func (u *UI) Draw() error {
}
if u.showDirAverageSize {
if averageSize > 0 {
extras += fmt.Sprintf("%9v ", fs.SizeSuffix(int64(averageSize)))
extras += fmt.Sprintf("%8v ", fs.SizeSuffix(int64(averageSize)))
} else {
extras += " "
extras += " "
}
}
@@ -406,7 +406,7 @@ func (u *UI) Draw() error {
}
extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
}
Linef(0, y, w, fg, bg, ' ', "%c %9v %s%c%s%s", fileFlag, fs.SizeSuffix(size), extras, mark, path.Base(entry.Remote()), message)
Linef(0, y, w, fg, bg, ' ', "%c %8v %s%c%s%s", fileFlag, fs.SizeSuffix(size), extras, mark, path.Base(entry.Remote()), message)
y++
}
}
@@ -485,15 +485,11 @@ func (u *UI) removeEntry(pos int) {
// delete the entry at the current position
func (u *UI) delete() {
if u.d == nil || len(u.entries) == 0 {
return
}
ctx := context.Background()
cursorPos := u.dirPosMap[u.path]
dirPos := u.sortPerm[cursorPos.entry]
dirEntry := u.entries[dirPos]
dirPos := u.sortPerm[u.dirPosMap[u.path].entry]
entry := u.entries[dirPos]
u.boxMenu = []string{"cancel", "confirm"}
if obj, isFile := dirEntry.(fs.Object); isFile {
if obj, isFile := entry.(fs.Object); isFile {
u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
if o != 1 {
return "Aborted!", nil
@@ -503,33 +499,27 @@ func (u *UI) delete() {
return "", err
}
u.removeEntry(dirPos)
if cursorPos.entry >= len(u.entries) {
u.move(-1) // move back onto a valid entry
}
return "Successfully deleted file!", nil
}
u.popupBox([]string{
"Delete this file?",
u.fsName + dirEntry.String()})
u.fsName + entry.String()})
} else {
u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
if o != 1 {
return "Aborted!", nil
}
err := operations.Purge(ctx, f, dirEntry.String())
err := operations.Purge(ctx, f, entry.String())
if err != nil {
return "", err
}
u.removeEntry(dirPos)
if cursorPos.entry >= len(u.entries) {
u.move(-1) // move back onto a valid entry
}
return "Successfully purged folder!", nil
}
u.popupBox([]string{
"Purge this directory?",
"ALL files in it will be deleted",
u.fsName + dirEntry.String()})
u.fsName + entry.String()})
}
}
+1 -15
View File
@@ -7,19 +7,12 @@ import (
"time"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
var (
size = int64(-1)
)
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.Int64VarP(cmdFlags, &size, "size", "", size, "File size hint to preallocate")
}
var commandDefinition = &cobra.Command{
@@ -44,13 +37,6 @@ must fit into RAM. The cutoff needs to be small enough to adhere
the limits of your remote, please see there. Generally speaking,
setting this cutoff too high will decrease your performance.
Use the |--size| flag to preallocate the file in advance at the remote end
and actually stream it, even if remote backend doesn't support streaming.
|--size| should be the exact size of the input stream in bytes. If the
size of the stream is different in length to the |--size| passed in
then the transfer will likely fail.
Note that the upload can also not be retried because the data is
not kept around until the upload succeeds. If you need to transfer
a lot of data, you're better off caching locally and then
@@ -65,7 +51,7 @@ a lot of data, you're better off caching locally and then
fdst, dstFileName := cmd.NewFsDstFile(args)
cmd.Run(false, false, command, func() error {
_, err := operations.RcatSize(context.Background(), fdst, dstFileName, os.Stdin, size, time.Now())
_, err := operations.Rcat(context.Background(), fdst, dstFileName, os.Stdin, time.Now())
return err
})
},
+1 -1
View File
@@ -41,7 +41,7 @@ func startServer(t *testing.T, f fs.Fs) {
}
func TestInit(t *testing.T) {
configfile.Install()
configfile.LoadConfig(context.Background())
f, err := fs.NewFs(context.Background(), "testdata/files")
l, _ := f.List(context.Background(), "")
+1 -1
View File
@@ -61,7 +61,7 @@ var (
func TestInit(t *testing.T) {
ctx := context.Background()
// Configure the remote
configfile.Install()
configfile.LoadConfig(context.Background())
// fs.Config.LogLevel = fs.LogLevelDebug
// fs.Config.DumpHeaders = true
// fs.Config.DumpBodies = true
@@ -367,7 +367,7 @@ footer {
}
};
function readableFileSize(size) {
var units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
var units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
var i = 0;
while(size >= 1024) {
size /= 1024;
+3 -1
View File
@@ -1,6 +1,7 @@
package restic
import (
"context"
"crypto/rand"
"encoding/hex"
"io"
@@ -64,7 +65,8 @@ func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest {
// TestResticHandler runs tests on the restic handler code, especially in append-only mode.
func TestResticHandler(t *testing.T) {
configfile.Install()
ctx := context.Background()
configfile.LoadConfig(ctx)
buf := make([]byte, 32)
_, err := io.ReadFull(rand.Reader, buf)
require.NoError(t, err)
+1 -1
View File
@@ -44,7 +44,7 @@ var commandDefinition = &cobra.Command{
}
fmt.Printf("Total objects: %d\n", results.Count)
fmt.Printf("Total size: %s (%d bytes)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
fmt.Printf("Total size: %s (%d Bytes)\n", fs.SizeSuffix(results.Bytes).Unit("Bytes"), results.Bytes)
return nil
})
-54
View File
@@ -1,54 +0,0 @@
// Package changenotify tests rclone's changenotify support
package changenotify
import (
"context"
"errors"
"time"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/test"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/spf13/cobra"
)
var (
pollInterval = 10 * time.Second
)
func init() {
test.Command.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.DurationVarP(cmdFlags, &pollInterval, "poll-interval", "", pollInterval, "Time to wait between polling for changes.")
}
var commandDefinition = &cobra.Command{
Use: "changenotify remote:",
Short: `Log any change notify requests for the remote passed in.`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)
ctx := context.Background()
// Start polling function
features := f.Features()
if do := features.ChangeNotify; do != nil {
pollChan := make(chan time.Duration)
do(ctx, changeNotify, pollChan)
pollChan <- pollInterval
fs.Logf(nil, "Waiting for changes, polling every %v", pollInterval)
} else {
return errors.New("poll-interval is not supported by this remote")
}
select {}
},
}
// changeNotify invalidates the directory cache for the relativePath
// passed in.
//
// if entryType is a directory it invalidates the parent of the directory too.
func changeNotify(relativePath string, entryType fs.EntryType) {
fs.Logf(nil, "%q: %v", relativePath, entryType)
}
+11 -24
View File
@@ -3,12 +3,12 @@
package makefiles
import (
cryptrand "crypto/rand"
"io"
"log"
"math/rand"
"os"
"path/filepath"
"time"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/test"
@@ -27,10 +27,8 @@ var (
maxFileSize = fs.SizeSuffix(100)
minFileNameLength = 4
maxFileNameLength = 12
seed = int64(1)
// Globals
randSource *rand.Rand
directoriesToCreate int
totalDirectories int
fileNames = map[string]struct{}{} // keep a note of which file name we've used already
@@ -46,7 +44,6 @@ func init() {
flags.FVarP(cmdFlags, &maxFileSize, "max-file-size", "", "Maximum size of files to create")
flags.IntVarP(cmdFlags, &minFileNameLength, "min-name-length", "", minFileNameLength, "Minimum size of file names")
flags.IntVarP(cmdFlags, &maxFileNameLength, "max-name-length", "", maxFileNameLength, "Maximum size of file names")
flags.Int64VarP(cmdFlags, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)")
}
var commandDefinition = &cobra.Command{
@@ -54,36 +51,28 @@ var commandDefinition = &cobra.Command{
Short: `Make a random file hierarchy in <dir>`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
if seed == 0 {
seed = time.Now().UnixNano()
fs.Logf(nil, "Using random seed = %d", seed)
}
randSource = rand.New(rand.NewSource(seed))
outputDirectory := args[0]
directoriesToCreate = numberOfFiles / averageFilesPerDirectory
averageSize := (minFileSize + maxFileSize) / 2
start := time.Now()
fs.Logf(nil, "Creating %d files of average size %v in %d directories in %q.", numberOfFiles, averageSize, directoriesToCreate, outputDirectory)
log.Printf("Creating %d files of average size %v in %d directories in %q.", numberOfFiles, averageSize, directoriesToCreate, outputDirectory)
root := &dir{name: outputDirectory, depth: 1}
for totalDirectories < directoriesToCreate {
root.createDirectories()
}
dirs := root.list("", []string{})
totalBytes := int64(0)
for i := 0; i < numberOfFiles; i++ {
dir := dirs[randSource.Intn(len(dirs))]
totalBytes += writeFile(dir, fileName())
dir := dirs[rand.Intn(len(dirs))]
writeFile(dir, fileName())
}
dt := time.Since(start)
fs.Logf(nil, "Written %viB in %v at %viB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
log.Printf("Done.")
},
}
// fileName creates a unique random file or directory name
func fileName() (name string) {
for {
length := randSource.Intn(maxFileNameLength-minFileNameLength) + minFileNameLength
name = random.StringFn(length, randSource.Intn)
length := rand.Intn(maxFileNameLength-minFileNameLength) + minFileNameLength
name = random.String(length)
if _, found := fileNames[name]; !found {
break
}
@@ -110,7 +99,7 @@ func (d *dir) createDirectories() {
}
d.children = append(d.children, newDir)
totalDirectories++
switch randSource.Intn(4) {
switch rand.Intn(4) {
case 0:
if d.depth < maxDepth {
newDir.createDirectories()
@@ -133,7 +122,7 @@ func (d *dir) list(path string, output []string) []string {
}
// writeFile writes a random file at dir/name
func writeFile(dir, name string) int64 {
func writeFile(dir, name string) {
err := os.MkdirAll(dir, 0777)
if err != nil {
log.Fatalf("Failed to make directory %q: %v", dir, err)
@@ -143,8 +132,8 @@ func writeFile(dir, name string) int64 {
if err != nil {
log.Fatalf("Failed to open file %q: %v", path, err)
}
size := randSource.Int63n(int64(maxFileSize-minFileSize)) + int64(minFileSize)
_, err = io.CopyN(fd, randSource, size)
size := rand.Int63n(int64(maxFileSize-minFileSize)) + int64(minFileSize)
_, err = io.CopyN(fd, cryptrand.Reader, size)
if err != nil {
log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err)
}
@@ -152,6 +141,4 @@ func writeFile(dir, name string) int64 {
if err != nil {
log.Fatalf("Failed to close file %q: %v", path, err)
}
fs.Infof(path, "Written file size %v", fs.SizeSuffix(size))
return size
}
+3 -3
View File
@@ -26,12 +26,12 @@ func TestVersionWorksWithoutAccessibleConfigFile(t *testing.T) {
}
// re-wire
oldOsStdout := os.Stdout
oldConfigPath := config.GetConfigPath()
assert.NoError(t, config.SetConfigPath(path))
oldConfigPath := config.ConfigPath
config.ConfigPath = path
os.Stdout = nil
defer func() {
os.Stdout = oldOsStdout
assert.NoError(t, config.SetConfigPath(oldConfigPath))
config.ConfigPath = oldConfigPath
}()
cmd.Root.SetArgs([]string{"version"})
-1
View File
@@ -152,7 +152,6 @@ WebDAV or S3, that work out of the box.)
{{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}}
{{< provider name="Tardigrade" home="https://tardigrade.io/" config="/tardigrade/" >}}
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
{{< provider name="Uptobox" home="https://uptobox.com" config="/uptobox/" >}}
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
{{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
{{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}
+7 -7
View File
@@ -227,16 +227,16 @@ Checkpoint for internal polling (debug).
#### --acd-upload-wait-per-gb
Additional time per GiB to wait after a failed complete upload to see if it appears.
Additional time per GB to wait after a failed complete upload to see if it appears.
Sometimes Amazon Drive gives an error when a file has been fully
uploaded but the file appears anyway after a little while. This
happens sometimes for files over 1 GiB in size and nearly every time for
files bigger than 10 GiB. This parameter controls the time rclone waits
happens sometimes for files over 1GB in size and nearly every time for
files bigger than 10GB. This parameter controls the time rclone waits
for the file to appear.
The default value for this parameter is 3 minutes per GiB, so by
default it will wait 3 minutes for every GiB uploaded to see if the
The default value for this parameter is 3 minutes per GB, so by
default it will wait 3 minutes for every GB uploaded to see if the
file appears.
You can disable this feature by setting it to 0. This may cause
@@ -260,7 +260,7 @@ Files >= this size will be downloaded via their tempLink.
Files this size or more will be downloaded via their "tempLink". This
is to work around a problem with Amazon Drive which blocks downloads
of files bigger than about 10 GiB. The default for this is 9 GiB which
of files bigger than about 10GB. The default for this is 9GB which
shouldn't need to be changed.
To download files above this threshold, rclone requests a "tempLink"
@@ -299,7 +299,7 @@ Amazon Drive has an internal limit of file sizes that can be uploaded
to the service. This limit is not officially published, but all files
larger than this will fail.
At the time of writing (Jan 2016) is in the area of 50 GiB per file.
At the time of writing (Jan 2016) is in the area of 50GB per file.
This means that larger files are likely to fail.
Unfortunately there is no way for rclone to see that this failure is
+1 -10
View File
@@ -372,7 +372,7 @@ put them back in again.` >}}
* Fred <fred@creativeprojects.tech>
* Sébastien Gross <renard@users.noreply.github.com>
* Maxime Suret <11944422+msuret@users.noreply.github.com>
* Caleb Case <caleb@storj.io> <calebcase@gmail.com>
* Caleb Case <caleb@storj.io>
* Ben Zenker <imbenzenker@gmail.com>
* Martin Michlmayr <tbm@cyrius.com>
* Brandon McNama <bmcnama@pagerduty.com>
@@ -479,12 +479,3 @@ put them back in again.` >}}
* x0b <x0bdev@gmail.com>
* CERN through the CS3MESH4EOSC Project
* Nick Gaya <nicholasgaya+github@gmail.com>
* Ashok Gelal <401055+ashokgelal@users.noreply.github.com>
* Dominik Mydlil <dominik.mydlil@outlook.com>
* Nazar Mishturak <nazarmx@gmail.com>
* Ansh Mittal <iamAnshMittal@gmail.com>
* noabody <noabody@yahoo.com>
* OleFrost <82263101+olefrost@users.noreply.github.com>
* Kenny Parsons <kennyparsons93@gmail.com>
* Jeffrey Tolar <tolar.jeffrey@gmail.com>
* jtagcat <git-514635f7@jtag.cat>
+2 -2
View File
@@ -269,7 +269,7 @@ Leave blank normally.
#### --azureblob-upload-cutoff
Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)
Cutoff for switching to chunked upload (<= 256MB). (Deprecated)
- Config: upload_cutoff
- Env Var: RCLONE_AZUREBLOB_UPLOAD_CUTOFF
@@ -278,7 +278,7 @@ Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)
#### --azureblob-chunk-size
Upload chunk size (<= 100 MiB).
Upload chunk size (<= 100MB).
Note that this is stored in memory and there may be up to
"--transfers" chunks stored at once in memory.
+9 -4
View File
@@ -155,8 +155,8 @@ depending on your hardware, how big the files are, how much you want
to load your computer, etc. The default of `--transfers 4` is
definitely too low for Backblaze B2 though.
Note that uploading big files (bigger than 200 MiB by default) will use
a 96 MiB RAM buffer by default. There can be at most `--transfers` of
Note that uploading big files (bigger than 200 MB by default) will use
a 96 MB RAM buffer by default. There can be at most `--transfers` of
these in use at any moment, so this sets the upper limit on the memory
used.
@@ -172,6 +172,11 @@ the file instead of hiding it.
Old versions of files, where available, are visible using the
`--b2-versions` flag.
**NB** Note that `--b2-versions` does not work with crypt at the
moment [#1627](https://github.com/rclone/rclone/issues/1627). Using
[--backup-dir](/docs/#backup-dir-dir) with rclone is the recommended
way of working around this.
If you wish to remove all the old versions then you can use the
`rclone cleanup remote:bucket` command which will delete all the old
versions of files, leaving the current ones intact. You can also
@@ -401,7 +406,7 @@ Cutoff for switching to chunked upload.
Files above this size will be uploaded in chunks of "--b2-chunk-size".
This value should be set no larger than 4.657 GiB (== 5 GB).
This value should be set no larger than 4.657GiB (== 5GB).
- Config: upload_cutoff
- Env Var: RCLONE_B2_UPLOAD_CUTOFF
@@ -415,7 +420,7 @@ Cutoff for switching to multipart copy
Any files larger than this that need to be server-side copied will be
copied in chunks of this size.
The minimum is 0 and the maximum is 4.6 GiB.
The minimum is 0 and the maximum is 4.6GB.
- Config: copy_cutoff
- Env Var: RCLONE_B2_COPY_CUTOFF
+3 -3
View File
@@ -225,10 +225,10 @@ as they can't be used in JSON strings.
### Transfers ###
For files above 50 MiB rclone will use a chunked transfer. Rclone will
For files above 50MB rclone will use a chunked transfer. Rclone will
upload up to `--transfers` chunks at the same time (shared among all
the multipart uploads). Chunks are buffered in memory and are
normally 8 MiB so increasing `--transfers` will increase memory use.
normally 8MB so increasing `--transfers` will increase memory use.
### Deleting files ###
@@ -369,7 +369,7 @@ Fill in for rclone to use a non root folder as its starting point.
#### --box-upload-cutoff
Cutoff for switching to multipart upload (>= 50 MiB).
Cutoff for switching to multipart upload (>= 50MB).
- Config: upload_cutoff
- Env Var: RCLONE_BOX_UPLOAD_CUTOFF
+13 -13
View File
@@ -70,11 +70,11 @@ password:
The size of a chunk. Lower value good for slow connections but can affect seamless reading.
Default: 5M
Choose a number from below, or type in your own value
1 / 1 MiB
\ "1M"
2 / 5 MiB
1 / 1MB
\ "1m"
2 / 5 MB
\ "5M"
3 / 10 MiB
3 / 10 MB
\ "10M"
chunk_size> 2
How much time should object info (file size, file hashes, etc.) be stored in cache. Use a very high value if you don't plan on changing the source FS from outside the cache.
@@ -91,11 +91,11 @@ info_age> 2
The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted.
Default: 10G
Choose a number from below, or type in your own value
1 / 500 MiB
1 / 500 MB
\ "500M"
2 / 1 GiB
2 / 1 GB
\ "1G"
3 / 10 GiB
3 / 10 GB
\ "10G"
chunk_total_size> 3
Remote config
@@ -364,11 +364,11 @@ will need to be cleared or unexpected EOF errors will occur.
- Default: 5M
- Examples:
- "1m"
- 1 MiB
- 1MB
- "5M"
- 5 MiB
- 5 MB
- "10M"
- 10 MiB
- 10 MB
#### --cache-info-age
@@ -401,11 +401,11 @@ oldest chunks until it goes under this value.
- Default: 10G
- Examples:
- "500M"
- 500 MiB
- 500 MB
- "1G"
- 1 GiB
- 1 GB
- "10G"
- 10 GiB
- 10 GB
### Advanced Options
-38
View File
@@ -5,44 +5,6 @@ description: "Rclone Changelog"
# Changelog
## v1.55.1 - 2021-04-26
[See commits](https://github.com/rclone/rclone/compare/v1.55.0...v1.55.1)
* Bug Fixes
* selfupdate
* Dont detect FUSE if build is static (Ivan Andreev)
* Add build tag noselfupdate (Ivan Andreev)
* sync: Fix incorrect error reported by graceful cutoff (Nick Craig-Wood)
* install.sh: fix macOS arm64 download (Nick Craig-Wood)
* build: Fix version numbers in android branch builds (Nick Craig-Wood)
* docs
* Contributing.md: update setup instructions for go1.16 (Nick Gaya)
* WinFsp 2021 is out of beta (albertony)
* Minor cleanup of space around code section (albertony)
* Fixed some typos (albertony)
* VFS
* Fix a code path which allows dirty data to be removed causing data loss (Nick Craig-Wood)
* Compress
* Fix compressed name regexp (buengese)
* Drive
* Fix backend copyid of google doc to directory (Nick Craig-Wood)
* Don't open browser when service account... (Ansh Mittal)
* Dropbox
* Add missing team_data.member scope for use with --impersonate (Nick Craig-Wood)
* Fix About after scopes changes - rclone config reconnect needed (Nick Craig-Wood)
* Fix Unable to decrypt returned paths from changeNotify (Nick Craig-Wood)
* FTP
* Fix implicit TLS (Ivan Andreev)
* Onedrive
* Work around for random "Unable to initialize RPS" errors (OleFrost)
* SFTP
* Revert sftp library to v1.12.0 from v1.13.0 to fix performance regression (Nick Craig-Wood)
* Fix Update ReadFrom failed: failed to send packet: EOF errors (Nick Craig-Wood)
* Zoho
* Fix error when region isn't set (buengese)
* Do not ask for mountpoint twice when using headless setup (buengese)
## v1.55.0 - 2021-03-31
[See commits](https://github.com/rclone/rclone/compare/v1.54.0...v1.55.0)
+1 -1
View File
@@ -43,7 +43,7 @@ Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
Enter a string value. Press Enter for the default ("").
remote> remote:path
Files larger than chunk size will be split in chunks.
Enter a size with suffix K,M,G,T. Press Enter for the default ("2G").
Enter a size with suffix k,M,G,T. Press Enter for the default ("2G").
chunk_size> 100M
Choose how chunker handles hash sums. All modes but "none" require metadata.
Enter a string value. Press Enter for the default ("md5").
+3 -3
View File
@@ -23,7 +23,7 @@ If you supply the `--rmdirs` flag, it will remove all empty directories along wi
You can also use the separate command `rmdir` or `rmdirs` to
delete empty directories only.
For example, to delete all files bigger than 100 MiByte, you may first want to check what
For example, to delete all files bigger than 100MBytes, you may first want to check what
would be deleted (use either):
rclone --min-size 100M lsl remote:path
@@ -33,8 +33,8 @@ Then proceed with the actual delete:
rclone --min-size 100M delete remote:path
That reads "delete everything with a minimum size of 100 MiB", hence
delete all files bigger than 100 MiByte.
That reads "delete everything with a minimum size of 100 MB", hence
delete all files bigger than 100MBytes.
**Important**: Since this can cause data loss, test first with the
`--dry-run` or the `--interactive`/`-i` flag.
+2 -2
View File
@@ -56,9 +56,9 @@ When that happens, it is the user's responsibility to stop the mount manually.
The size of the mounted file system will be set according to information retrieved
from the remote, the same as returned by the [rclone about](https://rclone.org/commands/rclone_about/)
command. Remotes with unlimited storage may report the used size only,
then an additional 1 PiB of free space is assumed. If the remote does not
then an additional 1PB of free space is assumed. If the remote does not
[support](https://rclone.org/overview/#optional-features) the about feature
at all, then 1 PiB is set as both the total and the free size.
at all, then 1PB is set as both the total and the free size.
**Note**: As of `rclone` 1.52.2, `rclone mount` now requires Go version 1.13
or newer on some platforms depending on the underlying FUSE library in use.
+3 -3
View File
@@ -627,7 +627,7 @@ approximately 2×10⁻³² of re-using a nonce.
#### Chunk
Each chunk will contain 64 KiB of data, except for the last one which
Each chunk will contain 64kB of data, except for the last one which
may have less data. The data chunk is in standard NaCl SecretBox
format. SecretBox uses XSalsa20 and Poly1305 to encrypt and
authenticate messages.
@@ -653,12 +653,12 @@ This uses a 32 byte (256 bit key) key derived from the user password.
49 bytes total
1 MiB (1048576 bytes) file will encrypt to
1MB (1048576 bytes) file will encrypt to
* 32 bytes header
* 16 chunks of 65568 bytes
1049120 bytes total (a 0.05% overhead). This is the overhead for big
1049120 bytes total (a 0.05% overhead). This is the overhead for big
files.
### Name encryption
+54 -89
View File
@@ -57,7 +57,6 @@ See the following for detailed instructions for
* [SugarSync](/sugarsync/)
* [Tardigrade](/tardigrade/)
* [Union](/union/)
* [Uptobox](/uptobox/)
* [WebDAV](/webdav/)
* [Yandex Disk](/yandex/)
* [Zoho WorkDrive](/zoho/)
@@ -228,7 +227,7 @@ adding the `--drive-shared-with-me` parameter to the remote `gdrive:`.
rclone lsf "gdrive,shared_with_me:path/to/dir"
The major advantage to using the connection string style syntax is
that it only applies to the remote, not to all the remotes of that
that it only applies the the remote, not to all the remotes of that
type of the command line. A common confusion is this attempt to copy a
file shared on google drive to the normal drive which **does not
work** because the `--drive-shared-with-me` flag applies to both the
@@ -421,10 +420,10 @@ possibly signed sequence of decimal numbers, each with optional
fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid
time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
Options which use SIZE use KiByte (multiples of 1024 bytes) by default.
However, a suffix of `B` for Byte, `K` for KiByte, `M` for MiByte,
`G` for GiByte, `T` for TiByte and `P` for PiByte may be used. These are
the binary units, e.g. 1, 2\*\*10, 2\*\*20, 2\*\*30 respectively.
Options which use SIZE use kByte by default. However, a suffix of `b`
for bytes, `k` for kBytes, `M` for MBytes, `G` for GBytes, `T` for
TBytes and `P` for PBytes may be used. These are the binary units, e.g.
1, 2\*\*10, 2\*\*20, 2\*\*30 respectively.
### --backup-dir=DIR ###
@@ -467,23 +466,23 @@ This option controls the bandwidth limit. For example
--bwlimit 10M
would mean limit the upload and download bandwidth to 10 MiByte/s.
would mean limit the upload and download bandwidth to 10 MByte/s.
**NB** this is **bytes** per second not **bits** per second. To use a
single limit, specify the desired bandwidth in KiByte/s, or use a
suffix B|K|M|G|T|P. The default is `0` which means to not limit bandwidth.
single limit, specify the desired bandwidth in kBytes/s, or use a
suffix b|k|M|G. The default is `0` which means to not limit bandwidth.
The upload and download bandwidth can be specified seperately, as
`--bwlimit UP:DOWN`, so
--bwlimit 10M:100k
would mean limit the upload bandwidth to 10 MiByte/s and the download
bandwidth to 100 KiByte/s. Either limit can be "off" meaning no limit, so
would mean limit the upload bandwidth to 10 MByte/s and the download
bandwidth to 100 kByte/s. Either limit can be "off" meaning no limit, so
to just limit the upload bandwidth you would use
--bwlimit 10M:off
this would limit the upload bandwidth to 10 MiByte/s but the download
this would limit the upload bandwidth to 10MByte/s but the download
bandwidth would be unlimited.
When specified as above the bandwidth limits last for the duration of
@@ -505,19 +504,19 @@ working hours could be:
`--bwlimit "08:00,512k 12:00,10M 13:00,512k 18:00,30M 23:00,off"`
In this example, the transfer bandwidth will be set to 512 KiByte/s
at 8am every day. At noon, it will rise to 10 MiByte/s, and drop back
to 512 KiByte/sec at 1pm. At 6pm, the bandwidth limit will be set to
30 MiByte/s, and at 11pm it will be completely disabled (full speed).
In this example, the transfer bandwidth will be set to 512kBytes/sec
at 8am every day. At noon, it will rise to 10MByte/s, and drop back
to 512kBytes/sec at 1pm. At 6pm, the bandwidth limit will be set to
30MByte/s, and at 11pm it will be completely disabled (full speed).
Anything between 11pm and 8am will remain unlimited.
An example of timetable with `WEEKDAY` could be:
`--bwlimit "Mon-00:00,512 Fri-23:59,10M Sat-10:00,1M Sun-20:00,off"`
It means that, the transfer bandwidth will be set to 512 KiByte/s on
Monday. It will rise to 10 MiByte/s before the end of Friday. At 10:00
on Saturday it will be set to 1 MiByte/s. From 20:00 on Sunday it will
It means that, the transfer bandwidth will be set to 512kBytes/sec on
Monday. It will rise to 10MByte/s before the end of Friday. At 10:00
on Saturday it will be set to 1MByte/s. From 20:00 on Sunday it will
be unlimited.
Timeslots without `WEEKDAY` are extended to the whole week. So this
@@ -533,10 +532,10 @@ Bandwidth limit apply to the data transfer for all backends. For most
backends the directory listing bandwidth is also included (exceptions
being the non HTTP backends, `ftp`, `sftp` and `tardigrade`).
Note that the units are **Byte/s**, not **bit/s**. Typically
connections are measured in bit/s - to convert divide by 8. For
Note that the units are **Bytes/s**, not **Bits/s**. Typically
connections are measured in Bits/s - to convert divide by 8. For
example, let's say you have a 10 Mbit/s connection and you wish rclone
to use half of it - 5 Mbit/s. This is 5/8 = 0.625 MiByte/s so you would
to use half of it - 5 Mbit/s. This is 5/8 = 0.625MByte/s so you would
use a `--bwlimit 0.625M` parameter for rclone.
On Unix systems (Linux, macOS, …) the bandwidth limiter can be toggled by
@@ -557,7 +556,7 @@ change the bwlimit dynamically:
This option controls per file bandwidth limit. For the options see the
`--bwlimit` flag.
For example use this to allow no transfers to be faster than 1 MiByte/s
For example use this to allow no transfers to be faster than 1MByte/s
--bwlimit-file 1M
@@ -640,54 +639,25 @@ See `--copy-dest` and `--backup-dir`.
### --config=CONFIG_FILE ###
Specify the location of the rclone configuration file, to override
the default. E.g. `rclone config --config="rclone.conf"`.
Specify the location of the rclone configuration file.
The exact default is a bit complex to describe, due to changes
introduced through different versions of rclone while preserving
backwards compatibility, but in most cases it is as simple as:
Normally the config file is in your home directory as a file called
`.config/rclone/rclone.conf` (or `.rclone.conf` if created with an
older version). If `$XDG_CONFIG_HOME` is set it will be at
`$XDG_CONFIG_HOME/rclone/rclone.conf`.
- `%APPDATA%/rclone/rclone.conf` on Windows
- `~/.config/rclone/rclone.conf` on other
The complete logic is as follows: Rclone will look for an existing
configuration file in any of the following locations, in priority order:
1. `rclone.conf` (in program directory, where rclone executable is)
2. `%APPDATA%/rclone/rclone.conf` (only on Windows)
3. `$XDG_CONFIG_HOME/rclone/rclone.conf` (on all systems, including Windows)
4. `~/.config/rclone/rclone.conf` (see below for explanation of ~ symbol)
5. `~/.rclone.conf`
If no existing configuration file is found, then a new one will be created
in the following location:
- On Windows: Location 2 listed above, except in the unlikely event
that `APPDATA` is not defined, then location 4 is used instead.
- On Unix: Location 3 if `XDG_CONFIG_HOME` is defined, else location 4.
- Fallback to location 5 (on all OS), when the rclone directory cannot be
created, but if also a home directory was not found then path
`.rclone.conf` relative to current working directory will be used as
a final resort.
The `~` symbol in paths above represent the home directory of the current user
on any OS, and the value is defined as following:
- On Windows: `%HOME%` if defined, else `%USERPROFILE%`, or else `%HOMEDRIVE%\%HOMEPATH%`.
- On Unix: `$HOME` if defined, else by looking up current user in OS-specific user database
(e.g. passwd file), or else use the result from shell command `cd && pwd`.
If there is a file `rclone.conf` in the same directory as the rclone
executable it will be preferred. This file must be created manually
for Rclone to use it, it will never be created automatically.
If you run `rclone config file` you will see where the default
location is for you.
The fact that an existing file `rclone.conf` in the same directory
as the rclone executable is always preferred, means that it is easy
to run in "portable" mode by downloading rclone executable to a
writable directory and then create an empty file `rclone.conf` in the
same directory.
Use this flag to override the config location, e.g. `rclone
--config=".myconfig" config`.
If the location is set to empty string `""` or path to a file
with name `notfound`, or the os null device represented by value `NUL` on
If the location is set to empty string `""` or the special value
`/notfound`, or the os null device represented by value `NUL` on
Windows and `/dev/null` on Unix systems, then rclone will keep the
config file in memory only.
@@ -770,7 +740,7 @@ which feature does what.
This flag can be useful for debugging and in exceptional circumstances
(e.g. Google Drive limiting the total volume of Server Side Copies to
100 GiB/day).
100GB/day).
### --dscp VALUE ###
@@ -1047,7 +1017,7 @@ This is the maximum allowable backlog of files in a sync/copy/move
queued for being checked or transferred.
This can be set arbitrarily large. It will only use memory when the
queue is in use. Note that it will use in the order of N KiB of memory
queue is in use. Note that it will use in the order of N kB of memory
when the backlog is in use.
Setting this large allows rclone to calculate how many files are
@@ -1176,13 +1146,13 @@ size of the file. To calculate the number of download streams Rclone
divides the size of the file by the `--multi-thread-cutoff` and rounds
up, up to the maximum set with `--multi-thread-streams`.
So if `--multi-thread-cutoff 250M` and `--multi-thread-streams 4` are
So if `--multi-thread-cutoff 250MB` and `--multi-thread-streams 4` are
in effect (the defaults):
- 0..250 MiB files will be downloaded with 1 stream
- 250..500 MiB files will be downloaded with 2 streams
- 500..750 MiB files will be downloaded with 3 streams
- 750+ MiB files will be downloaded with 4 streams
- 0MB..250MB files will be downloaded with 1 stream
- 250MB..500MB files will be downloaded with 2 streams
- 500MB..750MB files will be downloaded with 3 streams
- 750MB+ files will be downloaded with 4 streams
### --no-check-dest ###
@@ -1473,14 +1443,14 @@ date formatting syntax.
### --stats-unit=bits|bytes ###
By default, data transfer rates will be printed in bytes per second.
By default, data transfer rates will be printed in bytes/second.
This option allows the data rate to be printed in bits per second.
This option allows the data rate to be printed in bits/second.
Data transfer volume will still be reported in bytes.
The rate is reported as a binary unit, not SI unit. So 1 Mbit/s
equals 1,048,576 bit/s and not 1,000,000 bit/s.
equals 1,048,576 bits/s and not 1,000,000 bits/s.
The default is `bytes`.
@@ -1915,21 +1885,16 @@ password prompts. To do that, pass the parameter
of asking for a password if `RCLONE_CONFIG_PASS` doesn't contain
a valid password, and `--password-command` has not been supplied.
Whenever running commands that may be affected by options in a
configuration file, rclone will look for an existing file according
to the rules described [above](#config-config-file), and load any it
finds. If an encrypted file is found, this includes decrypting it,
with the possible consequence of a password prompt. When executing
a command line that you know are not actually using anything from such
a configuration file, you can avoid it being loaded by overriding the
location, e.g. with one of the documented special values for
memory-only configuration. Since only backend options can be stored
in configuration files, this is normally unnecessary for commands
that do not operate on backends, e.g. `genautocomplete`. However,
it will be relevant for commands that do operate on backends in
general, but are used without referencing a stored remote, e.g.
listing local filesystem paths, or
[connection strings](#connection-strings): `rclone --config="" ls .`
Some rclone commands, such as `genautocomplete`, do not require configuration.
Nevertheless, rclone will read any configuration file found
according to the rules described [above](https://rclone.org/docs/#config-config-file).
If an encrypted configuration file is found, this means you will be prompted for
password (unless using `--password-command`). To avoid this, you can bypass
the loading of the configuration file by overriding the location with an empty
string `""` or the special value `/notfound`, or the os null device represented
by value `NUL` on Windows and `/dev/null` on Unix systems (before rclone
version 1.55 only this null device alternative was supported).
E.g. `rclone --config="" genautocomplete bash`.
Developer options
-----------------
File diff suppressed because it is too large. Load diff
+3 -3
View File
@@ -993,7 +993,7 @@ See: https://github.com/rclone/rclone/issues/3631
Make upload limit errors be fatal
At the time of writing it is only possible to upload 750 GiB of data to
At the time of writing it is only possible to upload 750GB of data to
Google Drive a day (this is an undocumented limit). When this limit is
reached Google Drive produces a slightly different error message. When
this flag is set it causes these errors to be fatal. These will stop
@@ -1014,7 +1014,7 @@ See: https://github.com/rclone/rclone/issues/3857
Make download limit errors be fatal
At the time of writing it is only possible to download 10 TiB of data from
At the time of writing it is only possible to download 10TB of data from
Google Drive a day (this is an undocumented limit). When this limit is
reached Google Drive produces a slightly different error message. When
this flag is set it causes these errors to be fatal. These will stop
@@ -1226,7 +1226,7 @@ Use the -i flag to see what would be copied before copying.
Drive has quite a lot of rate limiting. This causes rclone to be
limited to transferring about 2 files per second only. Individual
files may be transferred much faster at 100s of MiByte/s but lots of
files may be transferred much faster at 100s of MBytes/s but lots of
small files can take a long time.
Server side copies are also subject to a separate rate limit. If you
+1 -1
View File
@@ -185,7 +185,7 @@ Any files larger than this will be uploaded in chunks of this size.
Note that chunks are buffered in memory (one at a time) so rclone can
deal with retries. Setting this larger will increase the speed
slightly (at most 10% for 128 MiB in tests) at the cost of using more
slightly (at most 10% for 128MB in tests) at the cost of using more
memory. It can be set smaller if you are tight on memory.
- Config: chunk_size
+6 -6
View File
@@ -586,17 +586,17 @@ remote or flag value. The fix then is to quote values containing spaces.
### `--min-size` - Don't transfer any file smaller than this
Controls the minimum size file within the scope of an rclone command.
Default units are `KiByte` but abbreviations `K`, `M`, `G`, `T` or `P` are valid.
Default units are `kBytes` but abbreviations `k`, `M`, or `G` are valid.
E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50 KiByte
E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50kByte
size or larger.
### `--max-size` - Don't transfer any file larger than this
Controls the maximum size file within the scope of an rclone command.
Default units are `KiByte` but abbreviations `K`, `M`, `G`, `T` or `P` are valid.
Default units are `kBytes` but abbreviations `k`, `M`, or `G` are valid.
E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1 GiByte
E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1GByte
size or smaller.
### `--max-age` - Don't transfer any file older than this
@@ -650,8 +650,8 @@ E.g. the scope of `rclone sync -i A: B:` can be restricted:
rclone --min-size 50k --delete-excluded sync A: B:
All files on `B:` which are less than 50 KiByte are deleted
because they are excluded from the rclone sync command.
All files on `B:` which are less than 50 kBytes are deleted
because they are excluded from the rclone sync command.
### `--dump filters` - dump the filters to the output
+8 -8
View File
@@ -18,8 +18,8 @@ These flags are available for every command.
--backup-dir string Make backups into hierarchy based in DIR.
--bind string Local address to bind to for outgoing connections, IPv4, IPv6 or name.
--buffer-size SizeSuffix In memory buffer size when reading files for each --transfer. (default 16M)
--bwlimit BwTimetable Bandwidth limit in KiByte/s, or use suffix B|K|M|G|T|P or a full timetable.
--bwlimit-file BwTimetable Bandwidth limit per file in KiByte/s, or use suffix B|K|M|G|T|P or a full timetable.
--bwlimit BwTimetable Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.
--bwlimit-file BwTimetable Bandwidth limit per file in kBytes/s, or use suffix b|k|M|G or a full timetable.
--ca-cert string CA certificate used to verify servers
--cache-dir string Directory rclone will use for caching. (default "$HOME/.cache/rclone")
--check-first Do all the checks before starting transfers.
@@ -79,12 +79,12 @@ These flags are available for every command.
--max-delete int When synchronizing, limit the number of deletes (default -1)
--max-depth int If set limits the recursion depth to this. (default -1)
--max-duration duration Maximum duration rclone will transfer data for.
--max-size SizeSuffix Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off)
--max-size SizeSuffix Only transfer files smaller than this in k or suffix b|k|M|G (default off)
--max-stats-groups int Maximum number of stats groups to keep in memory. On max oldest is discarded. (default 1000)
--max-transfer SizeSuffix Maximum size of data to transfer. (default off)
--memprofile string Write memory profile to file
--min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off)
--min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off)
--min-size SizeSuffix Only transfer files bigger than this in k or suffix b|k|M|G (default off)
--modify-window duration Max time diff to be considered the same (default 1ns)
--multi-thread-cutoff SizeSuffix Use multi-thread downloads for files above this size. (default 250M)
--multi-thread-streams int Max number of streams to use for multi-thread downloads. (default 4)
@@ -170,12 +170,12 @@ and may be set in the config file.
--acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink. (default 9G)
--acd-token string OAuth Access Token as a JSON blob.
--acd-token-url string Token server url.
--acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears. (default 3m0s)
--acd-upload-wait-per-gb Duration Additional time per GB to wait after a failed complete upload to see if it appears. (default 3m0s)
--alias-remote string Remote or path to alias.
--azureblob-access-tier string Access tier of blob: hot, cool or archive.
--azureblob-account string Storage Account Name (leave blank to use SAS URL or Emulator)
--azureblob-archive-tier-delete Delete archive tier blobs before overwriting.
--azureblob-chunk-size SizeSuffix Upload chunk size (<= 100 MiB). (default 4M)
--azureblob-chunk-size SizeSuffix Upload chunk size (<= 100MB). (default 4M)
--azureblob-disable-checksum Don't store MD5 checksum with object metadata.
--azureblob-encoding MultiEncoder This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8)
--azureblob-endpoint string Endpoint for the service
@@ -189,7 +189,7 @@ and may be set in the config file.
--azureblob-public-access string Public access level of a container: blob, container.
--azureblob-sas-url string SAS URL for container level access only
--azureblob-service-principal-file string Path to file containing credentials for use with a service principal.
--azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)
--azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256MB). (Deprecated)
--azureblob-use-emulator Uses local storage emulator if provided as 'true' (leave blank if using real azure storage endpoint)
--azureblob-use-msi Use a managed service identity to authenticate (only works in Azure)
--b2-account string Account ID or Application Key ID
@@ -218,7 +218,7 @@ and may be set in the config file.
--box-root-folder-id string Fill in for rclone to use a non root folder as its starting point.
--box-token string OAuth Access Token as a JSON blob.
--box-token-url string Token server url.
--box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB). (default 50M)
--box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50MB). (default 50M)
--cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage. (default 1m0s)
--cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming.
--cache-chunk-path string Directory to cache chunk files. (default "$HOME/.cache/rclone/cache-backend")
+2 -2
View File
@@ -168,7 +168,7 @@ Leave blank to use the provider defaults.
Above this size files will be chunked into a _segments container.
Above this size files will be chunked into a _segments container. The
default for this is 5 GiB which is its maximum value.
default for this is 5GB which is its maximum value.
- Config: chunk_size
- Env Var: RCLONE_HUBIC_CHUNK_SIZE
@@ -182,7 +182,7 @@ Don't chunk files during streaming upload.
When doing streaming uploads (e.g. using rcat or mount) setting this
flag will cause the swift backend to not upload chunked files.
This will limit the maximum upload size to 5 GiB. However non chunked
This will limit the maximum upload size to 5GB. However non chunked
files are easier to deal with and have an MD5SUM.
Rclone will still chunk files bigger than chunk_size when doing normal
-145
View File
@@ -12,7 +12,6 @@ Rclone is a Go program and comes as a single binary file.
* [Download](/downloads/) the relevant binary.
* Extract the `rclone` or `rclone.exe` binary from the archive
* Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
* Optionally configure [automatic execution](#autostart).
See below for some expanded Linux / macOS instructions.
@@ -227,147 +226,3 @@ Instructions
roles:
- rclone
```
# Autostart #
After installing and configuring rclone, as described above, you are ready to use rclone
as an interactive command line utility. If your goal is to perform *periodic* operations,
such as a regular [sync](https://rclone.org/commands/rclone_sync/), you will probably want
to configure your rclone command in your operating system's scheduler. If you need to
expose *service*-like features, such as [remote control](https://rclone.org/rc/),
[GUI](https://rclone.org/gui/), [serve](https://rclone.org/commands/rclone_serve/)
or [mount](https://rclone.org/commands/rclone_move/), you will often want an rclone
command always running in the background, and configuring it to run in a service infrastructure
may be a better option. Below are some alternatives on how to achieve this on
different operating systems.
NOTE: Before setting up autorun it is highly recommended that you have tested your command
manually from a Command Prompt first.
## Autostart on Windows ##
The most relevant alternatives for autostart on Windows are:
- Run at user log on using the Startup folder
- Run at user log on, at system startup or at schedule using Task Scheduler
- Run at system startup using Windows service
### Running in background
Rclone is a console application, so if not starting from an existing Command Prompt,
e.g. when starting rclone.exe from a shortcut, it will open a Command Prompt window.
When configuring rclone to run from task scheduler and windows service you are able
to set it to run hidden in background. From rclone version 1.54 you can also make it
run hidden from anywhere by adding option `--no-console` (it may still flash briefly
when the program starts). Since rclone normally writes information and any error
messages to the console, you must redirect this to a file to be able to see it.
Rclone has a built-in option `--log-file` for that.
Example command to run a sync in background:
```
c:\rclone\rclone.exe sync c:\files remote:/files --no-console --log-file c:\rclone\logs\sync_files.txt
```
### User account
As mentioned in the [mount](https://rclone.org/commands/rclone_move/) documentation,
mounted drives created as Administrator are not visible to other accounts, not even the
account that was elevated as Administrator. By running the mount command as the
built-in `SYSTEM` user account, it will create drives accessible for everyone on
the system. Both scheduled task and Windows service can be used to achieve this.
NOTE: Remember that when rclone runs as the `SYSTEM` user, the user profile
that it sees will not be yours. This means that if you normally run rclone with
configuration file in the default location, to be able to use the same configuration
when running as the system user you must explicitely tell rclone where to find
it with the [`--config`](https://rclone.org/docs/#config-config-file) option,
or else it will look in the system users profile path (`C:\Windows\System32\config\systemprofile`).
To test your command manually from a Command Prompt, you can run it with
the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
utility from Microsoft's Sysinternals suite, which takes option `-s` to
execute commands as the `SYSTEM` user.
### Start from Startup folder ###
To quickly execute an rclone command you can simply create a standard
Windows Explorer shortcut for the complete rclone command you want to run. If you
store this shortcut in the special "Startup" start-menu folder, Windows will
automatically run it at login. To open this folder in Windows Explorer,
enter path `%APPDATA%\Microsoft\Windows\Start Menu\Programs\Startup`,
or `C:\ProgramData\Microsoft\Windows\Start Menu\Programs\StartUp` if you want
the command to start for *every* user that logs in.
This is the easiest approach to autostarting of rclone, but it offers no
functionality to set it to run as different user, or to set conditions or
actions on certain events. Setting up a scheduled task as described below
will often give you better results.
### Start from Task Scheduler ###
Task Scheduler is an administrative tool built into Windows, and it can be used to
configure rclone to be started automatically in a highly configurable way, e.g.
periodically on a schedule, on user log on, or at system startup. It can run
be configured to run as the current user, or for a mount command that needs to
be available to all users it can run as the `SYSTEM` user.
For technical information, see
https://docs.microsoft.com/windows/win32/taskschd/task-scheduler-start-page.
### Run as service ###
For running rclone at system startup, you can create a Windows service that executes
your rclone command, as an alternative to scheduled task configured to run at startup.
#### Mount command built-in service integration ####
For mount commands, Rclone has a built-in Windows service integration via the third party
WinFsp library it uses. Registering as a regular Windows service easy, as you just have to
execute the built-in PowerShell command `New-Service` (requires administrative privileges).
Example of a PowerShell command that creates a Windows service for mounting
some `remote:/files` as drive letter `X:`, for *all* users (service will be running as the
local system account):
```
New-Service -Name Rclone -BinaryPathName 'c:\rclone\rclone.exe mount remote:/files X: --config c:\rclone\config\rclone.conf --log-file c:\rclone\logs\mount.txt'
```
The [WinFsp service infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)
supports incorporating services for file system implementations, such as rclone,
into its own launcher service, as kind of "child services". This has the additional
advantage that it also implements a network provider that integrates into
Windows standard methods for managing network drives. This is currently not
officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later
it should be possible through path rewriting as described [here](https://github.com/rclone/rclone/issues/3340).
#### Third party service integration ####
To Windows service running any rclone command, the excellent third party utility
[NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used.
It includes some advanced features such as adjusting process periority, defining
process environment variables, redirect to file anything written to stdout, and
customized response to different exit codes, with a GUI to configure everything from
(although it can also be used from command line ).
There are also several other alternatives. To mention one more,
[WinSW](https://github.com/winsw/winsw), "Windows Service Wrapper", is worth checking out.
It requires .NET Framework, but it is preinstalled on newer versions of Windows, and it
also provides alternative standalone distributions which includes necessary runtime (.NET 5).
WinSW is a command-line only utility, where you have to manually create an XML file with
service configuration. This may be a drawback for some, but it can also be an advantage
as it is easy to back up and re-use the configuration
settings, without having go through manual steps in a GUI. One thing to note is that
by default it does not restart the service on error, one have to explicit enable this
in the configuration file (via the "onfailure" parameter).
## Autostart on Linux
### Start as a service
To always run rclone in background, relevant for mount commands etc,
you can use systemd to set up rclone as a system or user service. Running as a
system service ensures that it is run at startup even if the user it is running as
has no active session. Running rclone as a user service ensures that it only
starts after the configured user has logged into the system.
### Run periodically from cron
To run a periodic command, such as a copy/sync, you can set up a cron job.
+3 -3
View File
@@ -53,9 +53,9 @@ export XDG_CONFIG_HOME=config
#check installed version of rclone to determine if update is necessary
version=$(rclone --version 2>>errors | head -n 1)
if [ -z "$install_beta" ]; then
current_version=$(curl -fsS https://downloads.rclone.org/version.txt)
current_version=$(curl -f https://downloads.rclone.org/version.txt)
else
current_version=$(curl -fsS https://beta.rclone.org/version.txt)
current_version=$(curl -f https://beta.rclone.org/version.txt)
fi
if [ "$version" = "$current_version" ]; then
@@ -123,7 +123,7 @@ else
rclone_zip="rclone-beta-latest-${OS}-${OS_type}.zip"
fi
curl -OfsS "$download_link"
curl -Of "$download_link"
unzip_dir="tmp_unzip_dir_for_rclone"
# there should be an entry in this switch for each element of unzip_tools_list
case "$unzip_tool" in
+1 -1
View File
@@ -172,7 +172,7 @@ like symlinks under Windows).
If you supply `--copy-links` or `-L` then rclone will follow the
symlink and copy the pointed to file or directory. Note that this
flag is incompatible with `--links` / `-l`.
flag is incompatible with `-links` / `-l`.
This flag applies to all commands.
+1 -1
View File
@@ -430,7 +430,7 @@ in it will be mapped to `?` instead.
#### File sizes ####
The largest allowed file size is 250 GiB for both OneDrive Personal and OneDrive for Business [(Updated 13 Jan 2021)](https://support.microsoft.com/en-us/office/invalid-file-names-and-file-types-in-onedrive-and-sharepoint-64883a5d-228e-48f5-b3d2-eb39e07630fa?ui=en-us&rs=en-us&ad=us#individualfilesize).
The largest allowed file size is 250GB for both OneDrive Personal and OneDrive for Business [(Updated 13 Jan 2021)](https://support.microsoft.com/en-us/office/invalid-file-names-and-file-types-in-onedrive-and-sharepoint-64883a5d-228e-48f5-b3d2-eb39e07630fa?ui=en-us&rs=en-us&ad=us#individualfilesize).
#### Path length ####
+1 -3
View File
@@ -48,7 +48,6 @@ Here is an overview of the major features of each cloud storage system.
| SFTP | MD5, SHA1 ² | Yes | Depends | No | - |
| SugarSync | - | No | No | No | - |
| Tardigrade | - | Yes | No | No | - |
| Uptobox | - | No | No | Yes | - |
| WebDAV | MD5, SHA1 ³ | Yes ⁴ | Depends | No | - |
| Yandex Disk | MD5 | Yes | No | No | R |
| Zoho WorkDrive | - | No | No | No | - |
@@ -58,7 +57,7 @@ Here is an overview of the major features of each cloud storage system.
¹ Dropbox supports [its own custom
hash](https://www.dropbox.com/developers/reference/content-hash).
This is an SHA256 sum of all the 4 MiB block SHA256s.
This is an SHA256 sum of all the 4MB block SHA256s.
² SFTP supports checksums if the same login has shell access and
`md5sum` or `sha1sum` as well as `echo` are in the remote's PATH.
@@ -362,7 +361,6 @@ upon backend specific capabilities.
| SFTP | No | No | Yes | Yes | No | No | Yes | No | Yes | Yes |
| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes |
| Tardigrade | Yes † | No | No | No | No | Yes | Yes | No | No | No |
| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No |
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes |
| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes |
| Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | Yes | Yes |
+2 -2
View File
@@ -101,7 +101,7 @@ docs](/docs/#fast-list) for more details.
### Multipart uploads ###
rclone supports multipart uploads with QingStor which means that it can
upload files bigger than 5 GiB. Note that files uploaded with multipart
upload files bigger than 5GB. Note that files uploaded with multipart
upload don't have an MD5SUM.
Note that incomplete multipart uploads older than 24 hours can be
@@ -227,7 +227,7 @@ Number of connection retries.
Cutoff for switching to chunked upload
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5 GiB.
The minimum is 0 and the maximum is 5GB.
- Config: upload_cutoff
- Env Var: RCLONE_QINGSTOR_UPLOAD_CUTOFF
+2 -2
View File
@@ -788,8 +788,8 @@ Returns the following values:
"eta": estimated time in seconds until file transfer completion
"name": name of the file,
"percentage": progress of the file transfer in percent,
"speed": average speed over the whole transfer in bytes per second,
"speedAvg": current speed in bytes per second as an exponentially weighted moving average,
"speed": average speed over the whole transfer in bytes/sec,
"speedAvg": current speed in bytes/sec as an exponentially weighted moving average,
"size": size of the file in bytes
}
],
+7 -7
View File
@@ -327,7 +327,7 @@ objects). See the [rclone docs](/docs/#fast-list) for more details.
`--fast-list` trades off API transactions for memory use. As a rough
guide rclone uses 1k of memory per object stored, so using
`--fast-list` on a sync of a million objects will use roughly 1 GiB of
`--fast-list` on a sync of a million objects will use roughly 1 GB of
RAM.
If you are only copying a small number of files into a big repository
@@ -407,13 +407,13 @@ work with the SDK properly:
### Multipart uploads ###
rclone supports multipart uploads with S3 which means that it can
upload files bigger than 5 GiB.
upload files bigger than 5GB.
Note that files uploaded *both* with multipart upload *and* through
crypt remotes do not have MD5 sums.
rclone switches from single part uploads to multipart uploads at the
point specified by `--s3-upload-cutoff`. This can be a maximum of 5 GiB
point specified by `--s3-upload-cutoff`. This can be a maximum of 5GB
and a minimum of 0 (ie always upload multipart files).
The chunk sizes used in the multipart upload are specified by
@@ -1412,7 +1412,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
Cutoff for switching to chunked upload
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5 GiB.
The minimum is 0 and the maximum is 5GB.
- Config: upload_cutoff
- Env Var: RCLONE_S3_UPLOAD_CUTOFF
@@ -1438,9 +1438,9 @@ Rclone will automatically increase the chunk size when uploading a
large file of known size to stay below the 10,000 chunks limit.
Files of unknown size are uploaded with the configured
chunk_size. Since the default chunk size is 5 MiB and there can be at
chunk_size. Since the default chunk size is 5MB and there can be at
most 10,000 chunks, this means that by default the maximum size of
a file you can stream upload is 48 GiB. If you wish to stream upload
a file you can stream upload is 48GB. If you wish to stream upload
larger files then you will need to increase chunk_size.
- Config: chunk_size
@@ -1474,7 +1474,7 @@ Cutoff for switching to multipart copy
Any files larger than this that need to be server-side copied will be
copied in chunks of this size.
The minimum is 0 and the maximum is 5 GiB.
The minimum is 0 and the maximum is 5GB.
- Config: copy_cutoff
- Env Var: RCLONE_S3_COPY_CUTOFF
+1 -13
View File
@@ -21,10 +21,7 @@ SSH installations.
Paths are specified as `remote:path`. If the path does not begin with
a `/` it is relative to the home directory of the user. An empty path
`remote:` refers to the user's home directory. For example, `rclone lsd remote:`
would list the home directory of the user cofigured in the rclone remote config
(`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root
directory for remote machine (i.e. `/`)
`remote:` refers to the user's home directory.
"Note that some SFTP servers will need the leading / - Synology is a
good example of this. rsync.net, on the other hand, requires users to
@@ -87,10 +84,6 @@ See all directories in the home directory
rclone lsd remote:
See all directories in the root directory
rclone lsd remote:/
Make a new directory
rclone mkdir remote:path/to/directory
@@ -104,11 +97,6 @@ excess files in the directory.
rclone sync -i /home/local/directory remote:directory
Mount the remote path `/srv/www-data/` to the local path
`/mnt/www-data`
rclone mount remote:/srv/www-data/ /mnt/www-data
### SSH Authentication ###
The SFTP remote supports three authentication methods:
Loaded 100 of 160 files, more files were not shown because too many files have changed in this diff. Show more