Compare commits
78
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1dc58a924c | ||
|
|
34627c5c7e | ||
|
|
e33303df94 | ||
|
|
665eceaec3 | ||
|
|
ba09ee18bb | ||
|
|
62bf63d36f | ||
|
|
f38c262471 | ||
|
|
5db88fed2b | ||
|
|
316e65589b | ||
|
|
4401d180aa | ||
|
|
9ccd870267 | ||
|
|
16d1da2c1e | ||
|
|
00a0ee1899 | ||
|
|
b78c9a65fa | ||
|
|
ef3c350686 | ||
|
|
742af80972 | ||
|
|
08a2df51be | ||
|
|
2925e1384c | ||
|
|
2ec0c8d45f | ||
|
|
98579608ec | ||
|
|
a1a41aa0c1 | ||
|
|
f8d56bebaf | ||
|
|
5d799431a7 | ||
|
|
8f23cae1c0 | ||
|
|
964088affa | ||
|
|
f4068d406b | ||
|
|
7511b6f4f1 | ||
|
|
e618ea83dd | ||
|
|
34dc257c55 | ||
|
|
4cacf5d30c | ||
|
|
0537791d14 | ||
|
|
4b1d28550a | ||
|
|
d27c35ee4a | ||
|
|
ffec0d4f03 | ||
|
|
89daa9efd1 | ||
|
|
ee502a757f | ||
|
|
386acaa110 | ||
|
|
efdee3a5fe | ||
|
|
5d85e6bc9c | ||
|
|
4a9469a3dc | ||
|
|
f8884a7200 | ||
|
|
2a40f00077 | ||
|
|
9799fdbae2 | ||
|
|
492504a601 | ||
|
|
0c03a7fead | ||
|
|
7afb4487ef | ||
|
|
b9d0ed4f5c | ||
|
|
baa4c039a0 | ||
|
|
31a8211afa | ||
|
|
3544e09e95 | ||
|
|
b456be4303 | ||
|
|
3e96752079 | ||
|
|
4a5cbf2a19 | ||
|
|
dcd4edc9f5 | ||
|
|
7f5e347d94 | ||
|
|
040677ab5b | ||
|
|
6366d3dfc5 | ||
|
|
60d376c323 | ||
|
|
7b1ca716bf | ||
|
|
d8711cf7f9 | ||
|
|
cd69f9e6e8 | ||
|
|
a737ff21af | ||
|
|
ad9aa693a3 | ||
|
|
964c3e0732 | ||
|
|
a46a3c0811 | ||
|
|
60dcafe04d | ||
|
|
813bf029d4 | ||
|
|
f2d3264054 | ||
|
|
23a0d4a1e6 | ||
|
|
b96ebfc40b | ||
|
|
3fe2aaf96c | ||
|
|
c163e6b250 | ||
|
|
c1492cfa28 | ||
|
|
38a8071a58 | ||
|
|
8c68a76a4a | ||
|
|
e7b736f8ca | ||
|
|
cb30a8c80e | ||
|
|
629a3eeca2 |
No files matched your search
@@ -37,7 +37,6 @@ The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
@@ -65,3 +64,11 @@ The Rclone Developers
|
||||
#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your bug report. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -26,7 +26,6 @@ The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
@@ -42,3 +41,11 @@ The Rclone Developers
|
||||
#### How do you think rclone should be changed to solve that?
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your feature request. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -12,9 +12,15 @@ on:
|
||||
tags:
|
||||
- '*'
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
manual:
|
||||
required: true
|
||||
default: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -30,6 +36,7 @@ jobs:
|
||||
check: true
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
librclonetest: true
|
||||
deploy: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
@@ -187,6 +194,14 @@ jobs:
|
||||
make racequicktest
|
||||
if: matrix.racequicktest
|
||||
|
||||
- name: Run librclone tests
|
||||
shell: bash
|
||||
run: |
|
||||
make -C librclone/ctest test
|
||||
make -C librclone/ctest clean
|
||||
librclone/python/test_rclone.py
|
||||
if: matrix.librclonetest
|
||||
|
||||
- name: Code quality test
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -214,6 +229,7 @@ jobs:
|
||||
if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
android:
|
||||
if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
|
||||
timeout-minutes: 30
|
||||
name: "android-all"
|
||||
runs-on: ubuntu-latest
|
||||
@@ -221,6 +237,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Upgrade together with NDK version
|
||||
- name: Set up Go 1.14
|
||||
|
||||
@@ -7,6 +7,7 @@ on:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'rclone/rclone'
|
||||
runs-on: ubuntu-latest
|
||||
name: Build image job
|
||||
steps:
|
||||
|
||||
@@ -6,6 +6,7 @@ on:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'rclone/rclone'
|
||||
runs-on: ubuntu-latest
|
||||
name: Build image job
|
||||
steps:
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
)
|
||||
|
||||
func prepare(t *testing.T, root string) {
|
||||
configfile.LoadConfig(context.Background())
|
||||
configfile.Install()
|
||||
|
||||
// Configure the remote
|
||||
config.FileSet(remoteName, "type", "alias")
|
||||
|
||||
@@ -41,6 +41,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/swift"
|
||||
_ "github.com/rclone/rclone/backend/tardigrade"
|
||||
_ "github.com/rclone/rclone/backend/union"
|
||||
_ "github.com/rclone/rclone/backend/uptobox"
|
||||
_ "github.com/rclone/rclone/backend/webdav"
|
||||
_ "github.com/rclone/rclone/backend/yandex"
|
||||
_ "github.com/rclone/rclone/backend/zoho"
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -70,11 +69,12 @@ func init() {
|
||||
Prefix: "acd",
|
||||
Description: "Amazon Drive",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
err := oauthutil.Config(ctx, "amazon cloud drive", name, m, acdConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "checkpoint",
|
||||
@@ -83,16 +83,16 @@ func init() {
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_wait_per_gb",
|
||||
Help: `Additional time per GB to wait after a failed complete upload to see if it appears.
|
||||
Help: `Additional time per GiB to wait after a failed complete upload to see if it appears.
|
||||
|
||||
Sometimes Amazon Drive gives an error when a file has been fully
|
||||
uploaded but the file appears anyway after a little while. This
|
||||
happens sometimes for files over 1GB in size and nearly every time for
|
||||
files bigger than 10GB. This parameter controls the time rclone waits
|
||||
happens sometimes for files over 1 GiB in size and nearly every time for
|
||||
files bigger than 10 GiB. This parameter controls the time rclone waits
|
||||
for the file to appear.
|
||||
|
||||
The default value for this parameter is 3 minutes per GB, so by
|
||||
default it will wait 3 minutes for every GB uploaded to see if the
|
||||
The default value for this parameter is 3 minutes per GiB, so by
|
||||
default it will wait 3 minutes for every GiB uploaded to see if the
|
||||
file appears.
|
||||
|
||||
You can disable this feature by setting it to 0. This may cause
|
||||
@@ -112,7 +112,7 @@ in this situation.`,
|
||||
|
||||
Files this size or more will be downloaded via their "tempLink". This
|
||||
is to work around a problem with Amazon Drive which blocks downloads
|
||||
of files bigger than about 10GB. The default for this is 9GB which
|
||||
of files bigger than about 10 GiB. The default for this is 9 GiB which
|
||||
shouldn't need to be changed.
|
||||
|
||||
To download files above this threshold, rclone requests a "tempLink"
|
||||
|
||||
@@ -47,8 +47,8 @@ const (
|
||||
timeFormatIn = time.RFC3339
|
||||
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
storageDefaultBaseURL = "blob.core.windows.net"
|
||||
defaultChunkSize = 4 * fs.MebiByte
|
||||
maxChunkSize = 100 * fs.MebiByte
|
||||
defaultChunkSize = 4 * fs.Mebi
|
||||
maxChunkSize = 100 * fs.Mebi
|
||||
uploadConcurrency = 4
|
||||
defaultAccessTier = azblob.AccessTierNone
|
||||
maxTryTimeout = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
|
||||
@@ -129,11 +129,11 @@ msi_client_id, or msi_mi_res_id parameters.`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: "Cutoff for switching to chunked upload (<= 256MB). (Deprecated)",
|
||||
Help: "Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Upload chunk size (<= 100MB).
|
||||
Help: `Upload chunk size (<= 100 MiB).
|
||||
|
||||
Note that this is stored in memory and there may be up to
|
||||
"--transfers" chunks stored at once in memory.`,
|
||||
@@ -404,7 +404,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
const minChunkSize = fs.Byte
|
||||
const minChunkSize = fs.SizeSuffixBase
|
||||
if cs < minChunkSize {
|
||||
return errors.Errorf("%s is less than %s", cs, minChunkSize)
|
||||
}
|
||||
|
||||
+12
-27
@@ -2,12 +2,11 @@ package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/version"
|
||||
)
|
||||
|
||||
// Error describes a B2 error response
|
||||
@@ -63,16 +62,17 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
const versionFormat = "-v2006-01-02-150405.000"
|
||||
// HasVersion returns true if it looks like the passed filename has a timestamp on it.
|
||||
//
|
||||
// Note that the passed filename's timestamp may still be invalid even if this
|
||||
// function returns true.
|
||||
func HasVersion(remote string) bool {
|
||||
return version.Match(remote)
|
||||
}
|
||||
|
||||
// AddVersion adds the timestamp as a version string into the filename passed in.
|
||||
func (t Timestamp) AddVersion(remote string) string {
|
||||
ext := path.Ext(remote)
|
||||
base := remote[:len(remote)-len(ext)]
|
||||
s := time.Time(t).Format(versionFormat)
|
||||
// Replace the '.' with a '-'
|
||||
s = strings.Replace(s, ".", "-", -1)
|
||||
return base + s + ext
|
||||
return version.Add(remote, time.Time(t))
|
||||
}
|
||||
|
||||
// RemoveVersion removes the timestamp from a filename as a version string.
|
||||
@@ -80,24 +80,9 @@ func (t Timestamp) AddVersion(remote string) string {
|
||||
// It returns the new file name and a timestamp, or the old filename
|
||||
// and a zero timestamp.
|
||||
func RemoveVersion(remote string) (t Timestamp, newRemote string) {
|
||||
newRemote = remote
|
||||
ext := path.Ext(remote)
|
||||
base := remote[:len(remote)-len(ext)]
|
||||
if len(base) < len(versionFormat) {
|
||||
return
|
||||
}
|
||||
versionStart := len(base) - len(versionFormat)
|
||||
// Check it ends in -xxx
|
||||
if base[len(base)-4] != '-' {
|
||||
return
|
||||
}
|
||||
// Replace with .xxx for parsing
|
||||
base = base[:len(base)-4] + "." + base[len(base)-3:]
|
||||
newT, err := time.Parse(versionFormat, base[versionStart:])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return Timestamp(newT), base[:versionStart] + ext
|
||||
time, newRemote := version.Remove(remote)
|
||||
t = Timestamp(time)
|
||||
return
|
||||
}
|
||||
|
||||
// IsZero returns true if the timestamp is uninitialized
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
var (
|
||||
emptyT api.Timestamp
|
||||
t0 = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123456789Z"))
|
||||
t0r = api.Timestamp(fstest.Time("1970-01-01T01:01:01.123000000Z"))
|
||||
t1 = api.Timestamp(fstest.Time("2001-02-03T04:05:06.123000000Z"))
|
||||
)
|
||||
|
||||
@@ -36,40 +35,6 @@ func TestTimestampUnmarshalJSON(t *testing.T) {
|
||||
assert.Equal(t, (time.Time)(t1), (time.Time)(tActual))
|
||||
}
|
||||
|
||||
func TestTimestampAddVersion(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
t api.Timestamp
|
||||
in string
|
||||
expected string
|
||||
}{
|
||||
{t0, "potato.txt", "potato-v1970-01-01-010101-123.txt"},
|
||||
{t1, "potato", "potato-v2001-02-03-040506-123"},
|
||||
{t1, "", "-v2001-02-03-040506-123"},
|
||||
} {
|
||||
actual := test.t.AddVersion(test.in)
|
||||
assert.Equal(t, test.expected, actual, test.in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimestampRemoveVersion(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
expectedT api.Timestamp
|
||||
expectedRemote string
|
||||
}{
|
||||
{"potato.txt", emptyT, "potato.txt"},
|
||||
{"potato-v1970-01-01-010101-123.txt", t0r, "potato.txt"},
|
||||
{"potato-v2001-02-03-040506-123", t1, "potato"},
|
||||
{"-v2001-02-03-040506-123", t1, ""},
|
||||
{"potato-v2A01-02-03-040506-123", emptyT, "potato-v2A01-02-03-040506-123"},
|
||||
{"potato-v2001-02-03-040506=123", emptyT, "potato-v2001-02-03-040506=123"},
|
||||
} {
|
||||
actualT, actualRemote := api.RemoveVersion(test.in)
|
||||
assert.Equal(t, test.expectedT, actualT, test.in)
|
||||
assert.Equal(t, test.expectedRemote, actualRemote, test.in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimestampIsZero(t *testing.T) {
|
||||
assert.True(t, emptyT.IsZero())
|
||||
assert.False(t, t0.IsZero())
|
||||
|
||||
+7
-7
@@ -54,10 +54,10 @@ const (
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
maxParts = 10000
|
||||
maxVersions = 100 // maximum number of versions we search in --b2-versions mode
|
||||
minChunkSize = 5 * fs.MebiByte
|
||||
defaultChunkSize = 96 * fs.MebiByte
|
||||
defaultUploadCutoff = 200 * fs.MebiByte
|
||||
largeFileCopyCutoff = 4 * fs.GibiByte // 5E9 is the max
|
||||
minChunkSize = 5 * fs.Mebi
|
||||
defaultChunkSize = 96 * fs.Mebi
|
||||
defaultUploadCutoff = 200 * fs.Mebi
|
||||
largeFileCopyCutoff = 4 * fs.Gibi // 5E9 is the max
|
||||
memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
|
||||
memoryPoolUseMmap = false
|
||||
)
|
||||
@@ -116,7 +116,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
|
||||
|
||||
Files above this size will be uploaded in chunks of "--b2-chunk-size".
|
||||
|
||||
This value should be set no larger than 4.657GiB (== 5GB).`,
|
||||
This value should be set no larger than 4.657 GiB (== 5 GB).`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -126,7 +126,7 @@ This value should be set no larger than 4.657GiB (== 5GB).`,
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 4.6GB.`,
|
||||
The minimum is 0 and the maximum is 4.6 GiB.`,
|
||||
Default: largeFileCopyCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -1353,7 +1353,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
|
||||
}
|
||||
var request = api.GetDownloadAuthorizationRequest{
|
||||
BucketID: bucketID,
|
||||
FileNamePrefix: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
|
||||
FileNamePrefix: f.opt.Enc.FromStandardPath(path.Join(f.rootDirectory, remote)),
|
||||
ValidDurationInSeconds: validDurationInSeconds,
|
||||
}
|
||||
var response api.GetDownloadAuthorizationResponse
|
||||
|
||||
@@ -230,14 +230,14 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
|
||||
//
|
||||
// The number of bytes in the file being uploaded. Note that
|
||||
// this header is required; you cannot leave it out and just
|
||||
// use chunked encoding. The minimum size of every part but
|
||||
// the last one is 100MB.
|
||||
// use chunked encoding. The minimum size of every part but
|
||||
// the last one is 100 MB (100,000,000 bytes)
|
||||
//
|
||||
// X-Bz-Content-Sha1
|
||||
//
|
||||
// The SHA1 checksum of the this part of the file. B2 will
|
||||
// check this when the part is uploaded, to make sure that the
|
||||
// data arrived correctly. The same SHA1 checksum must be
|
||||
// data arrived correctly. The same SHA1 checksum must be
|
||||
// passed to b2_finish_large_file.
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
|
||||
@@ -36,13 +36,13 @@ func (t *Time) UnmarshalJSON(data []byte) error {
|
||||
|
||||
// Error is returned from box when things go wrong
|
||||
type Error struct {
|
||||
Type string `json:"type"`
|
||||
Status int `json:"status"`
|
||||
Code string `json:"code"`
|
||||
ContextInfo json.RawMessage
|
||||
HelpURL string `json:"help_url"`
|
||||
Message string `json:"message"`
|
||||
RequestID string `json:"request_id"`
|
||||
Type string `json:"type"`
|
||||
Status int `json:"status"`
|
||||
Code string `json:"code"`
|
||||
ContextInfo json.RawMessage `json:"context_info"`
|
||||
HelpURL string `json:"help_url"`
|
||||
Message string `json:"message"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
// Error returns a string for the error and satisfies the error interface
|
||||
@@ -132,6 +132,38 @@ type UploadFile struct {
|
||||
ContentModifiedAt Time `json:"content_modified_at"`
|
||||
}
|
||||
|
||||
// PreUploadCheck is the request for upload preflight check
|
||||
type PreUploadCheck struct {
|
||||
Name string `json:"name"`
|
||||
Parent Parent `json:"parent"`
|
||||
Size *int64 `json:"size,omitempty"`
|
||||
}
|
||||
|
||||
// PreUploadCheckResponse is the response from upload preflight check
|
||||
// if successful
|
||||
type PreUploadCheckResponse struct {
|
||||
UploadToken string `json:"upload_token"`
|
||||
UploadURL string `json:"upload_url"`
|
||||
}
|
||||
|
||||
// PreUploadCheckConflict is returned in the ContextInfo error field
|
||||
// from PreUploadCheck when the error code is "item_name_in_use"
|
||||
type PreUploadCheckConflict struct {
|
||||
Conflicts struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
FileVersion struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Sha1 string `json:"sha1"`
|
||||
} `json:"file_version"`
|
||||
SequenceID string `json:"sequence_id"`
|
||||
Etag string `json:"etag"`
|
||||
Sha1 string `json:"sha1"`
|
||||
Name string `json:"name"`
|
||||
} `json:"conflicts"`
|
||||
}
|
||||
|
||||
// UpdateFileModTime is used in Update File Info
|
||||
type UpdateFileModTime struct {
|
||||
ContentModifiedAt Time `json:"content_modified_at"`
|
||||
|
||||
+75
-17
@@ -17,7 +17,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -84,7 +83,7 @@ func init() {
|
||||
Name: "box",
|
||||
Description: "Box",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
jsonFile, ok := m.Get("box_config_file")
|
||||
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
|
||||
boxAccessToken, boxAccessTokenOk := m.Get("access_token")
|
||||
@@ -93,15 +92,16 @@ func init() {
|
||||
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
|
||||
err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token with jwt authentication: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token with jwt authentication")
|
||||
}
|
||||
// Else, if not using an access token, use oauth2
|
||||
} else if boxAccessToken == "" || !boxAccessTokenOk {
|
||||
err = oauthutil.Config(ctx, "box", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token with oauth authentication: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token with oauth authentication")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "root_folder_id",
|
||||
@@ -126,7 +126,7 @@ func init() {
|
||||
}},
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: "Cutoff for switching to multipart upload (>= 50MB).",
|
||||
Help: "Cutoff for switching to multipart upload (>= 50 MiB).",
|
||||
Default: fs.SizeSuffix(defaultUploadCutoff),
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -157,15 +157,15 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
|
||||
jsonFile = env.ShellExpand(jsonFile)
|
||||
boxConfig, err := getBoxConfig(jsonFile)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "get box config")
|
||||
}
|
||||
privateKey, err := getDecryptedPrivateKey(boxConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "get decrypted private key")
|
||||
}
|
||||
claims, err := getClaims(boxConfig, boxSubType)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "get claims")
|
||||
}
|
||||
signingHeaders := getSigningHeaders(boxConfig)
|
||||
queryParams := getQueryParams(boxConfig)
|
||||
@@ -686,22 +686,80 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
|
||||
return o, leaf, directoryID, nil
|
||||
}
|
||||
|
||||
// preUploadCheck checks to see if a file can be uploaded
|
||||
//
|
||||
// It returns "", nil if the file is good to go
|
||||
// It returns "ID", nil if the file must be updated
|
||||
func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (ID string, err error) {
|
||||
check := api.PreUploadCheck{
|
||||
Name: f.opt.Enc.FromStandardName(leaf),
|
||||
Parent: api.Parent{
|
||||
ID: directoryID,
|
||||
},
|
||||
}
|
||||
if size >= 0 {
|
||||
check.Size = &size
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "OPTIONS",
|
||||
Path: "/files/content/",
|
||||
}
|
||||
var result api.PreUploadCheckResponse
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &check, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*api.Error); ok && apiErr.Code == "item_name_in_use" {
|
||||
var conflict api.PreUploadCheckConflict
|
||||
err = json.Unmarshal(apiErr.ContextInfo, &conflict)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "pre-upload check: JSON decode failed")
|
||||
}
|
||||
if conflict.Conflicts.Type != api.ItemTypeFile {
|
||||
return "", errors.Wrap(err, "pre-upload check: can't overwrite non file with file")
|
||||
}
|
||||
return conflict.Conflicts.ID, nil
|
||||
}
|
||||
return "", errors.Wrap(err, "pre-upload check")
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Put the object
|
||||
//
|
||||
// Copy the reader in to the new object which is returned
|
||||
//
|
||||
// The new object may have been created if an error is returned
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
|
||||
switch err {
|
||||
case nil:
|
||||
return existingObj, existingObj.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
// Not found so create it
|
||||
return f.PutUnchecked(ctx, in, src)
|
||||
default:
|
||||
// If directory doesn't exist, file doesn't exist so can upload
|
||||
remote := src.Remote()
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return f.PutUnchecked(ctx, in, src, options...)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Preflight check the upload, which returns the ID if the
|
||||
// object already exists
|
||||
ID, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ID == "" {
|
||||
return f.PutUnchecked(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// If object exists then create a skeleton one with just id
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
id: ID,
|
||||
}
|
||||
return o, o.Update(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
||||
@@ -1228,7 +1286,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
|
||||
// upload does a single non-multipart upload
|
||||
//
|
||||
// This is recommended for less than 50 MB of content
|
||||
// This is recommended for less than 50 MiB of content
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
upload := api.UploadFile{
|
||||
Name: o.fs.opt.Enc.FromStandardName(leaf),
|
||||
|
||||
Vendored
+7
-7
@@ -98,14 +98,14 @@ changed, any downloaded chunks will be invalid and cache-chunk-path
|
||||
will need to be cleared or unexpected EOF errors will occur.`,
|
||||
Default: DefCacheChunkSize,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "1m",
|
||||
Help: "1MB",
|
||||
Value: "1M",
|
||||
Help: "1 MiB",
|
||||
}, {
|
||||
Value: "5M",
|
||||
Help: "5 MB",
|
||||
Help: "5 MiB",
|
||||
}, {
|
||||
Value: "10M",
|
||||
Help: "10 MB",
|
||||
Help: "10 MiB",
|
||||
}},
|
||||
}, {
|
||||
Name: "info_age",
|
||||
@@ -132,13 +132,13 @@ oldest chunks until it goes under this value.`,
|
||||
Default: DefCacheTotalChunkSize,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "500M",
|
||||
Help: "500 MB",
|
||||
Help: "500 MiB",
|
||||
}, {
|
||||
Value: "1G",
|
||||
Help: "1 GB",
|
||||
Help: "1 GiB",
|
||||
}, {
|
||||
Value: "10G",
|
||||
Help: "10 GB",
|
||||
Help: "10 GiB",
|
||||
}},
|
||||
}, {
|
||||
Name: "db_path",
|
||||
|
||||
Vendored
+1
-1
@@ -836,7 +836,7 @@ func newRun() *run {
|
||||
if uploadDir == "" {
|
||||
r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create temp dir: %v", err)
|
||||
panic(fmt.Sprintf("Failed to create temp dir: %v", err))
|
||||
}
|
||||
} else {
|
||||
r.tmpUploadDir = uploadDir
|
||||
|
||||
@@ -155,7 +155,7 @@ Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Advanced: false,
|
||||
Default: fs.SizeSuffix(2147483648), // 2GB
|
||||
Default: fs.SizeSuffix(2147483648), // 2 GiB
|
||||
Help: `Files larger than chunk size will be split in chunks.`,
|
||||
}, {
|
||||
Name: "name_format",
|
||||
@@ -1448,7 +1448,7 @@ func (c *chunkingReader) dummyRead(in io.Reader, size int64) error {
|
||||
c.accountBytes(size)
|
||||
return nil
|
||||
}
|
||||
const bufLen = 1048576 // 1MB
|
||||
const bufLen = 1048576 // 1 MiB
|
||||
buf := make([]byte, bufLen)
|
||||
for size > 0 {
|
||||
n := size
|
||||
|
||||
@@ -33,7 +33,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
|
||||
fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
|
||||
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
|
||||
Path: fmt.Sprintf("chunker-upload-%dk", kilobytes),
|
||||
Size: int64(kilobytes) * int64(fs.KibiByte),
|
||||
Size: int64(kilobytes) * int64(fs.Kibi),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
// Globals
|
||||
const (
|
||||
initialChunkSize = 262144 // Initial and max sizes of chunks when reading parts of the file. Currently
|
||||
maxChunkSize = 8388608 // at 256KB and 8 MB.
|
||||
maxChunkSize = 8388608 // at 256 KiB and 8 MiB.
|
||||
|
||||
bufferSize = 8388608
|
||||
heuristicBytes = 1048576
|
||||
@@ -53,7 +53,7 @@ const (
|
||||
Gzip = 2
|
||||
)
|
||||
|
||||
var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9+_]{11})$")
|
||||
var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9-_]{11})$")
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
|
||||
+55
-3
@@ -12,12 +12,14 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/backend/crypt/pkcs7"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/version"
|
||||
"github.com/rfjakob/eme"
|
||||
"golang.org/x/crypto/nacl/secretbox"
|
||||
"golang.org/x/crypto/scrypt"
|
||||
@@ -442,11 +444,32 @@ func (c *Cipher) encryptFileName(in string) string {
|
||||
if !c.dirNameEncrypt && i != (len(segments)-1) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Strip version string so that only the non-versioned part
|
||||
// of the file name gets encrypted/obfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
if s != segments[i] {
|
||||
segments[i] = s
|
||||
hasVersion = true
|
||||
}
|
||||
}
|
||||
|
||||
if c.mode == NameEncryptionStandard {
|
||||
segments[i] = c.encryptSegment(segments[i])
|
||||
} else {
|
||||
segments[i] = c.obfuscateSegment(segments[i])
|
||||
}
|
||||
|
||||
// Add back a version to the encrypted/obfuscated
|
||||
// file name, if we stripped it off earlier
|
||||
if hasVersion {
|
||||
segments[i] = version.Add(segments[i], t)
|
||||
}
|
||||
}
|
||||
return strings.Join(segments, "/")
|
||||
}
|
||||
@@ -477,6 +500,21 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
if !c.dirNameEncrypt && i != (len(segments)-1) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Strip version string so that only the non-versioned part
|
||||
// of the file name gets decrypted/deobfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
if s != segments[i] {
|
||||
segments[i] = s
|
||||
hasVersion = true
|
||||
}
|
||||
}
|
||||
|
||||
if c.mode == NameEncryptionStandard {
|
||||
segments[i], err = c.decryptSegment(segments[i])
|
||||
} else {
|
||||
@@ -486,6 +524,12 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Add back a version to the decrypted/deobfuscated
|
||||
// file name, if we stripped it off earlier
|
||||
if hasVersion {
|
||||
segments[i] = version.Add(segments[i], t)
|
||||
}
|
||||
}
|
||||
return strings.Join(segments, "/"), nil
|
||||
}
|
||||
@@ -494,10 +538,18 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
func (c *Cipher) DecryptFileName(in string) (string, error) {
|
||||
if c.mode == NameEncryptionOff {
|
||||
remainingLength := len(in) - len(encryptedSuffix)
|
||||
if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
|
||||
return in[:remainingLength], nil
|
||||
if remainingLength == 0 || !strings.HasSuffix(in, encryptedSuffix) {
|
||||
return "", ErrorNotAnEncryptedFile
|
||||
}
|
||||
return "", ErrorNotAnEncryptedFile
|
||||
decrypted := in[:remainingLength]
|
||||
if version.Match(decrypted) {
|
||||
_, unversioned := version.Remove(decrypted)
|
||||
if unversioned == "" {
|
||||
return "", ErrorNotAnEncryptedFile
|
||||
}
|
||||
}
|
||||
// Leave the version string on, if it was there
|
||||
return decrypted, nil
|
||||
}
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
@@ -160,22 +160,29 @@ func TestEncryptFileName(t *testing.T) {
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
|
||||
// Standard mode with directory name encryption off
|
||||
c, _ = newCipher(NameEncryptionStandard, "", "", false)
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
|
||||
assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
|
||||
assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
|
||||
assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
|
||||
assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
|
||||
// Now off mode
|
||||
c, _ = newCipher(NameEncryptionOff, "", "", true)
|
||||
assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
|
||||
// Obfuscation mode
|
||||
c, _ = newCipher(NameEncryptionObfuscated, "", "", true)
|
||||
assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
|
||||
assert.Equal(t, "49.6/99.23/150.890/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
|
||||
assert.Equal(t, "49.6/99.23/150.890/162.uryyB-v2001-02-03-040506-123.GKG", c.EncryptFileName("1/12/123/hello-v2001-02-03-040506-123.txt"))
|
||||
assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
|
||||
assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
|
||||
// Obfuscation mode with directory name encryption off
|
||||
c, _ = newCipher(NameEncryptionObfuscated, "", "", false)
|
||||
assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
|
||||
assert.Equal(t, "1/12/123/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
|
||||
assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
|
||||
assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
|
||||
}
|
||||
@@ -194,14 +201,19 @@ func TestDecryptFileName(t *testing.T) {
|
||||
{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
|
||||
{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
|
||||
{NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
|
||||
{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", "1-v2001-02-03-040506-123", nil},
|
||||
{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
|
||||
{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
|
||||
{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
|
||||
{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
|
||||
{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
|
||||
{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
|
||||
{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
|
||||
{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
|
||||
{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
|
||||
{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
|
||||
{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
|
||||
{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
|
||||
} {
|
||||
c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
|
||||
actual, actualErr := c.DecryptFileName(test.in)
|
||||
|
||||
+11
-12
@@ -14,7 +14,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
"path"
|
||||
@@ -68,8 +67,8 @@ const (
|
||||
defaultScope = "drive"
|
||||
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
|
||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||
minChunkSize = 256 * fs.KibiByte
|
||||
defaultChunkSize = 8 * fs.MebiByte
|
||||
minChunkSize = 256 * fs.Kibi
|
||||
defaultChunkSize = 8 * fs.Mebi
|
||||
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
|
||||
listRGrouping = 50 // number of IDs to search at once when using ListR
|
||||
listRInputBuffer = 1000 // size of input buffer when using ListR
|
||||
@@ -183,13 +182,12 @@ func init() {
|
||||
Description: "Google Drive",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
|
||||
return
|
||||
return errors.Wrap(err, "couldn't parse config into struct")
|
||||
}
|
||||
|
||||
// Fill in the scopes
|
||||
@@ -199,16 +197,17 @@ func init() {
|
||||
m.Set("root_folder_id", "appDataFolder")
|
||||
}
|
||||
|
||||
if opt.ServiceAccountFile == "" {
|
||||
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" {
|
||||
err = oauthutil.Config(ctx, "drive", name, m, driveConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
}
|
||||
err = configTeamDrive(ctx, opt, m, name)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure Shared Drive: %v", err)
|
||||
return errors.Wrap(err, "failed to configure Shared Drive")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(driveOAuthOptions(), []fs.Option{{
|
||||
Name: "scope",
|
||||
@@ -467,7 +466,7 @@ See: https://github.com/rclone/rclone/issues/3631
|
||||
Default: false,
|
||||
Help: `Make upload limit errors be fatal
|
||||
|
||||
At the time of writing it is only possible to upload 750GB of data to
|
||||
At the time of writing it is only possible to upload 750 GiB of data to
|
||||
Google Drive a day (this is an undocumented limit). When this limit is
|
||||
reached Google Drive produces a slightly different error message. When
|
||||
this flag is set it causes these errors to be fatal. These will stop
|
||||
@@ -484,7 +483,7 @@ See: https://github.com/rclone/rclone/issues/3857
|
||||
Default: false,
|
||||
Help: `Make download limit errors be fatal
|
||||
|
||||
At the time of writing it is only possible to download 10TB of data from
|
||||
At the time of writing it is only possible to download 10 TiB of data from
|
||||
Google Drive a day (this is an undocumented limit). When this limit is
|
||||
reached Google Drive produces a slightly different error message. When
|
||||
this flag is set it causes these errors to be fatal. These will stop
|
||||
@@ -522,7 +521,7 @@ If this flag is set then rclone will ignore shortcut files completely.
|
||||
} {
|
||||
for mimeType, extension := range m {
|
||||
if err := mime.AddExtensionType(extension, mimeType); err != nil {
|
||||
log.Fatalf("Failed to register MIME type %q: %v", mimeType, err)
|
||||
fs.Errorf("Failed to register MIME type %q: %v", mimeType, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-16
@@ -25,7 +25,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -65,9 +64,9 @@ const (
|
||||
// Upload chunk size - setting too small makes uploads slow.
|
||||
// Chunks are buffered into memory for retries.
|
||||
//
|
||||
// Speed vs chunk size uploading a 1 GB file on 2017-11-22
|
||||
// Speed vs chunk size uploading a 1 GiB file on 2017-11-22
|
||||
//
|
||||
// Chunk Size MB, Speed Mbyte/s, % of max
|
||||
// Chunk Size MiB, Speed MiByte/s, % of max
|
||||
// 1 1.364 11%
|
||||
// 2 2.443 19%
|
||||
// 4 4.288 33%
|
||||
@@ -82,11 +81,11 @@ const (
|
||||
// 96 12.302 95%
|
||||
// 128 12.945 100%
|
||||
//
|
||||
// Choose 48MB which is 91% of Maximum speed. rclone by
|
||||
// default does 4 transfers so this should use 4*48MB = 192MB
|
||||
// Choose 48 MiB which is 91% of Maximum speed. rclone by
|
||||
// default does 4 transfers so this should use 4*48 MiB = 192 MiB
|
||||
// by default.
|
||||
defaultChunkSize = 48 * fs.MebiByte
|
||||
maxChunkSize = 150 * fs.MebiByte
|
||||
defaultChunkSize = 48 * fs.Mebi
|
||||
maxChunkSize = 150 * fs.Mebi
|
||||
// Max length of filename parts: https://help.dropbox.com/installs-integrations/sync-uploads/files-not-syncing
|
||||
maxFileNameLength = 255
|
||||
)
|
||||
@@ -99,8 +98,10 @@ var (
|
||||
"files.content.write",
|
||||
"files.content.read",
|
||||
"sharing.write",
|
||||
"account_info.read", // needed for About
|
||||
// "file_requests.write",
|
||||
// "members.read", // needed for impersonate - but causes app to need to be approved by Dropbox Team Admin during the flow
|
||||
// "team_data.member"
|
||||
},
|
||||
// Endpoint: oauth2.Endpoint{
|
||||
// AuthURL: "https://www.dropbox.com/1/oauth2/authorize",
|
||||
@@ -130,8 +131,8 @@ func getOauthConfig(m configmap.Mapper) *oauth2.Config {
|
||||
}
|
||||
// Make a copy of the config
|
||||
config := *dropboxConfig
|
||||
// Make a copy of the scopes with "members.read" appended
|
||||
config.Scopes = append(config.Scopes, "members.read")
|
||||
// Make a copy of the scopes with extra scopes requires appended
|
||||
config.Scopes = append(config.Scopes, "members.read", "team_data.member")
|
||||
return &config
|
||||
}
|
||||
|
||||
@@ -142,7 +143,7 @@ func init() {
|
||||
Name: "dropbox",
|
||||
Description: "Dropbox",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
opt := oauthutil.Options{
|
||||
NoOffline: true,
|
||||
OAuth2Opts: []oauth2.AuthCodeOption{
|
||||
@@ -151,8 +152,9 @@ func init() {
|
||||
}
|
||||
err := oauthutil.Config(ctx, "dropbox", name, m, getOauthConfig(m), &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "chunk_size",
|
||||
@@ -162,7 +164,7 @@ Any files larger than this will be uploaded in chunks of this size.
|
||||
|
||||
Note that chunks are buffered in memory (one at a time) so rclone can
|
||||
deal with retries. Setting this larger will increase the speed
|
||||
slightly (at most 10%% for 128MB in tests) at the cost of using more
|
||||
slightly (at most 10%% for 128 MiB in tests) at the cost of using more
|
||||
memory. It can be set smaller if you are tight on memory.`, maxChunkSize),
|
||||
Default: defaultChunkSize,
|
||||
Advanced: true,
|
||||
@@ -323,7 +325,7 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
const minChunkSize = fs.Byte
|
||||
const minChunkSize = fs.SizeSuffixBase
|
||||
if cs < minChunkSize {
|
||||
return errors.Errorf("%s is less than %s", cs, minChunkSize)
|
||||
}
|
||||
@@ -1351,13 +1353,13 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
switch info := entry.(type) {
|
||||
case *files.FolderMetadata:
|
||||
entryType = fs.EntryDirectory
|
||||
entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
case *files.FileMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
case *files.DeletedMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = strings.TrimLeft(info.PathDisplay, f.slashRootSlash)
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
default:
|
||||
fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
|
||||
continue
|
||||
|
||||
@@ -35,9 +35,7 @@ func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "fichier",
|
||||
Description: "1Fichier",
|
||||
Config: func(ctx context.Context, name string, config configmap.Mapper) {
|
||||
},
|
||||
NewFs: NewFs,
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
|
||||
Name: "api_key",
|
||||
|
||||
+16
-20
@@ -241,23 +241,6 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
type dialCtx struct {
|
||||
f *Fs
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// dial a new connection with fshttp dialer
|
||||
func (d *dialCtx) dial(network, address string) (net.Conn, error) {
|
||||
conn, err := fshttp.NewDialer(d.ctx).Dial(network, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d.f.tlsConf != nil {
|
||||
conn = tls.Client(conn, d.f.tlsConf)
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err deserve to be
|
||||
// retried. It returns the err as a convenience
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
@@ -277,9 +260,22 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
// Open a new connection to the FTP server.
|
||||
func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
|
||||
fs.Debugf(f, "Connecting to FTP server")
|
||||
dCtx := dialCtx{f, ctx}
|
||||
ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dCtx.dial)}
|
||||
if f.opt.ExplicitTLS {
|
||||
|
||||
// Make ftp library dial with fshttp dialer optionally using TLS
|
||||
dial := func(network, address string) (conn net.Conn, err error) {
|
||||
conn, err = fshttp.NewDialer(ctx).Dial(network, address)
|
||||
if f.tlsConf != nil && err == nil {
|
||||
conn = tls.Client(conn, f.tlsConf)
|
||||
}
|
||||
return
|
||||
}
|
||||
ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
|
||||
|
||||
if f.opt.TLS {
|
||||
// Our dialer takes care of TLS but ftp library also needs tlsConf
|
||||
// as a trigger for sending PSBZ and PROT options to server.
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
|
||||
} else if f.opt.ExplicitTLS {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
|
||||
// Initial connection needs to be cleartext for explicit TLS
|
||||
conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -76,17 +75,18 @@ func init() {
|
||||
Prefix: "gcs",
|
||||
Description: "Google Cloud Storage (this is not Google Drive)",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
saFile, _ := m.Get("service_account_file")
|
||||
saCreds, _ := m.Get("service_account_credentials")
|
||||
anonymous, _ := m.Get("anonymous")
|
||||
if saFile != "" || saCreds != "" || anonymous == "true" {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
err := oauthutil.Config(ctx, "google cloud storage", name, m, storageConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "project_number",
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
golog "log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -78,13 +77,12 @@ func init() {
|
||||
Prefix: "gphotos",
|
||||
Description: "Google Photos",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
|
||||
return
|
||||
return errors.Wrap(err, "couldn't parse config into struct")
|
||||
}
|
||||
|
||||
// Fill in the scopes
|
||||
@@ -97,7 +95,7 @@ func init() {
|
||||
// Do the oauth
|
||||
err = oauthutil.Config(ctx, "google photos", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
golog.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
|
||||
// Warn the user
|
||||
@@ -108,6 +106,7 @@ func init() {
|
||||
|
||||
`)
|
||||
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "read_only",
|
||||
|
||||
@@ -47,7 +47,7 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
|
||||
ts := httptest.NewServer(handler)
|
||||
|
||||
// Configure the remote
|
||||
configfile.LoadConfig(context.Background())
|
||||
configfile.Install()
|
||||
// fs.Config.LogLevel = fs.LogLevelDebug
|
||||
// fs.Config.DumpHeaders = true
|
||||
// fs.Config.DumpBodies = true
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -56,11 +55,12 @@ func init() {
|
||||
Name: "hubic",
|
||||
Description: "Hubic",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
err := oauthutil.Config(ctx, "hubic", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
|
||||
})
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -87,12 +86,12 @@ func init() {
|
||||
Name: "jottacloud",
|
||||
Description: "Jottacloud",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
refresh := false
|
||||
if version, ok := m.Get("configVersion"); ok {
|
||||
ver, err := strconv.Atoi(version)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to parse config version - corrupted config")
|
||||
return errors.Wrap(err, "failed to parse config version - corrupted config")
|
||||
}
|
||||
refresh = (ver != configVersion) && (ver != v1configVersion)
|
||||
}
|
||||
@@ -104,7 +103,7 @@ func init() {
|
||||
if ok && tokenString != "" {
|
||||
fmt.Printf("Already have a token - refresh?\n")
|
||||
if !config.Confirm(false) {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -116,11 +115,13 @@ func init() {
|
||||
|
||||
switch config.ChooseNumber("Your choice", 1, 3) {
|
||||
case 1:
|
||||
v2config(ctx, name, m)
|
||||
return v2config(ctx, name, m)
|
||||
case 2:
|
||||
v1config(ctx, name, m)
|
||||
return v1config(ctx, name, m)
|
||||
case 3:
|
||||
teliaCloudConfig(ctx, name, m)
|
||||
return teliaCloudConfig(ctx, name, m)
|
||||
default:
|
||||
return errors.New("unknown config choice")
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
@@ -242,7 +243,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
|
||||
func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
teliaCloudOauthConfig := &oauth2.Config{
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: teliaCloudAuthURL,
|
||||
@@ -255,15 +256,14 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
|
||||
|
||||
err := oauthutil.Config(ctx, "jottacloud", name, m, teliaCloudOauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
|
||||
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
|
||||
if config.Confirm(false) {
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, teliaCloudOauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load oAuthClient: %s", err)
|
||||
return errors.Wrap(err, "failed to load oAuthClient")
|
||||
}
|
||||
|
||||
srv := rest.NewClient(oAuthClient).SetRoot(rootURL)
|
||||
@@ -271,7 +271,7 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
|
||||
|
||||
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to setup mountpoint: %s", err)
|
||||
return errors.Wrap(err, "failed to setup mountpoint")
|
||||
}
|
||||
m.Set(configDevice, device)
|
||||
m.Set(configMountpoint, mountpoint)
|
||||
@@ -280,17 +280,18 @@ func teliaCloudConfig(ctx context.Context, name string, m configmap.Mapper) {
|
||||
m.Set("configVersion", strconv.Itoa(configVersion))
|
||||
m.Set(configClientID, teliaCloudClientID)
|
||||
m.Set(configTokenURL, teliaCloudTokenURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
// v1config configure a jottacloud backend using legacy authentication
|
||||
func v1config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
func v1config(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
srv := rest.NewClient(fshttp.NewClient(ctx))
|
||||
|
||||
fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
|
||||
if config.Confirm(false) {
|
||||
deviceRegistration, err := registerDevice(ctx, srv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to register device: %v", err)
|
||||
return errors.Wrap(err, "failed to register device")
|
||||
}
|
||||
|
||||
m.Set(configClientID, deviceRegistration.ClientID)
|
||||
@@ -318,18 +319,18 @@ func v1config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
|
||||
token, err := doAuthV1(ctx, srv, username, password)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get oauth token: %s", err)
|
||||
return errors.Wrap(err, "failed to get oauth token")
|
||||
}
|
||||
err = oauthutil.PutToken(name, m, &token, true)
|
||||
if err != nil {
|
||||
log.Fatalf("Error while saving token: %s", err)
|
||||
return errors.Wrap(err, "error while saving token")
|
||||
}
|
||||
|
||||
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
|
||||
if config.Confirm(false) {
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load oAuthClient: %s", err)
|
||||
return errors.Wrap(err, "failed to load oAuthClient")
|
||||
}
|
||||
|
||||
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
|
||||
@@ -337,13 +338,14 @@ func v1config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
|
||||
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to setup mountpoint: %s", err)
|
||||
return errors.Wrap(err, "failed to setup mountpoint")
|
||||
}
|
||||
m.Set(configDevice, device)
|
||||
m.Set(configMountpoint, mountpoint)
|
||||
}
|
||||
|
||||
m.Set("configVersion", strconv.Itoa(v1configVersion))
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerDevice register a new device for use with the jottacloud API
|
||||
@@ -418,7 +420,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
|
||||
}
|
||||
|
||||
// v2config configure a jottacloud backend using the modern JottaCli token based authentication
|
||||
func v2config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
func v2config(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
srv := rest.NewClient(fshttp.NewClient(ctx))
|
||||
|
||||
fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
|
||||
@@ -430,31 +432,32 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
|
||||
token, err := doAuthV2(ctx, srv, loginToken, m)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get oauth token: %s", err)
|
||||
return errors.Wrap(err, "failed to get oauth token")
|
||||
}
|
||||
err = oauthutil.PutToken(name, m, &token, true)
|
||||
if err != nil {
|
||||
log.Fatalf("Error while saving token: %s", err)
|
||||
return errors.Wrap(err, "error while saving token")
|
||||
}
|
||||
|
||||
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
|
||||
if config.Confirm(false) {
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load oAuthClient: %s", err)
|
||||
return errors.Wrap(err, "failed to load oAuthClient")
|
||||
}
|
||||
|
||||
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
|
||||
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
|
||||
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to setup mountpoint: %s", err)
|
||||
return errors.Wrap(err, "failed to setup mountpoint")
|
||||
}
|
||||
m.Set(configDevice, device)
|
||||
m.Set(configMountpoint, mountpoint)
|
||||
}
|
||||
|
||||
m.Set("configVersion", strconv.Itoa(configVersion))
|
||||
return nil
|
||||
}
|
||||
|
||||
// doAuthV2 runs the actual token request for V2 authentication
|
||||
|
||||
@@ -534,7 +534,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
return nil
|
||||
}
|
||||
|
||||
// About reports space usage (with a MB precision)
|
||||
// About reports space usage (with a MiB precision)
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
mount, err := f.client.MountsDetails(f.mountID)
|
||||
if err != nil {
|
||||
|
||||
@@ -1017,6 +1017,7 @@ func (file *localOpenFile) Close() (err error) {
|
||||
func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
|
||||
// Read the link and return the destination it as the contents of the object
|
||||
linkdst, err := os.Readlink(o.path)
|
||||
fs.Infof(o, "openTranslatedLink link = %q, offset = %d, limit = %d, err = %v", linkdst, offset, limit, err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1271,6 +1272,7 @@ func (o *Object) setMetadata(info os.FileInfo) {
|
||||
// Optionally, users can turn this feature on with the zero_size_links flag
|
||||
if (runtime.GOOS == "windows" || o.fs.opt.ZeroSizeLinks) && o.translatedLink {
|
||||
linkdst, err := os.Readlink(o.path)
|
||||
fs.Infof(o, "setMetadata link = %q, err = %v", linkdst, err)
|
||||
if err != nil {
|
||||
fs.Errorf(o, "Failed to read link size: %v", err)
|
||||
} else {
|
||||
|
||||
@@ -6,8 +6,8 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -48,7 +48,7 @@ func (w *BinWriter) Reader() io.Reader {
|
||||
// WritePu16 writes a short as unsigned varint
|
||||
func (w *BinWriter) WritePu16(val int) {
|
||||
if val < 0 || val > 65535 {
|
||||
log.Fatalf("Invalid UInt16 %v", val)
|
||||
panic(fmt.Sprintf("Invalid UInt16 %v", val))
|
||||
}
|
||||
w.WritePu64(int64(val))
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func (w *BinWriter) WritePu16(val int) {
|
||||
// WritePu32 writes a signed long as unsigned varint
|
||||
func (w *BinWriter) WritePu32(val int64) {
|
||||
if val < 0 || val > 4294967295 {
|
||||
log.Fatalf("Invalid UInt32 %v", val)
|
||||
panic(fmt.Sprintf("Invalid UInt32 %v", val))
|
||||
}
|
||||
w.WritePu64(val)
|
||||
}
|
||||
@@ -64,7 +64,7 @@ func (w *BinWriter) WritePu32(val int64) {
|
||||
// WritePu64 writes an unsigned (actually, signed) long as unsigned varint
|
||||
func (w *BinWriter) WritePu64(val int64) {
|
||||
if val < 0 {
|
||||
log.Fatalf("Invalid UInt64 %v", val)
|
||||
panic(fmt.Sprintf("Invalid UInt64 %v", val))
|
||||
}
|
||||
w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
|
||||
}
|
||||
@@ -123,7 +123,7 @@ func (r *BinReader) check(err error) bool {
|
||||
r.err = err
|
||||
}
|
||||
if err != io.EOF {
|
||||
log.Fatalf("Error parsing response: %v", err)
|
||||
panic(fmt.Sprintf("Error parsing response: %v", err))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -52,8 +51,8 @@ const (
|
||||
driveTypePersonal = "personal"
|
||||
driveTypeBusiness = "business"
|
||||
driveTypeSharepoint = "documentLibrary"
|
||||
defaultChunkSize = 10 * fs.MebiByte
|
||||
chunkSizeMultiple = 320 * fs.KibiByte
|
||||
defaultChunkSize = 10 * fs.Mebi
|
||||
chunkSizeMultiple = 320 * fs.Kibi
|
||||
|
||||
regionGlobal = "global"
|
||||
regionUS = "us"
|
||||
@@ -99,7 +98,7 @@ func init() {
|
||||
Name: "onedrive",
|
||||
Description: "Microsoft OneDrive",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
region, _ := m.Get("region")
|
||||
graphURL := graphAPIEndpoint[region] + "/v1.0"
|
||||
oauthConfig.Endpoint = oauth2.Endpoint{
|
||||
@@ -109,13 +108,12 @@ func init() {
|
||||
ci := fs.GetConfig(ctx)
|
||||
err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
|
||||
// Stop if we are running non-interactive config
|
||||
if ci.AutoConfirm {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
type driveResource struct {
|
||||
@@ -138,7 +136,7 @@ func init() {
|
||||
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure OneDrive: %v", err)
|
||||
return errors.Wrap(err, "failed to configure OneDrive")
|
||||
}
|
||||
srv := rest.NewClient(oAuthClient)
|
||||
|
||||
@@ -203,18 +201,17 @@ func init() {
|
||||
sites := siteResponse{}
|
||||
_, err := srv.CallJSON(ctx, &opts, nil, &sites)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to query available sites: %v", err)
|
||||
return errors.Wrap(err, "failed to query available sites")
|
||||
}
|
||||
|
||||
if len(sites.Sites) == 0 {
|
||||
log.Fatalf("Search for '%s' returned no results", searchTerm)
|
||||
} else {
|
||||
fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
|
||||
for index, site := range sites.Sites {
|
||||
fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
|
||||
}
|
||||
siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
|
||||
return errors.Errorf("search for %q returned no results", searchTerm)
|
||||
}
|
||||
fmt.Printf("Found %d sites, please select the one you want to use:\n", len(sites.Sites))
|
||||
for index, site := range sites.Sites {
|
||||
fmt.Printf("%d: %s (%s) id=%s\n", index, site.SiteName, site.SiteURL, site.SiteID)
|
||||
}
|
||||
siteID = sites.Sites[config.ChooseNumber("Chose drive to use:", 0, len(sites.Sites)-1)].SiteID
|
||||
}
|
||||
|
||||
// if we use server-relative URL for finding the drive
|
||||
@@ -227,7 +224,7 @@ func init() {
|
||||
site := siteResource{}
|
||||
_, err := srv.CallJSON(ctx, &opts, nil, &site)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to query available site by relative path: %v", err)
|
||||
return errors.Wrap(err, "failed to query available site by relative path")
|
||||
}
|
||||
siteID = site.SiteID
|
||||
}
|
||||
@@ -247,7 +244,7 @@ func init() {
|
||||
drives := drivesResponse{}
|
||||
_, err := srv.CallJSON(ctx, &opts, nil, &drives)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to query available drives: %v", err)
|
||||
return errors.Wrap(err, "failed to query available drives")
|
||||
}
|
||||
|
||||
// Also call /me/drive as sometimes /me/drives doesn't return it #4068
|
||||
@@ -256,7 +253,7 @@ func init() {
|
||||
meDrive := driveResource{}
|
||||
_, err := srv.CallJSON(ctx, &opts, nil, &meDrive)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to query available drives: %v", err)
|
||||
return errors.Wrap(err, "failed to query available drives")
|
||||
}
|
||||
found := false
|
||||
for _, drive := range drives.Drives {
|
||||
@@ -273,14 +270,13 @@ func init() {
|
||||
}
|
||||
|
||||
if len(drives.Drives) == 0 {
|
||||
log.Fatalf("No drives found")
|
||||
} else {
|
||||
fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
|
||||
for index, drive := range drives.Drives {
|
||||
fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
|
||||
}
|
||||
finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
|
||||
return errors.New("no drives found")
|
||||
}
|
||||
fmt.Printf("Found %d drives, please select the one you want to use:\n", len(drives.Drives))
|
||||
for index, drive := range drives.Drives {
|
||||
fmt.Printf("%d: %s (%s) id=%s\n", index, drive.DriveName, drive.DriveType, drive.DriveID)
|
||||
}
|
||||
finalDriveID = drives.Drives[config.ChooseNumber("Chose drive to use:", 0, len(drives.Drives)-1)].DriveID
|
||||
}
|
||||
|
||||
// Test the driveID and get drive type
|
||||
@@ -291,17 +287,18 @@ func init() {
|
||||
var rootItem api.Item
|
||||
_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err)
|
||||
return errors.Wrapf(err, "failed to query root for drive %s", finalDriveID)
|
||||
}
|
||||
|
||||
fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
|
||||
// This does not work, YET :)
|
||||
if !config.ConfirmWithConfig(ctx, m, "config_drive_ok", true) {
|
||||
log.Fatalf("Cancelled by user")
|
||||
return errors.New("cancelled by user")
|
||||
}
|
||||
|
||||
m.Set(configDriveID, finalDriveID)
|
||||
m.Set(configDriveType, rootItem.ParentReference.DriveType)
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "region",
|
||||
@@ -566,6 +563,9 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
if len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
|
||||
retry = true
|
||||
fs.Debugf(nil, "Should retry: %v", err)
|
||||
} else if err != nil && strings.Contains(err.Error(), "Unable to initialize RPS") {
|
||||
retry = true
|
||||
fs.Debugf(nil, "HTTP 401: Unable to initialize RPS. Trying again.")
|
||||
}
|
||||
case 429: // Too Many Requests.
|
||||
// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
|
||||
@@ -693,7 +693,7 @@ func errorHandler(resp *http.Response) error {
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
const minChunkSize = fs.Byte
|
||||
const minChunkSize = fs.SizeSuffixBase
|
||||
if cs%chunkSizeMultiple != 0 {
|
||||
return errors.Errorf("%s is not a multiple of %s", cs, chunkSizeMultiple)
|
||||
}
|
||||
@@ -1857,7 +1857,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
|
||||
fs.Debugf(o, "Cancelling multipart upload: %v", err)
|
||||
cancelErr := o.cancelUploadSession(ctx, uploadURL)
|
||||
if cancelErr != nil {
|
||||
fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
|
||||
fs.Logf(o, "Failed to cancel multipart upload: %v (upload failed due to: %v)", cancelErr, err)
|
||||
}
|
||||
})()
|
||||
|
||||
@@ -1882,11 +1882,11 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// Update the content of a remote file within 4MB size in one single request
|
||||
// Update the content of a remote file within 4 MiB size in one single request
|
||||
// This function will set modtime after uploading, which will create a new version for the remote file
|
||||
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, modTime time.Time, options ...fs.OpenOption) (info *api.Item, err error) {
|
||||
if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
|
||||
return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
|
||||
return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")
|
||||
}
|
||||
|
||||
fs.Debugf(o, "Starting singlepart upload")
|
||||
|
||||
@@ -88,7 +88,7 @@ func init() {
|
||||
|
||||
Note that these chunks are buffered in memory so increasing them will
|
||||
increase memory use.`,
|
||||
Default: 10 * fs.MebiByte,
|
||||
Default: 10 * fs.Mebi,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -72,7 +71,7 @@ func init() {
|
||||
Name: "pcloud",
|
||||
Description: "Pcloud",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
optc := new(Options)
|
||||
err := configstruct.Set(m, optc)
|
||||
if err != nil {
|
||||
@@ -100,8 +99,9 @@ func init() {
|
||||
}
|
||||
err = oauthutil.Config(ctx, "pcloud", name, m, oauthConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: config.ConfigEncoding,
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -78,11 +77,12 @@ func init() {
|
||||
Name: "premiumizeme",
|
||||
Description: "premiumize.me",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
err := oauthutil.Config(ctx, "premiumizeme", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: "api_key",
|
||||
|
||||
@@ -2,10 +2,10 @@ package putio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
@@ -35,7 +35,7 @@ const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
defaultChunkSize = 48 * fs.MebiByte
|
||||
defaultChunkSize = 48 * fs.Mebi
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -60,14 +60,15 @@ func init() {
|
||||
Name: "putio",
|
||||
Description: "Put.io",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
opt := oauthutil.Options{
|
||||
NoOffline: true,
|
||||
}
|
||||
err := oauthutil.Config(ctx, "putio", name, m, putioConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigEncoding,
|
||||
|
||||
@@ -80,7 +80,7 @@ func init() {
|
||||
Help: `Cutoff for switching to chunked upload
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5GB.`,
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
|
||||
+53
-27
@@ -26,7 +26,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/corehandlers"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
|
||||
"github.com/aws/aws-sdk-go/aws/defaults"
|
||||
"github.com/aws/aws-sdk-go/aws/ec2metadata"
|
||||
"github.com/aws/aws-sdk-go/aws/endpoints"
|
||||
@@ -1017,7 +1016,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
|
||||
Help: `Cutoff for switching to chunked upload
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5GB.`,
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -1039,9 +1038,9 @@ Rclone will automatically increase the chunk size when uploading a
|
||||
large file of known size to stay below the 10,000 chunks limit.
|
||||
|
||||
Files of unknown size are uploaded with the configured
|
||||
chunk_size. Since the default chunk size is 5MB and there can be at
|
||||
chunk_size. Since the default chunk size is 5 MiB and there can be at
|
||||
most 10,000 chunks, this means that by default the maximum size of
|
||||
a file you can stream upload is 48GB. If you wish to stream upload
|
||||
a file you can stream upload is 48 GiB. If you wish to stream upload
|
||||
larger files then you will need to increase chunk_size.`,
|
||||
Default: minChunkSize,
|
||||
Advanced: true,
|
||||
@@ -1067,7 +1066,7 @@ large file of a known size to stay below this number of chunks limit.
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 5GB.`,
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: fs.SizeSuffix(maxSizeForCopy),
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -1221,6 +1220,11 @@ very small even with this flag.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "no_head_object",
|
||||
Help: `If set, don't HEAD objects`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -1271,7 +1275,7 @@ See: https://github.com/rclone/rclone/issues/4673, https://github.com/rclone/rcl
|
||||
const (
|
||||
metaMtime = "Mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime
|
||||
metaMD5Hash = "Md5chksum" // the meta key to store md5hash in
|
||||
// The maximum size of object we can COPY - this should be 5GiB but is < 5GB for b2 compatibility
|
||||
// The maximum size of object we can COPY - this should be 5 GiB but is < 5 GB for b2 compatibility
|
||||
// See https://forum.rclone.org/t/copying-files-within-a-b2-bucket/16680/76
|
||||
maxSizeForCopy = 4768 * 1024 * 1024
|
||||
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
|
||||
@@ -1319,6 +1323,7 @@ type Options struct {
|
||||
ListChunk int64 `config:"list_chunk"`
|
||||
NoCheckBucket bool `config:"no_check_bucket"`
|
||||
NoHead bool `config:"no_head"`
|
||||
NoHeadObject bool `config:"no_head_object"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
MemoryPoolFlushTime fs.Duration `config:"memory_pool_flush_time"`
|
||||
MemoryPoolUseMmap bool `config:"memory_pool_use_mmap"`
|
||||
@@ -1511,11 +1516,6 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
}),
|
||||
ExpiryWindow: 3 * time.Minute,
|
||||
},
|
||||
|
||||
// Pick up IAM role if we are in EKS
|
||||
&stscreds.WebIdentityRoleProvider{
|
||||
ExpiryWindow: 3 * time.Minute,
|
||||
},
|
||||
}
|
||||
cred := credentials.NewChainCredentials(providers)
|
||||
|
||||
@@ -1693,7 +1693,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
GetTier: true,
|
||||
SlowModTime: true,
|
||||
}).Fill(ctx, f)
|
||||
if f.rootBucket != "" && f.rootDirectory != "" {
|
||||
if f.rootBucket != "" && f.rootDirectory != "" && !opt.NoHeadObject {
|
||||
// Check to see if the (bucket,directory) is actually an existing file
|
||||
oldRoot := f.root
|
||||
newRoot, leaf := path.Split(oldRoot)
|
||||
@@ -1730,7 +1730,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
|
||||
o.setMD5FromEtag(aws.StringValue(info.ETag))
|
||||
o.bytes = aws.Int64Value(info.Size)
|
||||
o.storageClass = aws.StringValue(info.StorageClass)
|
||||
} else {
|
||||
} else if !o.fs.opt.NoHeadObject {
|
||||
err := o.readMetaData(ctx) // reads info and meta, returning an error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2831,15 +2831,23 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.LastModified == nil {
|
||||
fs.Logf(o, "Failed to read last modified from HEAD: %v", err)
|
||||
}
|
||||
o.setMetaData(resp.ETag, resp.ContentLength, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Object) setMetaData(etag *string, contentLength *int64, lastModified *time.Time, meta map[string]*string, mimeType *string, storageClass *string) {
|
||||
var size int64
|
||||
// Ignore missing Content-Length assuming it is 0
|
||||
// Some versions of ceph do this due their apache proxies
|
||||
if resp.ContentLength != nil {
|
||||
size = *resp.ContentLength
|
||||
if contentLength != nil {
|
||||
size = *contentLength
|
||||
}
|
||||
o.setMD5FromEtag(aws.StringValue(resp.ETag))
|
||||
o.setMD5FromEtag(aws.StringValue(etag))
|
||||
o.bytes = size
|
||||
o.meta = resp.Metadata
|
||||
o.meta = meta
|
||||
if o.meta == nil {
|
||||
o.meta = map[string]*string{}
|
||||
}
|
||||
@@ -2854,15 +2862,13 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
o.md5 = hex.EncodeToString(md5sumBytes)
|
||||
}
|
||||
}
|
||||
o.storageClass = aws.StringValue(resp.StorageClass)
|
||||
if resp.LastModified == nil {
|
||||
fs.Logf(o, "Failed to read last modified from HEAD: %v", err)
|
||||
o.storageClass = aws.StringValue(storageClass)
|
||||
if lastModified == nil {
|
||||
o.lastModified = time.Now()
|
||||
} else {
|
||||
o.lastModified = *resp.LastModified
|
||||
o.lastModified = *lastModified
|
||||
}
|
||||
o.mimeType = aws.StringValue(resp.ContentType)
|
||||
return nil
|
||||
o.mimeType = aws.StringValue(mimeType)
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
@@ -2972,6 +2978,26 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.LastModified == nil {
|
||||
fs.Logf(o, "Failed to read last modified: %v", err)
|
||||
}
|
||||
// read size from ContentLength or ContentRange
|
||||
size := resp.ContentLength
|
||||
if resp.ContentRange != nil {
|
||||
var contentRange = *resp.ContentRange
|
||||
slash := strings.IndexRune(contentRange, '/')
|
||||
if slash >= 0 {
|
||||
i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
|
||||
if err == nil {
|
||||
size = &i
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find length in %q", contentRange)
|
||||
}
|
||||
}
|
||||
o.setMetaData(resp.ETag, size, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass)
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
@@ -2997,9 +3023,9 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
// calculate size of parts
|
||||
partSize := int(f.opt.ChunkSize)
|
||||
|
||||
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
|
||||
// buffers here (default 5MB). With a maximum number of parts (10,000) this will be a file of
|
||||
// 48GB which seems like a not too unreasonable limit.
|
||||
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
|
||||
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
|
||||
// 48 GiB which seems like a not too unreasonable limit.
|
||||
if size == -1 {
|
||||
warnStreamUpload.Do(func() {
|
||||
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
|
||||
@@ -3008,7 +3034,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
} else {
|
||||
// Adjust partSize until the number of parts is small enough.
|
||||
if size/int64(partSize) >= uploadParts {
|
||||
// Calculate partition size rounded up to the nearest MB
|
||||
// Calculate partition size rounded up to the nearest MiB
|
||||
partSize = int((((size / uploadParts) >> 20) + 1) << 20)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -296,36 +296,32 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
|
||||
// Config callback for 2FA
|
||||
func Config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
func Config(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
ci := fs.GetConfig(ctx)
|
||||
serverURL, ok := m.Get(configURL)
|
||||
if !ok || serverURL == "" {
|
||||
// If there's no server URL, it means we're trying an operation at the backend level, like a "rclone authorize seafile"
|
||||
fmt.Print("\nOperation not supported on this remote.\nIf you need a 2FA code on your account, use the command:\n\nrclone config reconnect <remote name>:\n\n")
|
||||
return
|
||||
return errors.New("operation not supported on this remote. If you need a 2FA code on your account, use the command: nrclone config reconnect <remote name>: ")
|
||||
}
|
||||
|
||||
// Stop if we are running non-interactive config
|
||||
if ci.AutoConfirm {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
u, err := url.Parse(serverURL)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Invalid server URL %s", serverURL)
|
||||
return
|
||||
return errors.Errorf("invalid server URL %s", serverURL)
|
||||
}
|
||||
|
||||
is2faEnabled, _ := m.Get(config2FA)
|
||||
if is2faEnabled != "true" {
|
||||
fmt.Println("Two-factor authentication is not enabled on this account.")
|
||||
return
|
||||
return errors.New("two-factor authentication is not enabled on this account")
|
||||
}
|
||||
|
||||
username, _ := m.Get(configUser)
|
||||
if username == "" {
|
||||
fs.Errorf(nil, "A username is required")
|
||||
return
|
||||
return errors.New("a username is required")
|
||||
}
|
||||
|
||||
password, _ := m.Get(configPassword)
|
||||
@@ -376,6 +372,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sets the AuthorizationToken up
|
||||
|
||||
+51
-26
@@ -224,6 +224,17 @@ have a server which returns
|
||||
Then you may need to enable this flag.
|
||||
|
||||
If concurrent reads are disabled, the use_fstat option is ignored.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "disable_concurrent_writes",
|
||||
Default: false,
|
||||
Help: `If set don't use concurrent writes
|
||||
|
||||
Normally rclone uses concurrent writes to upload files. This improves
|
||||
the performance greatly, especially for distant servers.
|
||||
|
||||
This option disables concurrent writes should that be necessary.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -244,29 +255,30 @@ Set to 0 to keep connections indefinitely.
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Host string `config:"host"`
|
||||
User string `config:"user"`
|
||||
Port string `config:"port"`
|
||||
Pass string `config:"pass"`
|
||||
KeyPem string `config:"key_pem"`
|
||||
KeyFile string `config:"key_file"`
|
||||
KeyFilePass string `config:"key_file_pass"`
|
||||
PubKeyFile string `config:"pubkey_file"`
|
||||
KnownHostsFile string `config:"known_hosts_file"`
|
||||
KeyUseAgent bool `config:"key_use_agent"`
|
||||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||||
DisableHashCheck bool `config:"disable_hashcheck"`
|
||||
AskPassword bool `config:"ask_password"`
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
Md5sumCommand string `config:"md5sum_command"`
|
||||
Sha1sumCommand string `config:"sha1sum_command"`
|
||||
SkipLinks bool `config:"skip_links"`
|
||||
Subsystem string `config:"subsystem"`
|
||||
ServerCommand string `config:"server_command"`
|
||||
UseFstat bool `config:"use_fstat"`
|
||||
DisableConcurrentReads bool `config:"disable_concurrent_reads"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
Host string `config:"host"`
|
||||
User string `config:"user"`
|
||||
Port string `config:"port"`
|
||||
Pass string `config:"pass"`
|
||||
KeyPem string `config:"key_pem"`
|
||||
KeyFile string `config:"key_file"`
|
||||
KeyFilePass string `config:"key_file_pass"`
|
||||
PubKeyFile string `config:"pubkey_file"`
|
||||
KnownHostsFile string `config:"known_hosts_file"`
|
||||
KeyUseAgent bool `config:"key_use_agent"`
|
||||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||||
DisableHashCheck bool `config:"disable_hashcheck"`
|
||||
AskPassword bool `config:"ask_password"`
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
Md5sumCommand string `config:"md5sum_command"`
|
||||
Sha1sumCommand string `config:"sha1sum_command"`
|
||||
SkipLinks bool `config:"skip_links"`
|
||||
Subsystem string `config:"subsystem"`
|
||||
ServerCommand string `config:"server_command"`
|
||||
UseFstat bool `config:"use_fstat"`
|
||||
DisableConcurrentReads bool `config:"disable_concurrent_reads"`
|
||||
DisableConcurrentWrites bool `config:"disable_concurrent_writes"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote SFTP files
|
||||
@@ -414,8 +426,8 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
|
||||
opts = opts[:len(opts):len(opts)] // make sure we don't overwrite the callers opts
|
||||
opts = append(opts,
|
||||
sftp.UseFstat(f.opt.UseFstat),
|
||||
// FIXME disabled after library reversion
|
||||
// sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
|
||||
sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads),
|
||||
sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites),
|
||||
)
|
||||
if f.opt.DisableConcurrentReads { // FIXME
|
||||
fs.Errorf(f, "Ignoring disable_concurrent_reads after library reversion - see #5197")
|
||||
@@ -1494,6 +1506,19 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return in, nil
|
||||
}
|
||||
|
||||
type sizeReader struct {
|
||||
io.Reader
|
||||
size int64
|
||||
}
|
||||
|
||||
// Size returns the expected size of the stream
|
||||
//
|
||||
// It is used in sftpFile.ReadFrom as a hint to work out the
|
||||
// concurrency needed
|
||||
func (sr *sizeReader) Size() int64 {
|
||||
return sr.size
|
||||
}
|
||||
|
||||
// Update a remote sftp file using the data <in> and ModTime from <src>
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
o.fs.addTransfer() // Show transfer in progress
|
||||
@@ -1525,7 +1550,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Debugf(src, "Removed after failed upload: %v", err)
|
||||
}
|
||||
}
|
||||
_, err = file.ReadFrom(in)
|
||||
_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
|
||||
if err != nil {
|
||||
remove()
|
||||
return errors.Wrap(err, "Update ReadFrom failed")
|
||||
|
||||
@@ -77,7 +77,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -110,10 +109,10 @@ const (
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
apiPath = "/sf/v3" // add to endpoint to get API path
|
||||
tokenPath = "/oauth/token" // add to endpoint to get Token path
|
||||
minChunkSize = 256 * fs.KibiByte
|
||||
maxChunkSize = 2 * fs.GibiByte
|
||||
defaultChunkSize = 64 * fs.MebiByte
|
||||
defaultUploadCutoff = 128 * fs.MebiByte
|
||||
minChunkSize = 256 * fs.Kibi
|
||||
maxChunkSize = 2 * fs.Gibi
|
||||
defaultChunkSize = 64 * fs.Mebi
|
||||
defaultUploadCutoff = 128 * fs.Mebi
|
||||
)
|
||||
|
||||
// Generate a new oauth2 config which we will update when we know the TokenURL
|
||||
@@ -136,7 +135,7 @@ func init() {
|
||||
Name: "sharefile",
|
||||
Description: "Citrix Sharefile",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
oauthConfig := newOauthConfig("")
|
||||
checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
|
||||
if auth == nil || auth.Form == nil {
|
||||
@@ -157,8 +156,9 @@ func init() {
|
||||
}
|
||||
err := oauthutil.Config(ctx, "sharefile", name, m, oauthConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: "upload_cutoff",
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -76,17 +75,17 @@ func init() {
|
||||
Name: "sugarsync",
|
||||
Description: "Sugarsync",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read options: %v", err)
|
||||
return errors.Wrap(err, "failed to read options")
|
||||
}
|
||||
|
||||
if opt.RefreshToken != "" {
|
||||
fmt.Printf("Already have a token - refresh?\n")
|
||||
if !config.ConfirmWithConfig(ctx, m, "config_refresh_token", true) {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
}
|
||||
fmt.Printf("Username (email address)> ")
|
||||
@@ -114,10 +113,11 @@ func init() {
|
||||
// return shouldRetry(ctx, resp, err)
|
||||
//})
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get token: %v", err)
|
||||
return errors.Wrap(err, "failed to get token")
|
||||
}
|
||||
opt.RefreshToken = resp.Header.Get("Location")
|
||||
m.Set("refresh_token", opt.RefreshToken)
|
||||
return nil
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: "app_id",
|
||||
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
const (
|
||||
directoryMarkerContentType = "application/directory" // content type of directory marker objects
|
||||
listChunks = 1000 // chunk size to read directory listings
|
||||
defaultChunkSize = 5 * fs.GibiByte
|
||||
defaultChunkSize = 5 * fs.Gibi
|
||||
minSleep = 10 * time.Millisecond // In case of error, start at 10ms sleep.
|
||||
)
|
||||
|
||||
@@ -46,7 +46,7 @@ var SharedOptions = []fs.Option{{
|
||||
Help: `Above this size files will be chunked into a _segments container.
|
||||
|
||||
Above this size files will be chunked into a _segments container. The
|
||||
default for this is 5GB which is its maximum value.`,
|
||||
default for this is 5 GiB which is its maximum value.`,
|
||||
Default: defaultChunkSize,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -56,7 +56,7 @@ default for this is 5GB which is its maximum value.`,
|
||||
When doing streaming uploads (e.g. using rcat or mount) setting this
|
||||
flag will cause the swift backend to not upload chunked files.
|
||||
|
||||
This will limit the maximum upload size to 5GB. However non chunked
|
||||
This will limit the maximum upload size to 5 GiB. However non chunked
|
||||
files are easier to deal with and have an MD5SUM.
|
||||
|
||||
Rclone will still chunk files bigger than chunk_size when doing normal
|
||||
@@ -419,7 +419,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
const minChunkSize = fs.Byte
|
||||
const minChunkSize = fs.SizeSuffixBase
|
||||
if cs < minChunkSize {
|
||||
return errors.Errorf("%s is less than %s", cs, minChunkSize)
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func (f *Fs) testWithChunk(t *testing.T) {
|
||||
preConfChunkSize := f.opt.ChunkSize
|
||||
preConfChunk := f.opt.NoChunk
|
||||
f.opt.NoChunk = false
|
||||
f.opt.ChunkSize = 1024 * fs.Byte
|
||||
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
|
||||
defer func() {
|
||||
//restore old config after test
|
||||
f.opt.ChunkSize = preConfChunkSize
|
||||
@@ -117,7 +117,7 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
|
||||
preConfChunkSize := f.opt.ChunkSize
|
||||
preConfChunk := f.opt.NoChunk
|
||||
f.opt.NoChunk = false
|
||||
f.opt.ChunkSize = 1024 * fs.Byte
|
||||
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
|
||||
segmentContainer := f.root + "_segments"
|
||||
defer func() {
|
||||
//restore config
|
||||
@@ -159,7 +159,7 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
|
||||
preConfChunkSize := f.opt.ChunkSize
|
||||
preConfChunk := f.opt.NoChunk
|
||||
f.opt.NoChunk = false
|
||||
f.opt.ChunkSize = 1024 * fs.Byte
|
||||
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
|
||||
defer func() {
|
||||
//restore old config after test
|
||||
f.opt.ChunkSize = preConfChunkSize
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -42,7 +41,7 @@ func init() {
|
||||
Name: "tardigrade",
|
||||
Description: "Tardigrade Decentralized Cloud Storage",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, configMapper configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, configMapper configmap.Mapper) error {
|
||||
provider, _ := configMapper.Get(fs.ConfigProvider)
|
||||
|
||||
config.FileDeleteKey(name, fs.ConfigProvider)
|
||||
@@ -54,7 +53,7 @@ func init() {
|
||||
|
||||
// satelliteString contains always default and passphrase can be empty
|
||||
if apiKey == "" {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
satellite, found := satMap[satelliteString]
|
||||
@@ -64,12 +63,12 @@ func init() {
|
||||
|
||||
access, err := uplink.RequestAccessWithPassphrase(context.TODO(), satellite, apiKey, passphrase)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't create access grant: %v", err)
|
||||
return errors.Wrap(err, "couldn't create access grant")
|
||||
}
|
||||
|
||||
serializedAccess, err := access.Serialize()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't serialize access grant: %v", err)
|
||||
return errors.Wrap(err, "couldn't serialize access grant")
|
||||
}
|
||||
configMapper.Set("satellite_address", satellite)
|
||||
configMapper.Set("access_grant", serializedAccess)
|
||||
@@ -78,8 +77,9 @@ func init() {
|
||||
config.FileDeleteKey(name, "api_key")
|
||||
config.FileDeleteKey(name, "passphrase")
|
||||
} else {
|
||||
log.Fatalf("Invalid provider type: %s", provider)
|
||||
return errors.Errorf("invalid provider type: %s", provider)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: []fs.Option{
|
||||
{
|
||||
|
||||
@@ -148,13 +148,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (_ io.ReadC
|
||||
case s && !e:
|
||||
offset = opt.Start
|
||||
case !s && e:
|
||||
object, err := o.fs.project.StatObject(ctx, bucketName, bucketPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offset = object.System.ContentLength - opt.End
|
||||
length = opt.End
|
||||
offset = -opt.End
|
||||
}
|
||||
case *fs.SeekOption:
|
||||
offset = opt.Offset
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package api
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Error contains the error code and message returned by the API
|
||||
type Error struct {
|
||||
Success bool `json:"success,omitempty"`
|
||||
StatusCode int `json:"statusCode,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Data string `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// Error returns a string for the error and satisfies the error interface
|
||||
func (e Error) Error() string {
|
||||
out := fmt.Sprintf("api error %d", e.StatusCode)
|
||||
if e.Message != "" {
|
||||
out += ": " + e.Message
|
||||
}
|
||||
if e.Data != "" {
|
||||
out += ": " + e.Data
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FolderEntry represents a Uptobox subfolder when listing folder contents
|
||||
type FolderEntry struct {
|
||||
FolderID uint64 `json:"fld_id"`
|
||||
Description string `json:"fld_descr"`
|
||||
Password string `json:"fld_password"`
|
||||
FullPath string `json:"fullPath"`
|
||||
Path string `json:"fld_name"`
|
||||
Name string `json:"name"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
// FolderInfo represents the current folder when listing folder contents
|
||||
type FolderInfo struct {
|
||||
FolderID uint64 `json:"fld_id"`
|
||||
Hash string `json:"hash"`
|
||||
FileCount uint64 `json:"fileCount"`
|
||||
TotalFileSize int64 `json:"totalFileSize"`
|
||||
}
|
||||
|
||||
// FileInfo represents a file when listing folder contents
|
||||
type FileInfo struct {
|
||||
Name string `json:"file_name"`
|
||||
Description string `json:"file_descr"`
|
||||
Created string `json:"file_created"`
|
||||
Size int64 `json:"file_size"`
|
||||
Downloads uint64 `json:"file_downloads"`
|
||||
Code string `json:"file_code"`
|
||||
Password string `json:"file_password"`
|
||||
Public int `json:"file_public"`
|
||||
LastDownload string `json:"file_last_download"`
|
||||
ID uint64 `json:"id"`
|
||||
}
|
||||
|
||||
// ReadMetadataResponse is the response when listing folder contents
|
||||
type ReadMetadataResponse struct {
|
||||
StatusCode int `json:"statusCode"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
CurrentFolder FolderInfo `json:"currentFolder"`
|
||||
Folders []FolderEntry `json:"folders"`
|
||||
Files []FileInfo `json:"files"`
|
||||
PageCount int `json:"pageCount"`
|
||||
TotalFileCount int `json:"totalFileCount"`
|
||||
TotalFileSize int64 `json:"totalFileSize"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// UploadInfo is the response when initiating an upload
|
||||
type UploadInfo struct {
|
||||
StatusCode int `json:"statusCode"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
UploadLink string `json:"uploadLink"`
|
||||
MaxUpload string `json:"maxUpload"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// UploadResponse is the respnse to a successful upload
|
||||
type UploadResponse struct {
|
||||
Files []struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
URL string `json:"url"`
|
||||
DeleteURL string `json:"deleteUrl"`
|
||||
} `json:"files"`
|
||||
}
|
||||
|
||||
// UpdateResponse is a generic response to various action on files (rename/copy/move)
|
||||
type UpdateResponse struct {
|
||||
Message string `json:"message"`
|
||||
StatusCode int `json:"statusCode"`
|
||||
}
|
||||
|
||||
// Download is the response when requesting a download link
|
||||
type Download struct {
|
||||
StatusCode int `json:"statusCode"`
|
||||
Message string `json:"message"`
|
||||
Data struct {
|
||||
DownloadLink string `json:"dlLink"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// MetadataRequestOptions represents all the options when listing folder contents
|
||||
type MetadataRequestOptions struct {
|
||||
Limit uint64
|
||||
Offset uint64
|
||||
SearchField string
|
||||
Search string
|
||||
}
|
||||
|
||||
// CreateFolderRequest is used for creating a folder
|
||||
type CreateFolderRequest struct {
|
||||
Token string `json:"token"`
|
||||
Path string `json:"path"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// DeleteFolderRequest is used for deleting a folder
|
||||
type DeleteFolderRequest struct {
|
||||
Token string `json:"token"`
|
||||
FolderID uint64 `json:"fld_id"`
|
||||
}
|
||||
|
||||
// CopyMoveFileRequest is used for moving/copying a file
|
||||
type CopyMoveFileRequest struct {
|
||||
Token string `json:"token"`
|
||||
FileCodes string `json:"file_codes"`
|
||||
DestinationFolderID uint64 `json:"destination_fld_id"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
|
||||
// MoveFolderRequest is used for moving a folder
|
||||
type MoveFolderRequest struct {
|
||||
Token string `json:"token"`
|
||||
FolderID uint64 `json:"fld_id"`
|
||||
DestinationFolderID uint64 `json:"destination_fld_id"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
|
||||
// RenameFolderRequest is used for renaming a folder
|
||||
type RenameFolderRequest struct {
|
||||
Token string `json:"token"`
|
||||
FolderID uint64 `json:"fld_id"`
|
||||
NewName string `json:"new_name"`
|
||||
}
|
||||
|
||||
// UpdateFileInformation is used for renaming a file
|
||||
type UpdateFileInformation struct {
|
||||
Token string `json:"token"`
|
||||
FileCode string `json:"file_code"`
|
||||
NewName string `json:"new_name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
Public string `json:"public,omitempty"`
|
||||
}
|
||||
|
||||
// RemoveFileRequest is used for deleting a file
|
||||
type RemoveFileRequest struct {
|
||||
Token string `json:"token"`
|
||||
FileCodes string `json:"file_codes"`
|
||||
}
|
||||
|
||||
// Token represents the authentication token
|
||||
type Token struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,21 @@
|
||||
// Test Uptobox filesystem interface
|
||||
package uptobox_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/uptobox"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
if *fstest.RemoteName == "" {
|
||||
*fstest.RemoteName = "TestUptobox:"
|
||||
}
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: *fstest.RemoteName,
|
||||
NilObject: (*uptobox.Object)(nil),
|
||||
})
|
||||
}
|
||||
@@ -125,7 +125,7 @@ func (ca *CookieAuth) getSPCookie(conf *SharepointSuccessResponse) (*CookieRespo
|
||||
return nil, errors.Wrap(err, "Error while constructing endpoint URL")
|
||||
}
|
||||
|
||||
u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
|
||||
u, err := url.Parse(spRoot.Scheme + "://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Error while constructing login URL")
|
||||
}
|
||||
|
||||
@@ -60,12 +60,12 @@ func init() {
|
||||
Name: "yandex",
|
||||
Description: "Yandex Disk",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
err := oauthutil.Config(ctx, "yandex", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: config.ConfigEncoding,
|
||||
@@ -251,22 +251,22 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
token, err := oauthutil.GetToken(name, m)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't read OAuth token (this should never happen).")
|
||||
return nil, errors.Wrap(err, "couldn't read OAuth token")
|
||||
}
|
||||
if token.RefreshToken == "" {
|
||||
log.Fatalf("Unable to get RefreshToken. If you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend.")
|
||||
return nil, errors.New("unable to get RefreshToken. If you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
|
||||
}
|
||||
if token.TokenType != "OAuth" {
|
||||
token.TokenType = "OAuth"
|
||||
err = oauthutil.PutToken(name, m, token, false)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't save OAuth token (this should never happen).")
|
||||
return nil, errors.Wrap(err, "couldn't save OAuth token")
|
||||
}
|
||||
log.Printf("Automatically upgraded OAuth config.")
|
||||
}
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure Yandex: %v", err)
|
||||
return nil, errors.Wrap(err, "failed to configure Yandex")
|
||||
}
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
|
||||
+19
-12
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -73,37 +72,41 @@ func init() {
|
||||
Name: "zoho",
|
||||
Description: "Zoho",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
// Need to setup region before configuring oauth
|
||||
setupRegion(m)
|
||||
err := setupRegion(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opt := oauthutil.Options{
|
||||
// No refresh token unless ApprovalForce is set
|
||||
OAuth2Opts: []oauth2.AuthCodeOption{oauth2.ApprovalForce},
|
||||
}
|
||||
if err := oauthutil.Config(ctx, "zoho", name, m, oauthConfig, &opt); err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
// We need to rewrite the token type to "Zoho-oauthtoken" because Zoho wants
|
||||
// it's own custom type
|
||||
token, err := oauthutil.GetToken(name, m)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read token: %v", err)
|
||||
return errors.Wrap(err, "failed to read token")
|
||||
}
|
||||
if token.TokenType != "Zoho-oauthtoken" {
|
||||
token.TokenType = "Zoho-oauthtoken"
|
||||
err = oauthutil.PutToken(name, m, token, false)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return errors.Wrap(err, "failed to configure token")
|
||||
}
|
||||
}
|
||||
|
||||
if fs.GetConfig(ctx).AutoConfirm {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = setupRoot(ctx, name, m); err != nil {
|
||||
log.Fatalf("Failed to configure root directory: %v", err)
|
||||
return errors.Wrap(err, "failed to configure root directory")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Options: append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "region",
|
||||
@@ -164,15 +167,16 @@ type Object struct {
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
func setupRegion(m configmap.Mapper) {
|
||||
func setupRegion(m configmap.Mapper) error {
|
||||
region, ok := m.Get("region")
|
||||
if !ok || region == "" {
|
||||
log.Fatalf("No region set\n")
|
||||
return errors.New("no region set")
|
||||
}
|
||||
rootURL = fmt.Sprintf("https://workdrive.zoho.%s/api/v1", region)
|
||||
accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region)
|
||||
oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
|
||||
oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -208,7 +212,7 @@ func listWorkspaces(ctx context.Context, teamID string, srv *rest.Client) ([]api
|
||||
func setupRoot(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load oAuthClient: %s", err)
|
||||
return errors.Wrap(err, "failed to load oAuthClient")
|
||||
}
|
||||
authSrv := rest.NewClient(oAuthClient).SetRoot(accountsURL)
|
||||
opts := rest.Opts{
|
||||
@@ -377,7 +381,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err := configstruct.Set(m, opt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setupRegion(m)
|
||||
err := setupRegion(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = parsePath(root)
|
||||
oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
|
||||
|
||||
@@ -62,6 +62,7 @@ docs = [
|
||||
"sftp.md",
|
||||
"sugarsync.md",
|
||||
"tardigrade.md",
|
||||
"uptobox.md",
|
||||
"union.md",
|
||||
"webdav.md",
|
||||
"yandex.md",
|
||||
|
||||
@@ -54,6 +54,7 @@ import (
|
||||
_ "github.com/rclone/rclone/cmd/size"
|
||||
_ "github.com/rclone/rclone/cmd/sync"
|
||||
_ "github.com/rclone/rclone/cmd/test"
|
||||
_ "github.com/rclone/rclone/cmd/test/changenotify"
|
||||
_ "github.com/rclone/rclone/cmd/test/histogram"
|
||||
_ "github.com/rclone/rclone/cmd/test/info"
|
||||
_ "github.com/rclone/rclone/cmd/test/makefiles"
|
||||
|
||||
+2
-2
@@ -49,7 +49,7 @@ var (
|
||||
cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file")
|
||||
memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file")
|
||||
statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)")
|
||||
dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s")
|
||||
dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second")
|
||||
version bool
|
||||
retries = flags.IntP("retries", "", 3, "Retry operations this many times if they fail")
|
||||
retriesInterval = flags.DurationP("retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)")
|
||||
@@ -400,7 +400,7 @@ func initConfig() {
|
||||
configflags.SetFlags(ci)
|
||||
|
||||
// Load the config
|
||||
configfile.LoadConfig(ctx)
|
||||
configfile.Install()
|
||||
|
||||
// Start accounting
|
||||
accounting.Start(ctx)
|
||||
|
||||
+13
-4
@@ -22,6 +22,7 @@ func init() {
|
||||
cmd.Root.AddCommand(configCommand)
|
||||
configCommand.AddCommand(configEditCommand)
|
||||
configCommand.AddCommand(configFileCommand)
|
||||
configCommand.AddCommand(configTouchCommand)
|
||||
configCommand.AddCommand(configShowCommand)
|
||||
configCommand.AddCommand(configDumpCommand)
|
||||
configCommand.AddCommand(configProvidersCommand)
|
||||
@@ -41,9 +42,9 @@ var configCommand = &cobra.Command{
|
||||
remotes and manage existing ones. You may also set or remove a
|
||||
password to protect your configuration.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
cmd.CheckArgs(0, 0, command, args)
|
||||
config.EditConfig(context.Background())
|
||||
return config.EditConfig(context.Background())
|
||||
},
|
||||
}
|
||||
|
||||
@@ -63,6 +64,15 @@ var configFileCommand = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var configTouchCommand = &cobra.Command{
|
||||
Use: "touch",
|
||||
Short: `Ensure configuration file exists.`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(0, 0, command, args)
|
||||
config.SaveConfig()
|
||||
},
|
||||
}
|
||||
|
||||
var configShowCommand = &cobra.Command{
|
||||
Use: "show [<remote>]",
|
||||
Short: `Print (decrypted) config file, or the config for a single remote.`,
|
||||
@@ -262,8 +272,7 @@ This normally means going through the interactive oauth flow again.
|
||||
if fsInfo.Config == nil {
|
||||
return errors.Errorf("%s: doesn't support Reconnect", configName)
|
||||
}
|
||||
fsInfo.Config(ctx, configName, config)
|
||||
return nil
|
||||
return fsInfo.Config(ctx, configName, config)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -36,8 +36,8 @@ If you supply the |--rmdirs| flag, it will remove all empty directories along wi
|
||||
You can also use the separate command |rmdir| or |rmdirs| to
|
||||
delete empty directories only.
|
||||
|
||||
For example, to delete all files bigger than 100MBytes, you may first want to check what
|
||||
would be deleted (use either):
|
||||
For example, to delete all files bigger than 100 MiB, you may first want to
|
||||
check what would be deleted (use either):
|
||||
|
||||
rclone --min-size 100M lsl remote:path
|
||||
rclone --dry-run --min-size 100M delete remote:path
|
||||
@@ -46,8 +46,8 @@ Then proceed with the actual delete:
|
||||
|
||||
rclone --min-size 100M delete remote:path
|
||||
|
||||
That reads "delete everything with a minimum size of 100 MB", hence
|
||||
delete all files bigger than 100MBytes.
|
||||
That reads "delete everything with a minimum size of 100 MiB", hence
|
||||
delete all files bigger than 100 MiB.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
|--dry-run| or the |--interactive|/|-i| flag.
|
||||
|
||||
+33
-14
@@ -206,9 +206,9 @@ When that happens, it is the user's responsibility to stop the mount manually.
|
||||
The size of the mounted file system will be set according to information retrieved
|
||||
from the remote, the same as returned by the [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
command. Remotes with unlimited storage may report the used size only,
|
||||
then an additional 1PB of free space is assumed. If the remote does not
|
||||
then an additional 1 PiB of free space is assumed. If the remote does not
|
||||
[support](https://rclone.org/overview/#optional-features) the about feature
|
||||
at all, then 1PB is set as both the total and the free size.
|
||||
at all, then 1 PiB is set as both the total and the free size.
|
||||
|
||||
**Note**: As of |rclone| 1.52.2, |rclone mount| now requires Go version 1.13
|
||||
or newer on some platforms depending on the underlying FUSE library in use.
|
||||
@@ -342,19 +342,38 @@ by specifying |-o FileSecurity="D:P(A;;FA;;;OW)"|, for file all access (FA) to t
|
||||
|
||||
#### Windows caveats
|
||||
|
||||
Note that drives created as Administrator are not visible by other
|
||||
accounts (including the account that was elevated as
|
||||
Administrator). So if you start a Windows drive from an Administrative
|
||||
Command Prompt and then try to access the same drive from Explorer
|
||||
(which does not run as Administrator), you will not be able to see the
|
||||
new drive.
|
||||
Drives created as Administrator are not visible to other accounts,
|
||||
not even an account that was elevated to Administrator with the
|
||||
User Account Control (UAC) feature. A result of this is that if you mount
|
||||
to a drive letter from a Command Prompt run as Administrator, and then try
|
||||
to access the same drive from Windows Explorer (which does not run as
|
||||
Administrator), you will not be able to see the mounted drive.
|
||||
|
||||
The easiest way around this is to start the drive from a normal
|
||||
command prompt. It is also possible to start a drive from the SYSTEM
|
||||
account (using [the WinFsp.Launcher
|
||||
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
|
||||
which creates drives accessible for everyone on the system or
|
||||
alternatively using [the nssm service manager](https://nssm.cc/usage).
|
||||
If you don't need to access the drive from applications running with
|
||||
administrative privileges, the easiest way around this is to always
|
||||
create the mount from a non-elevated command prompt.
|
||||
|
||||
To make mapped drives available to the user account that created them
|
||||
regardless if elevated or not, there is a special Windows setting called
|
||||
[linked connections](https://docs.microsoft.com/en-us/troubleshoot/windows-client/networking/mapped-drives-not-available-from-elevated-command#detail-to-configure-the-enablelinkedconnections-registry-entry)
|
||||
that can be enabled.
|
||||
|
||||
It is also possible to make a drive mount available to everyone on the system,
|
||||
by running the process creating it as the built-in SYSTEM account.
|
||||
There are several ways to do this: One is to use the command-line
|
||||
utility [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec),
|
||||
from Microsoft's Sysinternals suite, which has option |-s| to start
|
||||
processes as the SYSTEM account. Another alternative is to run the mount
|
||||
command from a Windows Scheduled Task, or a Windows Service, configured
|
||||
to run as the SYSTEM account. A third alternative is to use the
|
||||
[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)).
|
||||
Note that when running rclone as another user, it will not use
|
||||
the configuration file from your profile unless you tell it to
|
||||
with the [|--config|](https://rclone.org/docs/#config-config-file) option.
|
||||
Read more in the [install documentation](https://rclone.org/install/).
|
||||
|
||||
Note that mapping to a directory path, instead of a drive letter,
|
||||
does not suffer from the same limitations.
|
||||
|
||||
### Limitations
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
|
||||
func TestRc(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
configfile.LoadConfig(ctx)
|
||||
configfile.Install()
|
||||
mount := rc.Calls.Get("mount/mount")
|
||||
assert.NotNil(t, mount)
|
||||
unmount := rc.Calls.Get("mount/unmount")
|
||||
|
||||
+20
-10
@@ -373,7 +373,7 @@ func (u *UI) Draw() error {
|
||||
extras := ""
|
||||
if u.showCounts {
|
||||
if count > 0 {
|
||||
extras += fmt.Sprintf("%8v ", fs.SizeSuffix(count))
|
||||
extras += fmt.Sprintf("%8v ", fs.CountSuffix(count))
|
||||
} else {
|
||||
extras += " "
|
||||
}
|
||||
@@ -385,9 +385,9 @@ func (u *UI) Draw() error {
|
||||
}
|
||||
if u.showDirAverageSize {
|
||||
if averageSize > 0 {
|
||||
extras += fmt.Sprintf("%8v ", fs.SizeSuffix(int64(averageSize)))
|
||||
extras += fmt.Sprintf("%9v ", fs.SizeSuffix(int64(averageSize)))
|
||||
} else {
|
||||
extras += " "
|
||||
extras += " "
|
||||
}
|
||||
|
||||
}
|
||||
@@ -406,7 +406,7 @@ func (u *UI) Draw() error {
|
||||
}
|
||||
extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
|
||||
}
|
||||
Linef(0, y, w, fg, bg, ' ', "%c %8v %s%c%s%s", fileFlag, fs.SizeSuffix(size), extras, mark, path.Base(entry.Remote()), message)
|
||||
Linef(0, y, w, fg, bg, ' ', "%c %9v %s%c%s%s", fileFlag, fs.SizeSuffix(size), extras, mark, path.Base(entry.Remote()), message)
|
||||
y++
|
||||
}
|
||||
}
|
||||
@@ -485,11 +485,15 @@ func (u *UI) removeEntry(pos int) {
|
||||
|
||||
// delete the entry at the current position
|
||||
func (u *UI) delete() {
|
||||
if u.d == nil || len(u.entries) == 0 {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
dirPos := u.sortPerm[u.dirPosMap[u.path].entry]
|
||||
entry := u.entries[dirPos]
|
||||
cursorPos := u.dirPosMap[u.path]
|
||||
dirPos := u.sortPerm[cursorPos.entry]
|
||||
dirEntry := u.entries[dirPos]
|
||||
u.boxMenu = []string{"cancel", "confirm"}
|
||||
if obj, isFile := entry.(fs.Object); isFile {
|
||||
if obj, isFile := dirEntry.(fs.Object); isFile {
|
||||
u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
|
||||
if o != 1 {
|
||||
return "Aborted!", nil
|
||||
@@ -499,27 +503,33 @@ func (u *UI) delete() {
|
||||
return "", err
|
||||
}
|
||||
u.removeEntry(dirPos)
|
||||
if cursorPos.entry >= len(u.entries) {
|
||||
u.move(-1) // move back onto a valid entry
|
||||
}
|
||||
return "Successfully deleted file!", nil
|
||||
}
|
||||
u.popupBox([]string{
|
||||
"Delete this file?",
|
||||
u.fsName + entry.String()})
|
||||
u.fsName + dirEntry.String()})
|
||||
} else {
|
||||
u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) {
|
||||
if o != 1 {
|
||||
return "Aborted!", nil
|
||||
}
|
||||
err := operations.Purge(ctx, f, entry.String())
|
||||
err := operations.Purge(ctx, f, dirEntry.String())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u.removeEntry(dirPos)
|
||||
if cursorPos.entry >= len(u.entries) {
|
||||
u.move(-1) // move back onto a valid entry
|
||||
}
|
||||
return "Successfully purged folder!", nil
|
||||
}
|
||||
u.popupBox([]string{
|
||||
"Purge this directory?",
|
||||
"ALL files in it will be deleted",
|
||||
u.fsName + entry.String()})
|
||||
u.fsName + dirEntry.String()})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+15
-1
@@ -7,12 +7,19 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
size = int64(-1)
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.Int64VarP(cmdFlags, &size, "size", "", size, "File size hint to preallocate")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -37,6 +44,13 @@ must fit into RAM. The cutoff needs to be small enough to adhere
|
||||
the limits of your remote, please see there. Generally speaking,
|
||||
setting this cutoff too high will decrease your performance.
|
||||
|
||||
Use the |--size| flag to preallocate the file in advance at the remote end
|
||||
and actually stream it, even if remote backend doesn't support streaming.
|
||||
|
||||
|--size| should be the exact size of the input stream in bytes. If the
|
||||
size of the stream is different in length to the |--size| passed in
|
||||
then the transfer will likely fail.
|
||||
|
||||
Note that the upload can also not be retried because the data is
|
||||
not kept around until the upload succeeds. If you need to transfer
|
||||
a lot of data, you're better off caching locally and then
|
||||
@@ -51,7 +65,7 @@ a lot of data, you're better off caching locally and then
|
||||
|
||||
fdst, dstFileName := cmd.NewFsDstFile(args)
|
||||
cmd.Run(false, false, command, func() error {
|
||||
_, err := operations.Rcat(context.Background(), fdst, dstFileName, os.Stdin, time.Now())
|
||||
_, err := operations.RcatSize(context.Background(), fdst, dstFileName, os.Stdin, size, time.Now())
|
||||
return err
|
||||
})
|
||||
},
|
||||
|
||||
@@ -41,7 +41,7 @@ func startServer(t *testing.T, f fs.Fs) {
|
||||
}
|
||||
|
||||
func TestInit(t *testing.T) {
|
||||
configfile.LoadConfig(context.Background())
|
||||
configfile.Install()
|
||||
|
||||
f, err := fs.NewFs(context.Background(), "testdata/files")
|
||||
l, _ := f.List(context.Background(), "")
|
||||
|
||||
@@ -61,7 +61,7 @@ var (
|
||||
func TestInit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// Configure the remote
|
||||
configfile.LoadConfig(context.Background())
|
||||
configfile.Install()
|
||||
// fs.Config.LogLevel = fs.LogLevelDebug
|
||||
// fs.Config.DumpHeaders = true
|
||||
// fs.Config.DumpBodies = true
|
||||
|
||||
@@ -367,7 +367,7 @@ footer {
|
||||
}
|
||||
};
|
||||
function readableFileSize(size) {
|
||||
var units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
||||
var units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
|
||||
var i = 0;
|
||||
while(size >= 1024) {
|
||||
size /= 1024;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
@@ -65,8 +64,7 @@ func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest {
|
||||
|
||||
// TestResticHandler runs tests on the restic handler code, especially in append-only mode.
|
||||
func TestResticHandler(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
configfile.LoadConfig(ctx)
|
||||
configfile.Install()
|
||||
buf := make([]byte, 32)
|
||||
_, err := io.ReadFull(rand.Reader, buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ var commandDefinition = &cobra.Command{
|
||||
}
|
||||
|
||||
fmt.Printf("Total objects: %d\n", results.Count)
|
||||
fmt.Printf("Total size: %s (%d Bytes)\n", fs.SizeSuffix(results.Bytes).Unit("Bytes"), results.Bytes)
|
||||
fmt.Printf("Total size: %s (%d bytes)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// Package changenotify tests rclone's changenotify support
|
||||
package changenotify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/test"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
pollInterval = 10 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
test.Command.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.DurationVarP(cmdFlags, &pollInterval, "poll-interval", "", pollInterval, "Time to wait between polling for changes.")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "changenotify remote:",
|
||||
Short: `Log any change notify requests for the remote passed in.`,
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
f := cmd.NewFsSrc(args)
|
||||
ctx := context.Background()
|
||||
|
||||
// Start polling function
|
||||
features := f.Features()
|
||||
if do := features.ChangeNotify; do != nil {
|
||||
pollChan := make(chan time.Duration)
|
||||
do(ctx, changeNotify, pollChan)
|
||||
pollChan <- pollInterval
|
||||
fs.Logf(nil, "Waiting for changes, polling every %v", pollInterval)
|
||||
} else {
|
||||
return errors.New("poll-interval is not supported by this remote")
|
||||
}
|
||||
select {}
|
||||
},
|
||||
}
|
||||
|
||||
// changeNotify invalidates the directory cache for the relativePath
|
||||
// passed in.
|
||||
//
|
||||
// if entryType is a directory it invalidates the parent of the directory too.
|
||||
func changeNotify(relativePath string, entryType fs.EntryType) {
|
||||
fs.Logf(nil, "%q: %v", relativePath, entryType)
|
||||
}
|
||||
@@ -3,12 +3,12 @@
|
||||
package makefiles
|
||||
|
||||
import (
|
||||
cryptrand "crypto/rand"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/test"
|
||||
@@ -27,8 +27,10 @@ var (
|
||||
maxFileSize = fs.SizeSuffix(100)
|
||||
minFileNameLength = 4
|
||||
maxFileNameLength = 12
|
||||
seed = int64(1)
|
||||
|
||||
// Globals
|
||||
randSource *rand.Rand
|
||||
directoriesToCreate int
|
||||
totalDirectories int
|
||||
fileNames = map[string]struct{}{} // keep a note of which file name we've used already
|
||||
@@ -44,6 +46,7 @@ func init() {
|
||||
flags.FVarP(cmdFlags, &maxFileSize, "max-file-size", "", "Maximum size of files to create")
|
||||
flags.IntVarP(cmdFlags, &minFileNameLength, "min-name-length", "", minFileNameLength, "Minimum size of file names")
|
||||
flags.IntVarP(cmdFlags, &maxFileNameLength, "max-name-length", "", maxFileNameLength, "Maximum size of file names")
|
||||
flags.Int64VarP(cmdFlags, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -51,28 +54,36 @@ var commandDefinition = &cobra.Command{
|
||||
Short: `Make a random file hierarchy in <dir>`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
if seed == 0 {
|
||||
seed = time.Now().UnixNano()
|
||||
fs.Logf(nil, "Using random seed = %d", seed)
|
||||
}
|
||||
randSource = rand.New(rand.NewSource(seed))
|
||||
outputDirectory := args[0]
|
||||
directoriesToCreate = numberOfFiles / averageFilesPerDirectory
|
||||
averageSize := (minFileSize + maxFileSize) / 2
|
||||
log.Printf("Creating %d files of average size %v in %d directories in %q.", numberOfFiles, averageSize, directoriesToCreate, outputDirectory)
|
||||
start := time.Now()
|
||||
fs.Logf(nil, "Creating %d files of average size %v in %d directories in %q.", numberOfFiles, averageSize, directoriesToCreate, outputDirectory)
|
||||
root := &dir{name: outputDirectory, depth: 1}
|
||||
for totalDirectories < directoriesToCreate {
|
||||
root.createDirectories()
|
||||
}
|
||||
dirs := root.list("", []string{})
|
||||
totalBytes := int64(0)
|
||||
for i := 0; i < numberOfFiles; i++ {
|
||||
dir := dirs[rand.Intn(len(dirs))]
|
||||
writeFile(dir, fileName())
|
||||
dir := dirs[randSource.Intn(len(dirs))]
|
||||
totalBytes += writeFile(dir, fileName())
|
||||
}
|
||||
log.Printf("Done.")
|
||||
dt := time.Since(start)
|
||||
fs.Logf(nil, "Written %viB in %v at %viB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
|
||||
},
|
||||
}
|
||||
|
||||
// fileName creates a unique random file or directory name
|
||||
func fileName() (name string) {
|
||||
for {
|
||||
length := rand.Intn(maxFileNameLength-minFileNameLength) + minFileNameLength
|
||||
name = random.String(length)
|
||||
length := randSource.Intn(maxFileNameLength-minFileNameLength) + minFileNameLength
|
||||
name = random.StringFn(length, randSource.Intn)
|
||||
if _, found := fileNames[name]; !found {
|
||||
break
|
||||
}
|
||||
@@ -99,7 +110,7 @@ func (d *dir) createDirectories() {
|
||||
}
|
||||
d.children = append(d.children, newDir)
|
||||
totalDirectories++
|
||||
switch rand.Intn(4) {
|
||||
switch randSource.Intn(4) {
|
||||
case 0:
|
||||
if d.depth < maxDepth {
|
||||
newDir.createDirectories()
|
||||
@@ -122,7 +133,7 @@ func (d *dir) list(path string, output []string) []string {
|
||||
}
|
||||
|
||||
// writeFile writes a random file at dir/name
|
||||
func writeFile(dir, name string) {
|
||||
func writeFile(dir, name string) int64 {
|
||||
err := os.MkdirAll(dir, 0777)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to make directory %q: %v", dir, err)
|
||||
@@ -132,8 +143,8 @@ func writeFile(dir, name string) {
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open file %q: %v", path, err)
|
||||
}
|
||||
size := rand.Int63n(int64(maxFileSize-minFileSize)) + int64(minFileSize)
|
||||
_, err = io.CopyN(fd, cryptrand.Reader, size)
|
||||
size := randSource.Int63n(int64(maxFileSize-minFileSize)) + int64(minFileSize)
|
||||
_, err = io.CopyN(fd, randSource, size)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err)
|
||||
}
|
||||
@@ -141,4 +152,6 @@ func writeFile(dir, name string) {
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to close file %q: %v", path, err)
|
||||
}
|
||||
fs.Infof(path, "Written file size %v", fs.SizeSuffix(size))
|
||||
return size
|
||||
}
|
||||
@@ -26,12 +26,12 @@ func TestVersionWorksWithoutAccessibleConfigFile(t *testing.T) {
|
||||
}
|
||||
// re-wire
|
||||
oldOsStdout := os.Stdout
|
||||
oldConfigPath := config.ConfigPath
|
||||
config.ConfigPath = path
|
||||
oldConfigPath := config.GetConfigPath()
|
||||
assert.NoError(t, config.SetConfigPath(path))
|
||||
os.Stdout = nil
|
||||
defer func() {
|
||||
os.Stdout = oldOsStdout
|
||||
config.ConfigPath = oldConfigPath
|
||||
assert.NoError(t, config.SetConfigPath(oldConfigPath))
|
||||
}()
|
||||
|
||||
cmd.Root.SetArgs([]string{"version"})
|
||||
|
||||
@@ -152,6 +152,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}}
|
||||
{{< provider name="Tardigrade" home="https://tardigrade.io/" config="/tardigrade/" >}}
|
||||
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
|
||||
{{< provider name="Uptobox" home="https://uptobox.com" config="/uptobox/" >}}
|
||||
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
|
||||
{{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
|
||||
{{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}
|
||||
|
||||
@@ -227,16 +227,16 @@ Checkpoint for internal polling (debug).
|
||||
|
||||
#### --acd-upload-wait-per-gb
|
||||
|
||||
Additional time per GB to wait after a failed complete upload to see if it appears.
|
||||
Additional time per GiB to wait after a failed complete upload to see if it appears.
|
||||
|
||||
Sometimes Amazon Drive gives an error when a file has been fully
|
||||
uploaded but the file appears anyway after a little while. This
|
||||
happens sometimes for files over 1GB in size and nearly every time for
|
||||
files bigger than 10GB. This parameter controls the time rclone waits
|
||||
happens sometimes for files over 1 GiB in size and nearly every time for
|
||||
files bigger than 10 GiB. This parameter controls the time rclone waits
|
||||
for the file to appear.
|
||||
|
||||
The default value for this parameter is 3 minutes per GB, so by
|
||||
default it will wait 3 minutes for every GB uploaded to see if the
|
||||
The default value for this parameter is 3 minutes per GiB, so by
|
||||
default it will wait 3 minutes for every GiB uploaded to see if the
|
||||
file appears.
|
||||
|
||||
You can disable this feature by setting it to 0. This may cause
|
||||
@@ -260,7 +260,7 @@ Files >= this size will be downloaded via their tempLink.
|
||||
|
||||
Files this size or more will be downloaded via their "tempLink". This
|
||||
is to work around a problem with Amazon Drive which blocks downloads
|
||||
of files bigger than about 10GB. The default for this is 9GB which
|
||||
of files bigger than about 10 GiB. The default for this is 9 GiB which
|
||||
shouldn't need to be changed.
|
||||
|
||||
To download files above this threshold, rclone requests a "tempLink"
|
||||
@@ -299,7 +299,7 @@ Amazon Drive has an internal limit of file sizes that can be uploaded
|
||||
to the service. This limit is not officially published, but all files
|
||||
larger than this will fail.
|
||||
|
||||
At the time of writing (Jan 2016) is in the area of 50GB per file.
|
||||
At the time of writing (Jan 2016) is in the area of 50 GiB per file.
|
||||
This means that larger files are likely to fail.
|
||||
|
||||
Unfortunately there is no way for rclone to see that this failure is
|
||||
|
||||
+10
-1
@@ -372,7 +372,7 @@ put them back in again.` >}}
|
||||
* Fred <fred@creativeprojects.tech>
|
||||
* Sébastien Gross <renard@users.noreply.github.com>
|
||||
* Maxime Suret <11944422+msuret@users.noreply.github.com>
|
||||
* Caleb Case <caleb@storj.io>
|
||||
* Caleb Case <caleb@storj.io> <calebcase@gmail.com>
|
||||
* Ben Zenker <imbenzenker@gmail.com>
|
||||
* Martin Michlmayr <tbm@cyrius.com>
|
||||
* Brandon McNama <bmcnama@pagerduty.com>
|
||||
@@ -479,3 +479,12 @@ put them back in again.` >}}
|
||||
* x0b <x0bdev@gmail.com>
|
||||
* CERN through the CS3MESH4EOSC Project
|
||||
* Nick Gaya <nicholasgaya+github@gmail.com>
|
||||
* Ashok Gelal <401055+ashokgelal@users.noreply.github.com>
|
||||
* Dominik Mydlil <dominik.mydlil@outlook.com>
|
||||
* Nazar Mishturak <nazarmx@gmail.com>
|
||||
* Ansh Mittal <iamAnshMittal@gmail.com>
|
||||
* noabody <noabody@yahoo.com>
|
||||
* OleFrost <82263101+olefrost@users.noreply.github.com>
|
||||
* Kenny Parsons <kennyparsons93@gmail.com>
|
||||
* Jeffrey Tolar <tolar.jeffrey@gmail.com>
|
||||
* jtagcat <git-514635f7@jtag.cat>
|
||||
@@ -269,7 +269,7 @@ Leave blank normally.
|
||||
|
||||
#### --azureblob-upload-cutoff
|
||||
|
||||
Cutoff for switching to chunked upload (<= 256MB). (Deprecated)
|
||||
Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)
|
||||
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_AZUREBLOB_UPLOAD_CUTOFF
|
||||
@@ -278,7 +278,7 @@ Cutoff for switching to chunked upload (<= 256MB). (Deprecated)
|
||||
|
||||
#### --azureblob-chunk-size
|
||||
|
||||
Upload chunk size (<= 100MB).
|
||||
Upload chunk size (<= 100 MiB).
|
||||
|
||||
Note that this is stored in memory and there may be up to
|
||||
"--transfers" chunks stored at once in memory.
|
||||
|
||||
+4
-9
@@ -155,8 +155,8 @@ depending on your hardware, how big the files are, how much you want
|
||||
to load your computer, etc. The default of `--transfers 4` is
|
||||
definitely too low for Backblaze B2 though.
|
||||
|
||||
Note that uploading big files (bigger than 200 MB by default) will use
|
||||
a 96 MB RAM buffer by default. There can be at most `--transfers` of
|
||||
Note that uploading big files (bigger than 200 MiB by default) will use
|
||||
a 96 MiB RAM buffer by default. There can be at most `--transfers` of
|
||||
these in use at any moment, so this sets the upper limit on the memory
|
||||
used.
|
||||
|
||||
@@ -172,11 +172,6 @@ the file instead of hiding it.
|
||||
Old versions of files, where available, are visible using the
|
||||
`--b2-versions` flag.
|
||||
|
||||
**NB** Note that `--b2-versions` does not work with crypt at the
|
||||
moment [#1627](https://github.com/rclone/rclone/issues/1627). Using
|
||||
[--backup-dir](/docs/#backup-dir-dir) with rclone is the recommended
|
||||
way of working around this.
|
||||
|
||||
If you wish to remove all the old versions then you can use the
|
||||
`rclone cleanup remote:bucket` command which will delete all the old
|
||||
versions of files, leaving the current ones intact. You can also
|
||||
@@ -406,7 +401,7 @@ Cutoff for switching to chunked upload.
|
||||
|
||||
Files above this size will be uploaded in chunks of "--b2-chunk-size".
|
||||
|
||||
This value should be set no larger than 4.657GiB (== 5GB).
|
||||
This value should be set no larger than 4.657 GiB (== 5 GB).
|
||||
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_B2_UPLOAD_CUTOFF
|
||||
@@ -420,7 +415,7 @@ Cutoff for switching to multipart copy
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 4.6GB.
|
||||
The minimum is 0 and the maximum is 4.6 GiB.
|
||||
|
||||
- Config: copy_cutoff
|
||||
- Env Var: RCLONE_B2_COPY_CUTOFF
|
||||
|
||||
+3
-3
@@ -225,10 +225,10 @@ as they can't be used in JSON strings.
|
||||
|
||||
### Transfers ###
|
||||
|
||||
For files above 50MB rclone will use a chunked transfer. Rclone will
|
||||
For files above 50 MiB rclone will use a chunked transfer. Rclone will
|
||||
upload up to `--transfers` chunks at the same time (shared among all
|
||||
the multipart uploads). Chunks are buffered in memory and are
|
||||
normally 8MB so increasing `--transfers` will increase memory use.
|
||||
normally 8 MiB so increasing `--transfers` will increase memory use.
|
||||
|
||||
### Deleting files ###
|
||||
|
||||
@@ -369,7 +369,7 @@ Fill in for rclone to use a non root folder as its starting point.
|
||||
|
||||
#### --box-upload-cutoff
|
||||
|
||||
Cutoff for switching to multipart upload (>= 50MB).
|
||||
Cutoff for switching to multipart upload (>= 50 MiB).
|
||||
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_BOX_UPLOAD_CUTOFF
|
||||
|
||||
+13
-13
@@ -70,11 +70,11 @@ password:
|
||||
The size of a chunk. Lower value good for slow connections but can affect seamless reading.
|
||||
Default: 5M
|
||||
Choose a number from below, or type in your own value
|
||||
1 / 1MB
|
||||
\ "1m"
|
||||
2 / 5 MB
|
||||
1 / 1 MiB
|
||||
\ "1M"
|
||||
2 / 5 MiB
|
||||
\ "5M"
|
||||
3 / 10 MB
|
||||
3 / 10 MiB
|
||||
\ "10M"
|
||||
chunk_size> 2
|
||||
How much time should object info (file size, file hashes, etc.) be stored in cache. Use a very high value if you don't plan on changing the source FS from outside the cache.
|
||||
@@ -91,11 +91,11 @@ info_age> 2
|
||||
The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted.
|
||||
Default: 10G
|
||||
Choose a number from below, or type in your own value
|
||||
1 / 500 MB
|
||||
1 / 500 MiB
|
||||
\ "500M"
|
||||
2 / 1 GB
|
||||
2 / 1 GiB
|
||||
\ "1G"
|
||||
3 / 10 GB
|
||||
3 / 10 GiB
|
||||
\ "10G"
|
||||
chunk_total_size> 3
|
||||
Remote config
|
||||
@@ -364,11 +364,11 @@ will need to be cleared or unexpected EOF errors will occur.
|
||||
- Default: 5M
|
||||
- Examples:
|
||||
- "1m"
|
||||
- 1MB
|
||||
- 1 MiB
|
||||
- "5M"
|
||||
- 5 MB
|
||||
- 5 MiB
|
||||
- "10M"
|
||||
- 10 MB
|
||||
- 10 MiB
|
||||
|
||||
#### --cache-info-age
|
||||
|
||||
@@ -401,11 +401,11 @@ oldest chunks until it goes under this value.
|
||||
- Default: 10G
|
||||
- Examples:
|
||||
- "500M"
|
||||
- 500 MB
|
||||
- 500 MiB
|
||||
- "1G"
|
||||
- 1 GB
|
||||
- 1 GiB
|
||||
- "10G"
|
||||
- 10 GB
|
||||
- 10 GiB
|
||||
|
||||
### Advanced Options
|
||||
|
||||
|
||||
@@ -5,6 +5,44 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.55.1 - 2021-04-26
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.55.0...v1.55.1)
|
||||
|
||||
* Bug Fixes
|
||||
* selfupdate
|
||||
* Dont detect FUSE if build is static (Ivan Andreev)
|
||||
* Add build tag noselfupdate (Ivan Andreev)
|
||||
* sync: Fix incorrect error reported by graceful cutoff (Nick Craig-Wood)
|
||||
* install.sh: fix macOS arm64 download (Nick Craig-Wood)
|
||||
* build: Fix version numbers in android branch builds (Nick Craig-Wood)
|
||||
* docs
|
||||
* Contributing.md: update setup instructions for go1.16 (Nick Gaya)
|
||||
* WinFsp 2021 is out of beta (albertony)
|
||||
* Minor cleanup of space around code section (albertony)
|
||||
* Fixed some typos (albertony)
|
||||
* VFS
|
||||
* Fix a code path which allows dirty data to be removed causing data loss (Nick Craig-Wood)
|
||||
* Compress
|
||||
* Fix compressed name regexp (buengese)
|
||||
* Drive
|
||||
* Fix backend copyid of google doc to directory (Nick Craig-Wood)
|
||||
* Don't open browser when service account... (Ansh Mittal)
|
||||
* Dropbox
|
||||
* Add missing team_data.member scope for use with --impersonate (Nick Craig-Wood)
|
||||
* Fix About after scopes changes - rclone config reconnect needed (Nick Craig-Wood)
|
||||
* Fix Unable to decrypt returned paths from changeNotify (Nick Craig-Wood)
|
||||
* FTP
|
||||
* Fix implicit TLS (Ivan Andreev)
|
||||
* Onedrive
|
||||
* Work around for random "Unable to initialize RPS" errors (OleFrost)
|
||||
* SFTP
|
||||
* Revert sftp library to v1.12.0 from v1.13.0 to fix performance regression (Nick Craig-Wood)
|
||||
* Fix Update ReadFrom failed: failed to send packet: EOF errors (Nick Craig-Wood)
|
||||
* Zoho
|
||||
* Fix error when region isn't set (buengese)
|
||||
* Do not ask for mountpoint twice when using headless setup (buengese)
|
||||
|
||||
## v1.55.0 - 2021-03-31
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.54.0...v1.55.0)
|
||||
|
||||
@@ -43,7 +43,7 @@ Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
remote> remote:path
|
||||
Files larger than chunk size will be split in chunks.
|
||||
Enter a size with suffix k,M,G,T. Press Enter for the default ("2G").
|
||||
Enter a size with suffix K,M,G,T. Press Enter for the default ("2G").
|
||||
chunk_size> 100M
|
||||
Choose how chunker handles hash sums. All modes but "none" require metadata.
|
||||
Enter a string value. Press Enter for the default ("md5").
|
||||
|
||||
@@ -23,7 +23,7 @@ If you supply the `--rmdirs` flag, it will remove all empty directories along wi
|
||||
You can also use the separate command `rmdir` or `rmdirs` to
|
||||
delete empty directories only.
|
||||
|
||||
For example, to delete all files bigger than 100MBytes, you may first want to check what
|
||||
For example, to delete all files bigger than 100 MiByte, you may first want to check what
|
||||
would be deleted (use either):
|
||||
|
||||
rclone --min-size 100M lsl remote:path
|
||||
@@ -33,8 +33,8 @@ Then proceed with the actual delete:
|
||||
|
||||
rclone --min-size 100M delete remote:path
|
||||
|
||||
That reads "delete everything with a minimum size of 100 MB", hence
|
||||
delete all files bigger than 100MBytes.
|
||||
That reads "delete everything with a minimum size of 100 MiB", hence
|
||||
delete all files bigger than 100 MiByte.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
`--dry-run` or the `--interactive`/`-i` flag.
|
||||
|
||||
@@ -56,9 +56,9 @@ When that happens, it is the user's responsibility to stop the mount manually.
|
||||
The size of the mounted file system will be set according to information retrieved
|
||||
from the remote, the same as returned by the [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
command. Remotes with unlimited storage may report the used size only,
|
||||
then an additional 1PB of free space is assumed. If the remote does not
|
||||
then an additional 1 PiB of free space is assumed. If the remote does not
|
||||
[support](https://rclone.org/overview/#optional-features) the about feature
|
||||
at all, then 1PB is set as both the total and the free size.
|
||||
at all, then 1 PiB is set as both the total and the free size.
|
||||
|
||||
**Note**: As of `rclone` 1.52.2, `rclone mount` now requires Go version 1.13
|
||||
or newer on some platforms depending on the underlying FUSE library in use.
|
||||
|
||||
@@ -627,7 +627,7 @@ approximately 2×10⁻³² of re-using a nonce.
|
||||
|
||||
#### Chunk
|
||||
|
||||
Each chunk will contain 64kB of data, except for the last one which
|
||||
Each chunk will contain 64 KiB of data, except for the last one which
|
||||
may have less data. The data chunk is in standard NaCl SecretBox
|
||||
format. SecretBox uses XSalsa20 and Poly1305 to encrypt and
|
||||
authenticate messages.
|
||||
@@ -653,12 +653,12 @@ This uses a 32 byte (256 bit key) key derived from the user password.
|
||||
|
||||
49 bytes total
|
||||
|
||||
1MB (1048576 bytes) file will encrypt to
|
||||
1 MiB (1048576 bytes) file will encrypt to
|
||||
|
||||
* 32 bytes header
|
||||
* 16 chunks of 65568 bytes
|
||||
|
||||
1049120 bytes total (a 0.05% overhead). This is the overhead for big
|
||||
1049120 bytes total (a 0.05% overhead). This is the overhead for big
|
||||
files.
|
||||
|
||||
### Name encryption
|
||||
|
||||
+89
-54
@@ -57,6 +57,7 @@ See the following for detailed instructions for
|
||||
* [SugarSync](/sugarsync/)
|
||||
* [Tardigrade](/tardigrade/)
|
||||
* [Union](/union/)
|
||||
* [Uptobox](/uptobox/)
|
||||
* [WebDAV](/webdav/)
|
||||
* [Yandex Disk](/yandex/)
|
||||
* [Zoho WorkDrive](/zoho/)
|
||||
@@ -227,7 +228,7 @@ adding the `--drive-shared-with-me` parameter to the remote `gdrive:`.
|
||||
rclone lsf "gdrive,shared_with_me:path/to/dir"
|
||||
|
||||
The major advantage to using the connection string style syntax is
|
||||
that it only applies the the remote, not to all the remotes of that
|
||||
that it only applies to the remote, not to all the remotes of that
|
||||
type of the command line. A common confusion is this attempt to copy a
|
||||
file shared on google drive to the normal drive which **does not
|
||||
work** because the `--drive-shared-with-me` flag applies to both the
|
||||
@@ -420,10 +421,10 @@ possibly signed sequence of decimal numbers, each with optional
|
||||
fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid
|
||||
time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
|
||||
|
||||
Options which use SIZE use kByte by default. However, a suffix of `b`
|
||||
for bytes, `k` for kBytes, `M` for MBytes, `G` for GBytes, `T` for
|
||||
TBytes and `P` for PBytes may be used. These are the binary units, e.g.
|
||||
1, 2\*\*10, 2\*\*20, 2\*\*30 respectively.
|
||||
Options which use SIZE use KiByte (multiples of 1024 bytes) by default.
|
||||
However, a suffix of `B` for Byte, `K` for KiByte, `M` for MiByte,
|
||||
`G` for GiByte, `T` for TiByte and `P` for PiByte may be used. These are
|
||||
the binary units, e.g. 1, 2\*\*10, 2\*\*20, 2\*\*30 respectively.
|
||||
|
||||
### --backup-dir=DIR ###
|
||||
|
||||
@@ -466,23 +467,23 @@ This option controls the bandwidth limit. For example
|
||||
|
||||
--bwlimit 10M
|
||||
|
||||
would mean limit the upload and download bandwidth to 10 MByte/s.
|
||||
would mean limit the upload and download bandwidth to 10 MiByte/s.
|
||||
**NB** this is **bytes** per second not **bits** per second. To use a
|
||||
single limit, specify the desired bandwidth in kBytes/s, or use a
|
||||
suffix b|k|M|G. The default is `0` which means to not limit bandwidth.
|
||||
single limit, specify the desired bandwidth in KiByte/s, or use a
|
||||
suffix B|K|M|G|T|P. The default is `0` which means to not limit bandwidth.
|
||||
|
||||
The upload and download bandwidth can be specified seperately, as
|
||||
`--bwlimit UP:DOWN`, so
|
||||
|
||||
--bwlimit 10M:100k
|
||||
|
||||
would mean limit the upload bandwidth to 10 MByte/s and the download
|
||||
bandwidth to 100 kByte/s. Either limit can be "off" meaning no limit, so
|
||||
would mean limit the upload bandwidth to 10 MiByte/s and the download
|
||||
bandwidth to 100 KiByte/s. Either limit can be "off" meaning no limit, so
|
||||
to just limit the upload bandwidth you would use
|
||||
|
||||
--bwlimit 10M:off
|
||||
|
||||
this would limit the upload bandwidth to 10MByte/s but the download
|
||||
this would limit the upload bandwidth to 10 MiByte/s but the download
|
||||
bandwidth would be unlimited.
|
||||
|
||||
When specified as above the bandwidth limits last for the duration of
|
||||
@@ -504,19 +505,19 @@ working hours could be:
|
||||
|
||||
`--bwlimit "08:00,512k 12:00,10M 13:00,512k 18:00,30M 23:00,off"`
|
||||
|
||||
In this example, the transfer bandwidth will be set to 512kBytes/sec
|
||||
at 8am every day. At noon, it will rise to 10MByte/s, and drop back
|
||||
to 512kBytes/sec at 1pm. At 6pm, the bandwidth limit will be set to
|
||||
30MByte/s, and at 11pm it will be completely disabled (full speed).
|
||||
In this example, the transfer bandwidth will be set to 512 KiByte/s
|
||||
at 8am every day. At noon, it will rise to 10 MiByte/s, and drop back
|
||||
to 512 KiByte/sec at 1pm. At 6pm, the bandwidth limit will be set to
|
||||
30 MiByte/s, and at 11pm it will be completely disabled (full speed).
|
||||
Anything between 11pm and 8am will remain unlimited.
|
||||
|
||||
An example of timetable with `WEEKDAY` could be:
|
||||
|
||||
`--bwlimit "Mon-00:00,512 Fri-23:59,10M Sat-10:00,1M Sun-20:00,off"`
|
||||
|
||||
It means that, the transfer bandwidth will be set to 512kBytes/sec on
|
||||
Monday. It will rise to 10MByte/s before the end of Friday. At 10:00
|
||||
on Saturday it will be set to 1MByte/s. From 20:00 on Sunday it will
|
||||
It means that, the transfer bandwidth will be set to 512 KiByte/s on
|
||||
Monday. It will rise to 10 MiByte/s before the end of Friday. At 10:00
|
||||
on Saturday it will be set to 1 MiByte/s. From 20:00 on Sunday it will
|
||||
be unlimited.
|
||||
|
||||
Timeslots without `WEEKDAY` are extended to the whole week. So this
|
||||
@@ -532,10 +533,10 @@ Bandwidth limit apply to the data transfer for all backends. For most
|
||||
backends the directory listing bandwidth is also included (exceptions
|
||||
being the non HTTP backends, `ftp`, `sftp` and `tardigrade`).
|
||||
|
||||
Note that the units are **Bytes/s**, not **Bits/s**. Typically
|
||||
connections are measured in Bits/s - to convert divide by 8. For
|
||||
Note that the units are **Byte/s**, not **bit/s**. Typically
|
||||
connections are measured in bit/s - to convert divide by 8. For
|
||||
example, let's say you have a 10 Mbit/s connection and you wish rclone
|
||||
to use half of it - 5 Mbit/s. This is 5/8 = 0.625MByte/s so you would
|
||||
to use half of it - 5 Mbit/s. This is 5/8 = 0.625 MiByte/s so you would
|
||||
use a `--bwlimit 0.625M` parameter for rclone.
|
||||
|
||||
On Unix systems (Linux, macOS, …) the bandwidth limiter can be toggled by
|
||||
@@ -556,7 +557,7 @@ change the bwlimit dynamically:
|
||||
This option controls per file bandwidth limit. For the options see the
|
||||
`--bwlimit` flag.
|
||||
|
||||
For example use this to allow no transfers to be faster than 1MByte/s
|
||||
For example use this to allow no transfers to be faster than 1 MiByte/s
|
||||
|
||||
--bwlimit-file 1M
|
||||
|
||||
@@ -639,25 +640,54 @@ See `--copy-dest` and `--backup-dir`.
|
||||
|
||||
### --config=CONFIG_FILE ###
|
||||
|
||||
Specify the location of the rclone configuration file.
|
||||
Specify the location of the rclone configuration file, to override
|
||||
the default. E.g. `rclone config --config="rclone.conf"`.
|
||||
|
||||
Normally the config file is in your home directory as a file called
|
||||
`.config/rclone/rclone.conf` (or `.rclone.conf` if created with an
|
||||
older version). If `$XDG_CONFIG_HOME` is set it will be at
|
||||
`$XDG_CONFIG_HOME/rclone/rclone.conf`.
|
||||
The exact default is a bit complex to describe, due to changes
|
||||
introduced through different versions of rclone while preserving
|
||||
backwards compatibility, but in most cases it is as simple as:
|
||||
|
||||
If there is a file `rclone.conf` in the same directory as the rclone
|
||||
executable it will be preferred. This file must be created manually
|
||||
for Rclone to use it, it will never be created automatically.
|
||||
- `%APPDATA%/rclone/rclone.conf` on Windows
|
||||
- `~/.config/rclone/rclone.conf` on other
|
||||
|
||||
The complete logic is as follows: Rclone will look for an existing
|
||||
configuration file in any of the following locations, in priority order:
|
||||
|
||||
1. `rclone.conf` (in program directory, where rclone executable is)
|
||||
2. `%APPDATA%/rclone/rclone.conf` (only on Windows)
|
||||
3. `$XDG_CONFIG_HOME/rclone/rclone.conf` (on all systems, including Windows)
|
||||
4. `~/.config/rclone/rclone.conf` (see below for explanation of ~ symbol)
|
||||
5. `~/.rclone.conf`
|
||||
|
||||
If no existing configuration file is found, then a new one will be created
|
||||
in the following location:
|
||||
|
||||
- On Windows: Location 2 listed above, except in the unlikely event
|
||||
that `APPDATA` is not defined, then location 4 is used instead.
|
||||
- On Unix: Location 3 if `XDG_CONFIG_HOME` is defined, else location 4.
|
||||
- Fallback to location 5 (on all OS), when the rclone directory cannot be
|
||||
created, but if also a home directory was not found then path
|
||||
`.rclone.conf` relative to current working directory will be used as
|
||||
a final resort.
|
||||
|
||||
The `~` symbol in paths above represent the home directory of the current user
|
||||
on any OS, and the value is defined as following:
|
||||
|
||||
- On Windows: `%HOME%` if defined, else `%USERPROFILE%`, or else `%HOMEDRIVE%\%HOMEPATH%`.
|
||||
- On Unix: `$HOME` if defined, else by looking up current user in OS-specific user database
|
||||
(e.g. passwd file), or else use the result from shell command `cd && pwd`.
|
||||
|
||||
If you run `rclone config file` you will see where the default
|
||||
location is for you.
|
||||
|
||||
Use this flag to override the config location, e.g. `rclone
|
||||
--config=".myconfig" config`.
|
||||
The fact that an existing file `rclone.conf` in the same directory
|
||||
as the rclone executable is always preferred, means that it is easy
|
||||
to run in "portable" mode by downloading rclone executable to a
|
||||
writable directory and then create an empty file `rclone.conf` in the
|
||||
same directory.
|
||||
|
||||
If the location is set to empty string `""` or the special value
|
||||
`/notfound`, or the os null device represented by value `NUL` on
|
||||
If the location is set to empty string `""` or path to a file
|
||||
with name `notfound`, or the os null device represented by value `NUL` on
|
||||
Windows and `/dev/null` on Unix systems, then rclone will keep the
|
||||
config file in memory only.
|
||||
|
||||
@@ -740,7 +770,7 @@ which feature does what.
|
||||
|
||||
This flag can be useful for debugging and in exceptional circumstances
|
||||
(e.g. Google Drive limiting the total volume of Server Side Copies to
|
||||
100GB/day).
|
||||
100 GiB/day).
|
||||
|
||||
### --dscp VALUE ###
|
||||
|
||||
@@ -1017,7 +1047,7 @@ This is the maximum allowable backlog of files in a sync/copy/move
|
||||
queued for being checked or transferred.
|
||||
|
||||
This can be set arbitrarily large. It will only use memory when the
|
||||
queue is in use. Note that it will use in the order of N kB of memory
|
||||
queue is in use. Note that it will use in the order of N KiB of memory
|
||||
when the backlog is in use.
|
||||
|
||||
Setting this large allows rclone to calculate how many files are
|
||||
@@ -1146,13 +1176,13 @@ size of the file. To calculate the number of download streams Rclone
|
||||
divides the size of the file by the `--multi-thread-cutoff` and rounds
|
||||
up, up to the maximum set with `--multi-thread-streams`.
|
||||
|
||||
So if `--multi-thread-cutoff 250MB` and `--multi-thread-streams 4` are
|
||||
So if `--multi-thread-cutoff 250M` and `--multi-thread-streams 4` are
|
||||
in effect (the defaults):
|
||||
|
||||
- 0MB..250MB files will be downloaded with 1 stream
|
||||
- 250MB..500MB files will be downloaded with 2 streams
|
||||
- 500MB..750MB files will be downloaded with 3 streams
|
||||
- 750MB+ files will be downloaded with 4 streams
|
||||
- 0..250 MiB files will be downloaded with 1 stream
|
||||
- 250..500 MiB files will be downloaded with 2 streams
|
||||
- 500..750 MiB files will be downloaded with 3 streams
|
||||
- 750+ MiB files will be downloaded with 4 streams
|
||||
|
||||
### --no-check-dest ###
|
||||
|
||||
@@ -1443,14 +1473,14 @@ date formatting syntax.
|
||||
|
||||
### --stats-unit=bits|bytes ###
|
||||
|
||||
By default, data transfer rates will be printed in bytes/second.
|
||||
By default, data transfer rates will be printed in bytes per second.
|
||||
|
||||
This option allows the data rate to be printed in bits/second.
|
||||
This option allows the data rate to be printed in bits per second.
|
||||
|
||||
Data transfer volume will still be reported in bytes.
|
||||
|
||||
The rate is reported as a binary unit, not SI unit. So 1 Mbit/s
|
||||
equals 1,048,576 bits/s and not 1,000,000 bits/s.
|
||||
equals 1,048,576 bit/s and not 1,000,000 bit/s.
|
||||
|
||||
The default is `bytes`.
|
||||
|
||||
@@ -1885,16 +1915,21 @@ password prompts. To do that, pass the parameter
|
||||
of asking for a password if `RCLONE_CONFIG_PASS` doesn't contain
|
||||
a valid password, and `--password-command` has not been supplied.
|
||||
|
||||
Some rclone commands, such as `genautocomplete`, do not require configuration.
|
||||
Nevertheless, rclone will read any configuration file found
|
||||
according to the rules described [above](https://rclone.org/docs/#config-config-file).
|
||||
If an encrypted configuration file is found, this means you will be prompted for
|
||||
password (unless using `--password-command`). To avoid this, you can bypass
|
||||
the loading of the configuration file by overriding the location with an empty
|
||||
string `""` or the special value `/notfound`, or the os null device represented
|
||||
by value `NUL` on Windows and `/dev/null` on Unix systems (before rclone
|
||||
version 1.55 only this null device alternative was supported).
|
||||
E.g. `rclone --config="" genautocomplete bash`.
|
||||
Whenever running commands that may be affected by options in a
|
||||
configuration file, rclone will look for an existing file according
|
||||
to the rules described [above](#config-config-file), and load any it
|
||||
finds. If an encrypted file is found, this includes decrypting it,
|
||||
with the possible consequence of a password prompt. When executing
|
||||
a command line that you know are not actually using anything from such
|
||||
a configuration file, you can avoid it being loaded by overriding the
|
||||
location, e.g. with one of the documented special values for
|
||||
memory-only configuration. Since only backend options can be stored
|
||||
in configuration files, this is normally unnecessary for commands
|
||||
that do not operate on backends, e.g. `genautocomplete`. However,
|
||||
it will be relevant for commands that do operate on backends in
|
||||
general, but are used without referencing a stored remote, e.g.
|
||||
listing local filesystem paths, or
|
||||
[connection strings](#connection-strings): `rclone --config="" ls .`
|
||||
|
||||
Developer options
|
||||
-----------------
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -993,7 +993,7 @@ See: https://github.com/rclone/rclone/issues/3631
|
||||
|
||||
Make upload limit errors be fatal
|
||||
|
||||
At the time of writing it is only possible to upload 750GB of data to
|
||||
At the time of writing it is only possible to upload 750 GiB of data to
|
||||
Google Drive a day (this is an undocumented limit). When this limit is
|
||||
reached Google Drive produces a slightly different error message. When
|
||||
this flag is set it causes these errors to be fatal. These will stop
|
||||
@@ -1014,7 +1014,7 @@ See: https://github.com/rclone/rclone/issues/3857
|
||||
|
||||
Make download limit errors be fatal
|
||||
|
||||
At the time of writing it is only possible to download 10TB of data from
|
||||
At the time of writing it is only possible to download 10 TiB of data from
|
||||
Google Drive a day (this is an undocumented limit). When this limit is
|
||||
reached Google Drive produces a slightly different error message. When
|
||||
this flag is set it causes these errors to be fatal. These will stop
|
||||
@@ -1226,7 +1226,7 @@ Use the -i flag to see what would be copied before copying.
|
||||
|
||||
Drive has quite a lot of rate limiting. This causes rclone to be
|
||||
limited to transferring about 2 files per second only. Individual
|
||||
files may be transferred much faster at 100s of MBytes/s but lots of
|
||||
files may be transferred much faster at 100s of MiByte/s but lots of
|
||||
small files can take a long time.
|
||||
|
||||
Server side copies are also subject to a separate rate limit. If you
|
||||
|
||||
@@ -185,7 +185,7 @@ Any files larger than this will be uploaded in chunks of this size.
|
||||
|
||||
Note that chunks are buffered in memory (one at a time) so rclone can
|
||||
deal with retries. Setting this larger will increase the speed
|
||||
slightly (at most 10% for 128MB in tests) at the cost of using more
|
||||
slightly (at most 10% for 128 MiB in tests) at the cost of using more
|
||||
memory. It can be set smaller if you are tight on memory.
|
||||
|
||||
- Config: chunk_size
|
||||
|
||||
@@ -586,17 +586,17 @@ remote or flag value. The fix then is to quote values containing spaces.
|
||||
### `--min-size` - Don't transfer any file smaller than this
|
||||
|
||||
Controls the minimum size file within the scope of an rclone command.
|
||||
Default units are `kBytes` but abbreviations `k`, `M`, or `G` are valid.
|
||||
Default units are `KiByte` but abbreviations `K`, `M`, `G`, `T` or `P` are valid.
|
||||
|
||||
E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50kByte
|
||||
E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50 KiByte
|
||||
size or larger.
|
||||
|
||||
### `--max-size` - Don't transfer any file larger than this
|
||||
|
||||
Controls the maximum size file within the scope of an rclone command.
|
||||
Default units are `kBytes` but abbreviations `k`, `M`, or `G` are valid.
|
||||
Default units are `KiByte` but abbreviations `K`, `M`, `G`, `T` or `P` are valid.
|
||||
|
||||
E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1GByte
|
||||
E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1 GiByte
|
||||
size or smaller.
|
||||
|
||||
### `--max-age` - Don't transfer any file older than this
|
||||
@@ -650,8 +650,8 @@ E.g. the scope of `rclone sync -i A: B:` can be restricted:
|
||||
|
||||
rclone --min-size 50k --delete-excluded sync A: B:
|
||||
|
||||
All files on `B:` which are less than 50 kBytes are deleted
|
||||
because they are excluded from the rclone sync command.
|
||||
All files on `B:` which are less than 50 KiByte are deleted
|
||||
because they are excluded from the rclone sync command.
|
||||
|
||||
### `--dump filters` - dump the filters to the output
|
||||
|
||||
|
||||
@@ -18,8 +18,8 @@ These flags are available for every command.
|
||||
--backup-dir string Make backups into hierarchy based in DIR.
|
||||
--bind string Local address to bind to for outgoing connections, IPv4, IPv6 or name.
|
||||
--buffer-size SizeSuffix In memory buffer size when reading files for each --transfer. (default 16M)
|
||||
--bwlimit BwTimetable Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.
|
||||
--bwlimit-file BwTimetable Bandwidth limit per file in kBytes/s, or use suffix b|k|M|G or a full timetable.
|
||||
--bwlimit BwTimetable Bandwidth limit in KiByte/s, or use suffix B|K|M|G|T|P or a full timetable.
|
||||
--bwlimit-file BwTimetable Bandwidth limit per file in KiByte/s, or use suffix B|K|M|G|T|P or a full timetable.
|
||||
--ca-cert string CA certificate used to verify servers
|
||||
--cache-dir string Directory rclone will use for caching. (default "$HOME/.cache/rclone")
|
||||
--check-first Do all the checks before starting transfers.
|
||||
@@ -79,12 +79,12 @@ These flags are available for every command.
|
||||
--max-delete int When synchronizing, limit the number of deletes (default -1)
|
||||
--max-depth int If set limits the recursion depth to this. (default -1)
|
||||
--max-duration duration Maximum duration rclone will transfer data for.
|
||||
--max-size SizeSuffix Only transfer files smaller than this in k or suffix b|k|M|G (default off)
|
||||
--max-size SizeSuffix Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off)
|
||||
--max-stats-groups int Maximum number of stats groups to keep in memory. On max oldest is discarded. (default 1000)
|
||||
--max-transfer SizeSuffix Maximum size of data to transfer. (default off)
|
||||
--memprofile string Write memory profile to file
|
||||
--min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off)
|
||||
--min-size SizeSuffix Only transfer files bigger than this in k or suffix b|k|M|G (default off)
|
||||
--min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off)
|
||||
--modify-window duration Max time diff to be considered the same (default 1ns)
|
||||
--multi-thread-cutoff SizeSuffix Use multi-thread downloads for files above this size. (default 250M)
|
||||
--multi-thread-streams int Max number of streams to use for multi-thread downloads. (default 4)
|
||||
@@ -170,12 +170,12 @@ and may be set in the config file.
|
||||
--acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink. (default 9G)
|
||||
--acd-token string OAuth Access Token as a JSON blob.
|
||||
--acd-token-url string Token server url.
|
||||
--acd-upload-wait-per-gb Duration Additional time per GB to wait after a failed complete upload to see if it appears. (default 3m0s)
|
||||
--acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears. (default 3m0s)
|
||||
--alias-remote string Remote or path to alias.
|
||||
--azureblob-access-tier string Access tier of blob: hot, cool or archive.
|
||||
--azureblob-account string Storage Account Name (leave blank to use SAS URL or Emulator)
|
||||
--azureblob-archive-tier-delete Delete archive tier blobs before overwriting.
|
||||
--azureblob-chunk-size SizeSuffix Upload chunk size (<= 100MB). (default 4M)
|
||||
--azureblob-chunk-size SizeSuffix Upload chunk size (<= 100 MiB). (default 4M)
|
||||
--azureblob-disable-checksum Don't store MD5 checksum with object metadata.
|
||||
--azureblob-encoding MultiEncoder This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8)
|
||||
--azureblob-endpoint string Endpoint for the service
|
||||
@@ -189,7 +189,7 @@ and may be set in the config file.
|
||||
--azureblob-public-access string Public access level of a container: blob, container.
|
||||
--azureblob-sas-url string SAS URL for container level access only
|
||||
--azureblob-service-principal-file string Path to file containing credentials for use with a service principal.
|
||||
--azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256MB). (Deprecated)
|
||||
--azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB). (Deprecated)
|
||||
--azureblob-use-emulator Uses local storage emulator if provided as 'true' (leave blank if using real azure storage endpoint)
|
||||
--azureblob-use-msi Use a managed service identity to authenticate (only works in Azure)
|
||||
--b2-account string Account ID or Application Key ID
|
||||
@@ -218,7 +218,7 @@ and may be set in the config file.
|
||||
--box-root-folder-id string Fill in for rclone to use a non root folder as its starting point.
|
||||
--box-token string OAuth Access Token as a JSON blob.
|
||||
--box-token-url string Token server url.
|
||||
--box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50MB). (default 50M)
|
||||
--box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB). (default 50M)
|
||||
--cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage. (default 1m0s)
|
||||
--cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming.
|
||||
--cache-chunk-path string Directory to cache chunk files. (default "$HOME/.cache/rclone/cache-backend")
|
||||
|
||||
@@ -168,7 +168,7 @@ Leave blank to use the provider defaults.
|
||||
Above this size files will be chunked into a _segments container.
|
||||
|
||||
Above this size files will be chunked into a _segments container. The
|
||||
default for this is 5GB which is its maximum value.
|
||||
default for this is 5 GiB which is its maximum value.
|
||||
|
||||
- Config: chunk_size
|
||||
- Env Var: RCLONE_HUBIC_CHUNK_SIZE
|
||||
@@ -182,7 +182,7 @@ Don't chunk files during streaming upload.
|
||||
When doing streaming uploads (e.g. using rcat or mount) setting this
|
||||
flag will cause the swift backend to not upload chunked files.
|
||||
|
||||
This will limit the maximum upload size to 5GB. However non chunked
|
||||
This will limit the maximum upload size to 5 GiB. However non chunked
|
||||
files are easier to deal with and have an MD5SUM.
|
||||
|
||||
Rclone will still chunk files bigger than chunk_size when doing normal
|
||||
|
||||
@@ -12,6 +12,7 @@ Rclone is a Go program and comes as a single binary file.
|
||||
* [Download](/downloads/) the relevant binary.
|
||||
* Extract the `rclone` or `rclone.exe` binary from the archive
|
||||
* Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
|
||||
* Optionally configure [automatic execution](#autostart).
|
||||
|
||||
See below for some expanded Linux / macOS instructions.
|
||||
|
||||
@@ -226,3 +227,147 @@ Instructions
|
||||
roles:
|
||||
- rclone
|
||||
```
|
||||
|
||||
# Autostart #
|
||||
|
||||
After installing and configuring rclone, as described above, you are ready to use rclone
|
||||
as an interactive command line utility. If your goal is to perform *periodic* operations,
|
||||
such as a regular [sync](https://rclone.org/commands/rclone_sync/), you will probably want
|
||||
to configure your rclone command in your operating system's scheduler. If you need to
|
||||
expose *service*-like features, such as [remote control](https://rclone.org/rc/),
|
||||
[GUI](https://rclone.org/gui/), [serve](https://rclone.org/commands/rclone_serve/)
|
||||
or [mount](https://rclone.org/commands/rclone_move/), you will often want an rclone
|
||||
command always running in the background, and configuring it to run in a service infrastructure
|
||||
may be a better option. Below are some alternatives on how to achieve this on
|
||||
different operating systems.
|
||||
|
||||
NOTE: Before setting up autorun it is highly recommended that you have tested your command
|
||||
manually from a Command Prompt first.
|
||||
|
||||
## Autostart on Windows ##
|
||||
|
||||
The most relevant alternatives for autostart on Windows are:
|
||||
- Run at user log on using the Startup folder
|
||||
- Run at user log on, at system startup or at schedule using Task Scheduler
|
||||
- Run at system startup using Windows service
|
||||
|
||||
### Running in background
|
||||
|
||||
Rclone is a console application, so if not starting from an existing Command Prompt,
|
||||
e.g. when starting rclone.exe from a shortcut, it will open a Command Prompt window.
|
||||
When configuring rclone to run from task scheduler and windows service you are able
|
||||
to set it to run hidden in background. From rclone version 1.54 you can also make it
|
||||
run hidden from anywhere by adding option `--no-console` (it may still flash briefly
|
||||
when the program starts). Since rclone normally writes information and any error
|
||||
messages to the console, you must redirect this to a file to be able to see it.
|
||||
Rclone has a built-in option `--log-file` for that.
|
||||
|
||||
Example command to run a sync in background:
|
||||
```
|
||||
c:\rclone\rclone.exe sync c:\files remote:/files --no-console --log-file c:\rclone\logs\sync_files.txt
|
||||
```
|
||||
|
||||
### User account
|
||||
|
||||
As mentioned in the [mount](https://rclone.org/commands/rclone_move/) documentation,
|
||||
mounted drives created as Administrator are not visible to other accounts, not even the
|
||||
account that was elevated as Administrator. By running the mount command as the
|
||||
built-in `SYSTEM` user account, it will create drives accessible for everyone on
|
||||
the system. Both scheduled task and Windows service can be used to achieve this.
|
||||
|
||||
NOTE: Remember that when rclone runs as the `SYSTEM` user, the user profile
|
||||
that it sees will not be yours. This means that if you normally run rclone with
|
||||
configuration file in the default location, to be able to use the same configuration
|
||||
when running as the system user you must explicitely tell rclone where to find
|
||||
it with the [`--config`](https://rclone.org/docs/#config-config-file) option,
|
||||
or else it will look in the system users profile path (`C:\Windows\System32\config\systemprofile`).
|
||||
To test your command manually from a Command Prompt, you can run it with
|
||||
the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
|
||||
utility from Microsoft's Sysinternals suite, which takes option `-s` to
|
||||
execute commands as the `SYSTEM` user.
|
||||
|
||||
### Start from Startup folder ###
|
||||
|
||||
To quickly execute an rclone command you can simply create a standard
|
||||
Windows Explorer shortcut for the complete rclone command you want to run. If you
|
||||
store this shortcut in the special "Startup" start-menu folder, Windows will
|
||||
automatically run it at login. To open this folder in Windows Explorer,
|
||||
enter path `%APPDATA%\Microsoft\Windows\Start Menu\Programs\Startup`,
|
||||
or `C:\ProgramData\Microsoft\Windows\Start Menu\Programs\StartUp` if you want
|
||||
the command to start for *every* user that logs in.
|
||||
|
||||
This is the easiest approach to autostarting of rclone, but it offers no
|
||||
functionality to set it to run as different user, or to set conditions or
|
||||
actions on certain events. Setting up a scheduled task as described below
|
||||
will often give you better results.
|
||||
|
||||
### Start from Task Scheduler ###
|
||||
|
||||
Task Scheduler is an administrative tool built into Windows, and it can be used to
|
||||
configure rclone to be started automatically in a highly configurable way, e.g.
|
||||
periodically on a schedule, on user log on, or at system startup. It can run
|
||||
be configured to run as the current user, or for a mount command that needs to
|
||||
be available to all users it can run as the `SYSTEM` user.
|
||||
For technical information, see
|
||||
https://docs.microsoft.com/windows/win32/taskschd/task-scheduler-start-page.
|
||||
|
||||
### Run as service ###
|
||||
|
||||
For running rclone at system startup, you can create a Windows service that executes
|
||||
your rclone command, as an alternative to scheduled task configured to run at startup.
|
||||
|
||||
#### Mount command built-in service integration ####
|
||||
|
||||
For mount commands, Rclone has a built-in Windows service integration via the third party
|
||||
WinFsp library it uses. Registering as a regular Windows service easy, as you just have to
|
||||
execute the built-in PowerShell command `New-Service` (requires administrative privileges).
|
||||
|
||||
Example of a PowerShell command that creates a Windows service for mounting
|
||||
some `remote:/files` as drive letter `X:`, for *all* users (service will be running as the
|
||||
local system account):
|
||||
|
||||
```
|
||||
New-Service -Name Rclone -BinaryPathName 'c:\rclone\rclone.exe mount remote:/files X: --config c:\rclone\config\rclone.conf --log-file c:\rclone\logs\mount.txt'
|
||||
```
|
||||
|
||||
The [WinFsp service infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)
|
||||
supports incorporating services for file system implementations, such as rclone,
|
||||
into its own launcher service, as kind of "child services". This has the additional
|
||||
advantage that it also implements a network provider that integrates into
|
||||
Windows standard methods for managing network drives. This is currently not
|
||||
officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later
|
||||
it should be possible through path rewriting as described [here](https://github.com/rclone/rclone/issues/3340).
|
||||
|
||||
#### Third party service integration ####
|
||||
|
||||
To Windows service running any rclone command, the excellent third party utility
|
||||
[NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used.
|
||||
It includes some advanced features such as adjusting process periority, defining
|
||||
process environment variables, redirect to file anything written to stdout, and
|
||||
customized response to different exit codes, with a GUI to configure everything from
|
||||
(although it can also be used from command line ).
|
||||
|
||||
There are also several other alternatives. To mention one more,
|
||||
[WinSW](https://github.com/winsw/winsw), "Windows Service Wrapper", is worth checking out.
|
||||
It requires .NET Framework, but it is preinstalled on newer versions of Windows, and it
|
||||
also provides alternative standalone distributions which includes necessary runtime (.NET 5).
|
||||
WinSW is a command-line only utility, where you have to manually create an XML file with
|
||||
service configuration. This may be a drawback for some, but it can also be an advantage
|
||||
as it is easy to back up and re-use the configuration
|
||||
settings, without having go through manual steps in a GUI. One thing to note is that
|
||||
by default it does not restart the service on error, one have to explicit enable this
|
||||
in the configuration file (via the "onfailure" parameter).
|
||||
|
||||
## Autostart on Linux
|
||||
|
||||
### Start as a service
|
||||
|
||||
To always run rclone in background, relevant for mount commands etc,
|
||||
you can use systemd to set up rclone as a system or user service. Running as a
|
||||
system service ensures that it is run at startup even if the user it is running as
|
||||
has no active session. Running rclone as a user service ensures that it only
|
||||
starts after the configured user has logged into the system.
|
||||
|
||||
### Run periodically from cron
|
||||
|
||||
To run a periodic command, such as a copy/sync, you can set up a cron job.
|
||||
@@ -53,9 +53,9 @@ export XDG_CONFIG_HOME=config
|
||||
#check installed version of rclone to determine if update is necessary
|
||||
version=$(rclone --version 2>>errors | head -n 1)
|
||||
if [ -z "$install_beta" ]; then
|
||||
current_version=$(curl -f https://downloads.rclone.org/version.txt)
|
||||
current_version=$(curl -fsS https://downloads.rclone.org/version.txt)
|
||||
else
|
||||
current_version=$(curl -f https://beta.rclone.org/version.txt)
|
||||
current_version=$(curl -fsS https://beta.rclone.org/version.txt)
|
||||
fi
|
||||
|
||||
if [ "$version" = "$current_version" ]; then
|
||||
@@ -123,7 +123,7 @@ else
|
||||
rclone_zip="rclone-beta-latest-${OS}-${OS_type}.zip"
|
||||
fi
|
||||
|
||||
curl -Of "$download_link"
|
||||
curl -OfsS "$download_link"
|
||||
unzip_dir="tmp_unzip_dir_for_rclone"
|
||||
# there should be an entry in this switch for each element of unzip_tools_list
|
||||
case "$unzip_tool" in
|
||||
|
||||
@@ -172,7 +172,7 @@ like symlinks under Windows).
|
||||
|
||||
If you supply `--copy-links` or `-L` then rclone will follow the
|
||||
symlink and copy the pointed to file or directory. Note that this
|
||||
flag is incompatible with `-links` / `-l`.
|
||||
flag is incompatible with `--links` / `-l`.
|
||||
|
||||
This flag applies to all commands.
|
||||
|
||||
|
||||
@@ -430,7 +430,7 @@ in it will be mapped to `?` instead.
|
||||
|
||||
#### File sizes ####
|
||||
|
||||
The largest allowed file size is 250GB for both OneDrive Personal and OneDrive for Business [(Updated 13 Jan 2021)](https://support.microsoft.com/en-us/office/invalid-file-names-and-file-types-in-onedrive-and-sharepoint-64883a5d-228e-48f5-b3d2-eb39e07630fa?ui=en-us&rs=en-us&ad=us#individualfilesize).
|
||||
The largest allowed file size is 250 GiB for both OneDrive Personal and OneDrive for Business [(Updated 13 Jan 2021)](https://support.microsoft.com/en-us/office/invalid-file-names-and-file-types-in-onedrive-and-sharepoint-64883a5d-228e-48f5-b3d2-eb39e07630fa?ui=en-us&rs=en-us&ad=us#individualfilesize).
|
||||
|
||||
#### Path length ####
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ Here is an overview of the major features of each cloud storage system.
|
||||
| SFTP | MD5, SHA1 ² | Yes | Depends | No | - |
|
||||
| SugarSync | - | No | No | No | - |
|
||||
| Tardigrade | - | Yes | No | No | - |
|
||||
| Uptobox | - | No | No | Yes | - |
|
||||
| WebDAV | MD5, SHA1 ³ | Yes ⁴ | Depends | No | - |
|
||||
| Yandex Disk | MD5 | Yes | No | No | R |
|
||||
| Zoho WorkDrive | - | No | No | No | - |
|
||||
@@ -57,7 +58,7 @@ Here is an overview of the major features of each cloud storage system.
|
||||
|
||||
¹ Dropbox supports [its own custom
|
||||
hash](https://www.dropbox.com/developers/reference/content-hash).
|
||||
This is an SHA256 sum of all the 4MB block SHA256s.
|
||||
This is an SHA256 sum of all the 4 MiB block SHA256s.
|
||||
|
||||
² SFTP supports checksums if the same login has shell access and
|
||||
`md5sum` or `sha1sum` as well as `echo` are in the remote's PATH.
|
||||
@@ -361,6 +362,7 @@ upon backend specific capabilities.
|
||||
| SFTP | No | No | Yes | Yes | No | No | Yes | No | Yes | Yes |
|
||||
| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes |
|
||||
| Tardigrade | Yes † | No | No | No | No | Yes | Yes | No | No | No |
|
||||
| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No |
|
||||
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes |
|
||||
| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes |
|
||||
| Zoho WorkDrive | Yes | Yes | Yes | Yes | No | No | No | No | Yes | Yes |
|
||||
|
||||
@@ -101,7 +101,7 @@ docs](/docs/#fast-list) for more details.
|
||||
### Multipart uploads ###
|
||||
|
||||
rclone supports multipart uploads with QingStor which means that it can
|
||||
upload files bigger than 5GB. Note that files uploaded with multipart
|
||||
upload files bigger than 5 GiB. Note that files uploaded with multipart
|
||||
upload don't have an MD5SUM.
|
||||
|
||||
Note that incomplete multipart uploads older than 24 hours can be
|
||||
@@ -227,7 +227,7 @@ Number of connection retries.
|
||||
Cutoff for switching to chunked upload
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5GB.
|
||||
The minimum is 0 and the maximum is 5 GiB.
|
||||
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_QINGSTOR_UPLOAD_CUTOFF
|
||||
|
||||
+2
-2
@@ -788,8 +788,8 @@ Returns the following values:
|
||||
"eta": estimated time in seconds until file transfer completion
|
||||
"name": name of the file,
|
||||
"percentage": progress of the file transfer in percent,
|
||||
"speed": average speed over the whole transfer in bytes/sec,
|
||||
"speedAvg": current speed in bytes/sec as an exponentially weighted moving average,
|
||||
"speed": average speed over the whole transfer in bytes per second,
|
||||
"speedAvg": current speed in bytes per second as an exponentially weighted moving average,
|
||||
"size": size of the file in bytes
|
||||
}
|
||||
],
|
||||
|
||||
+7
-7
@@ -327,7 +327,7 @@ objects). See the [rclone docs](/docs/#fast-list) for more details.
|
||||
|
||||
`--fast-list` trades off API transactions for memory use. As a rough
|
||||
guide rclone uses 1k of memory per object stored, so using
|
||||
`--fast-list` on a sync of a million objects will use roughly 1 GB of
|
||||
`--fast-list` on a sync of a million objects will use roughly 1 GiB of
|
||||
RAM.
|
||||
|
||||
If you are only copying a small number of files into a big repository
|
||||
@@ -407,13 +407,13 @@ work with the SDK properly:
|
||||
### Multipart uploads ###
|
||||
|
||||
rclone supports multipart uploads with S3 which means that it can
|
||||
upload files bigger than 5GB.
|
||||
upload files bigger than 5 GiB.
|
||||
|
||||
Note that files uploaded *both* with multipart upload *and* through
|
||||
crypt remotes do not have MD5 sums.
|
||||
|
||||
rclone switches from single part uploads to multipart uploads at the
|
||||
point specified by `--s3-upload-cutoff`. This can be a maximum of 5GB
|
||||
point specified by `--s3-upload-cutoff`. This can be a maximum of 5 GiB
|
||||
and a minimum of 0 (ie always upload multipart files).
|
||||
|
||||
The chunk sizes used in the multipart upload are specified by
|
||||
@@ -1412,7 +1412,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
|
||||
Cutoff for switching to chunked upload
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5GB.
|
||||
The minimum is 0 and the maximum is 5 GiB.
|
||||
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_S3_UPLOAD_CUTOFF
|
||||
@@ -1438,9 +1438,9 @@ Rclone will automatically increase the chunk size when uploading a
|
||||
large file of known size to stay below the 10,000 chunks limit.
|
||||
|
||||
Files of unknown size are uploaded with the configured
|
||||
chunk_size. Since the default chunk size is 5MB and there can be at
|
||||
chunk_size. Since the default chunk size is 5 MiB and there can be at
|
||||
most 10,000 chunks, this means that by default the maximum size of
|
||||
a file you can stream upload is 48GB. If you wish to stream upload
|
||||
a file you can stream upload is 48 GiB. If you wish to stream upload
|
||||
larger files then you will need to increase chunk_size.
|
||||
|
||||
- Config: chunk_size
|
||||
@@ -1474,7 +1474,7 @@ Cutoff for switching to multipart copy
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 5GB.
|
||||
The minimum is 0 and the maximum is 5 GiB.
|
||||
|
||||
- Config: copy_cutoff
|
||||
- Env Var: RCLONE_S3_COPY_CUTOFF
|
||||
|
||||
+13
-1
@@ -21,7 +21,10 @@ SSH installations.
|
||||
|
||||
Paths are specified as `remote:path`. If the path does not begin with
|
||||
a `/` it is relative to the home directory of the user. An empty path
|
||||
`remote:` refers to the user's home directory.
|
||||
`remote:` refers to the user's home directory. For example, `rclone lsd remote:`
|
||||
would list the home directory of the user cofigured in the rclone remote config
|
||||
(`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root
|
||||
directory for remote machine (i.e. `/`)
|
||||
|
||||
"Note that some SFTP servers will need the leading / - Synology is a
|
||||
good example of this. rsync.net, on the other hand, requires users to
|
||||
@@ -84,6 +87,10 @@ See all directories in the home directory
|
||||
|
||||
rclone lsd remote:
|
||||
|
||||
See all directories in the root directory
|
||||
|
||||
rclone lsd remote:/
|
||||
|
||||
Make a new directory
|
||||
|
||||
rclone mkdir remote:path/to/directory
|
||||
@@ -97,6 +104,11 @@ excess files in the directory.
|
||||
|
||||
rclone sync -i /home/local/directory remote:directory
|
||||
|
||||
Mount the remote path `/srv/www-data/` to the local path
|
||||
`/mnt/www-data`
|
||||
|
||||
rclone mount remote:/srv/www-data/ /mnt/www-data
|
||||
|
||||
### SSH Authentication ###
|
||||
|
||||
The SFTP remote supports three authentication methods:
|
||||
|
||||
Loaded 100 of 160 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user