Compare commits
102
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ab60a77aba | ||
|
|
09535a06f7 | ||
|
|
173b720173 | ||
|
|
6660e6ec7c | ||
|
|
14c604335e | ||
|
|
bfcb23b7b2 | ||
|
|
46dbdb8cb7 | ||
|
|
17932fcc38 | ||
|
|
77faa787e1 | ||
|
|
0701dd55cd | ||
|
|
1903b4c1a2 | ||
|
|
f7cbcf556f | ||
|
|
3581e628c0 | ||
|
|
62c41bf449 | ||
|
|
c5864e113b | ||
|
|
39259a5bd1 | ||
|
|
2e376eb3b9 | ||
|
|
de8e9d4693 | ||
|
|
710cf49bc6 | ||
|
|
8dacac60ea | ||
|
|
3a80d4d4b4 | ||
|
|
a531f987a8 | ||
|
|
e906b8d0c4 | ||
|
|
a5932ef91a | ||
|
|
3afa563eaf | ||
|
|
9d9654b31f | ||
|
|
cfe257f13d | ||
|
|
0375efbd35 | ||
|
|
cad1954213 | ||
|
|
604e37caa5 | ||
|
|
b249d384b9 | ||
|
|
04e91838db | ||
|
|
94829aaec5 | ||
|
|
f574e3395c | ||
|
|
2bc155a96a | ||
|
|
adc8ea3427 | ||
|
|
068eea025c | ||
|
|
4510aa679a | ||
|
|
79281354c7 | ||
|
|
f57a178719 | ||
|
|
44f2e2ed39 | ||
|
|
13e1752d94 | ||
|
|
bb82c0e43b | ||
|
|
1af7151e73 | ||
|
|
fd63478ed6 | ||
|
|
5133b05c74 | ||
|
|
6ba96ede4b | ||
|
|
2896973964 | ||
|
|
be123d85ff | ||
|
|
b1b9562ab7 | ||
|
|
5146b66569 | ||
|
|
8898372d5a | ||
|
|
091fe9e453 | ||
|
|
8fdb68e41a | ||
|
|
c124aa2ed3 | ||
|
|
54e8bb89f7 | ||
|
|
50c1b594ab | ||
|
|
72437a9ca2 | ||
|
|
8ed55c61e1 | ||
|
|
bd598c1ceb | ||
|
|
7e30665102 | ||
|
|
d44957a09c | ||
|
|
37524e2dea | ||
|
|
2f6a6c8233 | ||
|
|
4ad40b6554 | ||
|
|
4f33d64f25 | ||
|
|
519623d9f1 | ||
|
|
913278327b | ||
|
|
a9b05e4c7a | ||
|
|
5d6d79e7d4 | ||
|
|
11de074cbf | ||
|
|
e9ab177a32 | ||
|
|
f3f4fba98d | ||
|
|
03fccdd67b | ||
|
|
231083647e | ||
|
|
0e203a7546 | ||
|
|
a7dd787569 | ||
|
|
689555033e | ||
|
|
4fc4898287 | ||
|
|
b003169088 | ||
|
|
babd112665 | ||
|
|
71b9b4ad7a | ||
|
|
4368863fcb | ||
|
|
04d49bf0ea | ||
|
|
d7aa37d263 | ||
|
|
379dffa61c | ||
|
|
5fd4ece31f | ||
|
|
fc3f95190b | ||
|
|
d6f5652b65 | ||
|
|
b5cbb7520d | ||
|
|
a170dfa55b | ||
|
|
1449c5b5ba | ||
|
|
35fe609722 | ||
|
|
cce399515f | ||
|
|
8c5af2f51c | ||
|
|
c639d3656e | ||
|
|
d9fbbba5c3 | ||
|
|
fd87560388 | ||
|
|
d87720a787 | ||
|
|
d541caa52b | ||
|
|
fd1665ae93 | ||
|
|
457d80e8a9 |
No files matched your search
@@ -291,7 +291,7 @@ jobs:
|
||||
README.md
|
||||
RELEASE.md
|
||||
CODE_OF_CONDUCT.md
|
||||
docs/content/{authors,bugs,changelog,docs,downloads,faq,filtering,gui,install,licence,overview,privacy}.md
|
||||
docs/content/{authors,bugs,changelog,cluster,docs,downloads,faq,filtering,gui,install,licence,overview,privacy}.md
|
||||
|
||||
- name: Scan edits of autogenerated files
|
||||
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
|
||||
|
||||
+1
-1
@@ -628,7 +628,7 @@ You'll need to modify the following files
|
||||
- `backend/s3/s3.go`
|
||||
- Add the provider to `providerOption` at the top of the file
|
||||
- Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
|
||||
- Exclude your provider from generic config questions (eg `region` and `endpoint).
|
||||
- Exclude your provider from generic config questions (eg `region` and `endpoint`).
|
||||
- Add the provider to the `setQuirks` function - see the documentation there.
|
||||
- `docs/content/s3.md`
|
||||
- Add the provider at the top of the page.
|
||||
|
||||
@@ -114,21 +114,21 @@ release_dep_linux:
|
||||
# Update dependencies
|
||||
showupdates:
|
||||
@echo "*** Direct dependencies that could be updated ***"
|
||||
@GO111MODULE=on go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
|
||||
@go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
|
||||
|
||||
# Update direct dependencies only
|
||||
updatedirect:
|
||||
GO111MODULE=on go get -d $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
GO111MODULE=on go mod tidy
|
||||
go get $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
go mod tidy
|
||||
|
||||
# Update direct and indirect dependencies and test dependencies
|
||||
update:
|
||||
GO111MODULE=on go get -d -u -t ./...
|
||||
GO111MODULE=on go mod tidy
|
||||
go get -u -t ./...
|
||||
go mod tidy
|
||||
|
||||
# Tidy the module dependencies
|
||||
tidy:
|
||||
GO111MODULE=on go mod tidy
|
||||
go mod tidy
|
||||
|
||||
doc: rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ directories to and from different cloud storage providers.
|
||||
- Google Drive [:page_facing_up:](https://rclone.org/drive/)
|
||||
- Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
|
||||
- HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
|
||||
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
|
||||
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
|
||||
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
|
||||
- HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
@@ -95,6 +96,7 @@ directories to and from different cloud storage providers.
|
||||
- Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)
|
||||
- QingStor [:page_facing_up:](https://rclone.org/qingstor/)
|
||||
- Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
|
||||
- Rabata Cloud Storage [:page_facing_up:](https://rclone.org/s3/#Rabata)
|
||||
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
|
||||
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
|
||||
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
|
||||
@@ -106,6 +108,7 @@ directories to and from different cloud storage providers.
|
||||
- Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
|
||||
- SFTP [:page_facing_up:](https://rclone.org/sftp/)
|
||||
- SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
|
||||
- Spectra Logic [:page_facing_up:](https://rclone.org/s3/#spectralogic)
|
||||
- StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
|
||||
- Storj [:page_facing_up:](https://rclone.org/storj/)
|
||||
- SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
|
||||
|
||||
@@ -2152,7 +2152,6 @@ func (o *Object) getMetadata() (metadata map[string]*string) {
|
||||
}
|
||||
metadata = make(map[string]*string, len(o.meta))
|
||||
for k, v := range o.meta {
|
||||
v := v
|
||||
metadata[k] = &v
|
||||
}
|
||||
return metadata
|
||||
@@ -2798,8 +2797,6 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) {
|
||||
blockList blockblob.GetBlockListResponse
|
||||
properties *blob.GetPropertiesResponse
|
||||
options *blockblob.CommitBlockListOptions
|
||||
// Use temporary pacer as this can be called recursively which can cause a deadlock with --max-connections
|
||||
pacer = fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
|
||||
)
|
||||
|
||||
properties, err = o.readMetaDataAlways(ctx)
|
||||
@@ -2811,7 +2808,7 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) {
|
||||
|
||||
if objectExists {
|
||||
// Get the committed block list
|
||||
err = pacer.Call(func() (bool, error) {
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
blockList, err = blockBlobSVC.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -2853,7 +2850,7 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) {
|
||||
|
||||
// Commit only the committed blocks
|
||||
fs.Debugf(o, "Committing %d blocks to remove uncommitted blocks", len(blockIDs))
|
||||
err = pacer.Call(func() (bool, error) {
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := blockBlobSVC.CommitBlockList(ctx, blockIDs, options)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -1313,10 +1313,29 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
srcURL := srcObj.fileClient().URL()
|
||||
fc := f.fileClient(remote)
|
||||
_, err = fc.StartCopyFromURL(ctx, srcURL, &opt)
|
||||
startCopy, err := fc.StartCopyFromURL(ctx, srcURL, &opt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Copy failed: %w", err)
|
||||
}
|
||||
|
||||
// Poll for completion if necessary
|
||||
//
|
||||
// The for loop is never executed for same storage account copies.
|
||||
copyStatus := startCopy.CopyStatus
|
||||
var properties file.GetPropertiesResponse
|
||||
pollTime := 100 * time.Millisecond
|
||||
|
||||
for copyStatus != nil && string(*copyStatus) == string(file.CopyStatusTypePending) {
|
||||
time.Sleep(pollTime)
|
||||
|
||||
properties, err = fc.GetProperties(ctx, &file.GetPropertiesOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copyStatus = properties.CopyStatus
|
||||
pollTime = min(2*pollTime, time.Second)
|
||||
}
|
||||
|
||||
dstObj, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Copy: NewObject failed: %w", err)
|
||||
|
||||
+7
-3
@@ -2224,13 +2224,17 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
|
||||
if err != nil {
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(f.opt.ChunkSize),
|
||||
ChunkSize: up.chunkSize,
|
||||
Concurrency: o.fs.opt.UploadConcurrency,
|
||||
//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
|
||||
}
|
||||
up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
|
||||
return info, up, err
|
||||
return info, up, nil
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
|
||||
Vendored
+1
-1
@@ -684,7 +684,7 @@ func (f *Fs) rcFetch(ctx context.Context, in rc.Params) (rc.Params, error) {
|
||||
start, end int64
|
||||
}
|
||||
parseChunks := func(ranges string) (crs []chunkRange, err error) {
|
||||
for _, part := range strings.Split(ranges, ",") {
|
||||
for part := range strings.SplitSeq(ranges, ",") {
|
||||
var start, end int64 = 0, math.MaxInt64
|
||||
switch ints := strings.Split(part, ":"); len(ints) {
|
||||
case 1:
|
||||
|
||||
@@ -187,7 +187,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var mu sync.Mutex
|
||||
for _, upstream := range opt.Upstreams {
|
||||
upstream := upstream
|
||||
g.Go(func() (err error) {
|
||||
equal := strings.IndexRune(upstream, '=')
|
||||
if equal < 0 {
|
||||
@@ -370,7 +369,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream) error) error {
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
for _, u := range f.upstreams {
|
||||
u := u
|
||||
g.Go(func() (err error) {
|
||||
return fn(gCtx, u)
|
||||
})
|
||||
@@ -637,7 +635,6 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
var uChans []chan time.Duration
|
||||
|
||||
for _, u := range f.upstreams {
|
||||
u := u
|
||||
if do := u.f.Features().ChangeNotify; do != nil {
|
||||
ch := make(chan time.Duration)
|
||||
uChans = append(uChans, ch)
|
||||
|
||||
+2
-2
@@ -598,7 +598,7 @@ It doesn't return anything.
|
||||
// The result should be capable of being JSON encoded
|
||||
// If it is a string or a []string it will be shown to the user
|
||||
// otherwise it will be JSON encoded and shown to the user like that
|
||||
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
|
||||
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
|
||||
switch name {
|
||||
case "metadata":
|
||||
return f.ShowMetadata(ctx)
|
||||
@@ -625,7 +625,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
}
|
||||
|
||||
// ShowMetadata returns some metadata about the corresponding DOI
|
||||
func (f *Fs) ShowMetadata(ctx context.Context) (metadata interface{}, err error) {
|
||||
func (f *Fs) ShowMetadata(ctx context.Context) (metadata any, err error) {
|
||||
doiURL, err := url.Parse("https://doi.org/" + f.opt.Doi)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -18,7 +18,7 @@ type headerLink struct {
|
||||
}
|
||||
|
||||
func parseLinkHeader(header string) (links []headerLink) {
|
||||
for _, link := range strings.Split(header, ",") {
|
||||
for link := range strings.SplitSeq(header, ",") {
|
||||
link = strings.TrimSpace(link)
|
||||
parsed := parseLink(link)
|
||||
if parsed != nil {
|
||||
@@ -30,7 +30,7 @@ func parseLinkHeader(header string) (links []headerLink) {
|
||||
|
||||
func parseLink(link string) (parsedLink *headerLink) {
|
||||
var parts []string
|
||||
for _, part := range strings.Split(link, ";") {
|
||||
for part := range strings.SplitSeq(link, ";") {
|
||||
parts = append(parts, strings.TrimSpace(part))
|
||||
}
|
||||
|
||||
|
||||
@@ -191,7 +191,7 @@ func driveScopes(scopesString string) (scopes []string) {
|
||||
if scopesString == "" {
|
||||
scopesString = defaultScope
|
||||
}
|
||||
for _, scope := range strings.Split(scopesString, ",") {
|
||||
for scope := range strings.SplitSeq(scopesString, ",") {
|
||||
scope = strings.TrimSpace(scope)
|
||||
scopes = append(scopes, scopePrefix+scope)
|
||||
}
|
||||
@@ -1220,7 +1220,7 @@ func isLinkMimeType(mimeType string) bool {
|
||||
// into a list of unique extensions with leading "." and a list of associated MIME types
|
||||
func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, err error) {
|
||||
for _, extensionText := range extensionsIn {
|
||||
for _, extension := range strings.Split(extensionText, ",") {
|
||||
for extension := range strings.SplitSeq(extensionText, ",") {
|
||||
extension = strings.ToLower(strings.TrimSpace(extension))
|
||||
if extension == "" {
|
||||
continue
|
||||
|
||||
@@ -386,7 +386,6 @@ func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err e
|
||||
g.SetLimit(o.fs.ci.Checkers)
|
||||
var mu sync.Mutex // protect the info.Permissions from concurrent writes
|
||||
for _, permissionID := range info.PermissionIds {
|
||||
permissionID := permissionID
|
||||
g.Go(func() error {
|
||||
// must fetch the team drive ones individually to check the inherited flag
|
||||
perm, inherited, err := o.fs.getPermission(gCtx, actualID(info.Id), permissionID, !o.fs.isTeamDrive)
|
||||
@@ -520,7 +519,6 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
|
||||
}
|
||||
// merge metadata into request and user metadata
|
||||
for k, v := range meta {
|
||||
k, v := k, v
|
||||
// parse a boolean from v and write into out
|
||||
parseBool := func(out *bool) error {
|
||||
b, err := strconv.ParseBool(v)
|
||||
|
||||
@@ -8,7 +8,7 @@ type CreateFolderResponse struct {
|
||||
Status int `json:"status"`
|
||||
Msg string `json:"msg"`
|
||||
Result struct {
|
||||
FldID interface{} `json:"fld_id"`
|
||||
FldID any `json:"fld_id"`
|
||||
} `json:"result"`
|
||||
}
|
||||
|
||||
|
||||
@@ -252,6 +252,9 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
|
||||
}, {
|
||||
Value: "us-east4",
|
||||
Help: "Northern Virginia",
|
||||
}, {
|
||||
Value: "us-east5",
|
||||
Help: "Ohio",
|
||||
}, {
|
||||
Value: "us-west1",
|
||||
Help: "Oregon",
|
||||
|
||||
+126
-117
@@ -17,6 +17,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -59,31 +60,43 @@ const (
|
||||
configVersion = 1
|
||||
|
||||
defaultTokenURL = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
|
||||
defaultClientID = "jottacli"
|
||||
defaultClientID = "jottacli" // Identified as "Jottacloud CLI" in "My logged in devices"
|
||||
|
||||
legacyTokenURL = "https://api.jottacloud.com/auth/v1/token"
|
||||
legacyRegisterURL = "https://api.jottacloud.com/auth/v1/register"
|
||||
legacyClientID = "nibfk8biu12ju7hpqomr8b1e40"
|
||||
legacyEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
|
||||
legacyConfigVersion = 0
|
||||
|
||||
teliaseCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
|
||||
teliaseCloudAuthURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
|
||||
teliaseCloudClientID = "desktop"
|
||||
|
||||
telianoCloudTokenURL = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/token"
|
||||
telianoCloudAuthURL = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/auth"
|
||||
telianoCloudClientID = "desktop"
|
||||
|
||||
tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
|
||||
tele2CloudAuthURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
|
||||
tele2CloudClientID = "desktop"
|
||||
|
||||
onlimeCloudTokenURL = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/token"
|
||||
onlimeCloudAuthURL = "https://cloud-auth.onlime.dk/auth/realms/onlime_wl/protocol/openid-connect/auth"
|
||||
onlimeCloudClientID = "desktop"
|
||||
)
|
||||
|
||||
type service struct {
|
||||
key string
|
||||
name string
|
||||
domain string
|
||||
realm string
|
||||
clientID string
|
||||
scopes []string
|
||||
}
|
||||
|
||||
// The list of services and their settings for supporting traditional OAuth.
|
||||
// Please keep these in alphabetical order, but with jottacloud first.
|
||||
func getServices() []service {
|
||||
return []service{
|
||||
{"jottacloud", "Jottacloud", "id.jottacloud.com", "jottacloud", "desktop", []string{"openid", "jotta-default", "offline_access"}}, // Chose client id "desktop" here, will be identified as "Jottacloud for Desktop" in "My logged in devices", but could have used "jottacli" here as well.
|
||||
{"elgiganten_dk", "Elgiganten Cloud (Denmark)", "cloud.elgiganten.dk", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"elgiganten_se", "Elgiganten Cloud (Sweden)", "cloud.elgiganten.se", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"gigantti", "Gigantti Cloud (Finland)", "cloud.gigantti.fi", "gigantti", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
|
||||
{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"onlime", "Onlime (Denmark)", "cloud-auth.onlime.dk", "onlime_wl", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"tele2", "Tele2 Cloud (Sweden)", "mittcloud-auth.tele2.se", "comhem", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"telia_no", "Telia Sky (Norway)", "sky-auth.telia.no", "get", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"telia_se", "Telia Cloud (Sweden)", "cloud-auth.telia.se", "telia_se", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
}
|
||||
}
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
// needs to be done early so we can use oauth during config
|
||||
@@ -159,36 +172,44 @@ func init() {
|
||||
}
|
||||
|
||||
// Config runs the backend configuration protocol
|
||||
func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
switch config.State {
|
||||
func Config(ctx context.Context, name string, m configmap.Mapper, conf fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
switch conf.State {
|
||||
case "":
|
||||
return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Select authentication type.`, []fs.OptionExample{{
|
||||
if isAuthorize, _ := m.Get(config.ConfigAuthorize); isAuthorize == "true" {
|
||||
return nil, errors.New("not supported by this backend")
|
||||
}
|
||||
return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Type of authentication.`, []fs.OptionExample{{
|
||||
Value: "standard",
|
||||
Help: "Standard authentication.\nUse this if you're a normal Jottacloud user.",
|
||||
Help: `Standard authentication.
|
||||
This is primarily supported by the official service, but may also be
|
||||
supported by some white-label services. It is designed for command-line
|
||||
applications, and you will be asked to enter a single-use personal login
|
||||
token which you must manually generate from the account security settings
|
||||
in the web interface of your service.`,
|
||||
}, {
|
||||
Value: "traditional",
|
||||
Help: `Traditional authentication.
|
||||
This is supported by the official service and all white-label services
|
||||
that rclone knows about. You will be asked which service to connect to.
|
||||
It has a limitation of only a single active authentication at a time. You
|
||||
need to be on, or have access to, a machine with an internet-connected
|
||||
web browser.`,
|
||||
}, {
|
||||
Value: "legacy",
|
||||
Help: "Legacy authentication.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.",
|
||||
}, {
|
||||
Value: "telia_se",
|
||||
Help: "Telia Cloud authentication.\nUse this if you are using Telia Cloud (Sweden).",
|
||||
}, {
|
||||
Value: "telia_no",
|
||||
Help: "Telia Sky authentication.\nUse this if you are using Telia Sky (Norway).",
|
||||
}, {
|
||||
Value: "tele2",
|
||||
Help: "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
|
||||
}, {
|
||||
Value: "onlime",
|
||||
Help: "Onlime Cloud authentication.\nUse this if you are using Onlime Cloud.",
|
||||
Help: `Legacy authentication.
|
||||
This is no longer supported by any known services and not recommended
|
||||
used. You will be asked for your account's username and password.`,
|
||||
}})
|
||||
case "auth_type_done":
|
||||
// Jump to next state according to config chosen
|
||||
return fs.ConfigGoto(config.Result)
|
||||
return fs.ConfigGoto(conf.Result)
|
||||
case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication
|
||||
m.Set("configVersion", fmt.Sprint(configVersion))
|
||||
return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\nGenerate here: https://www.jottacloud.com/web/secure")
|
||||
return fs.ConfigInput("standard_token", "config_login_token", `Personal login token.
|
||||
Generate it from the account security settings in the web interface of your
|
||||
service, for the official service on https://www.jottacloud.com/web/secure.`)
|
||||
case "standard_token":
|
||||
loginToken := config.Result
|
||||
loginToken := conf.Result
|
||||
m.Set(configClientID, defaultClientID)
|
||||
m.Set(configClientSecret, "")
|
||||
|
||||
@@ -203,10 +224,50 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
|
||||
return nil, fmt.Errorf("error while saving token: %w", err)
|
||||
}
|
||||
return fs.ConfigGoto("choose_device")
|
||||
case "traditional":
|
||||
services := getServices()
|
||||
options := make([]fs.OptionExample, 0, len(services))
|
||||
for _, service := range services {
|
||||
options = append(options, fs.OptionExample{
|
||||
Value: service.key,
|
||||
Help: service.name,
|
||||
})
|
||||
}
|
||||
return fs.ConfigChooseExclusiveFixed("traditional_type", "config_traditional",
|
||||
"White-label service. This decides the domain name to connect to and\nthe authentication configuration to use.",
|
||||
options)
|
||||
case "traditional_type":
|
||||
services := getServices()
|
||||
i := slices.IndexFunc(services, func(s service) bool { return s.key == conf.Result })
|
||||
if i == -1 {
|
||||
return nil, fmt.Errorf("unexpected service %q", conf.Result)
|
||||
}
|
||||
service := services[i]
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: "https://" + service.domain + "/auth/realms/" + service.realm + "/.well-known/openid-configuration",
|
||||
}
|
||||
var wellKnown api.WellKnown
|
||||
srv := rest.NewClient(fshttp.NewClient(ctx))
|
||||
_, err := srv.CallJSON(ctx, &opts, nil, &wellKnown)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get authentication provider configuration: %w", err)
|
||||
}
|
||||
m.Set("configVersion", fmt.Sprint(configVersion))
|
||||
m.Set(configClientID, service.clientID)
|
||||
m.Set(configTokenURL, wellKnown.TokenEndpoint)
|
||||
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
|
||||
OAuth2Config: &oauthutil.Config{
|
||||
AuthURL: wellKnown.AuthorizationEndpoint,
|
||||
TokenURL: wellKnown.TokenEndpoint,
|
||||
ClientID: service.clientID,
|
||||
Scopes: service.scopes,
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
},
|
||||
})
|
||||
case "legacy": // configure a jottacloud backend using legacy authentication
|
||||
m.Set("configVersion", fmt.Sprint(legacyConfigVersion))
|
||||
return fs.ConfigConfirm("legacy_api", false, "config_machine_specific", `Do you want to create a machine specific API key?
|
||||
|
||||
Rclone has it's own Jottacloud API KEY which works fine as long as one
|
||||
only uses rclone on a single machine. When you want to use rclone with
|
||||
this account on more than one machine it's recommended to create a
|
||||
@@ -214,7 +275,7 @@ machine specific API key. These keys can NOT be shared between
|
||||
machines.`)
|
||||
case "legacy_api":
|
||||
srv := rest.NewClient(fshttp.NewClient(ctx))
|
||||
if config.Result == "true" {
|
||||
if conf.Result == "true" {
|
||||
deviceRegistration, err := registerDevice(ctx, srv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to register device: %w", err)
|
||||
@@ -223,16 +284,16 @@ machines.`)
|
||||
m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
|
||||
fs.Debugf(nil, "Got clientID %q and clientSecret %q", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
|
||||
}
|
||||
return fs.ConfigInput("legacy_username", "config_username", "Username (e-mail address)")
|
||||
return fs.ConfigInput("legacy_username", "config_username", "Username (e-mail address) of your account.")
|
||||
case "legacy_username":
|
||||
m.Set(configUsername, config.Result)
|
||||
return fs.ConfigPassword("legacy_password", "config_password", "Password (only used in setup, will not be stored)")
|
||||
m.Set(configUsername, conf.Result)
|
||||
return fs.ConfigPassword("legacy_password", "config_password", "Password of your account. This is only used in setup, it will not be stored.")
|
||||
case "legacy_password":
|
||||
m.Set("password", config.Result)
|
||||
m.Set("password", conf.Result)
|
||||
m.Set("auth_code", "")
|
||||
return fs.ConfigGoto("legacy_do_auth")
|
||||
case "legacy_auth_code":
|
||||
authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
|
||||
authCode := strings.ReplaceAll(conf.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
|
||||
m.Set("auth_code", authCode)
|
||||
return fs.ConfigGoto("legacy_do_auth")
|
||||
case "legacy_do_auth":
|
||||
@@ -242,12 +303,12 @@ machines.`)
|
||||
authCode, _ := m.Get("auth_code")
|
||||
|
||||
srv := rest.NewClient(fshttp.NewClient(ctx))
|
||||
clientID, ok := m.Get(configClientID)
|
||||
if !ok {
|
||||
clientID, _ := m.Get(configClientID)
|
||||
if clientID == "" {
|
||||
clientID = legacyClientID
|
||||
}
|
||||
clientSecret, ok := m.Get(configClientSecret)
|
||||
if !ok {
|
||||
clientSecret, _ := m.Get(configClientSecret)
|
||||
if clientSecret == "" {
|
||||
clientSecret = legacyEncryptedClientSecret
|
||||
}
|
||||
|
||||
@@ -260,7 +321,7 @@ machines.`)
|
||||
}
|
||||
token, err := doLegacyAuth(ctx, srv, oauthConfig, username, password, authCode)
|
||||
if err == errAuthCodeRequired {
|
||||
return fs.ConfigInput("legacy_auth_code", "config_auth_code", "Verification Code\nThis account uses 2 factor authentication you will receive a verification code via SMS.")
|
||||
return fs.ConfigInput("legacy_auth_code", "config_auth_code", "Verification code.\nThis account uses 2 factor authentication you will receive a verification code via SMS.")
|
||||
}
|
||||
m.Set("password", "")
|
||||
m.Set("auth_code", "")
|
||||
@@ -272,58 +333,6 @@ machines.`)
|
||||
return nil, fmt.Errorf("error while saving token: %w", err)
|
||||
}
|
||||
return fs.ConfigGoto("choose_device")
|
||||
case "telia_se": // telia_se cloud config
|
||||
m.Set("configVersion", fmt.Sprint(configVersion))
|
||||
m.Set(configClientID, teliaseCloudClientID)
|
||||
m.Set(configTokenURL, teliaseCloudTokenURL)
|
||||
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
|
||||
OAuth2Config: &oauthutil.Config{
|
||||
AuthURL: teliaseCloudAuthURL,
|
||||
TokenURL: teliaseCloudTokenURL,
|
||||
ClientID: teliaseCloudClientID,
|
||||
Scopes: []string{"openid", "jotta-default", "offline_access"},
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
},
|
||||
})
|
||||
case "telia_no": // telia_no cloud config
|
||||
m.Set("configVersion", fmt.Sprint(configVersion))
|
||||
m.Set(configClientID, telianoCloudClientID)
|
||||
m.Set(configTokenURL, telianoCloudTokenURL)
|
||||
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
|
||||
OAuth2Config: &oauthutil.Config{
|
||||
AuthURL: telianoCloudAuthURL,
|
||||
TokenURL: telianoCloudTokenURL,
|
||||
ClientID: telianoCloudClientID,
|
||||
Scopes: []string{"openid", "jotta-default", "offline_access"},
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
},
|
||||
})
|
||||
case "tele2": // tele2 cloud config
|
||||
m.Set("configVersion", fmt.Sprint(configVersion))
|
||||
m.Set(configClientID, tele2CloudClientID)
|
||||
m.Set(configTokenURL, tele2CloudTokenURL)
|
||||
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
|
||||
OAuth2Config: &oauthutil.Config{
|
||||
AuthURL: tele2CloudAuthURL,
|
||||
TokenURL: tele2CloudTokenURL,
|
||||
ClientID: tele2CloudClientID,
|
||||
Scopes: []string{"openid", "jotta-default", "offline_access"},
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
},
|
||||
})
|
||||
case "onlime": // onlime cloud config
|
||||
m.Set("configVersion", fmt.Sprint(configVersion))
|
||||
m.Set(configClientID, onlimeCloudClientID)
|
||||
m.Set(configTokenURL, onlimeCloudTokenURL)
|
||||
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
|
||||
OAuth2Config: &oauthutil.Config{
|
||||
AuthURL: onlimeCloudAuthURL,
|
||||
TokenURL: onlimeCloudTokenURL,
|
||||
ClientID: onlimeCloudClientID,
|
||||
Scopes: []string{"openid", "jotta-default", "offline_access"},
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
},
|
||||
})
|
||||
case "choose_device":
|
||||
return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", `Use a non-standard device/mountpoint?
|
||||
Choosing no, the default, will let you access the storage used for the archive
|
||||
@@ -331,7 +340,7 @@ section of the official Jottacloud client. If you instead want to access the
|
||||
sync or the backup section, for example, you must choose yes.`)
|
||||
|
||||
case "choose_device_query":
|
||||
if config.Result != "true" {
|
||||
if conf.Result != "true" {
|
||||
m.Set(configDevice, "")
|
||||
m.Set(configMountpoint, "")
|
||||
return fs.ConfigGoto("end")
|
||||
@@ -372,7 +381,7 @@ a new by entering a unique name.`, defaultDevice)
|
||||
return deviceNames[i], ""
|
||||
})
|
||||
case "choose_device_result":
|
||||
device := config.Result
|
||||
device := conf.Result
|
||||
|
||||
oAuthClient, _, err := getOAuthClient(ctx, name, m)
|
||||
if err != nil {
|
||||
@@ -432,7 +441,7 @@ You may create a new by entering a unique name.`, device)
|
||||
return dev.MountPoints[i].Name, ""
|
||||
})
|
||||
case "choose_device_mountpoint":
|
||||
mountpoint := config.Result
|
||||
mountpoint := conf.Result
|
||||
|
||||
oAuthClient, _, err := getOAuthClient(ctx, name, m)
|
||||
if err != nil {
|
||||
@@ -463,7 +472,7 @@ You may create a new by entering a unique name.`, device)
|
||||
|
||||
if isNew {
|
||||
if device == defaultDevice {
|
||||
return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err)
|
||||
return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device", defaultDevice)
|
||||
}
|
||||
fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint)
|
||||
_, err := createMountPoint(ctx, jfsSrv, path.Join(cust.Username, device, mountpoint))
|
||||
@@ -478,7 +487,7 @@ You may create a new by entering a unique name.`, device)
|
||||
// All the config flows end up here in case we need to carry on with something
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown state %q", config.State)
|
||||
return nil, fmt.Errorf("unknown state %q", conf.State)
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
@@ -929,12 +938,12 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth
|
||||
oauthConfig.AuthURL = tokenURL
|
||||
}
|
||||
} else if ver == legacyConfigVersion {
|
||||
clientID, ok := m.Get(configClientID)
|
||||
if !ok {
|
||||
clientID, _ := m.Get(configClientID)
|
||||
if clientID == "" {
|
||||
clientID = legacyClientID
|
||||
}
|
||||
clientSecret, ok := m.Get(configClientSecret)
|
||||
if !ok {
|
||||
clientSecret, _ := m.Get(configClientSecret)
|
||||
if clientSecret == "" {
|
||||
clientSecret = legacyEncryptedClientSecret
|
||||
}
|
||||
oauthConfig.ClientID = clientID
|
||||
@@ -1000,6 +1009,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.features.ListR = nil
|
||||
}
|
||||
|
||||
cust, err := getCustomerInfo(ctx, f.apiSrv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.user = cust.Username
|
||||
f.setEndpoints()
|
||||
|
||||
// Renew the token in the background
|
||||
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
|
||||
_, err := f.readMetaDataForPath(ctx, "")
|
||||
@@ -1009,13 +1025,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return err
|
||||
})
|
||||
|
||||
cust, err := getCustomerInfo(ctx, f.apiSrv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.user = cust.Username
|
||||
f.setEndpoints()
|
||||
|
||||
if root != "" && !rootIsDir {
|
||||
// Check to see if the root actually an existing file
|
||||
remote := path.Base(root)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
iofs "io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -841,7 +842,13 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
} else if !fi.IsDir() {
|
||||
return fs.ErrorIsFile
|
||||
}
|
||||
return os.Remove(localPath)
|
||||
err := os.Remove(localPath)
|
||||
if runtime.GOOS == "windows" && errors.Is(err, iofs.ErrPermission) { // https://github.com/golang/go/issues/26295
|
||||
if os.Chmod(localPath, 0o600) == nil {
|
||||
err = os.Remove(localPath)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Precision of the file system
|
||||
|
||||
@@ -334,7 +334,7 @@ func TestMetadata(t *testing.T) {
|
||||
|
||||
func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
|
||||
ctx := context.Background()
|
||||
whenRFC := when.Format(time.RFC3339Nano)
|
||||
whenRFC := when.Local().Format(time.RFC3339Nano)
|
||||
const dayLength = len("2001-01-01")
|
||||
|
||||
f := r.Flocal.(*Fs)
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
//go:build windows
|
||||
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestRmdirWindows tests that FILE_ATTRIBUTE_READONLY does not block Rmdir on windows.
|
||||
// Microsoft docs indicate that "This attribute is not honored on directories."
|
||||
// See https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants#file_attribute_readonly
|
||||
// and https://github.com/golang/go/issues/26295
|
||||
func TestRmdirWindows(t *testing.T) {
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skipf("windows only")
|
||||
}
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
err := operations.Mkdir(context.Background(), r.Flocal, "testdir")
|
||||
require.NoError(t, err)
|
||||
|
||||
ptr, err := syscall.UTF16PtrFromString(filepath.Join(r.Flocal.Root(), "testdir"))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = syscall.SetFileAttributes(ptr, uint32(syscall.FILE_ATTRIBUTE_DIRECTORY+syscall.FILE_ATTRIBUTE_READONLY))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = operations.Rmdir(context.Background(), r.Flocal, "testdir")
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
@@ -400,7 +400,7 @@ type quirks struct {
|
||||
}
|
||||
|
||||
func (q *quirks) parseQuirks(option string) {
|
||||
for _, flag := range strings.Split(option, ",") {
|
||||
for flag := range strings.SplitSeq(option, ",") {
|
||||
switch strings.ToLower(strings.TrimSpace(flag)) {
|
||||
case "binlist":
|
||||
// The official client sometimes uses a so called "bin" protocol,
|
||||
@@ -1770,7 +1770,7 @@ func (f *Fs) parseSpeedupPatterns(patternString string) (err error) {
|
||||
f.speedupAny = false
|
||||
uniqueValidPatterns := make(map[string]any)
|
||||
|
||||
for _, pattern := range strings.Split(patternString, ",") {
|
||||
for pattern := range strings.SplitSeq(patternString, ",") {
|
||||
pattern = strings.ToLower(strings.TrimSpace(pattern))
|
||||
if pattern == "" {
|
||||
continue
|
||||
|
||||
@@ -243,7 +243,6 @@ func (m *Metadata) Get(ctx context.Context) (metadata fs.Metadata, err error) {
|
||||
func (m *Metadata) Set(ctx context.Context, metadata fs.Metadata) (numSet int, err error) {
|
||||
numSet = 0
|
||||
for k, v := range metadata {
|
||||
k, v := k, v
|
||||
switch k {
|
||||
case "mtime":
|
||||
t, err := time.Parse(timeFormatIn, v)
|
||||
@@ -422,12 +421,7 @@ func (m *Metadata) orderPermissions(xs []*api.PermissionsType) {
|
||||
if hasUserIdentity(p.GetGrantedTo(m.fs.driveType)) {
|
||||
return true
|
||||
}
|
||||
for _, identity := range p.GetGrantedToIdentities(m.fs.driveType) {
|
||||
if hasUserIdentity(identity) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return slices.ContainsFunc(p.GetGrantedToIdentities(m.fs.driveType), hasUserIdentity)
|
||||
}
|
||||
// Put Permissions with a user first, leaving unsorted otherwise
|
||||
slices.SortStableFunc(xs, func(a, b *api.PermissionsType) int {
|
||||
|
||||
@@ -172,8 +172,8 @@ func BenchmarkQuickXorHash(b *testing.B) {
|
||||
require.NoError(b, err)
|
||||
require.Equal(b, len(buf), n)
|
||||
h := New()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
|
||||
for b.Loop() {
|
||||
h.Reset()
|
||||
h.Write(buf)
|
||||
h.Sum(nil)
|
||||
|
||||
@@ -5,6 +5,7 @@ package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -136,8 +137,25 @@ type Link struct {
|
||||
}
|
||||
|
||||
// Valid reports whether l is non-nil, has an URL, and is not expired.
|
||||
// It primarily checks the URL's expire query parameter, falling back to the Expire field.
|
||||
func (l *Link) Valid() bool {
|
||||
return l != nil && l.URL != "" && time.Now().Add(10*time.Second).Before(time.Time(l.Expire))
|
||||
if l == nil || l.URL == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Primary validation: check URL's expire query parameter
|
||||
if u, err := url.Parse(l.URL); err == nil {
|
||||
if expireStr := u.Query().Get("expire"); expireStr != "" {
|
||||
// Try parsing as Unix timestamp (seconds)
|
||||
if expireInt, err := strconv.ParseInt(expireStr, 10, 64); err == nil {
|
||||
expireTime := time.Unix(expireInt, 0)
|
||||
return time.Now().Add(10 * time.Second).Before(expireTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback validation: use the Expire field if URL parsing didn't work
|
||||
return time.Now().Add(10 * time.Second).Before(time.Time(l.Expire))
|
||||
}
|
||||
|
||||
// URL is a basic form of URL
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLinkValid tests the Link.Valid method for various scenarios
|
||||
func TestLinkValid(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
link *Link
|
||||
expected bool
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
name: "nil link",
|
||||
link: nil,
|
||||
expected: false,
|
||||
desc: "nil link should be invalid",
|
||||
},
|
||||
{
|
||||
name: "empty URL",
|
||||
link: &Link{URL: ""},
|
||||
expected: false,
|
||||
desc: "empty URL should be invalid",
|
||||
},
|
||||
{
|
||||
name: "valid URL with future expire parameter",
|
||||
link: &Link{
|
||||
URL: fmt.Sprintf("https://example.com/file?expire=%d", time.Now().Add(time.Hour).Unix()),
|
||||
},
|
||||
expected: true,
|
||||
desc: "URL with future expire parameter should be valid",
|
||||
},
|
||||
{
|
||||
name: "expired URL with past expire parameter",
|
||||
link: &Link{
|
||||
URL: fmt.Sprintf("https://example.com/file?expire=%d", time.Now().Add(-time.Hour).Unix()),
|
||||
},
|
||||
expected: false,
|
||||
desc: "URL with past expire parameter should be invalid",
|
||||
},
|
||||
{
|
||||
name: "URL expire parameter takes precedence over Expire field",
|
||||
link: &Link{
|
||||
URL: fmt.Sprintf("https://example.com/file?expire=%d", time.Now().Add(time.Hour).Unix()),
|
||||
Expire: Time(time.Now().Add(-time.Hour)), // Fallback is expired
|
||||
},
|
||||
expected: true,
|
||||
desc: "URL expire parameter should take precedence over Expire field",
|
||||
},
|
||||
{
|
||||
name: "URL expire parameter within 10 second buffer should be invalid",
|
||||
link: &Link{
|
||||
URL: fmt.Sprintf("https://example.com/file?expire=%d", time.Now().Add(5*time.Second).Unix()),
|
||||
},
|
||||
expected: false,
|
||||
desc: "URL expire parameter within 10 second buffer should be invalid",
|
||||
},
|
||||
{
|
||||
name: "fallback to Expire field when no URL expire parameter",
|
||||
link: &Link{
|
||||
URL: "https://example.com/file",
|
||||
Expire: Time(time.Now().Add(time.Hour)),
|
||||
},
|
||||
expected: true,
|
||||
desc: "should fallback to Expire field when URL has no expire parameter",
|
||||
},
|
||||
{
|
||||
name: "fallback to Expire field when URL expire parameter is invalid",
|
||||
link: &Link{
|
||||
URL: "https://example.com/file?expire=invalid",
|
||||
Expire: Time(time.Now().Add(time.Hour)),
|
||||
},
|
||||
expected: true,
|
||||
desc: "should fallback to Expire field when URL expire parameter is unparseable",
|
||||
},
|
||||
{
|
||||
name: "invalid when both URL expire and Expire field are expired",
|
||||
link: &Link{
|
||||
URL: fmt.Sprintf("https://example.com/file?expire=%d", time.Now().Add(-time.Hour).Unix()),
|
||||
Expire: Time(time.Now().Add(-time.Hour)),
|
||||
},
|
||||
expected: false,
|
||||
desc: "should be invalid when both URL expire and Expire field are expired",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tt.link.Valid()
|
||||
if result != tt.expected {
|
||||
t.Errorf("Link.Valid() = %v, expected %v. %s", result, tt.expected, tt.desc)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
protonDriveAPI "github.com/henrybear327/Proton-API-Bridge"
|
||||
"github.com/henrybear327/go-proton-api"
|
||||
|
||||
"github.com/pquerna/otp/totp"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
@@ -87,6 +89,17 @@ The value can also be provided with --protondrive-2fa=000000
|
||||
The 2FA code of your proton drive account if the account is set up with
|
||||
two-factor authentication`,
|
||||
Required: false,
|
||||
}, {
|
||||
Name: "otp_secret_key",
|
||||
Help: `The OTP secret key
|
||||
|
||||
The value can also be provided with --protondrive-otp-secret-key=ABCDEFGHIJKLMNOPQRSTUVWXYZ234567
|
||||
|
||||
The OTP secret key of your proton drive account if the account is set up with
|
||||
two-factor authentication`,
|
||||
Required: false,
|
||||
Sensitive: true,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: clientUIDKey,
|
||||
Help: "Client uid key (internal use only)",
|
||||
@@ -191,6 +204,7 @@ type Options struct {
|
||||
Password string `config:"password"`
|
||||
MailboxPassword string `config:"mailbox_password"`
|
||||
TwoFA string `config:"2fa"`
|
||||
OtpSecretKey string `config:"otp_secret_key"`
|
||||
|
||||
// advanced
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
@@ -356,7 +370,15 @@ func newProtonDrive(ctx context.Context, f *Fs, opt *Options, m configmap.Mapper
|
||||
config.FirstLoginCredential.Username = opt.Username
|
||||
config.FirstLoginCredential.Password = opt.Password
|
||||
config.FirstLoginCredential.MailboxPassword = opt.MailboxPassword
|
||||
// if 2FA code is provided, use it; otherwise, generate one using the OTP secret key if provided
|
||||
config.FirstLoginCredential.TwoFA = opt.TwoFA
|
||||
if opt.TwoFA == "" && opt.OtpSecretKey != "" {
|
||||
code, err := totp.GenerateCode(opt.OtpSecretKey, time.Now())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't generate 2FA code: %w", err)
|
||||
}
|
||||
config.FirstLoginCredential.TwoFA = code
|
||||
}
|
||||
protonDrive, auth, err := protonDriveAPI.NewProtonDrive(ctx, config, authHandler, deAuthHandler)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't initialize a new proton drive instance: %w", err)
|
||||
@@ -395,6 +417,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
}
|
||||
|
||||
if opt.OtpSecretKey != "" {
|
||||
var err error
|
||||
opt.OtpSecretKey, err = obscure.Reveal(opt.OtpSecretKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't decrypt OtpSecretKey: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
@@ -59,11 +59,7 @@ func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed f
|
||||
|
||||
defer func() { u.fileUsage[fileID] = borrowed }()
|
||||
|
||||
effectiveChunkSize := max(int64(speed*u.effectiveTime.Seconds()), u.reserved)
|
||||
|
||||
if neededMemory < effectiveChunkSize {
|
||||
effectiveChunkSize = neededMemory
|
||||
}
|
||||
effectiveChunkSize := min(neededMemory, max(int64(speed*u.effectiveTime.Seconds()), u.reserved))
|
||||
|
||||
if effectiveChunkSize <= u.reserved {
|
||||
return effectiveChunkSize
|
||||
|
||||
+588
-457
File diff suppressed because it is too large.
Load diff
@@ -200,7 +200,7 @@ func TestFilePool_ConcurrentAccess(t *testing.T) {
|
||||
pool := newFilePool(ctx, fs, "testshare", "/test/path")
|
||||
|
||||
const numGoroutines = 10
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
for range numGoroutines {
|
||||
mockFile := newMockFile()
|
||||
pool.pool = append(pool.pool, mockFile)
|
||||
}
|
||||
@@ -208,7 +208,7 @@ func TestFilePool_ConcurrentAccess(t *testing.T) {
|
||||
// Test concurrent get operations
|
||||
done := make(chan bool, numGoroutines)
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
for range numGoroutines {
|
||||
go func() {
|
||||
defer func() { done <- true }()
|
||||
|
||||
@@ -219,7 +219,7 @@ func TestFilePool_ConcurrentAccess(t *testing.T) {
|
||||
}()
|
||||
}
|
||||
|
||||
for i := 0; i < numGoroutines; i++ {
|
||||
for range numGoroutines {
|
||||
<-done
|
||||
}
|
||||
|
||||
|
||||
@@ -192,6 +192,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if root is empty or ends with / (must be a directory)
|
||||
isRootDir := isPathDir(root)
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &Fs{
|
||||
@@ -218,6 +221,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if share == "" || dir == "" {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Skip stat check if root is already a directory
|
||||
if isRootDir {
|
||||
return f, nil
|
||||
}
|
||||
cn, err := f.getConnection(ctx, share)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -894,6 +902,11 @@ func ensureSuffix(s, suffix string) string {
|
||||
return s + suffix
|
||||
}
|
||||
|
||||
// isPathDir determines if a path represents a directory based on trailing slash
|
||||
func isPathDir(path string) bool {
|
||||
return path == "" || strings.HasSuffix(path, "/")
|
||||
}
|
||||
|
||||
func trimPathPrefix(s, prefix string) string {
|
||||
// we need to clean the paths to make tests pass!
|
||||
s = betterPathClean(s)
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
// Unit tests for internal SMB functions
|
||||
package smb
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestIsPathDir tests the isPathDir function logic
|
||||
func TestIsPathDir(t *testing.T) {
|
||||
tests := []struct {
|
||||
path string
|
||||
expected bool
|
||||
}{
|
||||
// Empty path should be considered a directory
|
||||
{"", true},
|
||||
|
||||
// Paths with trailing slash should be directories
|
||||
{"/", true},
|
||||
{"share/", true},
|
||||
{"share/dir/", true},
|
||||
{"share/dir/subdir/", true},
|
||||
|
||||
// Paths without trailing slash should not be directories
|
||||
{"share", false},
|
||||
{"share/dir", false},
|
||||
{"share/dir/file", false},
|
||||
{"share/dir/subdir/file", false},
|
||||
|
||||
// Edge cases
|
||||
{"share//", true},
|
||||
{"share///", true},
|
||||
{"share/dir//", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
result := isPathDir(tt.path)
|
||||
if result != tt.expected {
|
||||
t.Errorf("isPathDir(%q) = %v, want %v", tt.path, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -561,6 +561,21 @@ func (f *Fs) setRoot(root string) {
|
||||
f.rootContainer, f.rootDirectory = bucket.Split(f.root)
|
||||
}
|
||||
|
||||
// Fetch the base container's policy to be used if/when we need to create a
|
||||
// segments container to ensure we use the same policy.
|
||||
func (f *Fs) fetchStoragePolicy(ctx context.Context, container string) (fs.Fs, error) {
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var rxHeaders swift.Headers
|
||||
_, rxHeaders, err := f.c.Container(ctx, container)
|
||||
|
||||
f.opt.StoragePolicy = rxHeaders["X-Storage-Policy"]
|
||||
fs.Debugf(f, "Auto set StoragePolicy to %s", f.opt.StoragePolicy)
|
||||
|
||||
return shouldRetryHeaders(ctx, rxHeaders, err)
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// NewFsWithConnection constructs an Fs from the path, container:path
|
||||
// and authenticated connection.
|
||||
//
|
||||
@@ -590,6 +605,7 @@ func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c
|
||||
f.opt.UseSegmentsContainer.Valid = true
|
||||
fs.Debugf(f, "Auto set use_segments_container to %v", f.opt.UseSegmentsContainer.Value)
|
||||
}
|
||||
|
||||
if f.rootContainer != "" && f.rootDirectory != "" {
|
||||
// Check to see if the object exists - ignoring directory markers
|
||||
var info swift.Object
|
||||
@@ -1132,6 +1148,13 @@ func (f *Fs) newSegmentedUpload(ctx context.Context, dstContainer string, dstPat
|
||||
container: dstContainer,
|
||||
}
|
||||
if f.opt.UseSegmentsContainer.Value {
|
||||
if f.opt.StoragePolicy == "" {
|
||||
_, err = f.fetchStoragePolicy(ctx, dstContainer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
su.container += segmentsContainerSuffix
|
||||
err = f.makeContainer(ctx, su.container)
|
||||
if err != nil {
|
||||
|
||||
@@ -76,6 +76,7 @@ func (f *Fs) testNoChunk(t *testing.T) {
|
||||
|
||||
// Additional tests that aren't in the framework
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("PolicyDiscovery", f.testPolicyDiscovery)
|
||||
t.Run("NoChunk", f.testNoChunk)
|
||||
t.Run("WithChunk", f.testWithChunk)
|
||||
t.Run("WithChunkFail", f.testWithChunkFail)
|
||||
@@ -195,4 +196,50 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
|
||||
require.Equal(t, obj.Size(), objTarget.Size())
|
||||
}
|
||||
|
||||
func (f *Fs) testPolicyDiscovery(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
container := "testPolicyDiscovery-1"
|
||||
// Reset the policy so we can test if it is populated.
|
||||
f.opt.StoragePolicy = ""
|
||||
err := f.makeContainer(ctx, container)
|
||||
require.NoError(t, err)
|
||||
_, err = f.fetchStoragePolicy(ctx, container)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Default policy for SAIO image is 1replica.
|
||||
assert.Equal(t, "1replica", f.opt.StoragePolicy)
|
||||
|
||||
// Create a container using a non-default policy, and check to ensure
|
||||
// that the created segments container uses the same non-default policy.
|
||||
policy := "Policy-1"
|
||||
container = "testPolicyDiscovery-2"
|
||||
|
||||
f.opt.StoragePolicy = policy
|
||||
err = f.makeContainer(ctx, container)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Reset the policy so we can test if it is populated, and set to the
|
||||
// non-default policy.
|
||||
f.opt.StoragePolicy = ""
|
||||
_, err = f.fetchStoragePolicy(ctx, container)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, policy, f.opt.StoragePolicy)
|
||||
|
||||
// Test that when a segmented upload container is made, the newly
|
||||
// created container inherits the non-default policy of the base
|
||||
// container.
|
||||
f.opt.StoragePolicy = ""
|
||||
f.opt.UseSegmentsContainer.Value = true
|
||||
su, err := f.newSegmentedUpload(ctx, container, "")
|
||||
require.NoError(t, err)
|
||||
// The container name we expected?
|
||||
segmentsContainer := container + segmentsContainerSuffix
|
||||
assert.Equal(t, segmentsContainer, su.container)
|
||||
// The policy we expected?
|
||||
f.opt.StoragePolicy = ""
|
||||
_, err = f.fetchStoragePolicy(ctx, su.container)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, policy, f.opt.StoragePolicy)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
@@ -21,7 +21,6 @@ func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath stri
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
for _, u := range upstreams {
|
||||
u := u // Closure
|
||||
go func() {
|
||||
rfs := u.RootFs
|
||||
remote := path.Join(u.RootPath, filePath)
|
||||
|
||||
@@ -123,7 +123,7 @@ func (p *Prop) Hashes() (hashes map[hash.Type]string) {
|
||||
hashes = make(map[hash.Type]string)
|
||||
for _, checksums := range p.Checksums {
|
||||
checksums = strings.ToLower(checksums)
|
||||
for _, checksum := range strings.Split(checksums, " ") {
|
||||
for checksum := range strings.SplitSeq(checksums, " ") {
|
||||
switch {
|
||||
case strings.HasPrefix(checksum, "sha1:"):
|
||||
hashes[hash.SHA1] = checksum[5:]
|
||||
|
||||
@@ -33,7 +33,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
|
||||
}
|
||||
logMap = map[string]string{}
|
||||
logs = []string{}
|
||||
for _, line := range bytes.Split(out, []byte{'\n'}) {
|
||||
for line := range bytes.SplitSeq(out, []byte{'\n'}) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -23,15 +23,15 @@ func init() {
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "authorize <fs name> [base64_json_blob | client_id client_secret]",
|
||||
Use: "authorize <backendname> [base64_json_blob | client_id client_secret]",
|
||||
Short: `Remote authorization.`,
|
||||
Long: `Remote authorization. Used to authorize a remote or headless
|
||||
rclone from a machine with a browser - use as instructed by
|
||||
rclone config.
|
||||
rclone from a machine with a browser. Use as instructed by rclone config.
|
||||
See also the [remote setup documentation](/remote_setup).
|
||||
|
||||
The command requires 1-3 arguments:
|
||||
|
||||
- fs name (e.g., "drive", "s3", etc.)
|
||||
- Name of a backend (e.g. "drive", "s3")
|
||||
- Either a base64 encoded JSON blob obtained from a previous rclone config session
|
||||
- Or a client_id and client_secret pair obtained from the remote service
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
func TestAuthorizeCommand(t *testing.T) {
|
||||
// Test that the Use string is correctly formatted
|
||||
if commandDefinition.Use != "authorize <fs name> [base64_json_blob | client_id client_secret]" {
|
||||
if commandDefinition.Use != "authorize <backendname> [base64_json_blob | client_id client_secret]" {
|
||||
t.Errorf("Command Use string doesn't match expected format: %s", commandDefinition.Use)
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ func TestAuthorizeCommand(t *testing.T) {
|
||||
}
|
||||
|
||||
helpOutput := buf.String()
|
||||
if !strings.Contains(helpOutput, "authorize <fs name>") {
|
||||
if !strings.Contains(helpOutput, "authorize <backendname>") {
|
||||
t.Errorf("Help output doesn't contain correct usage information")
|
||||
}
|
||||
}
|
||||
@@ -4,15 +4,19 @@ package bilib
|
||||
import (
|
||||
"bytes"
|
||||
"log/slog"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
)
|
||||
|
||||
// CaptureOutput runs a function capturing its output at log level INFO.
|
||||
func CaptureOutput(fun func()) []byte {
|
||||
var mu sync.Mutex
|
||||
buf := &bytes.Buffer{}
|
||||
oldLevel := log.Handler.SetLevel(slog.LevelInfo)
|
||||
log.Handler.SetOutput(func(level slog.Level, text string) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
buf.WriteString(text)
|
||||
})
|
||||
defer func() {
|
||||
@@ -20,5 +24,7 @@ func CaptureOutput(fun func()) []byte {
|
||||
log.Handler.SetLevel(oldLevel)
|
||||
}()
|
||||
fun()
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -522,7 +522,7 @@ func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase str
|
||||
require.NoError(b.t, err)
|
||||
b.step = 0
|
||||
b.stopped = false
|
||||
for _, line := range strings.Split(string(scenBuf), "\n") {
|
||||
for line := range strings.SplitSeq(string(scenBuf), "\n") {
|
||||
comment := strings.Index(line, "#")
|
||||
if comment != -1 {
|
||||
line = line[:comment]
|
||||
@@ -936,7 +936,7 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
|
||||
// splitLine splits scenario line into tokens and performs
|
||||
// substitutions that involve whitespace or control chars.
|
||||
func splitLine(line string) (args []string) {
|
||||
for _, s := range strings.Fields(line) {
|
||||
for s := range strings.FieldsSeq(line) {
|
||||
b := []byte(whitespaceReplacer.Replace(s))
|
||||
b = regexChar.ReplaceAllFunc(b, func(b []byte) []byte {
|
||||
c, _ := strconv.ParseUint(string(b[5:7]), 16, 8)
|
||||
@@ -1513,7 +1513,7 @@ func (b *bisyncTest) compareResults() int {
|
||||
|
||||
fs.Log(nil, divider)
|
||||
fs.Logf(nil, color(terminal.RedFg, "| MISCOMPARE -Golden vs +Results for %s"), file)
|
||||
for _, line := range strings.Split(strings.TrimSpace(text), "\n") {
|
||||
for line := range strings.SplitSeq(strings.TrimSpace(text), "\n") {
|
||||
fs.Logf(nil, "| %s", strings.TrimSpace(line))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,8 +219,8 @@ func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
var CompareFlag CompareOpt // for exclusions
|
||||
opts := strings.Split(b.opt.CompareFlag, ",")
|
||||
for _, opt := range opts {
|
||||
opts := strings.SplitSeq(b.opt.CompareFlag, ",")
|
||||
for opt := range opts {
|
||||
switch strings.ToLower(strings.TrimSpace(opt)) {
|
||||
case "size":
|
||||
b.opt.Compare.Size = true
|
||||
|
||||
@@ -434,7 +434,6 @@ func (b *bisyncRun) listDirsOnly(listingNum int) (*fileList, error) {
|
||||
}
|
||||
|
||||
fulllisting, err = b.loadListingNum(listingNum)
|
||||
|
||||
if err != nil {
|
||||
b.critical = true
|
||||
b.retryable = true
|
||||
@@ -610,6 +609,11 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
|
||||
}
|
||||
}
|
||||
if srcNewName != "" { // if it was renamed and not deleted
|
||||
if new == nil { // should not happen. log error and debug info
|
||||
b.handleErr(b.renames, "internal error", fmt.Errorf("missing info for %q. Please report a bug at https://github.com/rclone/rclone/issues", srcNewName), true, true)
|
||||
fs.PrettyPrint(srcList, "srcList for debugging", fs.LogLevelNotice)
|
||||
continue
|
||||
}
|
||||
srcList.put(srcNewName, new.size, new.time, new.hash, new.id, new.flags)
|
||||
dstList.put(srcNewName, new.size, new.time, new.hash, new.id, new.flags)
|
||||
}
|
||||
@@ -703,8 +707,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
|
||||
prettyprint(dstList.list, "dstList", fs.LogLevelDebug)
|
||||
|
||||
// clear stats so we only do this once
|
||||
accounting.MaxCompletedTransfers = 0
|
||||
accounting.Stats(ctx).PruneTransfers()
|
||||
accounting.Stats(ctx).RemoveDoneTransfers()
|
||||
}
|
||||
|
||||
if b.DebugName != "" {
|
||||
|
||||
+2
-4
@@ -245,10 +245,8 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
|
||||
}
|
||||
}
|
||||
|
||||
b.SyncCI = fs.GetConfig(ctxCopy) // allows us to request graceful shutdown
|
||||
if accounting.MaxCompletedTransfers != -1 {
|
||||
accounting.MaxCompletedTransfers = -1 // we need a complete list in the event of graceful shutdown
|
||||
}
|
||||
b.SyncCI = fs.GetConfig(ctxCopy) // allows us to request graceful shutdown
|
||||
accounting.Stats(ctxCopy).SetMaxCompletedTransfers(-1) // we need a complete list in the event of graceful shutdown
|
||||
ctxCopy, b.CancelSync = context.WithCancel(ctxCopy)
|
||||
b.testFn()
|
||||
err := sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
|
||||
|
||||
+17
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/cluster"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/config/configflags"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
@@ -481,6 +482,22 @@ func initConfig() {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Run as a cluster worker if configured, otherwise ignoring
|
||||
// the command given on the command line
|
||||
if ci.Cluster != "" {
|
||||
if ci.ClusterID == "" || ci.ClusterID == "0" {
|
||||
fs.Infof(nil, "Running in cluster mode %q as controller", ci.ClusterID)
|
||||
} else {
|
||||
fs.Infof(nil, "Running in cluster mode %q as worker with id %q", ci.ClusterID, ci.ClusterID)
|
||||
worker, err := cluster.NewWorker(ctx)
|
||||
if err != nil || worker == nil {
|
||||
fs.Fatalf(nil, "Failed to start cluster worker: %v", err)
|
||||
}
|
||||
// Do not continue with the main thread
|
||||
select {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resolveExitCode(err error) {
|
||||
|
||||
@@ -37,6 +37,7 @@ func init() {
|
||||
configCommand.AddCommand(configDisconnectCommand)
|
||||
configCommand.AddCommand(configUserInfoCommand)
|
||||
configCommand.AddCommand(configEncryptionCommand)
|
||||
configCommand.AddCommand(configStringCommand)
|
||||
}
|
||||
|
||||
var configCommand = &cobra.Command{
|
||||
@@ -613,3 +614,55 @@ If the config file is not encrypted it will return a non zero exit code.`, "|",
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var configStringCommand = &cobra.Command{
|
||||
Use: "string <remote>",
|
||||
Short: `Print connection string for a single remote.`,
|
||||
Long: strings.ReplaceAll(`Print a connection string for a single remote.
|
||||
|
||||
The [connection strings](/docs/#connection-strings) can be used
|
||||
wherever a remote is needed and can be more convenient than using the
|
||||
config file, especially if using the RC API.
|
||||
|
||||
Backend parameters may be provided to the command also.
|
||||
|
||||
Example:
|
||||
|
||||
|||sh
|
||||
$ rclone config string s3:rclone --s3-no-check-bucket
|
||||
:s3,access_key_id=XXX,no_check_bucket,provider=AWS,region=eu-west-2,secret_access_key=YYY:rclone
|
||||
|||
|
||||
|
||||
**NB** the strings are not quoted for use in shells (eg bash,
|
||||
powershell, windows cmd). Most will work if enclosed in "double
|
||||
quotes", however connection strings that contain double quotes will
|
||||
require further quoting which is very shell dependent.
|
||||
|
||||
`, "|", "`"),
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.72",
|
||||
},
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
remote := args[0]
|
||||
fsInfo, _, fsPath, m, err := fs.ConfigFs(remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Find the overridden options and construct the string
|
||||
overridden := fsInfo.Options.NonDefault(m)
|
||||
var out strings.Builder
|
||||
out.WriteRune(':')
|
||||
out.WriteString(fsInfo.Name)
|
||||
config := overridden.Human()
|
||||
if config != "" {
|
||||
out.WriteRune(',')
|
||||
out.WriteString(config)
|
||||
}
|
||||
out.WriteRune(':')
|
||||
out.WriteString(fsPath)
|
||||
fmt.Println(out.String())
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item {
|
||||
items := []fstest.Item{}
|
||||
for _, c := range alphabet {
|
||||
var out strings.Builder
|
||||
for i := rune(0); i < 7; i++ {
|
||||
for i := range rune(7) {
|
||||
out.WriteRune(c + i)
|
||||
}
|
||||
fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
|
||||
|
||||
+100
-23
@@ -3,6 +3,7 @@ package copyurl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -12,7 +13,9 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/errcount"
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -21,6 +24,7 @@ var (
|
||||
printFilename = false
|
||||
stdout = false
|
||||
noClobber = false
|
||||
urls = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -31,6 +35,7 @@ func init() {
|
||||
flags.BoolVarP(cmdFlags, &printFilename, "print-filename", "p", printFilename, "Print the resulting name from --auto-filename", "")
|
||||
flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name", "")
|
||||
flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file", "")
|
||||
flags.BoolVarP(cmdFlags, &urls, "urls", "", stdout, "Use a CSV file of links to process multiple URLs", "")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -54,6 +59,17 @@ destination if there is one with the same name.
|
||||
Setting |--stdout| or making the output file name |-|
|
||||
will cause the output to be written to standard output.
|
||||
|
||||
Setting |--urls| allows you to input a CSV file of URLs in format: URL,
|
||||
FILENAME. If |--urls| is in use then replace the URL in the arguments with the
|
||||
file containing the URLs, e.g.:
|
||||
|||sh
|
||||
rclone copyurl --urls myurls.csv remote:dir
|
||||
|||
|
||||
Missing filenames will be autogenerated equivalent to using |--auto-filename|.
|
||||
Note that |--stdout| and |--print-filename| are incompatible with |--urls|.
|
||||
This will do |--transfers| copies in parallel. Note that if |--auto-filename|
|
||||
is desired for all URLs then a file with only URLs and no filename can be used.
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
If you can't get |rclone copyurl| to work then here are some things you can try:
|
||||
@@ -70,32 +86,93 @@ If you can't get |rclone copyurl| to work then here are some things you can try:
|
||||
RunE: func(command *cobra.Command, args []string) (err error) {
|
||||
cmd.CheckArgs(1, 2, command, args)
|
||||
|
||||
var dstFileName string
|
||||
var fsdst fs.Fs
|
||||
if !stdout {
|
||||
if len(args) < 2 {
|
||||
return errors.New("need 2 arguments if not using --stdout")
|
||||
}
|
||||
if args[1] == "-" {
|
||||
stdout = true
|
||||
} else if autoFilename {
|
||||
fsdst = cmd.NewFsDir(args[1:])
|
||||
} else {
|
||||
fsdst, dstFileName = cmd.NewFsDstFile(args[1:])
|
||||
}
|
||||
}
|
||||
cmd.Run(true, true, command, func() error {
|
||||
var dst fs.Object
|
||||
if stdout {
|
||||
err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout)
|
||||
} else {
|
||||
dst, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, headerFilename, noClobber)
|
||||
if printFilename && err == nil && dst != nil {
|
||||
fmt.Println(dst.Remote())
|
||||
}
|
||||
if !urls {
|
||||
return run(args)
|
||||
}
|
||||
return err
|
||||
return runURLS(args)
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var copyURL = operations.CopyURL // for testing
|
||||
|
||||
// runURLS processes a .csv file of urls and filenames
|
||||
func runURLS(args []string) (err error) {
|
||||
if stdout {
|
||||
return errors.New("can't use --stdout with --urls")
|
||||
}
|
||||
if printFilename {
|
||||
return errors.New("can't use --print-filename with --urls")
|
||||
}
|
||||
dstFs := cmd.NewFsDir(args[1:])
|
||||
|
||||
f, err := os.Open(args[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open .csv file: %w", err)
|
||||
}
|
||||
defer fs.CheckClose(f, &err)
|
||||
reader := csv.NewReader(f)
|
||||
reader.FieldsPerRecord = -1
|
||||
urlList, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed reading .csv file: %w", err)
|
||||
}
|
||||
|
||||
ec := errcount.New()
|
||||
g, gCtx := errgroup.WithContext(context.Background())
|
||||
ci := fs.GetConfig(gCtx)
|
||||
g.SetLimit(ci.Transfers)
|
||||
|
||||
for _, urlEntry := range urlList {
|
||||
if len(urlEntry) == 0 {
|
||||
continue
|
||||
}
|
||||
g.Go(func() error {
|
||||
url := urlEntry[0]
|
||||
var filename string
|
||||
if len(urlEntry) > 1 {
|
||||
filename = urlEntry[1]
|
||||
}
|
||||
_, err := copyURL(gCtx, dstFs, filename, url, filename == "", headerFilename, noClobber)
|
||||
if err != nil {
|
||||
fs.Errorf(filename, "failed to copy URL %q: %v", url, err)
|
||||
ec.Add(err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
ec.Add(g.Wait())
|
||||
return ec.Err("not all URLs copied successfully")
|
||||
}
|
||||
|
||||
// run runs the command for a single URL
|
||||
func run(args []string) error {
|
||||
var err error
|
||||
var dstFileName string
|
||||
var fsdst fs.Fs
|
||||
if !stdout {
|
||||
if len(args) < 2 {
|
||||
return errors.New("need 2 arguments if not using --stdout")
|
||||
}
|
||||
if args[1] == "-" {
|
||||
stdout = true
|
||||
} else if autoFilename {
|
||||
fsdst = cmd.NewFsDir(args[1:])
|
||||
} else {
|
||||
fsdst, dstFileName = cmd.NewFsDstFile(args[1:])
|
||||
}
|
||||
}
|
||||
|
||||
var dst fs.Object
|
||||
if stdout {
|
||||
err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout)
|
||||
} else {
|
||||
dst, err = copyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, headerFilename, noClobber)
|
||||
if printFilename && err == nil && dst != nil {
|
||||
fmt.Println(dst.Remote())
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package copyurl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func resetGlobals() {
|
||||
autoFilename = false
|
||||
headerFilename = false
|
||||
printFilename = false
|
||||
stdout = false
|
||||
noClobber = false
|
||||
urls = false
|
||||
copyURL = operations.CopyURL
|
||||
}
|
||||
|
||||
func TestRun_RequiresTwoArgsWhenNotStdout(t *testing.T) {
|
||||
t.Cleanup(resetGlobals)
|
||||
resetGlobals()
|
||||
|
||||
err := run([]string{"https://example.com/foo"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "need 2 arguments if not using --stdout")
|
||||
}
|
||||
|
||||
func TestRun_CallsCopyURL_WithExplicitFilename_Success(t *testing.T) {
|
||||
t.Cleanup(resetGlobals)
|
||||
resetGlobals()
|
||||
|
||||
tmp := t.TempDir()
|
||||
dstPath := filepath.Join(tmp, "out.txt")
|
||||
|
||||
var called int32
|
||||
|
||||
copyURL = func(_ctx context.Context, _dst fs.Fs, dstFileName, url string, auto, header, noclobber bool) (fs.Object, error) {
|
||||
atomic.AddInt32(&called, 1)
|
||||
assert.Equal(t, "https://example.com/file", url)
|
||||
assert.Equal(t, "out.txt", dstFileName)
|
||||
assert.False(t, auto)
|
||||
assert.False(t, header)
|
||||
assert.False(t, noclobber)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
err := run([]string{"https://example.com/file", dstPath})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&called))
|
||||
}
|
||||
|
||||
func TestRun_CallsCopyURL_WithAutoFilename_AndPropagatesError(t *testing.T) {
|
||||
t.Cleanup(resetGlobals)
|
||||
resetGlobals()
|
||||
|
||||
tmp := t.TempDir()
|
||||
autoFilename = true
|
||||
|
||||
want := errors.New("boom")
|
||||
var called int32
|
||||
|
||||
copyURL = func(_ctx context.Context, _dst fs.Fs, dstFileName, url string, auto, header, noclobber bool) (fs.Object, error) {
|
||||
atomic.AddInt32(&called, 1)
|
||||
assert.Equal(t, "", dstFileName) // auto filename -> empty
|
||||
assert.True(t, auto)
|
||||
return nil, want
|
||||
}
|
||||
|
||||
err := run([]string{"https://example.com/auto/name", tmp})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, want, err)
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&called))
|
||||
}
|
||||
|
||||
func TestRunURLS_ErrorsWithStdoutAndWithPrintFilename(t *testing.T) {
|
||||
t.Cleanup(resetGlobals)
|
||||
resetGlobals()
|
||||
|
||||
stdout = true
|
||||
err := runURLS([]string{"dummy.csv", "destDir"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "can't use --stdout with --urls")
|
||||
|
||||
resetGlobals()
|
||||
printFilename = true
|
||||
err = runURLS([]string{"dummy.csv", "destDir"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "can't use --print-filename with --urls")
|
||||
}
|
||||
|
||||
func TestRunURLS_ProcessesCSV_ParallelCalls_AndAggregatesError(t *testing.T) {
|
||||
t.Cleanup(resetGlobals)
|
||||
resetGlobals()
|
||||
|
||||
tmp := t.TempDir()
|
||||
csvPath := filepath.Join(tmp, "urls.csv")
|
||||
csvContent := []byte(
|
||||
"https://example.com/a,aaa.txt\n" + // success
|
||||
"https://example.com/b\n" + // auto filename
|
||||
"https://example.com/c,ccc.txt\n") // error
|
||||
require.NoError(t, os.WriteFile(csvPath, csvContent, 0o600))
|
||||
|
||||
// destination dir (local backend)
|
||||
dest := t.TempDir()
|
||||
|
||||
// mock copyURL: succeed for /a and /b, fail for /c
|
||||
|
||||
var calls int32
|
||||
var mu sync.Mutex
|
||||
var seen []string
|
||||
|
||||
copyURL = func(_ctx context.Context, _dst fs.Fs, dstFileName, url string, auto, header, noclobber bool) (fs.Object, error) {
|
||||
atomic.AddInt32(&calls, 1)
|
||||
mu.Lock()
|
||||
seen = append(seen, url+"|"+dstFileName)
|
||||
mu.Unlock()
|
||||
|
||||
switch {
|
||||
case url == "https://example.com/a":
|
||||
require.Equal(t, "aaa.txt", dstFileName)
|
||||
return nil, nil
|
||||
case url == "https://example.com/b":
|
||||
require.Equal(t, "", dstFileName) // auto-name path
|
||||
return nil, nil
|
||||
case url == "https://example.com/c":
|
||||
return nil, errors.New("network down")
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
err := runURLS([]string{csvPath, dest})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not all URLs copied successfully")
|
||||
// 3 lines => 3 calls
|
||||
assert.Equal(t, int32(3), atomic.LoadInt32(&calls))
|
||||
|
||||
// sanity: all expected URLs were seen
|
||||
assert.ElementsMatch(t,
|
||||
[]string{
|
||||
"https://example.com/a|aaa.txt",
|
||||
"https://example.com/b|",
|
||||
"https://example.com/c|ccc.txt",
|
||||
},
|
||||
seen,
|
||||
)
|
||||
}
|
||||
@@ -229,7 +229,6 @@ func TestEndToEnd(t *testing.T) {
|
||||
skipE2eTestIfNecessary(t)
|
||||
|
||||
for _, mode := range allLayoutModes() {
|
||||
mode := mode
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -258,7 +257,6 @@ func TestEndToEndMigration(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, mode := range allLayoutModes() {
|
||||
mode := mode
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -318,7 +316,6 @@ func TestEndToEndRepoLayoutCompat(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, mode := range allLayoutModes() {
|
||||
mode := mode
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+1
-1
@@ -344,7 +344,7 @@ func showBackend(name string) {
|
||||
}
|
||||
for _, ex := range opt.Examples {
|
||||
fmt.Printf(" - %s\n", quoteString(ex.Value))
|
||||
for _, line := range strings.Split(ex.Help, "\n") {
|
||||
for line := range strings.SplitSeq(ex.Help, "\n") {
|
||||
fmt.Printf(" - %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cluster"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/fs/rc/rcflags"
|
||||
"github.com/rclone/rclone/fs/rc/rcserver"
|
||||
@@ -38,6 +39,8 @@ See the [rc documentation](/rc/) for more info on the rc flags.
|
||||
"groups": "RC",
|
||||
},
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
ctx := context.Background()
|
||||
|
||||
cmd.CheckArgs(0, 1, command, args)
|
||||
if rc.Opt.Enabled {
|
||||
fs.Fatalf(nil, "Don't supply --rc flag when using rcd")
|
||||
@@ -49,6 +52,12 @@ See the [rc documentation](/rc/) for more info on the rc flags.
|
||||
rc.Opt.Files = args[0]
|
||||
}
|
||||
|
||||
// Start the cluster worker if configured
|
||||
_, err := cluster.NewWorker(ctx)
|
||||
if err != nil {
|
||||
fs.Fatalf(nil, "Failed to start cluster worker: %v", err)
|
||||
}
|
||||
|
||||
s, err := rcserver.Start(context.Background(), &rc.Opt)
|
||||
if err != nil {
|
||||
fs.Fatalf(nil, "Failed to start remote control: %v", err)
|
||||
|
||||
+26
-3
@@ -41,9 +41,10 @@ var OptionsInfo = fs.Options{}.
|
||||
|
||||
// Options required for http server
|
||||
type Options struct {
|
||||
Auth libhttp.AuthConfig
|
||||
HTTP libhttp.Config
|
||||
Template libhttp.TemplateConfig
|
||||
Auth libhttp.AuthConfig
|
||||
HTTP libhttp.Config
|
||||
Template libhttp.TemplateConfig
|
||||
DisableZip bool
|
||||
}
|
||||
|
||||
// DefaultOpt is the default values used for Options
|
||||
@@ -69,6 +70,7 @@ func init() {
|
||||
flags.AddFlagsFromOptions(flagSet, "", OptionsInfo)
|
||||
vfsflags.AddFlags(flagSet)
|
||||
proxyflags.AddFlags(flagSet)
|
||||
flagSet.BoolVar(&Opt.DisableZip, "disable-zip", false, "Disable zip download of directories")
|
||||
cmdserve.Command.AddCommand(Command)
|
||||
cmdserve.AddRc("http", func(ctx context.Context, f fs.Fs, in rc.Params) (cmdserve.Handle, error) {
|
||||
// Read VFS Opts
|
||||
@@ -208,6 +210,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
// Serve HTTP until the server is shutdown
|
||||
func (s *HTTP) Serve() error {
|
||||
s.server.Serve()
|
||||
fs.Logf(s.f, "HTTP Server started on %s", s.server.URLs())
|
||||
s.server.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -256,6 +259,24 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string
|
||||
return
|
||||
}
|
||||
dir := node.(*vfs.Dir)
|
||||
|
||||
if r.URL.Query().Get("download") == "zip" && !s.opt.DisableZip {
|
||||
fs.Infof(dirRemote, "%s: Zipping directory", r.RemoteAddr)
|
||||
zipName := path.Base(dirRemote)
|
||||
if dirRemote == "" {
|
||||
zipName = "root"
|
||||
}
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=\""+zipName+".zip\"")
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
w.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat))
|
||||
err := vfs.CreateZip(ctx, dir, w)
|
||||
if err != nil {
|
||||
serve.Error(ctx, dirRemote, w, "Failed to create zip", err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
dirEntries, err := dir.ReadDirAll()
|
||||
if err != nil {
|
||||
serve.Error(ctx, dirRemote, w, "Failed to list directory", err)
|
||||
@@ -279,6 +300,8 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string
|
||||
// Set the Last-Modified header to the timestamp
|
||||
w.Header().Set("Last-Modified", dir.ModTime().UTC().Format(http.TimeFormat))
|
||||
|
||||
directory.DisableZip = s.opt.DisableZip
|
||||
|
||||
directory.Serve(w, r)
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"flag"
|
||||
"io"
|
||||
stdfs "io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -75,6 +76,16 @@ func start(ctx context.Context, t *testing.T, f fs.Fs) (s *HTTP, testURL string)
|
||||
return s, testURL
|
||||
}
|
||||
|
||||
// setAllModTimes walks root and sets atime/mtime to t for every file & directory.
|
||||
func setAllModTimes(root string, t time.Time) error {
|
||||
return filepath.WalkDir(root, func(path string, d stdfs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Chtimes(path, t, t)
|
||||
})
|
||||
}
|
||||
|
||||
var (
|
||||
datedObject = "two.txt"
|
||||
expectedTime = time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
@@ -123,6 +134,8 @@ func testGET(t *testing.T, useProxy bool) {
|
||||
|
||||
f = nil
|
||||
} else {
|
||||
// set all the mod times to expectedTime
|
||||
require.NoError(t, setAllModTimes("testdata/files", expectedTime))
|
||||
// Create a test Fs
|
||||
var err error
|
||||
f, err = fs.NewFs(context.Background(), "testdata/files")
|
||||
@@ -233,6 +246,16 @@ func testGET(t *testing.T, useProxy bool) {
|
||||
Range: "bytes=3-",
|
||||
Golden: "testdata/golden/two3-.txt",
|
||||
},
|
||||
{
|
||||
URL: "/?download=zip",
|
||||
Status: http.StatusOK,
|
||||
Golden: "testdata/golden/root.zip",
|
||||
},
|
||||
{
|
||||
URL: "/three/?download=zip",
|
||||
Status: http.StatusOK,
|
||||
Golden: "testdata/golden/three.zip",
|
||||
},
|
||||
} {
|
||||
method := test.Method
|
||||
if method == "" {
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -66,7 +66,6 @@ func testCacheCRUD(t *testing.T, h *Handler, c Cache, fileName string) {
|
||||
func testCacheThrashDifferent(t *testing.T, h *Handler, c Cache) {
|
||||
var wg sync.WaitGroup
|
||||
for i := range 100 {
|
||||
i := i
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
@@ -125,7 +124,6 @@ func TestCache(t *testing.T) {
|
||||
}()
|
||||
billyFS := &FS{nil} // place holder billyFS
|
||||
for _, cacheType := range []handleCache{cacheMemory, cacheDisk, cacheSymlink} {
|
||||
cacheType := cacheType
|
||||
t.Run(cacheType.String(), func(t *testing.T) {
|
||||
h := &Handler{
|
||||
vfs: vfs.New(object.MemoryFs, nil),
|
||||
|
||||
@@ -182,7 +182,7 @@ func (p *Proxy) run(in map[string]string) (config configmap.Simple, err error) {
|
||||
// Obscure any values in the config map that need it
|
||||
obscureFields, ok := config.Get("_obscure")
|
||||
if ok {
|
||||
for _, key := range strings.Split(obscureFields, ",") {
|
||||
for key := range strings.SplitSeq(obscureFields, ",") {
|
||||
value, ok := config.Get(key)
|
||||
if ok {
|
||||
obscuredValue, err := obscure.Obscure(value)
|
||||
|
||||
+11
-12
@@ -2,6 +2,7 @@ package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/gofakes3"
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -12,25 +13,23 @@ type logger struct{}
|
||||
|
||||
// print log message
|
||||
func (l logger) Print(level gofakes3.LogLevel, v ...any) {
|
||||
var s string
|
||||
if len(v) == 0 {
|
||||
s = ""
|
||||
} else {
|
||||
var ok bool
|
||||
s, ok = v[0].(string)
|
||||
if !ok {
|
||||
s = fmt.Sprint(v[0])
|
||||
var b strings.Builder
|
||||
for i := range v {
|
||||
if i > 0 {
|
||||
fmt.Fprintf(&b, " ")
|
||||
}
|
||||
v = v[1:]
|
||||
fmt.Fprint(&b, v[i])
|
||||
}
|
||||
s := b.String()
|
||||
|
||||
switch level {
|
||||
default:
|
||||
fallthrough
|
||||
case gofakes3.LogErr:
|
||||
fs.Errorf("serve s3", s, v...)
|
||||
fs.Errorf("serve s3", s)
|
||||
case gofakes3.LogWarn:
|
||||
fs.Infof("serve s3", s, v...)
|
||||
fs.Infof("serve s3", s)
|
||||
case gofakes3.LogInfo:
|
||||
fs.Debugf("serve s3", s, v...)
|
||||
fs.Debugf("serve s3", s)
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ func (r *results) checkBase32768() {
|
||||
// Create test files
|
||||
for _, c := range safeAlphabet {
|
||||
var out strings.Builder
|
||||
for i := rune(0); i < 32; i++ {
|
||||
for i := range rune(32) {
|
||||
out.WriteRune(c + i)
|
||||
}
|
||||
fileName := filepath.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
|
||||
|
||||
@@ -292,7 +292,7 @@ func (r *results) checkControls() {
|
||||
tokens <- struct{}{}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for i := rune(0); i < 128; i++ {
|
||||
for i := range rune(128) {
|
||||
s := string(i)
|
||||
if i == 0 || i == '/' {
|
||||
// We're not even going to check NULL or /
|
||||
|
||||
@@ -95,7 +95,7 @@ func (e *Position) UnmarshalText(text []byte) error {
|
||||
switch s := strings.ToLower(string(text)); s {
|
||||
default:
|
||||
*e = PositionNone
|
||||
for _, p := range strings.Split(s, ",") {
|
||||
for p := range strings.SplitSeq(s, ",") {
|
||||
switch p {
|
||||
case "left":
|
||||
*e |= PositionLeft
|
||||
|
||||
@@ -351,7 +351,7 @@ func TestEnvironmentVariables(t *testing.T) {
|
||||
parseFileFilters := func(out string) (extensions []string) {
|
||||
// Match: - (^|/)[^/]*\.jpg$
|
||||
find := regexp.MustCompile(`^- \(\^\|\/\)\[\^\/\]\*\\\.(.*?)\$$`)
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
for line := range strings.SplitSeq(out, "\n") {
|
||||
if m := find.FindStringSubmatch(line); m != nil {
|
||||
extensions = append(extensions, m[1])
|
||||
}
|
||||
|
||||
@@ -125,6 +125,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="Exaba" home="https://exaba.com/" config="/s3/#exaba" >}}
|
||||
{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
|
||||
{{< provider name="FileLu Cloud Storage" home="https://filelu.com/" config="/filelu/" >}}
|
||||
{{< provider name="FileLu S5 (S3-Compatible Object Storage)" home="https://s5lu.com/" config="/s3/#filelu-s5" >}}
|
||||
{{< provider name="Files.com" home="https://www.files.com/" config="/filescom/" >}}
|
||||
{{< provider name="FlashBlade" home="https://www.purestorage.com/products/unstructured-data-storage.html" config="/s3/#pure-storage-flashblade" >}}
|
||||
{{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
|
||||
@@ -133,9 +134,11 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}}
|
||||
{{< provider name="Google Photos" home="https://www.google.com/photos/about/" config="/googlephotos/" >}}
|
||||
{{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}}
|
||||
{{< provider name="Hetzner Object Storage" home="https://www.hetzner.com/storage/object-storage/" config="/s3/#hetzner" >}}
|
||||
{{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}}
|
||||
{{< provider name="HiDrive" home="https://www.strato.de/cloud-speicher/" config="/hidrive/" >}}
|
||||
{{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
|
||||
{{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}}
|
||||
{{< provider name="iCloud Drive" home="https://icloud.com/" config="/iclouddrive/" >}}
|
||||
{{< provider name="ImageKit" home="https://imagekit.io" config="/imagekit/" >}}
|
||||
{{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}
|
||||
@@ -179,7 +182,10 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
|
||||
{{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}}
|
||||
{{< provider name="Quatrix by Maytech" home="https://www.maytech.net/products/quatrix-business" config="/quatrix/" >}}
|
||||
{{< provider name="Rabata Cloud Storage" home="https://rabata.io" config="/s3/#Rabata" >}}
|
||||
{{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}}
|
||||
{{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}
|
||||
{{< provider name="Rclone Serve S3" home="/commands/rclone_serve_s3/" config="/s3/#rclone" >}}
|
||||
{{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}}
|
||||
{{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}}
|
||||
{{< provider name="Seafile" home="https://www.seafile.com/" config="/seafile/" >}}
|
||||
@@ -189,6 +195,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}}
|
||||
{{< provider name="Sia" home="https://sia.tech/" config="/sia/" >}}
|
||||
{{< provider name="SMB / CIFS" home="https://en.wikipedia.org/wiki/Server_Message_Block" config="/smb/" >}}
|
||||
{{< provider name="Spectra Logic" home="https://spectralogic.com/blackpearl-nearline-object-gateway/" config="/s3/#spectralogic" >}}
|
||||
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
|
||||
{{< provider name="Storj" home="https://storj.io/" config="/storj/" >}}
|
||||
{{< provider name="Synology" home="https://c2.synology.com/en-global/object-storage/overview" config="/s3/#synology-c2" >}}
|
||||
|
||||
+11
-1
@@ -990,7 +990,7 @@ put them back in again.` >}}
|
||||
- Ross Smith II <ross@smithii.com>
|
||||
- Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com>
|
||||
- Sudipto Baral <sudiptobaral.me@gmail.com>
|
||||
- Sam Pegg <samrpegg@gmail.com>
|
||||
- Sam Pegg <samrpegg@gmail.com> <70067376+S-Pegg1@users.noreply.github.com>
|
||||
- liubingrun <liubr1@chinatelecom.cn>
|
||||
- Albin Parou <fumesover@gmail.com>
|
||||
- n4n5 <56606507+Its-Just-Nans@users.noreply.github.com>
|
||||
@@ -1011,3 +1011,13 @@ put them back in again.` >}}
|
||||
- dougal <dougal.craigwood@gmail.com> <147946567+roucc@users.noreply.github.com>
|
||||
- anon-pradip <pradipsubedi360@gmail.com>
|
||||
- Robin Rolf <imer@imer.cc>
|
||||
- Jean-Christophe Cura <jcaspes@gmail.com>
|
||||
- russcoss <russcoss@outlook.com>
|
||||
- Matt LaPaglia <mlapaglia@gmail.com>
|
||||
- Youfu Zhang <1315097+zhangyoufu@users.noreply.github.com>
|
||||
- juejinyuxitu <juejinyuxitu@outlook.com>
|
||||
- iTrooz <hey@itrooz.fr>
|
||||
- Microscotch <github.com@microscotch.net>
|
||||
- Andrew Ruthven <andrew@etc.gen.nz>
|
||||
- spiffytech <git@spiffy.tech>
|
||||
- Dulani Woods <Dulani@gmail.com>
|
||||
+1
-1
@@ -84,7 +84,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Box. This only runs from the moment it opens
|
||||
|
||||
@@ -6,6 +6,31 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.71.1 - 2025-09-24
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.71.0...v1.71.1)
|
||||
|
||||
- Bug Fixes
|
||||
- bisync: Fix error handling for renamed conflicts (nielash)
|
||||
- march: Fix deadlock when using --fast-list on syncs (Nick Craig-Wood)
|
||||
- operations: Fix partial name collisions for non --inplace copies (Nick Craig-Wood)
|
||||
- pacer: Fix deadlock with --max-connections (Nick Craig-Wood)
|
||||
- doc fixes (albertony, anon-pradip, Claudius Ellsel, dougal, Jean-Christophe Cura, Nick Craig-Wood, nielash)
|
||||
- Mount
|
||||
- Do not log successful unmount as an error (Tilman Vogel)
|
||||
- VFS
|
||||
- Fix SIGHUP killing serve instead of flushing directory caches (dougal)
|
||||
- Local
|
||||
- Fix rmdir "Access is denied" on windows (nielash)
|
||||
- Box
|
||||
- Fix about after change in API return (Nick Craig-Wood)
|
||||
- Combine
|
||||
- Propagate SlowHash feature (skbeh)
|
||||
- Drive
|
||||
- Update making your own client ID instructions (Ed Craig-Wood)
|
||||
- Internet Archive
|
||||
- Fix server side copy files with spaces (Nick Craig-Wood)
|
||||
|
||||
## v1.71.0 - 2025-08-22
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.0...v1.71.0)
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
---
|
||||
title: "Cluster"
|
||||
description: "Clustering rclone"
|
||||
versionIntroduced: "v1.72"
|
||||
---
|
||||
|
||||
# Cluster
|
||||
|
||||
Rclone has a cluster mode invoked with the `--cluster` flag. This
|
||||
enables a group of rclone instances to work together on doing a sync.
|
||||
|
||||
This is controlled by a group of flags starting with `--cluster-` and
|
||||
enabled with the `--cluster` flag.
|
||||
|
||||
```text
|
||||
--cluster string Enable cluster mode with remote to use as shared storage
|
||||
--cluster-batch-files int Max number of files for a cluster batch (default 1000)
|
||||
--cluster-batch-size SizeSuffix Max size of files for a cluster batch (default 1Ti)
|
||||
--cluster-cleanup ClusterCleanup Control which cluster files get cleaned up (default full)
|
||||
--cluster-id string Set to an ID for the cluster. An ID of 0 or empty becomes the controller
|
||||
--cluster-quit-workers Set to cause the controller to quit the workers when it finished
|
||||
```
|
||||
|
||||
The command might look something like this which is a normal rclone
|
||||
command but with a new `--cluster` flag which points at an rclone
|
||||
remote defining the cluster storage. This is the signal to rclone that
|
||||
it should engage the cluster mode with a controller and workers.
|
||||
|
||||
```sh
|
||||
rclone copy source: destination: --flags --cluster /work
|
||||
rclone copy source: destination: --flags --cluster s3:bucket
|
||||
```
|
||||
|
||||
This works only with the `rclone sync`, `copy` and `move` commands.
|
||||
|
||||
If the remote specified by the `--cluster` command is inside the
|
||||
`source:` or `destination:` it must be excluded with the filter flags.
|
||||
|
||||
Any rclone remotes used in the transfer must be defined in all cluster
|
||||
nodes. Defining remotes with connection strings will get around that
|
||||
problem.
|
||||
|
||||
## Terminology
|
||||
|
||||
The cluster has two logical groups, the controller and the workers.
|
||||
There is one controller and many workers.
|
||||
|
||||
The controller and the workers will communicate with each other by
|
||||
creating files in the remote pointed to by the `--cluster` flag. This
|
||||
could be for example an S3 bucket or a Kubernetes PVC.
|
||||
|
||||
The files are JSON serialized rc commands. Multiple commands are sent
|
||||
using `rc/batch`. The commands flow `pending` →`processing` → `done` →
|
||||
`finished`
|
||||
|
||||
```text
|
||||
└── queue
|
||||
├── pending ← pending task files created by the controller
|
||||
├── processing ← claimed tasks being executed by a worker
|
||||
├── done ← finished tasks awaiting the controller to read the result
|
||||
└── finished ← completed task files
|
||||
```
|
||||
|
||||
The cluster can be set up in two ways as a persistent cluster or as a
|
||||
transient cluster.
|
||||
|
||||
### Persistent cluster
|
||||
|
||||
Run a cluster of workers using
|
||||
|
||||
```sh
|
||||
rclone rcd --cluster /work
|
||||
```
|
||||
|
||||
Then run rclone commands when required on the cluster:
|
||||
|
||||
```sh
|
||||
rclone copy source: destination: --flags --cluster /work
|
||||
```
|
||||
|
||||
In this mode there can be many rclone commands executing at once.
|
||||
|
||||
### Transient cluster
|
||||
|
||||
Run many copies of rclone simultaneously, for example in a Kubernetes
|
||||
indexed job.
|
||||
|
||||
The rclone with `--cluster-id 0` becomes the controller and the others
|
||||
become the workers. For a Kubernetes indexed job, setting
|
||||
`--cluster-id $(JOB_COMPLETION_INDEX)` would work well.
|
||||
|
||||
Add the `--cluster-quit-workers` flag - this will cause the controller
|
||||
to make sure the workers exit when it has finished.
|
||||
|
||||
All instances of rclone run a command like this so the whole cluster
|
||||
can only run one rclone command:
|
||||
|
||||
```sh
|
||||
rclone copy source: destination: --flags --cluster /work --cluster-id $(JOB_COMPLETION_INDEX) --cluster-quit-workers
|
||||
```
|
||||
|
||||
## Controller
|
||||
|
||||
The controller runs the sync and work distribution.
|
||||
|
||||
- It does the listing of the source and destination directories
|
||||
comparing files in order to find files which need to be transferred.
|
||||
- Files which need to be transferred are then batched into jobs of
|
||||
`--cluster-batch-files` files to transfer or `--cluster-batch-size`
|
||||
max size in `queue/pending` for the workers to pick up.
|
||||
- It watches `queue/done` for finished jobs and updates the transfer
|
||||
statistics and logs any errors, accordingly moving the job to
|
||||
`queue/finished`.
|
||||
|
||||
Once the sync is complete, if `--cluster-quit-workers` is set, then it
|
||||
sends the workers a special command which causes them all to exit.
|
||||
|
||||
The controller only sends transfer jobs to the workers. All the other
|
||||
tasks (eg listing, comparing) are done by the controller. The
|
||||
controller does not execute any transfer tasks itself.
|
||||
|
||||
The controller reads worker status as written to `queue/status` and
|
||||
will detect workers which have stopped. If it detects a failed worker
|
||||
then it will re-assign any outstanding work.
|
||||
|
||||
## Workers
|
||||
|
||||
The workers job is entirely to act as API endpoints that receive their
|
||||
work via files in `/work`. Then
|
||||
|
||||
- Read work in `queue/pending`
|
||||
- Attempt to rename into `queue/processing`
|
||||
- If the cluster work directory supports atomic renames, then use
|
||||
those, otherwise read the file, write the copy, delete the original.
|
||||
If the delete fails then the rename was not successful (possible on
|
||||
s3 backends).
|
||||
- If successful then do that item of work. If not successful another
|
||||
worker got there first and sleep for a bit then retry.
|
||||
- After the copy is complete then remove the `queue/processing` file
|
||||
or rename it into `queue/finished` if the `--cluster-cleanup` flag
|
||||
allows it.
|
||||
- Repeat
|
||||
|
||||
Every second the worker will write a status file in `queue/status` to
|
||||
be read by the controller.
|
||||
|
||||
## Layout of the work directory
|
||||
|
||||
The format of the files in this directory may change without notice
|
||||
but the layout is documented here as it can help debugging.
|
||||
|
||||
```text
|
||||
/work - root of the work directory
|
||||
└── queue - files to control the queue
|
||||
├── done - job files that are finished and read
|
||||
├── finished - job files that are finished but not yet read
|
||||
├── pending - job files that are not started yet
|
||||
├── processing - job files that are running
|
||||
└── status - worker status files
|
||||
```
|
||||
|
||||
If debugging use `--cluster-cleanup none` to leave the completed files
|
||||
in the directory layout.
|
||||
|
||||
## Flags
|
||||
|
||||
### --cluster string
|
||||
|
||||
This enables the cluster mode. Without this flag, all the other
|
||||
cluster flags are ignored. This should be given a remote which can be
|
||||
a local directory, eg `/work` or a remote directory, eg `s3:bucket`.
|
||||
|
||||
### --cluster-batch-files int
|
||||
|
||||
This controls the number of files copied in a cluster batch. Setting
|
||||
this larger may be more efficient but it means the statistics will be
|
||||
less accurate on the controller (default 1000).
|
||||
|
||||
### --cluster-batch-size SizeSuffix
|
||||
|
||||
This controls the total size of files in a cluster batch. If the size
|
||||
of the files in a batch exceeds this number then the batch will be
|
||||
sent to the workers. Setting this larger may be more efficient but it
|
||||
means the statistics will be less accurate on the controller. (default
|
||||
1TiB)
|
||||
|
||||
### --cluster-cleanup ClusterCleanup
|
||||
|
||||
Controls which cluster files get cleaned up.
|
||||
|
||||
- `full` - clean all work files (default)
|
||||
- `completed` - clean completed work files but leave the errors and status
|
||||
- `none` - leave all the file (useful for debugging)
|
||||
|
||||
### --cluster-id string
|
||||
|
||||
Set an ID for the rclone instance. This can be a string or a number.
|
||||
An ID of 0 will become the controller otherwise the instance will
|
||||
become a worker. If this flag isn't supplied or the value is empty,
|
||||
then a random string will be used instead.
|
||||
|
||||
### --cluster-quit-workers
|
||||
|
||||
If this flag is set, then when the controller finishes its sync task
|
||||
it will quit all the workers before it exits.
|
||||
|
||||
## Not implemented
|
||||
|
||||
Here are some features from the original design which are not
|
||||
implemented yet:
|
||||
|
||||
- the controller will not notice if workers die or fail to complete
|
||||
their tasks
|
||||
- the controller does not re-assign the workers work if necessary
|
||||
- the controller does not restart the sync
|
||||
- the workers do not write any status files (but the stats are
|
||||
correctly accounted)
|
||||
@@ -384,6 +384,9 @@ does not work on Windows.)
|
||||
rclone copy ':http,url="https://example.com":path/to/dir' /tmp/dir
|
||||
```
|
||||
|
||||
You can use [rclone config string](/commands/rclone_config_string/) to
|
||||
convert a remote into a connection string.
|
||||
|
||||
#### Connection strings, config and logging
|
||||
|
||||
If you supply extra configuration to a backend by command line flag,
|
||||
@@ -3312,6 +3315,15 @@ For the remote control options and for instructions on how to remote control rcl
|
||||
|
||||
See [the remote control section](/rc/).
|
||||
|
||||
## Cluster
|
||||
|
||||
For the cluster options and for instructions on how to cluster rclone:
|
||||
|
||||
- `--cluster`
|
||||
- Anything starting with `--cluster-`
|
||||
|
||||
See the [cluster section](/cluster/).
|
||||
|
||||
## Logging
|
||||
|
||||
rclone has 4 levels of logging, `ERROR`, `NOTICE`, `INFO` and `DEBUG`.
|
||||
|
||||
@@ -96,7 +96,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Google if using web browser to automatically
|
||||
|
||||
@@ -60,7 +60,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Dropbox. This only
|
||||
|
||||
@@ -100,7 +100,9 @@ Choose a number from below, or type in your own value
|
||||
\ "us-east1"
|
||||
13 / Northern Virginia.
|
||||
\ "us-east4"
|
||||
14 / Oregon.
|
||||
14 / Ohio.
|
||||
\ "us-east5"
|
||||
15 / Oregon.
|
||||
\ "us-west1"
|
||||
location> 12
|
||||
The storage class to use when storing objects in Google Cloud Storage.
|
||||
@@ -147,7 +149,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Google if using web browser to automatically
|
||||
|
||||
@@ -97,7 +97,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Google if using web browser to automatically
|
||||
|
||||
@@ -257,5 +257,6 @@ Properties:
|
||||
|
||||
## Limitations
|
||||
|
||||
- Erasure coding not supported, see [issue #8808](https://github.com/rclone/rclone/issues/8808)
|
||||
- No server-side `Move` or `DirMove`.
|
||||
- Checksums not implemented.
|
||||
@@ -72,7 +72,7 @@ and hence should not be shared with other persons.**
|
||||
See the [below section](#keeping-your-tokens-safe) for more information.
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from HiDrive. This only runs from the moment it opens
|
||||
|
||||
+176
-105
@@ -7,106 +7,171 @@ versionIntroduced: "v1.43"
|
||||
# {{< icon "fa fa-cloud" >}} Jottacloud
|
||||
|
||||
Jottacloud is a cloud storage service provider from a Norwegian company, using
|
||||
its own datacenters in Norway. In addition to the official service at
|
||||
[jottacloud.com](https://www.jottacloud.com/), it also provides white-label
|
||||
solutions to different companies, such as:
|
||||
its own datacenters in Norway.
|
||||
|
||||
In addition to the official service at [jottacloud.com](https://www.jottacloud.com/),
|
||||
it also provides white-label solutions to different companies. The following
|
||||
are currently supported by this backend, using a different authentication setup
|
||||
as described [below](#whitelabel-authentication):
|
||||
|
||||
- Elkjøp (with subsidiaries):
|
||||
- Elkjøp Cloud (cloud.elkjop.no)
|
||||
- Elgiganten Cloud (cloud.elgiganten.dk)
|
||||
- Elgiganten Cloud (cloud.elgiganten.se)
|
||||
- ELKO Cloud (cloud.elko.is)
|
||||
- Gigantti Cloud (cloud.gigantti.fi)
|
||||
- Telia
|
||||
- Telia Cloud (cloud.telia.se)
|
||||
- Telia Sky (sky.telia.no)
|
||||
- Tele2
|
||||
- Tele2 Cloud (mittcloud.tele2.se)
|
||||
- Onlime
|
||||
- Onlime Cloud Storage (onlime.dk)
|
||||
- Elkjøp (with subsidiaries):
|
||||
- Elkjøp Cloud (cloud.elkjop.no)
|
||||
- Elgiganten Sweden (cloud.elgiganten.se)
|
||||
- Elgiganten Denmark (cloud.elgiganten.dk)
|
||||
- Giganti Cloud (cloud.gigantti.fi)
|
||||
- ELKO Cloud (cloud.elko.is)
|
||||
|
||||
Most of the white-label versions are supported by this backend, although may
|
||||
require different authentication setup - described below.
|
||||
- Onlime (onlime.dk)
|
||||
- MediaMarkt
|
||||
- MediaMarkt Cloud (mediamarkt.jottacloud.com)
|
||||
- Let's Go Cloud (letsgo.jotta.cloud)
|
||||
|
||||
Paths are specified as `remote:path`
|
||||
|
||||
Paths may be as deep as required, e.g. `remote:directory/subdirectory`.
|
||||
|
||||
## Authentication types
|
||||
## Authentication
|
||||
|
||||
Some of the whitelabel versions uses a different authentication method than the
|
||||
official service, and you have to choose the correct one when setting up the remote.
|
||||
Authentication in Jottacloud is in general based on OAuth and OpenID Connect
|
||||
(OIDC). There are different variants to choose from, depending on which service
|
||||
you are using, e.g. a white-label service may only support one of them. Note
|
||||
that there is no documentation to rely on, so the descriptions provided here
|
||||
are based on observations and may not be accurate.
|
||||
|
||||
### Standard authentication
|
||||
Jottacloud uses two optional OAuth security mechanisms, referred to as "Refresh
|
||||
Token Rotation" and "Automatic Reuse Detection", which has some implications.
|
||||
Access tokens normally have one hour expiry, after which they need to be
|
||||
refreshed (rotated), an operation that requires the refresh token to be
|
||||
supplied. Rclone does this automatically. This is standard OAuth. But in
|
||||
Jottacloud, such a refresh operation not only creates a new access token, but
|
||||
also refresh token, and invalidates the existing refresh token, the one that
|
||||
was supplied. It keeps track of the history of refresh tokens, sometimes
|
||||
referred to as a token family, descending from the original refresh token that
|
||||
was issued after the initial authentication. This is used to detect any
|
||||
attempts at reusing old refresh tokens, and trigger an immedate invalidation of
|
||||
the current refresh token, and effectively the entire refresh token family.
|
||||
|
||||
The standard authentication method used by the official service (jottacloud.com),
|
||||
as well as some of the whitelabel services, requires you to generate a single-use
|
||||
personal login token from the account security settings in the service's web
|
||||
interface. Log in to your account, go to "Settings" and then "Security", or use
|
||||
the direct link presented to you by rclone when configuring the remote:
|
||||
<https://www.jottacloud.com/web/secure>. Scroll down to the section "Personal login
|
||||
token", and click the "Generate" button. Note that if you are using a whitelabel
|
||||
service you probably can't use the direct link, you need to find the same page in
|
||||
their dedicated web interface, and also it may be in a different location than
|
||||
described above.
|
||||
|
||||
To access your account from multiple instances of rclone, you need to configure
|
||||
each of them with a separate personal login token. E.g. you create a Jottacloud
|
||||
remote with rclone in one location, and copy the configuration file to a second
|
||||
location where you also want to run rclone and access the same remote. Then you
|
||||
need to replace the token for one of them, using the [config reconnect](https://rclone.org/commands/rclone_config_reconnect/)
|
||||
command, which requires you to generate a new personal login token and supply
|
||||
as input. If you do not do this, the token may easily end up being invalidated,
|
||||
resulting in both instances failing with an error message something along the
|
||||
When the current refresh token has been invalidated, next time rclone tries to
|
||||
perform a token refresh, it will fail with an error message something along the
|
||||
lines of:
|
||||
|
||||
```text
|
||||
oauth2: cannot fetch token: 400 Bad Request
|
||||
Response: {"error":"invalid_grant","error_description":"Stale token"}
|
||||
CRITICAL: Failed to create file system for "remote:": (...): couldn't fetch token: invalid_grant: maybe token expired? - try refreshing with "rclone config reconnect remote:"
|
||||
```
|
||||
|
||||
When this happens, you need to replace the token as described above to be able
|
||||
to use your remote again.
|
||||
If you run rclone with verbosity level 2 (`-vv`), you will see a debug message
|
||||
with an additional error description from the OAuth response:
|
||||
|
||||
All personal login tokens you have taken into use will be listed in the web
|
||||
interface under "My logged in devices", and from the right side of that list
|
||||
you can click the "X" button to revoke individual tokens.
|
||||
```text
|
||||
DEBUG : remote: got fatal oauth error: oauth2: "invalid_grant" "Session doesn't have required client"
|
||||
```
|
||||
|
||||
### Legacy authentication
|
||||
(The error description used to be "Stale token" instead of "Session doesn't
|
||||
have required client", so you may see references to that in older descriptions
|
||||
of this situation.)
|
||||
|
||||
If you are using one of the whitelabel versions (e.g. from Elkjøp) you may not
|
||||
have the option to generate a CLI token. In this case you'll have to use the
|
||||
legacy authentication. To do this select yes when the setup asks for legacy
|
||||
authentication and enter your username and password. The rest of the setup is
|
||||
identical to the default setup.
|
||||
When this happens, you need to re-authenticate to be able to use your remote
|
||||
again, e.g. using the [config reconnect](/commands/rclone_config_reconnect/)
|
||||
command as suggested in the error message. This will create an entirely new
|
||||
refresh token (family).
|
||||
|
||||
### Telia Cloud authentication
|
||||
A typical example of how you may end up in this situation, is if you create
|
||||
a Jottacloud remote with rclone in one location, and then copy the
|
||||
configuration file to a second location where you start using rclone to access
|
||||
the same remote. Eventually there will now be a token refresh attempt with an
|
||||
invalidated token, i.e. refresh token reuse, resulting in both instances
|
||||
starting to fail with the "invalid_grant" error. It is possible to copy remote
|
||||
configurations, but you must then replace the token for one of them using the
|
||||
[config reconnect](https://rclone.org/commands/rclone_config_reconnect/)
|
||||
command.
|
||||
|
||||
Similar to other whitelabel versions Telia Cloud doesn't offer the option of
|
||||
creating a CLI token, and additionally uses a separate authentication flow
|
||||
where the username is generated internally. To setup rclone to use Telia Cloud,
|
||||
choose Telia Cloud authentication in the setup. The rest of the setup is
|
||||
identical to the default setup.
|
||||
You can get some overview of your active tokens in your service's web user
|
||||
interface, if you navigate to "Settings" and then "Security" (in which case
|
||||
you end up at <https://www.jottacloud.com/web/secure> or similar). Down on
|
||||
that page you have a section "My logged in devices". This contains a list
|
||||
of entries which seemingly represents currently valid refresh tokens, or
|
||||
refresh token families. From the right side of that list you can click a
|
||||
button ("X") to revoke (invalidate) it, which means you will still have access
|
||||
using an existing access token until that expires, but you will not be able to
|
||||
perform a token refresh. Note that this entire "My logged in devices" feature
|
||||
seem to behave a bit differently with different authentication variants and
|
||||
with use of the different (white-label) services.
|
||||
|
||||
### Tele2 Cloud authentication
|
||||
### Standard
|
||||
|
||||
As Tele2-Com Hem merger was completed this authentication can be used for former
|
||||
Com Hem Cloud and Tele2 Cloud customers as no support for creating a CLI token
|
||||
exists, and additionally uses a separate authentication flow where the username
|
||||
is generated internally. To setup rclone to use Tele2 Cloud, choose Tele2 Cloud
|
||||
authentication in the setup. The rest of the setup is identical to the default setup.
|
||||
This is an OAuth variant designed for command-line applications. It is
|
||||
primarily supported by the official service (jottacloud.com), but may also be
|
||||
supported by some of the white-label services. The information necessary to be
|
||||
able to perform authentication, like domain name and endpoint to connect to,
|
||||
are found automatically (it is encoded into the supplied login token, described
|
||||
next), so you do not need to specify which service to configure.
|
||||
|
||||
### Onlime Cloud Storage authentication
|
||||
When configuring a remote, you are asked to enter a single-use personal login
|
||||
token, which you must manually generate from the account security settings in
|
||||
the service's web interface. You do not need a web browser on the same machine
|
||||
like with traditional OAuth, but need to use a web browser somewhere, and be
|
||||
able to be copy the generated string into your rclone configuration session.
|
||||
Log in to your service's web user interface, navigate to "Settings" and then
|
||||
"Security", or, for the official service, use the direct link presented to you
|
||||
by rclone when configuring the remote: <https://www.jottacloud.com/web/secure>.
|
||||
Scroll down to the section "Personal login token", and click the "Generate"
|
||||
button. Copy the presented string and paste it where rclone asks for it. Rclone
|
||||
will then use this to perform an initial token request, and receive a regular
|
||||
OAuth token which it stores in your remote configuration. There will then also
|
||||
be a new entry in the "My logged in devices" list in the web interface, with
|
||||
device name and application name "Jottacloud CLI".
|
||||
|
||||
Onlime has sold access to Jottacloud proper, while providing localized support
|
||||
to Danish Customers, but have recently set up their own hosting, transferring
|
||||
their customers from Jottacloud servers to their own ones.
|
||||
Each time a new token is created this way, i.e. a new personal login token is
|
||||
generated and traded in for an OAuth token, you get an entirely new refresh
|
||||
token family, with a new entry in the "My logged in devices". You can create as
|
||||
many remotes as you want, and use multiple instances of rclone on same or
|
||||
different machine, as long as you configure them separately like this, and not
|
||||
get your self into the refresh token reuse issue described above.
|
||||
|
||||
This, of course, necessitates using their servers for authentication, but
|
||||
otherwise functionality and architecture seems equivalent to Jottacloud.
|
||||
### Traditional
|
||||
|
||||
To setup rclone to use Onlime Cloud Storage, choose Onlime Cloud authentication
|
||||
in the setup. The rest of the setup is identical to the default setup.
|
||||
Jottacloud also supports a more traditional OAuth variant. Most of the
|
||||
white-label services support this, and for many of them this is the only
|
||||
alternative because they do not support personal login tokens. This method
|
||||
relies on pre-defined service-specific domain names and endpoints, and rclone
|
||||
need you to specify which service to configure. This also means that any
|
||||
changes to existing or additions of new white-label services needs an update
|
||||
in the rclone backend implementation.
|
||||
|
||||
When configuring a remote, you must interactively login to an OAuth
|
||||
authorization web site, and a one-time authorization code is sent back to
|
||||
rclone behind the scene, which it uses to request an OAuth token. This means
|
||||
that you need to be on a machine with an internet-connected web browser. If you
|
||||
need it on a machine where this is not the case, then you will have to create
|
||||
the configuration on a different machine and copy it from there. The Jottacloud
|
||||
backend does not support the `rclone authorize` command. See the
|
||||
[remote setup docs](/remote_setup) for details.
|
||||
|
||||
Jottacloud exerts some form of strict session management when authenticating
|
||||
using this method. This leads to some unexpected cases of the "invalid_grant"
|
||||
error described above, and effectively limits you to only use of a single
|
||||
active authentication on the same machine. I.e. you can only create a single
|
||||
rclone remote, and you can't even log in with the service's official desktop
|
||||
client while having a rclone remote configured, or else you will eventually get
|
||||
all sessions invalidated and are forced to re-authenticate.
|
||||
|
||||
When you have successfully authenticated, there will be an entry in the
|
||||
"My logged in devices" list in the web interface representing your session. It
|
||||
will typically be listed with application name "Jottacloud for Desktop" or
|
||||
similar (it depends on the white-label service configuration).
|
||||
|
||||
### Legacy
|
||||
|
||||
Originally Jottacloud used an OAuth variant which required your account's
|
||||
username and password to be specified. When Jottacloud migrated to the newer
|
||||
methods, some white-label versions (those from Elkjøp) still used this legacy
|
||||
method for a long time. Currently there are no known uses of this, it is still
|
||||
supported by rclone, but the support will be removed in a future version.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -125,7 +190,10 @@ n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> remote
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
@@ -134,60 +202,63 @@ XX / Jottacloud
|
||||
\ (jottacloud)
|
||||
[snip]
|
||||
Storage> jottacloud
|
||||
|
||||
Option client_id.
|
||||
OAuth Client Id.
|
||||
Leave blank normally.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
client_id>
|
||||
|
||||
Option client_secret.
|
||||
OAuth Client Secret.
|
||||
Leave blank normally.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
client_secret>
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
|
||||
Option config_type.
|
||||
Select authentication type.
|
||||
Choose a number from below, or type in an existing string value.
|
||||
Type of authentication.
|
||||
Choose a number from below, or type in an existing value of type string.
|
||||
Press Enter for the default (standard).
|
||||
/ Standard authentication.
|
||||
1 | Use this if you're a normal Jottacloud user.
|
||||
| This is primarily supported by the official service, but may also be
|
||||
| supported by some white-label services. It is designed for command-line
|
||||
1 | applications, and you will be asked to enter a single-use personal login
|
||||
| token which you must manually generate from the account security settings
|
||||
| in the web interface of your service.
|
||||
\ (standard)
|
||||
/ Traditional authentication.
|
||||
| This is supported by the official service and all white-label services
|
||||
| that rclone knows about. You will be asked which service to connect to.
|
||||
2 | It has a limitation of only a single active authentication at a time. You
|
||||
| need to be on, or have access to, a machine with an internet-connected
|
||||
| web browser.
|
||||
\ (traditional)
|
||||
/ Legacy authentication.
|
||||
2 | This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.
|
||||
3 | This is no longer supported by any known services and not recommended
|
||||
| used. You will be asked for your account's username and password.
|
||||
\ (legacy)
|
||||
/ Telia Cloud authentication.
|
||||
3 | Use this if you are using Telia Cloud.
|
||||
\ (telia)
|
||||
/ Tele2 Cloud authentication.
|
||||
4 | Use this if you are using Tele2 Cloud.
|
||||
\ (tele2)
|
||||
/ Onlime Cloud authentication.
|
||||
5 | Use this if you are using Onlime Cloud.
|
||||
\ (onlime)
|
||||
config_type> 1
|
||||
|
||||
Option config_login_token.
|
||||
Personal login token.
|
||||
Generate here: https://www.jottacloud.com/web/secure
|
||||
Login Token> <your token here>
|
||||
Generate it from the account security settings in the web interface of your
|
||||
service, for the official service on https://www.jottacloud.com/web/secure.
|
||||
Enter a value.
|
||||
config_login_token> <your token here>
|
||||
|
||||
Use a non-standard device/mountpoint?
|
||||
Choosing no, the default, will let you access the storage used for the archive
|
||||
section of the official Jottacloud client. If you instead want to access the
|
||||
sync or the backup section, for example, you must choose yes.
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> y
|
||||
Option config_device.
|
||||
The device to use. In standard setup the built-in Jotta device is used,
|
||||
which contains predefined mountpoints for archive, sync etc. All other devices
|
||||
are treated as backup devices by the official Jottacloud client. You may create
|
||||
a new by entering a unique name.
|
||||
Choose a number from below, or type in your own string value.
|
||||
Press Enter for the default (DESKTOP-3H31129).
|
||||
1 > DESKTOP-3H31129
|
||||
2 > Jotta
|
||||
config_device> 2
|
||||
Option config_mountpoint.
|
||||
The mountpoint to use for the built-in device Jotta.
|
||||
The standard setup is to use the Archive mountpoint. Most other mountpoints
|
||||
have very limited support in rclone and should generally be avoided.
|
||||
Choose a number from below, or type in an existing string value.
|
||||
Press Enter for the default (Archive).
|
||||
1 > Archive
|
||||
2 > Shared
|
||||
3 > Sync
|
||||
config_mountpoint> 1
|
||||
y/n> n
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: jottacloud
|
||||
|
||||
@@ -100,7 +100,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Microsoft. This only runs from the moment it
|
||||
|
||||
+17
-8
@@ -71,7 +71,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note if you are using remote config with rclone authorize while your pcloud
|
||||
server is the EU region, you will need to set the hostname in 'Edit advanced
|
||||
@@ -152,14 +152,23 @@ However you can set this to restrict rclone to a specific folder
|
||||
hierarchy.
|
||||
|
||||
In order to do this you will have to find the `Folder ID` of the
|
||||
directory you wish rclone to display. This will be the `folder` field
|
||||
of the URL when you open the relevant folder in the pCloud web
|
||||
interface.
|
||||
directory you wish rclone to display. This can be accomplished by executing
|
||||
the ```rclone lsf``` command using a basic configuration setup that does not
|
||||
include the ```root_folder_id``` parameter.
|
||||
|
||||
So if the folder you want rclone to use has a URL which looks like
|
||||
`https://my.pcloud.com/#page=filemanager&folder=5xxxxxxxx8&tpl=foldergrid`
|
||||
in the browser, then you use `5xxxxxxxx8` as
|
||||
the `root_folder_id` in the config.
|
||||
The command will enumerate available directories, allowing you to locate the
|
||||
appropriate Folder ID for subsequent use.
|
||||
|
||||
Example:
|
||||
```
|
||||
$ rclone lsf --dirs-only -Fip --csv TestPcloud:
|
||||
dxxxxxxxx2,My Music/
|
||||
dxxxxxxxx3,My Pictures/
|
||||
dxxxxxxxx4,My Videos/
|
||||
```
|
||||
|
||||
So if the folder you want rclone to use your is "My Music/", then use the returned id from ```rclone lsf``` command (ex. `dxxxxxxxx2`) as
|
||||
the `root_folder_id` variable value in the config file.
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/pcloud/pcloud.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
@@ -65,7 +65,7 @@ y/e/d>
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from premiumize.me. This only runs from the moment it opens
|
||||
|
||||
@@ -131,6 +131,109 @@ section and followed by the privacy policy of Rclone.
|
||||
local configuration file.
|
||||
- Rclone does not share any user data with third parties.
|
||||
|
||||
## User Data Collection and Storage
|
||||
|
||||
This section outlines how rclone accesses, uses, stores, and shares
|
||||
user data obtained from service provider APIs. Our use of information
|
||||
received from provider APIs will adhere to the provider API Services
|
||||
User Data Policy, including the Limited Use requirements.
|
||||
|
||||
Rclone is a client-side command-line program that users run on their
|
||||
own computers to manage their files on cloud storage services. The
|
||||
rclone project does not operate any servers that store or process your
|
||||
personal data. All data access and processing occurs directly on the
|
||||
user's machine and between the user's machine and the provider API
|
||||
servers.
|
||||
|
||||
### Data Accessed
|
||||
|
||||
When you authorize rclone to access your files on your provider, it
|
||||
may access the following types of data, depending on the permissions
|
||||
you grant:
|
||||
|
||||
- Files: Rclone accesses the metadata (filenames, sizes, modification
|
||||
times, etc.) and content of your files and folders on your provider.
|
||||
This is necessary for rclone to perform file management tasks like
|
||||
copying, syncing, moving, and listing files.
|
||||
|
||||
- Authentication Tokens: Rclone requests OAuth 2.0 access tokens from
|
||||
the provider. These tokens are used to authenticate your requests to
|
||||
the provider's APIs and prove that you have granted rclone
|
||||
permission to access your data.
|
||||
|
||||
- Basic Profile Information: As part of the authentication process,
|
||||
rclone may receive your email address to identify the connected
|
||||
account within the rclone configuration.
|
||||
|
||||
### Data Usage
|
||||
|
||||
Rclone uses the user data it accesses solely to provide its core
|
||||
functionality, which is initiated and controlled entirely by you, the
|
||||
user. Specifically:
|
||||
|
||||
- The data is used to perform file transfer and management operations
|
||||
(such as `copy`, `sync`, `move`, `list`, `delete`) between your
|
||||
local machine and your provider account as per your direct commands.
|
||||
|
||||
- Authentication tokens are used exclusively to make authorized API
|
||||
calls to the provider's services on your behalf.
|
||||
|
||||
- Your email address is used locally to help you identify which
|
||||
provider account is configured.
|
||||
|
||||
Rclone does not use your data for any other purpose, such as
|
||||
advertising, marketing, or analysis by the rclone project developers.
|
||||
|
||||
### Data Sharing
|
||||
|
||||
Rclone does not share your user data with any third parties.
|
||||
|
||||
All data transfers initiated by the user occur directly between the
|
||||
machine where rclone is running and the provider's servers. The rclone
|
||||
project and its developers **never** have access to your
|
||||
authentication tokens or your file data.
|
||||
|
||||
### Data Storage & Protection
|
||||
|
||||
- Configuration Data: Rclone stores its configuration, including the
|
||||
OAuth 2.0 tokens required to access your provider account, in a
|
||||
configuration file (`rclone.conf`) located on your local machine.
|
||||
|
||||
- Security: You are responsible for securing this configuration
|
||||
file on your own computer. Rclone provides a built-in option to
|
||||
encrypt the configuration file with a password for an added layer of
|
||||
security. We strongly recommend using this feature.
|
||||
|
||||
- File Data: Your file data is only held in your computer's memory
|
||||
(RAM) temporarily during transfer operations. Rclone does not
|
||||
permanently store your file content on your local disk unless you
|
||||
explicitly command it to do so (e.g., by running a `copy` command
|
||||
from the provider to a local directory).
|
||||
|
||||
### Data Retention & Deletion
|
||||
|
||||
Rclone gives you full control over your data.
|
||||
|
||||
- Data Retention: Rclone retains the configuration data, including
|
||||
authentication tokens, on your local machine for as long as you keep
|
||||
the configuration file. This allows you to use rclone without having
|
||||
to re-authenticate for every session.
|
||||
|
||||
- Data Deletion: You can delete your data and revoke rclone's
|
||||
access at any time through one of the following methods:
|
||||
|
||||
1. Local Deletion: You can delete the specific provider
|
||||
configuration from your `rclone.conf` file or delete the entire
|
||||
file itself. This will permanently remove the authentication
|
||||
tokens from your machine.
|
||||
|
||||
2. Revoking Access via the provider: You can revoke rclone's
|
||||
access to your provider directly from your the providers's
|
||||
security settings page. This will invalidate the authentication
|
||||
tokens, and rclone will no longer be able to access your data.
|
||||
For example, if you are using Google you can manage your permissions
|
||||
[on the Google permissions page](https://myaccount.google.com/permissions).
|
||||
|
||||
## Resources & Further Information
|
||||
|
||||
- [Data Protection Act 1998](http://www.legislation.gov.uk/ukpga/1998/29/contents)
|
||||
|
||||
@@ -80,7 +80,7 @@ e/n/d/r/c/s/q> q
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from put.io if using web browser to automatically
|
||||
|
||||
@@ -6,22 +6,23 @@ description: "Configuring rclone up on a remote / headless machine"
|
||||
# Configuring rclone on a remote / headless machine
|
||||
|
||||
Some of the configurations (those involving oauth2) require an
|
||||
Internet connected web browser.
|
||||
internet-connected web browser.
|
||||
|
||||
If you are trying to set rclone up on a remote or headless box with no
|
||||
browser available on it (e.g. a NAS or a server in a datacenter) then
|
||||
you will need to use an alternative means of configuration. There are
|
||||
two ways of doing it, described below.
|
||||
If you are trying to set rclone up on a remote or headless machine with no
|
||||
browser available on it (e.g. a NAS or a server in a datacenter), then
|
||||
you will need to use an alternative means of configuration. There are
|
||||
three ways of doing it, described below.
|
||||
|
||||
## Configuring using rclone authorize
|
||||
|
||||
On the headless box run `rclone` config but answer `N` to the `Use auto config?`
|
||||
question.
|
||||
On the headless machine run [rclone config](/commands/rclone_config), but
|
||||
answer `N` to the question `Use web browser to automatically authenticate rclone with remote?`.
|
||||
|
||||
```text
|
||||
Use auto config?
|
||||
* Say Y if not sure
|
||||
* Say N if you are working on a remote or headless machine
|
||||
Use web browser to automatically authenticate rclone with remote?
|
||||
* Say Y if the machine running rclone has a web browser you can use
|
||||
* Say N if running rclone on a (remote) machine without web browser access
|
||||
If not sure try Y. If Y failed, try N.
|
||||
|
||||
y) Yes (default)
|
||||
n) No
|
||||
@@ -33,33 +34,35 @@ a web browser available.
|
||||
For more help and alternate methods see: https://rclone.org/remote_setup/
|
||||
Execute the following on the machine with the web browser (same rclone
|
||||
version recommended):
|
||||
rclone authorize "onedrive"
|
||||
rclone authorize "onedrive"
|
||||
Then paste the result.
|
||||
Enter a value.
|
||||
config_token>
|
||||
```
|
||||
|
||||
Then on your main desktop machine
|
||||
Then on your main desktop machine, run [rclone authorize](/commands/rclone_authorize/).
|
||||
|
||||
```text
|
||||
rclone authorize "onedrive"
|
||||
If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
|
||||
Log in and authorize rclone for access
|
||||
Waiting for code...
|
||||
NOTICE: Make sure your Redirect URL is set to "http://localhost:53682/" in your custom config.
|
||||
NOTICE: If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx
|
||||
NOTICE: Log in and authorize rclone for access
|
||||
NOTICE: Waiting for code...
|
||||
|
||||
Got code
|
||||
Paste the following into your remote machine --->
|
||||
SECRET_TOKEN
|
||||
<---End paste
|
||||
```
|
||||
|
||||
Then back to the headless box, paste in the code
|
||||
Then back to the headless machine, paste in the code.
|
||||
|
||||
```text
|
||||
config_token> SECRET_TOKEN
|
||||
--------------------
|
||||
[acd12]
|
||||
client_id =
|
||||
client_secret =
|
||||
client_id =
|
||||
client_secret =
|
||||
token = SECRET_TOKEN
|
||||
--------------------
|
||||
y) Yes this is OK
|
||||
@@ -70,18 +73,19 @@ y/e/d>
|
||||
|
||||
## Configuring by copying the config file
|
||||
|
||||
Rclone stores all of its config in a single configuration file. This
|
||||
can easily be copied to configure a remote rclone.
|
||||
Rclone stores all of its configuration in a single file. This can easily be
|
||||
copied to configure a remote rclone (although some backends does not support
|
||||
reusing the same configuration, consult your backend documentation to be
|
||||
sure).
|
||||
|
||||
So first configure rclone on your desktop machine with
|
||||
Start by running [rclone config](/commands/rclone_config) to create the
|
||||
configuration file on your desktop machine.
|
||||
|
||||
```sh
|
||||
rclone config
|
||||
```
|
||||
|
||||
to set up the config file.
|
||||
|
||||
Find the config file by running `rclone config file`, for example
|
||||
Then locate the file by running [rclone config file](/commands/rclone_config_file).
|
||||
|
||||
```sh
|
||||
$ rclone config file
|
||||
@@ -89,31 +93,37 @@ Configuration file is stored at:
|
||||
/home/user/.rclone.conf
|
||||
```
|
||||
|
||||
Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and
|
||||
place it in the correct place (use `rclone config file` on the remote
|
||||
box to find out where).
|
||||
Finally, transfer the file to the remote machine (scp, cut paste, ftp, sftp, etc.)
|
||||
and place it in the correct location (use [rclone config file](/commands/rclone_config_file)
|
||||
on the remote machine to find out where).
|
||||
|
||||
## Configuring using SSH Tunnel
|
||||
|
||||
Linux and MacOS users can utilize SSH Tunnel to redirect the headless box
|
||||
port 53682 to local machine by using the following command:
|
||||
If you have an SSH client installed on your local machine, you can set up an
|
||||
SSH tunnel to redirect the port 53682 into the headless machine by using the
|
||||
following command:
|
||||
|
||||
```sh
|
||||
ssh -L localhost:53682:localhost:53682 username@remote_server
|
||||
```
|
||||
|
||||
Then on the headless box run `rclone config` and answer `Y` to the
|
||||
`Use auto config?` question.
|
||||
Then on the headless machine run [rclone config](/commands/rclone_config) and
|
||||
answer `Y` to the question `Use web browser to automatically authenticate rclone with remote?`.
|
||||
|
||||
```text
|
||||
Use auto config?
|
||||
* Say Y if not sure
|
||||
* Say N if you are working on a remote or headless machine
|
||||
Use web browser to automatically authenticate rclone with remote?
|
||||
* Say Y if the machine running rclone has a web browser you can use
|
||||
* Say N if running rclone on a (remote) machine without web browser access
|
||||
If not sure try Y. If Y failed, try N.
|
||||
|
||||
y) Yes (default)
|
||||
n) No
|
||||
y/n> y
|
||||
NOTICE: Make sure your Redirect URL is set to "http://localhost:53682/" in your custom config.
|
||||
NOTICE: If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx
|
||||
NOTICE: Log in and authorize rclone for access
|
||||
NOTICE: Waiting for code...
|
||||
```
|
||||
|
||||
Then copy and paste the auth url `http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx`
|
||||
to the browser on your local machine, complete the auth and it is done.
|
||||
Finally, copy and paste the presented URL `http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx`
|
||||
to the browser on your local machine, complete the auth and you are done.
|
||||
@@ -21,7 +21,9 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
|
||||
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
|
||||
{{< provider name="Exaba" home="https://exaba.com/" config="/s3/#exaba" >}}
|
||||
{{< provider name="FileLu S5 (S3-Compatible Object Storage)" home="https://s5lu.com/" config="/s3/#filelu-s5" >}}
|
||||
{{< provider name="GCS" home="https://cloud.google.com/storage/docs" config="/s3/#google-cloud-storage" >}}
|
||||
{{< provider name="Hetzner" home="https://www.hetzner.com/storage/object-storage/" config="/s3/#hetzner" >}}
|
||||
{{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}}
|
||||
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
|
||||
{{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}}
|
||||
@@ -38,12 +40,14 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="Petabox" home="https://petabox.io/" config="/s3/#petabox" >}}
|
||||
{{< provider name="Pure Storage FlashBlade" home="https://www.purestorage.com/products/unstructured-data-storage.html" config="/s3/#pure-storage-flashblade" >}}
|
||||
{{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}}
|
||||
{{< provider name="Rabata Cloud Storage" home="https://rabata.io" config="/s3/#Rabata" >}}
|
||||
{{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}}
|
||||
{{< provider name="Rclone Serve S3" home="/commands/rclone_serve_s3/" config="/s3/#rclone" >}}
|
||||
{{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}}
|
||||
{{< provider name="Seagate Lyve Cloud" home="https://www.seagate.com/gb/en/services/cloud/storage/" config="/s3/#lyve" >}}
|
||||
{{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
|
||||
{{< provider name="Selectel" home="https://selectel.ru/services/cloud/storage/" config="/s3/#selectel" >}}
|
||||
{{< provider name="Spectra Logic" home="https://spectralogic.com/blackpearl-nearline-object-gateway" config="/s3/#spectralogic" >}}
|
||||
{{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
|
||||
{{< provider name="Storj" home="https://storj.io/" config="/s3/#storj" >}}
|
||||
{{< provider name="Synology C2 Object Storage" home="https://c2.synology.com/en-global/object-storage/overview" config="/s3/#synology-c2" >}}
|
||||
@@ -3399,6 +3403,150 @@ endpoint = https://storage.googleapis.com
|
||||
|
||||
This is Google bug [#312292516](https://issuetracker.google.com/u/0/issues/312292516).
|
||||
|
||||
### Hetzner Object Storage {#hetzner}
|
||||
|
||||
Here is an example of making a [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)
|
||||
configuration. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> my-hetzner
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, GCS, Hetzner, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu and others
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
[snip]
|
||||
XX / Hetzner Object Storage
|
||||
\ (Hetzner)
|
||||
[snip]
|
||||
provider> Hetzner
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth>
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> ACCESS_KEY
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> SECRET_KEY
|
||||
Option region.
|
||||
Region to connect to.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / Helsinki
|
||||
\ (hel1)
|
||||
2 / Falkenstein
|
||||
\ (fsn1)
|
||||
3 / Nuremberg
|
||||
\ (nbg1)
|
||||
region>
|
||||
Option endpoint.
|
||||
Endpoint for Hetzner Object Storage
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / Helsinki
|
||||
\ (hel1.your-objectstorage.com)
|
||||
2 / Falkenstein
|
||||
\ (fsn1.your-objectstorage.com)
|
||||
3 / Nuremberg
|
||||
\ (nbg1.your-objectstorage.com)
|
||||
endpoint>
|
||||
Option location_constraint.
|
||||
Location constraint - must be set to match the Region.
|
||||
Leave blank if not sure. Used when creating buckets only.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
location_constraint>
|
||||
Option acl.
|
||||
Canned ACL used when creating buckets and storing or copying objects.
|
||||
This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
|
||||
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
Note that this ACL is applied when server-side copying objects as S3
|
||||
doesn't copy the ACL from the source but rather writes a fresh one.
|
||||
If the acl is an empty string then no X-Amz-Acl: header is added and
|
||||
the default (private) will be used.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
/ Owner gets FULL_CONTROL.
|
||||
1 | No one else has access rights (default).
|
||||
\ (private)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
2 | The AllUsers group gets READ access.
|
||||
\ (public-read)
|
||||
acl>
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n>
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: Hetzner
|
||||
- access_key_id: ACCESS_KEY
|
||||
- secret_access_key: SECRET_KEY
|
||||
Keep this "my-hetzner" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d>
|
||||
Current remotes:
|
||||
|
||||
Name Type
|
||||
==== ====
|
||||
my-hetzner s3
|
||||
|
||||
e) Edit existing remote
|
||||
n) New remote
|
||||
d) Delete remote
|
||||
r) Rename remote
|
||||
c) Copy remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
e/n/d/r/c/s/q>
|
||||
```
|
||||
|
||||
This will leave the config file looking like this.
|
||||
|
||||
```
|
||||
[my-hetzner]
|
||||
type = s3
|
||||
provider = Hetzner
|
||||
access_key_id = ACCESS_KEY
|
||||
secret_access_key = SECRET_KEY
|
||||
region = hel1
|
||||
endpoint = hel1.your-objectstorage.com
|
||||
acl = private
|
||||
```
|
||||
|
||||
|
||||
### Huawei OBS {#huawei-obs}
|
||||
|
||||
Object Storage Service (OBS) provides stable, secure, efficient, and easy-to-use cloud storage that lets you store virtually any volume of unstructured data in any format and access it from anywhere.
|
||||
@@ -5634,6 +5782,244 @@ Name Type
|
||||
qiniu s3
|
||||
```
|
||||
|
||||
### FileLu S5 {#filelu-s5}
|
||||
|
||||
[FileLu S5 Object Storage](https://s5lu.com) is an S3-compatible object storage system.
|
||||
It provides multiple region options (Global, US-East, EU-Central, AP-Southeast, and ME-Central) while using a single endpoint (`s5lu.com`).
|
||||
FileLu S5 is designed for scalability, security, and simplicity, with predictable pricing and no hidden charges for data transfers or API requests.
|
||||
|
||||
Here is an example of making a configuration. First run:
|
||||
|
||||
```sh
|
||||
rclone config
|
||||
```
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```text
|
||||
No remotes found, make a new one\?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> s5lu
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS,... FileLu, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
[snip]
|
||||
XX / FileLu S5 Object Storage
|
||||
\ (FileLu)
|
||||
[snip]
|
||||
provider> FileLu
|
||||
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth>
|
||||
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> XXX
|
||||
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> XXX
|
||||
|
||||
Option endpoint.
|
||||
Endpoint for S3 API.
|
||||
Required when using an S3 clone.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / Global
|
||||
\ (global)
|
||||
2 / North America (US-East)
|
||||
\ (us-east)
|
||||
3 / Europe (EU-Central)
|
||||
\ (eu-central)
|
||||
4 / Asia Pacific (AP-Southeast)
|
||||
\ (ap-southeast)
|
||||
5 / Middle East (ME-Central)
|
||||
\ (me-central)
|
||||
region> 1
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: FileLu
|
||||
- access_key_id: XXX
|
||||
- secret_access_key: XXX
|
||||
- endpoint: s5lu.com
|
||||
Keep this "s5lu" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This will leave the config file looking like this.
|
||||
|
||||
```
|
||||
[s5lu]
|
||||
type = s3
|
||||
provider = FileLu
|
||||
access_key_id = XXX
|
||||
secret_access_key = XXX
|
||||
endpoint = s5lu.com
|
||||
```
|
||||
|
||||
### Rabata {#Rabata}
|
||||
|
||||
[Rabata](https://rabata.io) is an S3-compatible secure cloud storage service that offers flat, transparent pricing (no API request fees)
|
||||
while supporting standard S3 APIs. It is suitable for backup, application storage,media workflows, and archive use cases.
|
||||
|
||||
Server side copy is not implemented with Rabata, also meaning modification time of objects cannot be updated.
|
||||
|
||||
Rclone config:
|
||||
|
||||
```
|
||||
rclone config
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> Rabata
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
[snip]
|
||||
XX / Rabata Cloud Storage
|
||||
\ (Rabata)
|
||||
[snip]
|
||||
provider> Rabata
|
||||
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth>
|
||||
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> ACCESS_KEY_ID
|
||||
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> SECRET_ACCESS_KEY
|
||||
|
||||
Option region.
|
||||
Region where your bucket will be created and your data stored.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / US East (N. Virginia)
|
||||
\ (us-east-1)
|
||||
2 / EU (Ireland)
|
||||
\ (eu-west-1)
|
||||
3 / EU (London)
|
||||
\ (eu-west-2)
|
||||
region> 3
|
||||
|
||||
Option endpoint.
|
||||
Endpoint for Rabata Object Storage.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / US East (N. Virginia)
|
||||
\ (s3.us-east-1.rabata.io)
|
||||
2 / EU West (Ireland)
|
||||
\ (s3.eu-west-1.rabata.io)
|
||||
3 / EU West (London)
|
||||
\ (s3.eu-west-2.rabata.io)
|
||||
endpoint> 3
|
||||
|
||||
Option location_constraint.
|
||||
location where your bucket will be created and your data stored.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / US East (N. Virginia)
|
||||
\ (us-east-1)
|
||||
2 / EU (Ireland)
|
||||
\ (eu-west-1)
|
||||
3 / EU (London)
|
||||
\ (eu-west-2)
|
||||
location_constraint> 3
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: Rabata
|
||||
- access_key_id: ACCESS_KEY_ID
|
||||
- secret_access_key: SECRET_ACCESS_KEY
|
||||
- region: eu-west-2
|
||||
- endpoint: s3.eu-west-2.rabata.io
|
||||
- location_constraint: eu-west-2
|
||||
Keep this "rabata" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
|
||||
Current remotes:
|
||||
|
||||
Name Type
|
||||
==== ====
|
||||
rabata s3
|
||||
```
|
||||
|
||||
### RackCorp {#RackCorp}
|
||||
|
||||
[RackCorp Object Storage](https://www.rackcorp.com/storage/s3storage) is an S3 compatible object storage platform from your friendly cloud provider RackCorp.
|
||||
@@ -6018,6 +6404,105 @@ region = ru-1
|
||||
endpoint = s3.ru-1.storage.selcloud.ru
|
||||
```
|
||||
|
||||
### Spectra Logic {#spectralogic}
|
||||
|
||||
[Spectra Logic](https://www.spectralogic.com/blackpearl-nearline-object-gateway)
|
||||
is an on-prem S3-compatible object storage gateway that exposes local object storage and
|
||||
policy-tiers data to Spectra tape and public clouds under a single namespace for
|
||||
backup and archiving.
|
||||
|
||||
The S3 compatible gateway is configured using `rclone config` with a
|
||||
type of `s3` and with a provider name of `SpectraLogic`. Here is an example
|
||||
run of the configurator.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> spectralogic
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including ..., SpectraLogic, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
[snip]
|
||||
XX / SpectraLogic BlackPearl
|
||||
\ (SpectraLogic)
|
||||
[snip]
|
||||
provider> SpectraLogic
|
||||
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth> 1
|
||||
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> ACCESS_KEY
|
||||
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> SECRET_ACCESS_KEY
|
||||
|
||||
Option endpoint.
|
||||
Endpoint for S3 API.
|
||||
Required when using an S3 clone.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
endpoint> https://bps3.example.com
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: SpectraLogic
|
||||
- access_key_id: ACCESS_KEY
|
||||
- secret_access_key: SECRET_ACCESS_KEY
|
||||
- endpoint: https://bps3.example.com
|
||||
Keep this "spectratest" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
And your config should end up looking like this:
|
||||
|
||||
```
|
||||
[spectratest]
|
||||
type = s3
|
||||
provider = SpectraLogic
|
||||
access_key_id = ACCESS_KEY
|
||||
secret_access_key = SECRET_ACCESS_KEY
|
||||
endpoint = https://bps3.example.com
|
||||
```
|
||||
|
||||
### Storj
|
||||
|
||||
Storj is a decentralized cloud storage which can be used through its
|
||||
|
||||
@@ -84,7 +84,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Citrix ShareFile. This only runs from the moment it opens
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ you started to share on Windows. On smbd, it's the section title in `smb.conf`
|
||||
(usually in `/etc/samba/`) file.
|
||||
You can find shares by querying the root if you're unsure (e.g. `rclone lsd remote:`).
|
||||
|
||||
You can't access to the shared printers from rclone, obviously.
|
||||
You can't access the shared printers from rclone, obviously.
|
||||
|
||||
You can't use Anonymous access for logging in. You have to use the `guest` user
|
||||
with an empty password instead. The rclone client tries to avoid 8.3 names when
|
||||
|
||||
@@ -62,11 +62,12 @@ Thank you very much to our sponsors:
|
||||
{{< sponsor src="/img/logos/backblaze.svg" width="300" height="200" title="Visit our sponsor Backblaze" link="https://www.backblaze.com/cloud-storage-rclonead?utm_source=rclone&utm_medium=paid&utm_campaign=rclone-website-20250715">}}
|
||||
{{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}}
|
||||
{{< sponsor src="/img/logos/filescom-enterprise-grade-workflows.png" width="300" height="200" title="Start Your Free Trial Today" link="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone">}}
|
||||
{{< sponsor src="/img/logos/mega-s4.svg" width="300" height="200" title="MEGA S4: New S3 compatible object storage. High scale. Low cost. Free egress." link="https://mega.io/objectstorage?utm_source=rclone&utm_medium=referral&utm_campaign=rclone-mega-s4&mct=rclonepromo">}}
|
||||
{{< sponsor src="/img/logos/sia.svg" width="200" height="200" title="Visit our sponsor sia" link="https://sia.tech">}}
|
||||
{{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}}
|
||||
{{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
|
||||
{{< sponsor src="/img/logos/rcloneui.svg" width="300" height="200" title="Visit our sponsor RcloneUI" link="https://github.com/rclone-ui/rclone-ui">}}
|
||||
{{< sponsor src="/img/logos/filelu-rclone.svg" width="330" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}}
|
||||
{{< sponsor src="/img/logos/filelu-rclone.svg" width="250" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}}
|
||||
{{< sponsor src="/img/logos/torbox.png" width="200" height="200" title="Visit our sponsor TORBOX" link="https://www.torbox.app/">}}
|
||||
|
||||
<!-- markdownlint-restore -->
|
||||
@@ -61,7 +61,7 @@ y/e/d> y
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Note that rclone runs a webserver on your local machine to collect the
|
||||
token as returned from Yandex Disk. This only runs from the moment it
|
||||
|
||||
@@ -80,7 +80,7 @@ y/e/d>
|
||||
```
|
||||
|
||||
See the [remote setup docs](/remote_setup/) for how to set it up on a
|
||||
machine with no Internet browser available.
|
||||
machine without an internet-connected web browser available.
|
||||
|
||||
Rclone runs a webserver on your local computer to collect the
|
||||
authorization token from Zoho Workdrive. This is only from the moment
|
||||
|
||||
@@ -19,6 +19,8 @@
|
||||
<a class="dropdown-item" href="/filtering/"><i class="fa fa-book fa-fw"></i> Filtering</a>
|
||||
<a class="dropdown-item" href="/gui/"><i class="fa fa-book fa-fw"></i> GUI</a>
|
||||
<a class="dropdown-item" href="/rc/"><i class="fa fa-book fa-fw"></i> Remote Control</a>
|
||||
<a class="dropdown-item" href="/remote_setup/"><i class="fa fa-book fa-fw"></i> Remote Setup</a>
|
||||
<a class="dropdown-item" href="/cluster/"><i class="fa fa-book fa-fw"></i> Cluster</a>
|
||||
<a class="dropdown-item" href="/changelog/"><i class="fa fa-book fa-fw"></i> Changelog</a>
|
||||
<a class="dropdown-item" href="/bugs/"><i class="fa fa-book fa-fw"></i> Bugs</a>
|
||||
<a class="dropdown-item" href="/faq/"><i class="fa fa-book fa-fw"></i> FAQ</a>
|
||||
|
||||
@@ -82,7 +82,7 @@ type accountValues struct {
|
||||
max int64 // if >=0 the max number of bytes to transfer
|
||||
start time.Time // Start time of first read
|
||||
lpTime time.Time // Time of last average measurement
|
||||
lpBytes int // Number of bytes read since last measurement
|
||||
lpBytes int64 // Number of bytes read since last measurement
|
||||
avg float64 // Moving average of last few measurements in Byte/s
|
||||
}
|
||||
|
||||
@@ -344,15 +344,20 @@ func (acc *Account) limitPerFileBandwidth(n int) {
|
||||
}
|
||||
}
|
||||
|
||||
// Account the read and limit bandwidth
|
||||
func (acc *Account) accountRead(n int) {
|
||||
// Account the read
|
||||
func (acc *Account) accountReadN(n int64) {
|
||||
// Update Stats
|
||||
acc.values.mu.Lock()
|
||||
acc.values.lpBytes += n
|
||||
acc.values.bytes += int64(n)
|
||||
acc.values.bytes += n
|
||||
acc.values.mu.Unlock()
|
||||
|
||||
acc.stats.Bytes(int64(n))
|
||||
acc.stats.Bytes(n)
|
||||
}
|
||||
|
||||
// Account the read and limit bandwidth
|
||||
func (acc *Account) accountRead(n int) {
|
||||
acc.accountReadN(int64(n))
|
||||
|
||||
TokenBucket.LimitBandwidth(TokenBucketSlotAccounting, n)
|
||||
acc.limitPerFileBandwidth(n)
|
||||
@@ -427,6 +432,15 @@ func (acc *Account) AccountRead(n int) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
// AccountReadN account having read n bytes
|
||||
//
|
||||
// Does not obey any transfer limits, bandwidth limits, etc.
|
||||
func (acc *Account) AccountReadN(n int64) {
|
||||
acc.mu.Lock()
|
||||
defer acc.mu.Unlock()
|
||||
acc.accountReadN(n)
|
||||
}
|
||||
|
||||
// Close the object
|
||||
func (acc *Account) Close() error {
|
||||
acc.mu.Lock()
|
||||
|
||||
@@ -100,7 +100,7 @@ func TestAccountRead(t *testing.T) {
|
||||
|
||||
assert.True(t, acc.values.start.IsZero())
|
||||
acc.values.mu.Lock()
|
||||
assert.Equal(t, 0, acc.values.lpBytes)
|
||||
assert.Equal(t, int64(0), acc.values.lpBytes)
|
||||
assert.Equal(t, int64(0), acc.values.bytes)
|
||||
acc.values.mu.Unlock()
|
||||
assert.Equal(t, int64(0), stats.bytes)
|
||||
@@ -113,7 +113,7 @@ func TestAccountRead(t *testing.T) {
|
||||
|
||||
assert.False(t, acc.values.start.IsZero())
|
||||
acc.values.mu.Lock()
|
||||
assert.Equal(t, 2, acc.values.lpBytes)
|
||||
assert.Equal(t, int64(2), acc.values.lpBytes)
|
||||
assert.Equal(t, int64(2), acc.values.bytes)
|
||||
acc.values.mu.Unlock()
|
||||
assert.Equal(t, int64(2), stats.bytes)
|
||||
@@ -145,7 +145,7 @@ func testAccountWriteTo(t *testing.T, withBuffer bool) {
|
||||
|
||||
assert.True(t, acc.values.start.IsZero())
|
||||
acc.values.mu.Lock()
|
||||
assert.Equal(t, 0, acc.values.lpBytes)
|
||||
assert.Equal(t, int64(0), acc.values.lpBytes)
|
||||
assert.Equal(t, int64(0), acc.values.bytes)
|
||||
acc.values.mu.Unlock()
|
||||
assert.Equal(t, int64(0), stats.bytes)
|
||||
@@ -159,7 +159,7 @@ func testAccountWriteTo(t *testing.T, withBuffer bool) {
|
||||
|
||||
assert.False(t, acc.values.start.IsZero())
|
||||
acc.values.mu.Lock()
|
||||
assert.Equal(t, len(buf), acc.values.lpBytes)
|
||||
assert.Equal(t, int64(len(buf)), acc.values.lpBytes)
|
||||
assert.Equal(t, int64(len(buf)), acc.values.bytes)
|
||||
acc.values.mu.Unlock()
|
||||
assert.Equal(t, int64(len(buf)), stats.bytes)
|
||||
|
||||
+72
-50
@@ -22,48 +22,52 @@ const (
|
||||
averageStopAfter = time.Minute
|
||||
)
|
||||
|
||||
// MaxCompletedTransfers specifies maximum number of completed transfers in startedTransfers list
|
||||
// MaxCompletedTransfers specifies the default maximum number of
|
||||
// completed transfers in startedTransfers list. This can be adjusted
|
||||
// for a given StatsInfo by calling the SetMaxCompletedTransfers
|
||||
// method.
|
||||
var MaxCompletedTransfers = 100
|
||||
|
||||
// StatsInfo accounts all transfers
|
||||
// N.B.: if this struct is modified, please remember to also update sum() function in stats_groups
|
||||
// to correctly count the updated fields
|
||||
type StatsInfo struct {
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
ci *fs.ConfigInfo
|
||||
bytes int64
|
||||
errors int64
|
||||
lastError error
|
||||
fatalError bool
|
||||
retryError bool
|
||||
retryAfter time.Time
|
||||
checks int64
|
||||
checking *transferMap
|
||||
checkQueue int
|
||||
checkQueueSize int64
|
||||
transfers int64
|
||||
transferring *transferMap
|
||||
transferQueue int
|
||||
transferQueueSize int64
|
||||
listed int64
|
||||
renames int64
|
||||
renameQueue int
|
||||
renameQueueSize int64
|
||||
deletes int64
|
||||
deletesSize int64
|
||||
deletedDirs int64
|
||||
inProgress *inProgress
|
||||
startedTransfers []*Transfer // currently active transfers
|
||||
oldTimeRanges timeRanges // a merged list of time ranges for the transfers
|
||||
oldDuration time.Duration // duration of transfers we have culled
|
||||
group string
|
||||
startTime time.Time // the moment these stats were initialized or reset
|
||||
average averageValues
|
||||
serverSideCopies int64
|
||||
serverSideCopyBytes int64
|
||||
serverSideMoves int64
|
||||
serverSideMoveBytes int64
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
ci *fs.ConfigInfo
|
||||
bytes int64
|
||||
errors int64
|
||||
lastError error
|
||||
fatalError bool
|
||||
retryError bool
|
||||
retryAfter time.Time
|
||||
checks int64
|
||||
checking *transferMap
|
||||
checkQueue int
|
||||
checkQueueSize int64
|
||||
transfers int64
|
||||
transferring *transferMap
|
||||
transferQueue int
|
||||
transferQueueSize int64
|
||||
listed int64
|
||||
renames int64
|
||||
renameQueue int
|
||||
renameQueueSize int64
|
||||
deletes int64
|
||||
deletesSize int64
|
||||
deletedDirs int64
|
||||
inProgress *inProgress
|
||||
startedTransfers []*Transfer // currently active transfers
|
||||
oldTimeRanges timeRanges // a merged list of time ranges for the transfers
|
||||
oldDuration time.Duration // duration of transfers we have culled
|
||||
group string
|
||||
startTime time.Time // the moment these stats were initialized or reset
|
||||
average averageValues
|
||||
serverSideCopies int64
|
||||
serverSideCopyBytes int64
|
||||
serverSideMoves int64
|
||||
serverSideMoveBytes int64
|
||||
maxCompletedTransfers int
|
||||
}
|
||||
|
||||
type averageValues struct {
|
||||
@@ -81,17 +85,26 @@ type averageValues struct {
|
||||
func NewStats(ctx context.Context) *StatsInfo {
|
||||
ci := fs.GetConfig(ctx)
|
||||
s := &StatsInfo{
|
||||
ctx: ctx,
|
||||
ci: ci,
|
||||
checking: newTransferMap(ci.Checkers, "checking"),
|
||||
transferring: newTransferMap(ci.Transfers, "transferring"),
|
||||
inProgress: newInProgress(ctx),
|
||||
startTime: time.Now(),
|
||||
average: averageValues{},
|
||||
ctx: ctx,
|
||||
ci: ci,
|
||||
checking: newTransferMap(ci.Checkers, "checking"),
|
||||
transferring: newTransferMap(ci.Transfers, "transferring"),
|
||||
inProgress: newInProgress(ctx),
|
||||
startTime: time.Now(),
|
||||
average: averageValues{},
|
||||
maxCompletedTransfers: MaxCompletedTransfers,
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// SetMaxCompletedTransfers sets the maximum number of completed transfers to keep.
|
||||
func (s *StatsInfo) SetMaxCompletedTransfers(n int) *StatsInfo {
|
||||
s.mu.Lock()
|
||||
s.maxCompletedTransfers = n
|
||||
s.mu.Unlock()
|
||||
return s
|
||||
}
|
||||
|
||||
// RemoteStats returns stats for rc
|
||||
//
|
||||
// If short is true then the transfers and checkers won't be added.
|
||||
@@ -912,22 +925,31 @@ func (s *StatsInfo) RemoveTransfer(transfer *Transfer) {
|
||||
}
|
||||
|
||||
// PruneTransfers makes sure there aren't too many old transfers by removing
|
||||
// single finished transfer.
|
||||
func (s *StatsInfo) PruneTransfers() {
|
||||
if MaxCompletedTransfers < 0 {
|
||||
return
|
||||
}
|
||||
// a single finished transfer. Returns true if it removed a transfer.
|
||||
func (s *StatsInfo) PruneTransfers() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.maxCompletedTransfers < 0 {
|
||||
return false
|
||||
}
|
||||
removed := false
|
||||
// remove a transfer from the start if we are over quota
|
||||
if len(s.startedTransfers) > MaxCompletedTransfers+s.ci.Transfers {
|
||||
if len(s.startedTransfers) > s.maxCompletedTransfers+s.ci.Transfers {
|
||||
for i, tr := range s.startedTransfers {
|
||||
if tr.IsDone() {
|
||||
s._removeTransfer(tr, i)
|
||||
removed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return removed
|
||||
}
|
||||
|
||||
// RemoveDoneTransfers removes all Done transfers.
|
||||
func (s *StatsInfo) RemoveDoneTransfers() {
|
||||
for s.PruneTransfers() {
|
||||
}
|
||||
}
|
||||
|
||||
// AddServerSideMove counts a server side move
|
||||
|
||||
@@ -465,3 +465,27 @@ func TestPruneTransfers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveDoneTransfers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewStats(ctx)
|
||||
const transfers = 10
|
||||
for i := int64(1); i <= int64(transfers); i++ {
|
||||
s.AddTransfer(&Transfer{
|
||||
startedAt: time.Unix(i, 0),
|
||||
completedAt: time.Unix(i+1, 0),
|
||||
})
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
assert.Equal(t, time.Duration(transfers)*time.Second, s._totalDuration())
|
||||
assert.Equal(t, transfers, len(s.startedTransfers))
|
||||
s.mu.Unlock()
|
||||
|
||||
s.RemoveDoneTransfers()
|
||||
|
||||
s.mu.Lock()
|
||||
assert.Equal(t, time.Duration(transfers)*time.Second, s._totalDuration())
|
||||
assert.Equal(t, transfers, len(s.startedTransfers))
|
||||
s.mu.Unlock()
|
||||
}
|
||||
+1
-1
@@ -151,7 +151,7 @@ func (x *BwTimetable) Set(s string) error {
|
||||
}
|
||||
|
||||
// Split the timetable string by both spaces and semicolons
|
||||
for _, tok := range strings.FieldsFunc(s, func(r rune) bool {
|
||||
for tok := range strings.FieldsFuncSeq(s, func(r rune) bool {
|
||||
return r == ' ' || r == ';'
|
||||
}) {
|
||||
tv := strings.Split(tok, ",")
|
||||
|
||||
@@ -0,0 +1,598 @@
|
||||
// Package cluster implements a machanism to distribute work over a
|
||||
// cluster of rclone instances.
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/errcount"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// ErrClusterNotConfigured is returned from creation functions.
|
||||
var ErrClusterNotConfigured = errors.New("cluster is not configured")
|
||||
|
||||
// If we don't hear from workers in this time we assume they have timed out
|
||||
// and re-assign their jobs.
|
||||
const workerTimeout = 2 * time.Second
|
||||
|
||||
// Cluster describes the workings of the current cluster.
|
||||
type Cluster struct {
|
||||
jobs *Jobs
|
||||
id string
|
||||
batchFiles int
|
||||
batchSize fs.SizeSuffix
|
||||
cleanup fs.ClusterCleanup // how we cleanup cluster files
|
||||
_config rc.Params // for rc
|
||||
_filter rc.Params // for rc
|
||||
cancel func() // stop bg job
|
||||
wg sync.WaitGroup // bg job finished
|
||||
quit chan struct{} // signal graceful stop
|
||||
sync chan chan<- struct{} // sync the current jobs
|
||||
quitWorkers bool // if set, send workers a stop signal on Shutdown
|
||||
|
||||
workers map[string]*WorkerStatus // worker ID => status
|
||||
deadWorkers map[string]struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
currentBatch Batch
|
||||
inflight map[string]Batch
|
||||
shutdown bool
|
||||
}
|
||||
|
||||
// Batch is a collection of rc tasks to do
|
||||
type Batch struct {
|
||||
size int64 // size in batch
|
||||
Path string `json:"_path"`
|
||||
Inputs []rc.Params `json:"inputs"`
|
||||
Config rc.Params `json:"_config,omitempty"`
|
||||
Filter rc.Params `json:"_filter,omitempty"`
|
||||
|
||||
trs []*accounting.Transfer // transfer for each Input
|
||||
sizes []int64 // sizes for each Input
|
||||
}
|
||||
|
||||
// BatchResult has the results of the batch as received.
|
||||
type BatchResult struct {
|
||||
Results []rc.Params `json:"results"`
|
||||
|
||||
// Error returns
|
||||
Error string `json:"error"`
|
||||
Status int `json:"status"`
|
||||
Input string `json:"input"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
// NewCluster creates a new cluster from the config in ctx.
|
||||
//
|
||||
// It may return nil for no cluster is configured.
|
||||
func NewCluster(ctx context.Context) (*Cluster, error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.Cluster == "" {
|
||||
return nil, nil
|
||||
}
|
||||
jobs, err := NewJobs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c := &Cluster{
|
||||
jobs: jobs,
|
||||
id: ci.ClusterID,
|
||||
quitWorkers: ci.ClusterQuitWorkers,
|
||||
batchFiles: ci.ClusterBatchFiles,
|
||||
batchSize: ci.ClusterBatchSize,
|
||||
cleanup: ci.ClusterCleanup,
|
||||
quit: make(chan struct{}),
|
||||
sync: make(chan chan<- struct{}),
|
||||
inflight: make(map[string]Batch),
|
||||
workers: make(map[string]*WorkerStatus),
|
||||
deadWorkers: make(map[string]struct{}),
|
||||
}
|
||||
|
||||
// Configure _config
|
||||
configParams, err := fs.ConfigOptionsInfo.NonDefaultRC(ci)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read global config: %w", err)
|
||||
}
|
||||
// Remove any global cluster config
|
||||
for k := range configParams {
|
||||
if strings.HasPrefix(k, "Cluster") {
|
||||
delete(configParams, k)
|
||||
}
|
||||
}
|
||||
if len(configParams) != 0 {
|
||||
fs.Debugf(nil, "Overridden global config: %#v", configParams)
|
||||
}
|
||||
c._config = rc.Params(configParams)
|
||||
|
||||
// Configure _filter
|
||||
fi := filter.GetConfig(ctx)
|
||||
if !fi.InActive() {
|
||||
filterParams, err := filter.OptionsInfo.NonDefaultRC(fi)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read filter config: %w", err)
|
||||
}
|
||||
fs.Debugf(nil, "Overridden filter config: %#v", filterParams)
|
||||
c._filter = rc.Params(filterParams)
|
||||
}
|
||||
|
||||
err = c.jobs.createDirectoryStructure(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Start the background worker
|
||||
bgCtx, cancel := context.WithCancel(context.Background())
|
||||
c.cancel = cancel
|
||||
c.wg.Add(1)
|
||||
go c.run(bgCtx)
|
||||
|
||||
fs.Logf(c.jobs.f, "Started cluster master")
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
var (
|
||||
globalClusterMu sync.Mutex
|
||||
globalCluster *Cluster
|
||||
)
|
||||
|
||||
// GetCluster starts or gets a cluster.
|
||||
//
|
||||
// If no cluster is configured or the cluster can't be started then it
|
||||
// returns nil.
|
||||
func GetCluster(ctx context.Context) *Cluster {
|
||||
globalClusterMu.Lock()
|
||||
defer globalClusterMu.Unlock()
|
||||
|
||||
if globalCluster != nil {
|
||||
return globalCluster
|
||||
}
|
||||
|
||||
cluster, err := NewCluster(ctx)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to start cluster: %v", err)
|
||||
return nil
|
||||
}
|
||||
if cluster != nil {
|
||||
atexit.Register(func() {
|
||||
err := cluster.Shutdown(context.Background())
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to shutdown cluster: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
globalCluster = cluster
|
||||
return globalCluster
|
||||
}
|
||||
|
||||
// Send the current batch for processing
|
||||
//
|
||||
// call with c.mu held
|
||||
func (c *Cluster) sendBatch(ctx context.Context) (err error) {
|
||||
// Do nothing if the batch is empty
|
||||
if len(c.currentBatch.Inputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get and reset current batch
|
||||
b := c.currentBatch
|
||||
c.currentBatch = Batch{}
|
||||
|
||||
b.Path = "job/batch"
|
||||
b.Config = c._config
|
||||
b.Filter = c._filter
|
||||
|
||||
// write the pending job
|
||||
name, err := c.jobs.writeJob(ctx, clusterPending, &b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fs.Infof(name, "written cluster batch file")
|
||||
c.inflight[name] = b
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add the command to the current batch
|
||||
func (c *Cluster) addToBatch(ctx context.Context, obj fs.Object, in rc.Params, size int64, tr *accounting.Transfer) (err error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.shutdown {
|
||||
return errors.New("internal error: can't add file to Shutdown cluster")
|
||||
}
|
||||
|
||||
c.currentBatch.Inputs = append(c.currentBatch.Inputs, in)
|
||||
c.currentBatch.size += size
|
||||
c.currentBatch.trs = append(c.currentBatch.trs, tr)
|
||||
c.currentBatch.sizes = append(c.currentBatch.sizes, size)
|
||||
|
||||
if c.currentBatch.size >= int64(c.batchSize) || len(c.currentBatch.Inputs) >= c.batchFiles {
|
||||
err = c.sendBatch(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move does operations.Move via the cluster.
|
||||
//
|
||||
// Move src object to dst or fdst if nil. If dst is nil then it uses
|
||||
// remote as the name of the new object.
|
||||
func (c *Cluster) Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (err error) {
|
||||
tr := accounting.Stats(ctx).NewTransfer(src, fdst)
|
||||
if operations.SkipDestructive(ctx, src, "cluster move") {
|
||||
in := tr.Account(ctx, nil)
|
||||
in.DryRun(src.Size())
|
||||
tr.Done(ctx, nil)
|
||||
return nil
|
||||
}
|
||||
fsrc, ok := src.Fs().(fs.Fs)
|
||||
if !ok {
|
||||
err = errors.New("internal error: cluster move: can't cast src.Fs() to fs.Fs")
|
||||
tr.Done(ctx, err)
|
||||
return err
|
||||
}
|
||||
in := rc.Params{
|
||||
"_path": "operations/movefile",
|
||||
"dstFs": fs.ConfigStringFull(fdst),
|
||||
"dstRemote": remote,
|
||||
"srcFs": fs.ConfigStringFull(fsrc),
|
||||
"srcRemote": src.Remote(),
|
||||
}
|
||||
if dst != nil {
|
||||
in["dstRemote"] = dst.Remote()
|
||||
}
|
||||
return c.addToBatch(ctx, src, in, src.Size(), tr)
|
||||
}
|
||||
|
||||
// Copy does operations.Copy via the cluster.
|
||||
//
|
||||
// Copy src object to dst or fdst if nil. If dst is nil then it uses
|
||||
// remote as the name of the new object.
|
||||
func (c *Cluster) Copy(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (err error) {
|
||||
tr := accounting.Stats(ctx).NewTransfer(src, fdst)
|
||||
if operations.SkipDestructive(ctx, src, "cluster copy") {
|
||||
in := tr.Account(ctx, nil)
|
||||
in.DryRun(src.Size())
|
||||
tr.Done(ctx, nil)
|
||||
return nil
|
||||
}
|
||||
fsrc, ok := src.Fs().(fs.Fs)
|
||||
if !ok {
|
||||
err = errors.New("internal error: cluster copy: can't cast src.Fs() to fs.Fs")
|
||||
tr.Done(ctx, err)
|
||||
return err
|
||||
}
|
||||
in := rc.Params{
|
||||
"_path": "operations/copyfile",
|
||||
"dstFs": fs.ConfigStringFull(fdst),
|
||||
"dstRemote": remote,
|
||||
"srcFs": fs.ConfigStringFull(fsrc),
|
||||
"srcRemote": src.Remote(),
|
||||
}
|
||||
if dst != nil {
|
||||
in["dstRemote"] = dst.Remote()
|
||||
}
|
||||
return c.addToBatch(ctx, src, in, src.Size(), tr)
|
||||
}
|
||||
|
||||
// DeleteFile does operations.DeleteFile via the cluster
|
||||
//
|
||||
// If useBackupDir is set and --backup-dir is in effect then it moves
|
||||
// the file to there instead of deleting
|
||||
func (c *Cluster) DeleteFile(ctx context.Context, dst fs.Object) (err error) {
|
||||
tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "deleting")
|
||||
err = accounting.Stats(ctx).DeleteFile(ctx, dst.Size())
|
||||
if err != nil {
|
||||
tr.Done(ctx, err)
|
||||
return err
|
||||
}
|
||||
if operations.SkipDestructive(ctx, dst, "cluster delete") {
|
||||
tr.Done(ctx, nil)
|
||||
return
|
||||
}
|
||||
fdst, ok := dst.Fs().(fs.Fs)
|
||||
if !ok {
|
||||
err = errors.New("internal error: cluster delete: can't cast dst.Fs() to fs.Fs")
|
||||
tr.Done(ctx, nil)
|
||||
return err
|
||||
}
|
||||
in := rc.Params{
|
||||
"_path": "operations/deletefile",
|
||||
"fs": fs.ConfigStringFull(fdst),
|
||||
"remote": dst.Remote(),
|
||||
}
|
||||
return c.addToBatch(ctx, dst, in, 0, tr)
|
||||
}
|
||||
|
||||
// processCompletedJob loads the job and checks it off
|
||||
func (c *Cluster) processCompletedJob(ctx context.Context, obj fs.Object) error {
|
||||
name := path.Base(obj.Remote())
|
||||
name, _ = strings.CutSuffix(name, ".json")
|
||||
fs.Debugf(nil, "cluster: processing completed job %q", name)
|
||||
|
||||
var output BatchResult
|
||||
err := c.jobs.readJob(ctx, obj, &output)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check jobs read: %w", err)
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
input, ok := c.inflight[name]
|
||||
// FIXME delete or save job
|
||||
if !ok {
|
||||
for k := range c.inflight {
|
||||
fs.Debugf(nil, "key %q", k)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return fmt.Errorf("check jobs: job %q not found", name)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
|
||||
// Delete the inflight entry when batch is processed
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
delete(c.inflight, name)
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
|
||||
// Check job
|
||||
if output.Error != "" {
|
||||
return fmt.Errorf("cluster: failed to run batch job: %s (%d)", output.Error, output.Status)
|
||||
}
|
||||
if len(input.Inputs) != len(output.Results) {
|
||||
return fmt.Errorf("cluster: input had %d jobs but output had %d", len(input.Inputs), len(output.Results))
|
||||
}
|
||||
|
||||
// Run through the batch and mark operations as successful or not
|
||||
for i := range input.Inputs {
|
||||
in := input.Inputs[i]
|
||||
tr := input.trs[i]
|
||||
size := input.sizes[i]
|
||||
out := output.Results[i]
|
||||
errorString, hasError := out["error"]
|
||||
var err error
|
||||
if hasError && errorString != "" {
|
||||
err = fmt.Errorf("cluster: worker error: %s (%v)", errorString, out["status"])
|
||||
}
|
||||
if err == nil && in["_path"] == "operations/movefile" {
|
||||
accounting.Stats(ctx).Renames(1)
|
||||
}
|
||||
acc := tr.Account(ctx, nil)
|
||||
acc.AccountReadN(size)
|
||||
tr.Done(ctx, err)
|
||||
remote, ok := in["dstRemote"]
|
||||
if !ok {
|
||||
remote = in["remote"]
|
||||
}
|
||||
if err == nil {
|
||||
fs.Infof(remote, "cluster %s successful", in["_path"])
|
||||
} else {
|
||||
fs.Errorf(remote, "cluster %s failed: %v", in["_path"], err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadWorkerStatus updates the worker status
|
||||
func (c *Cluster) loadWorkerStatus(ctx context.Context) error {
|
||||
objs, err := c.jobs.listDir(ctx, clusterStatus)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cluster: get job status list failed: %w", err)
|
||||
}
|
||||
ec := errcount.New()
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var mu sync.Mutex
|
||||
for _, obj := range objs {
|
||||
g.Go(func() error {
|
||||
buf, err := c.jobs.readFile(gCtx, obj)
|
||||
if err != nil {
|
||||
ec.Add(fmt.Errorf("read object: %w", err))
|
||||
return nil
|
||||
}
|
||||
workerStatus := new(WorkerStatus)
|
||||
err = json.Unmarshal(buf, workerStatus)
|
||||
if err != nil {
|
||||
ec.Add(fmt.Errorf("status json: %w", err))
|
||||
return nil
|
||||
}
|
||||
mu.Lock()
|
||||
c.workers[workerStatus.ID] = workerStatus
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return ec.Err("cluster: load status")
|
||||
}
|
||||
|
||||
// checkWorkers loads the worker status
|
||||
func (c *Cluster) checkWorkers(ctx context.Context) {
|
||||
err := c.loadWorkerStatus(ctx)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "failed to read some worker status: %v", err)
|
||||
}
|
||||
for workerID, status := range c.workers {
|
||||
timeSinceUpdated := time.Since(status.Updated)
|
||||
if timeSinceUpdated > workerTimeout {
|
||||
if _, isDead := c.deadWorkers[workerID]; isDead {
|
||||
continue
|
||||
}
|
||||
fs.Errorf(nil, "cluster: haven't heard from worker %q for %v - assuming dead", workerID, timeSinceUpdated)
|
||||
// Find any jobs claimed by worker and restart
|
||||
objs, err := c.jobs.listDir(ctx, clusterProcessing)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: failed to find pending jobs: %v", err)
|
||||
continue
|
||||
}
|
||||
for _, obj := range objs {
|
||||
fs.Errorf(obj, "cluster: checking job")
|
||||
// Jobs are named {jobID}-{workerID}.json
|
||||
name := strings.TrimSuffix(path.Base(obj.Remote()), ".json")
|
||||
dash := strings.LastIndex(name, "-")
|
||||
if dash < 0 {
|
||||
fs.Errorf(nil, "cluster: failed to find dash in job %q", name)
|
||||
continue
|
||||
}
|
||||
jobID, jobWorkerID := name[:dash], name[dash+1:]
|
||||
fs.Errorf(obj, "cluster: checking jobID %q, workerID %q", jobID, jobWorkerID)
|
||||
if workerID != jobWorkerID {
|
||||
fs.Debugf(nil, "cluster: job %q doesn't match %q", jobWorkerID, workerID)
|
||||
continue
|
||||
}
|
||||
// Found a job running on worker - rename it back to Pending
|
||||
newRemote := path.Join(clusterPending, jobID+".json")
|
||||
_, err = c.jobs.rename(ctx, obj, newRemote)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: failed to restart job %q: %v", jobID, err)
|
||||
continue
|
||||
}
|
||||
fs.Errorf(nil, "cluster: restarted job %q", jobID)
|
||||
}
|
||||
c.deadWorkers[workerID] = struct{}{}
|
||||
} else {
|
||||
if _, isDead := c.deadWorkers[workerID]; isDead {
|
||||
fs.Errorf(nil, "cluster: dead worker %q came back to life!", workerID)
|
||||
delete(c.deadWorkers, workerID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkJobs sees if there are any completed jobs
|
||||
func (c *Cluster) checkJobs(ctx context.Context) {
|
||||
objs, err := c.jobs.listDir(ctx, clusterDone)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: get completed job list failed: %v", err)
|
||||
return
|
||||
}
|
||||
for _, obj := range objs {
|
||||
err := c.processCompletedJob(ctx, obj)
|
||||
status := "output-ok"
|
||||
ok := true
|
||||
if err != nil {
|
||||
status = "output-failed"
|
||||
ok = false
|
||||
fs.Errorf(nil, "cluster: process completed job failed: %v", err)
|
||||
}
|
||||
c.jobs.finish(ctx, obj, status, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// Run the background process
|
||||
func (c *Cluster) run(ctx context.Context) {
|
||||
defer c.wg.Done()
|
||||
checkJobs := time.NewTicker(clusterCheckJobsInterval)
|
||||
defer checkJobs.Stop()
|
||||
checkWorkers := time.NewTicker(clusterCheckWorkersInterval)
|
||||
defer checkWorkers.Stop()
|
||||
var syncedChans []chan<- struct{}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-c.quit:
|
||||
fs.Debugf(nil, "cluster: quit request received")
|
||||
return
|
||||
case synced := <-c.sync:
|
||||
syncedChans = append(syncedChans, synced)
|
||||
fs.Debugf(nil, "cluster: sync request received")
|
||||
case <-checkWorkers.C:
|
||||
c.checkWorkers(ctx)
|
||||
case <-checkJobs.C:
|
||||
}
|
||||
c.checkJobs(ctx)
|
||||
if len(syncedChans) > 0 {
|
||||
c.mu.Lock()
|
||||
n := len(c.inflight)
|
||||
c.mu.Unlock()
|
||||
if n == 0 {
|
||||
fs.Debugf(nil, "cluster: synced")
|
||||
for _, synced := range syncedChans {
|
||||
synced <- struct{}{}
|
||||
}
|
||||
syncedChans = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sync the cluster.
|
||||
//
|
||||
// Call this when all job items have been added to the cluster.
|
||||
//
|
||||
// This will wait for any outstanding jobs to finish regardless of who
|
||||
// put them in
|
||||
func (c *Cluster) Sync(ctx context.Context) error {
|
||||
// Flush any outstanding
|
||||
c.mu.Lock()
|
||||
err := c.sendBatch(ctx)
|
||||
c.mu.Unlock()
|
||||
|
||||
// Wait for the cluster to be empty
|
||||
done := make(chan struct{})
|
||||
c.sync <- done
|
||||
<-done
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Shutdown the cluster.
|
||||
//
|
||||
// Call this when all job items have been added to the cluster.
|
||||
//
|
||||
// This will wait for any outstanding jobs to finish.
|
||||
func (c *Cluster) Shutdown(ctx context.Context) (err error) {
|
||||
c.mu.Lock()
|
||||
inBatch := len(c.currentBatch.Inputs)
|
||||
inFlight := len(c.inflight)
|
||||
shutdown := c.shutdown
|
||||
c.shutdown = true
|
||||
c.mu.Unlock()
|
||||
|
||||
if inBatch > 0 {
|
||||
err = errors.Join(nil, fmt.Errorf("%d items batched on cluster shutdown", inBatch))
|
||||
}
|
||||
if inFlight > 0 {
|
||||
err = errors.Join(nil, fmt.Errorf("%d items in flight on cluster shutdown", inFlight))
|
||||
}
|
||||
if shutdown {
|
||||
fs.Debugf(nil, "cluster: already shutdown")
|
||||
return nil
|
||||
}
|
||||
c.quit <- struct{}{}
|
||||
fs.Debugf(nil, "Waiting for cluster to finish")
|
||||
c.wg.Wait()
|
||||
|
||||
// Send a quit job
|
||||
if c.quitWorkers {
|
||||
fs.Logf(nil, "Sending quit to workers")
|
||||
quitErr := c.jobs.writeQuitJob(ctx, clusterPending)
|
||||
if quitErr != nil {
|
||||
err = errors.Join(err, fmt.Errorf("shutdown quit: %w", quitErr))
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Abort the cluster and any outstanding jobs.
|
||||
func (c *Cluster) Abort() {
|
||||
c.cancel()
|
||||
c.wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
)
|
||||
|
||||
// Batches flow from queue/pending to queue/processing/
|
||||
const (
|
||||
clusterQueue = "queue"
|
||||
clusterPending = clusterQueue + "/pending"
|
||||
clusterProcessing = clusterQueue + "/processing"
|
||||
clusterDone = clusterQueue + "/done"
|
||||
clusterFinished = clusterQueue + "/finished"
|
||||
clusterStatus = clusterQueue + "/status"
|
||||
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
|
||||
// Read the queue this often
|
||||
clusterCheckJobsInterval = time.Second
|
||||
|
||||
// Write the worker status this often
|
||||
clusterWriteStatusInterval = time.Second
|
||||
|
||||
// Read the worker status this often
|
||||
clusterCheckWorkersInterval = time.Second
|
||||
|
||||
// Name of job which signals to the workers to quit
|
||||
quitJob = "QUIT"
|
||||
)
|
||||
|
||||
// Jobs is a container for sending and receiving jobs to the cluster.
|
||||
type Jobs struct {
|
||||
remote string // remote for job storage
|
||||
f fs.Fs // cluster remote storage
|
||||
partial bool // do we need to write and rename
|
||||
hasMove bool // set if f has server side move otherwise has server side copy
|
||||
cleanup fs.ClusterCleanup // how we cleanup the cluster files
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
}
|
||||
|
||||
// NewJobs creates a Jobs source from the config in ctx.
|
||||
//
|
||||
// It may return nil for no cluster is configured.
|
||||
func NewJobs(ctx context.Context) (*Jobs, error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.Cluster == "" {
|
||||
return nil, nil
|
||||
}
|
||||
f, err := cache.Get(ctx, ci.Cluster)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cluster remote creation: %w", err)
|
||||
}
|
||||
features := f.Features()
|
||||
if features.Move == nil && features.Copy == nil {
|
||||
return nil, fmt.Errorf("cluster remote must have server side move and %q doesn't", ci.Cluster)
|
||||
}
|
||||
jobs := &Jobs{
|
||||
remote: ci.Cluster,
|
||||
f: f,
|
||||
partial: features.PartialUploads,
|
||||
hasMove: features.Move != nil,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
cleanup: ci.ClusterCleanup,
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// Create the cluster directory structure
|
||||
func (jobs *Jobs) createDirectoryStructure(ctx context.Context) (err error) {
|
||||
for _, dir := range []string{clusterPending, clusterProcessing, clusterDone, clusterFinished, clusterStatus} {
|
||||
err = jobs.f.Mkdir(ctx, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cluster mkdir %q: %w", dir, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rename a file
|
||||
//
|
||||
// if this returns fs.ErrorObjectNotFound then the file has already been renamed.
|
||||
func (jobs *Jobs) rename(ctx context.Context, src fs.Object, dstRemote string) (dst fs.Object, err error) {
|
||||
features := jobs.f.Features()
|
||||
if jobs.hasMove {
|
||||
dst, err = features.Move(ctx, src, dstRemote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cluster: failed to rename job file: %w", err)
|
||||
}
|
||||
} else {
|
||||
dst, err = features.Copy(ctx, src, dstRemote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cluster: failed to rename (copy phase) job file: %w", err)
|
||||
}
|
||||
err = src.Remove(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cluster: failed to rename (delete phase) job file: %w", err)
|
||||
}
|
||||
}
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// Finish with a jobs file
|
||||
func (jobs *Jobs) finish(ctx context.Context, obj fs.Object, status string, ok bool) {
|
||||
var err error
|
||||
if (ok && jobs.cleanup == fs.ClusterCleanupCompleted) || jobs.cleanup == fs.ClusterCleanupFull {
|
||||
err = obj.Remove(ctx)
|
||||
} else {
|
||||
name := path.Join(clusterFinished, status, path.Base(obj.Remote()))
|
||||
_, err = jobs.rename(ctx, obj, name)
|
||||
}
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: removing completed job failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// write buf into remote
|
||||
func (jobs *Jobs) writeFile(ctx context.Context, remote string, modTime time.Time, buf []byte) error {
|
||||
partialRemote := remote
|
||||
if jobs.partial {
|
||||
partialRemote = remote + ".partial"
|
||||
}
|
||||
// Calculate hashes
|
||||
w, err := hash.NewMultiHasherTypes(jobs.f.Hashes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
obji := object.NewStaticObjectInfo(partialRemote, modTime, int64(len(buf)), true, w.Sums(), jobs.f)
|
||||
var obj fs.Object
|
||||
err = jobs.pacer.Call(func() (bool, error) {
|
||||
in := bytes.NewBuffer(buf)
|
||||
obj, err = jobs.f.Put(ctx, in, obji)
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("cluster: failed to write %q: %q", remote, err)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if jobs.partial {
|
||||
obj, err = jobs.rename(ctx, obj, remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove the file if it exists
|
||||
func (jobs *Jobs) removeFile(ctx context.Context, remote string) error {
|
||||
obj, err := jobs.f.NewObject(ctx, remote)
|
||||
if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorDirNotFound) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return obj.Remove(ctx)
|
||||
}
|
||||
|
||||
// write a job to a file returning the name
|
||||
func (jobs *Jobs) writeJob(ctx context.Context, where string, job any) (name string, err error) {
|
||||
now := time.Now().UTC()
|
||||
name = fmt.Sprintf("%s-%s", now.Format(time.RFC3339Nano), random.String(20))
|
||||
remote := path.Join(where, name+".json")
|
||||
buf, err := json.MarshalIndent(job, "", "\t")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cluster: job json: %w", err)
|
||||
}
|
||||
err = jobs.writeFile(ctx, remote, now, buf)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cluster: job write: %w", err)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// write a quit job to a file
|
||||
func (jobs *Jobs) writeQuitJob(ctx context.Context, where string) (err error) {
|
||||
now := time.Now().UTC()
|
||||
remote := path.Join(where, quitJob+".json")
|
||||
err = jobs.writeFile(ctx, remote, now, []byte("{}"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cluster: quit job write: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// read buf from object
|
||||
func (jobs *Jobs) readFile(ctx context.Context, o fs.Object) (buf []byte, err error) {
|
||||
err = jobs.pacer.Call(func() (bool, error) {
|
||||
in, err := operations.Open(ctx, o)
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("cluster: failed to open %q: %w", o, err)
|
||||
}
|
||||
buf, err = io.ReadAll(in)
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("cluster: failed to read %q: %w", o, err)
|
||||
}
|
||||
err = in.Close()
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("cluster: failed to close %q: %w", o, err)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// read a job from a file
|
||||
//
|
||||
// job should be a pointer to something to be unmarshalled
|
||||
func (jobs *Jobs) readJob(ctx context.Context, obj fs.Object, job any) error {
|
||||
buf, err := jobs.readFile(ctx, obj)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cluster: job read: %w", err)
|
||||
}
|
||||
err = json.Unmarshal(buf, job)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cluster: job read json: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lists the json files in a cluster directory
|
||||
func (jobs *Jobs) listDir(ctx context.Context, dir string) (objects []fs.Object, err error) {
|
||||
entries, err := jobs.f.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cluster: failed to list %q: %w", dir, err)
|
||||
}
|
||||
entries.ForObject(func(o fs.Object) {
|
||||
if strings.HasSuffix(o.Remote(), ".json") {
|
||||
objects = append(objects, o)
|
||||
}
|
||||
})
|
||||
slices.SortStableFunc(objects, func(a, b fs.Object) int {
|
||||
return cmp.Compare(a.Remote(), b.Remote())
|
||||
})
|
||||
return objects, nil
|
||||
}
|
||||
|
||||
// get a job from pending if there is one available.
|
||||
//
|
||||
// Returns a nil object if no jobs are available.
|
||||
//
|
||||
// FIXME should mark jobs as error jobs in here if they can't be read properly?
|
||||
func (jobs *Jobs) getJob(ctx context.Context, id string) (name string, obj fs.Object, err error) {
|
||||
objs, err := jobs.listDir(ctx, clusterPending)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("get job list: %w", err)
|
||||
}
|
||||
quit := false
|
||||
for len(objs) > 0 {
|
||||
obj = objs[0]
|
||||
objs = objs[1:]
|
||||
name = path.Base(obj.Remote())
|
||||
name, _ = strings.CutSuffix(name, ".json")
|
||||
|
||||
// See if we have been asked to quit
|
||||
if name == quitJob {
|
||||
quit = true
|
||||
continue
|
||||
}
|
||||
|
||||
// claim the job
|
||||
newName := fmt.Sprintf("%s-%s.json", name, id)
|
||||
newRemote := path.Join(clusterProcessing, newName)
|
||||
obj, err = jobs.rename(ctx, obj, newRemote)
|
||||
if errors.Is(err, fs.ErrorObjectNotFound) {
|
||||
// claim failed - try again
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("get job claim: %w", err)
|
||||
}
|
||||
return name, obj, nil
|
||||
}
|
||||
// No jobs found
|
||||
if quit {
|
||||
fs.Logf(nil, "Exiting cluster worker on command")
|
||||
atexit.Run()
|
||||
os.Exit(0)
|
||||
}
|
||||
return "", nil, nil
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/fs/rc/jobs"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
)
|
||||
|
||||
const maxWorkersDone = 16 // maximum jobs in the done list
|
||||
|
||||
// Worker describes a single instance of a cluster worker.
|
||||
type Worker struct {
|
||||
jobs *Jobs
|
||||
cancel func() // stop bg job
|
||||
wg sync.WaitGroup // bg job finished
|
||||
id string // id of this worker
|
||||
status string // place it stores it status
|
||||
|
||||
jobsMu sync.Mutex
|
||||
running map[string]struct{} // IDs of the jobs being processed
|
||||
done []string // IDs of finished jobs
|
||||
}
|
||||
|
||||
// WorkerStatus shows the status of this worker including jobs
|
||||
// running.
|
||||
type WorkerStatus struct {
|
||||
ID string `json:"id"`
|
||||
Running map[string]rc.Params `json:"running"` // Job ID => accounting.RemoteStats
|
||||
Done map[string]bool `json:"done"` // Job ID => finished status
|
||||
Updated time.Time `json:"updated"`
|
||||
}
|
||||
|
||||
// NewWorker creates a new cluster from the config in ctx.
|
||||
//
|
||||
// It may return nil for no cluster is configured.
|
||||
func NewWorker(ctx context.Context) (*Worker, error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.Cluster == "" {
|
||||
return nil, nil
|
||||
}
|
||||
jobs, err := NewJobs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w := &Worker{
|
||||
jobs: jobs,
|
||||
id: ci.ClusterID,
|
||||
running: make(map[string]struct{}),
|
||||
}
|
||||
if w.id == "" {
|
||||
w.id = random.String(10)
|
||||
}
|
||||
w.status = path.Join(clusterStatus, w.id+".json")
|
||||
|
||||
// Start the background workers
|
||||
bgCtx, cancel := context.WithCancel(context.Background())
|
||||
w.cancel = cancel
|
||||
w.wg.Add(1)
|
||||
go w.runJobs(bgCtx)
|
||||
w.wg.Add(1)
|
||||
go w.runStatus(bgCtx)
|
||||
|
||||
fs.Logf(w.jobs.f, "Started cluster worker")
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Check to see if a job exists and run it if available
|
||||
func (w *Worker) checkJobs(ctx context.Context) {
|
||||
name, obj, err := w.jobs.getJob(ctx, w.id)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "check jobs get: %v", err)
|
||||
return
|
||||
}
|
||||
if obj == nil {
|
||||
return // no jobs available
|
||||
}
|
||||
|
||||
// make a stats group for this job
|
||||
ctx = accounting.WithStatsGroup(ctx, name)
|
||||
|
||||
// Add job ID
|
||||
w.jobsMu.Lock()
|
||||
w.running[name] = struct{}{}
|
||||
w.jobsMu.Unlock()
|
||||
fs.Infof(nil, "write jobID %q", name)
|
||||
|
||||
// Remove job ID on exit
|
||||
defer func() {
|
||||
w.jobsMu.Lock()
|
||||
delete(w.running, name)
|
||||
w.done = append(w.done, name)
|
||||
if len(w.done) > maxWorkersDone {
|
||||
w.done = w.done[len(w.done)-maxWorkersDone : len(w.done)]
|
||||
}
|
||||
w.jobsMu.Unlock()
|
||||
}()
|
||||
|
||||
fs.Debugf(nil, "cluster: processing pending job %q", name)
|
||||
inBuf, err := w.jobs.readFile(ctx, obj)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "check jobs read: %v", err)
|
||||
w.jobs.finish(ctx, obj, "input-error", false)
|
||||
return
|
||||
}
|
||||
outBuf := jobs.NewJobFromBytes(ctx, inBuf)
|
||||
remote := path.Join(clusterDone, name+".json")
|
||||
err = w.jobs.writeFile(ctx, remote, time.Now(), outBuf)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "check jobs failed to write output: %v", err)
|
||||
return
|
||||
}
|
||||
w.jobs.finish(ctx, obj, "input-ok", true)
|
||||
fs.Debugf(nil, "cluster: processed pending job %q", name)
|
||||
}
|
||||
|
||||
// Run the background process to pick up jobs
|
||||
func (w *Worker) runJobs(ctx context.Context) {
|
||||
defer w.wg.Done()
|
||||
checkJobs := time.NewTicker(clusterCheckJobsInterval)
|
||||
defer checkJobs.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-checkJobs.C:
|
||||
w.checkJobs(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write the worker status
|
||||
func (w *Worker) writeStatus(ctx context.Context) {
|
||||
// Create the worker status from the jobIDs and the short stats
|
||||
status := WorkerStatus{
|
||||
ID: w.id,
|
||||
Running: make(map[string]rc.Params),
|
||||
Updated: time.Now(),
|
||||
Done: make(map[string]bool),
|
||||
}
|
||||
w.jobsMu.Lock()
|
||||
for _, jobID := range w.done {
|
||||
status.Done[jobID] = true
|
||||
}
|
||||
for jobID := range w.running {
|
||||
fs.Infof(nil, "read jobID %q", jobID)
|
||||
si := accounting.StatsGroup(ctx, jobID)
|
||||
out, err := si.RemoteStats(true)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: write status: stats: %v", err)
|
||||
status.Running[jobID] = rc.Params{}
|
||||
} else {
|
||||
status.Running[jobID] = out
|
||||
}
|
||||
status.Done[jobID] = false
|
||||
}
|
||||
w.jobsMu.Unlock()
|
||||
|
||||
// Write the stats to a file
|
||||
buf, err := json.MarshalIndent(status, "", "\t")
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: write status: json: %w", err)
|
||||
return
|
||||
}
|
||||
err = w.jobs.writeFile(ctx, w.status, status.Updated, buf)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: write status: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the worker status
|
||||
func (w *Worker) clearStatus(ctx context.Context) {
|
||||
err := w.jobs.removeFile(ctx, w.status)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "cluster: clear status: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Run the background process to write status
|
||||
func (w *Worker) runStatus(ctx context.Context) {
|
||||
defer w.wg.Done()
|
||||
w.writeStatus(ctx)
|
||||
defer w.clearStatus(ctx)
|
||||
writeStatus := time.NewTicker(clusterWriteStatusInterval)
|
||||
defer writeStatus.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-writeStatus.C:
|
||||
t0 := time.Now()
|
||||
w.writeStatus(ctx)
|
||||
fs.Debugf(nil, "write status took %v at %v", time.Since(t0), t0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the worker regardless of whether it has work to process or not.
|
||||
func (w *Worker) Shutdown(ctx context.Context) error {
|
||||
w.cancel()
|
||||
w.wg.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -50,6 +50,34 @@ var (
|
||||
ConfigEdit = "config_fs_edit"
|
||||
)
|
||||
|
||||
// ClusterCleanup describes the cluster cleanup choices.
|
||||
type ClusterCleanup = Enum[clusterCleanupChoices]
|
||||
|
||||
// Cluster cleanup choices.
|
||||
//
|
||||
// ClusterCleanupNone don't remove any cluster files
|
||||
// ClusterCleanupCompleted remove successfully completed jobs
|
||||
// ClusterCleanupFull remove everything on exit
|
||||
const (
|
||||
ClusterCleanupNone ClusterCleanup = iota
|
||||
ClusterCleanupCompleted
|
||||
ClusterCleanupFull
|
||||
)
|
||||
|
||||
type clusterCleanupChoices struct{}
|
||||
|
||||
func (clusterCleanupChoices) Choices() []string {
|
||||
return []string{
|
||||
ClusterCleanupNone: "none",
|
||||
ClusterCleanupCompleted: "completed",
|
||||
ClusterCleanupFull: "full",
|
||||
}
|
||||
}
|
||||
|
||||
func (clusterCleanupChoices) Type() string {
|
||||
return "ClusterCleanup"
|
||||
}
|
||||
|
||||
// ConfigOptionsInfo describes the Options in use
|
||||
var ConfigOptionsInfo = Options{{
|
||||
Name: "modify_window",
|
||||
@@ -566,6 +594,36 @@ var ConfigOptionsInfo = Options{{
|
||||
Default: "",
|
||||
Help: "HTTP proxy URL.",
|
||||
Groups: "Networking",
|
||||
}, {
|
||||
Name: "cluster",
|
||||
Default: "",
|
||||
Help: "Enable cluster mode with remote to use as shared storage.",
|
||||
Groups: "Networking",
|
||||
}, {
|
||||
Name: "cluster_id",
|
||||
Default: "",
|
||||
Help: "Set to an ID for the cluster. An ID of 0 or empty becomes the controller.",
|
||||
Groups: "Networking",
|
||||
}, {
|
||||
Name: "cluster_quit_workers",
|
||||
Default: false,
|
||||
Help: "Set to cause the controller to quit the workers when it finished.",
|
||||
Groups: "Networking",
|
||||
}, {
|
||||
Name: "cluster_batch_files",
|
||||
Default: 1000,
|
||||
Help: "Max number of files for a cluster batch.",
|
||||
Groups: "Networking",
|
||||
}, {
|
||||
Name: "cluster_batch_size",
|
||||
Default: Tebi,
|
||||
Help: "Max size of files for a cluster batch.",
|
||||
Groups: "Networking",
|
||||
}, {
|
||||
Name: "cluster_cleanup",
|
||||
Default: ClusterCleanupFull,
|
||||
Help: "Control which cluster files get cleaned up.",
|
||||
Groups: "Networking",
|
||||
}}
|
||||
|
||||
// ConfigInfo is filesystem config options
|
||||
@@ -680,6 +738,12 @@ type ConfigInfo struct {
|
||||
MaxConnections int `config:"max_connections"`
|
||||
NameTransform []string `config:"name_transform"`
|
||||
HTTPProxy string `config:"http_proxy"`
|
||||
Cluster string `config:"cluster"`
|
||||
ClusterID string `config:"cluster_id"`
|
||||
ClusterQuitWorkers bool `config:"cluster_quit_workers"`
|
||||
ClusterBatchFiles int `config:"cluster_batch_files"`
|
||||
ClusterBatchSize SizeSuffix `config:"cluster_batch_size"`
|
||||
ClusterCleanup ClusterCleanup `config:"cluster_cleanup"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
//
|
||||
// It expects 1, 2 or 3 arguments
|
||||
//
|
||||
// rclone authorize "fs name"
|
||||
// rclone authorize "fs name" "base64 encoded JSON blob"
|
||||
// rclone authorize "fs name" "client id" "client secret"
|
||||
// rclone authorize "backend name"
|
||||
// rclone authorize "backend name" "base64 encoded JSON blob"
|
||||
// rclone authorize "backend name" "client id" "client secret"
|
||||
func Authorize(ctx context.Context, args []string, noAutoBrowser bool, templateFile string) error {
|
||||
ctx = suppressConfirm(ctx)
|
||||
ctx = fs.ConfigOAuthOnly(ctx)
|
||||
|
||||
@@ -136,9 +136,11 @@ func (c Simple) Set(key, value string) {
|
||||
c[key] = value
|
||||
}
|
||||
|
||||
// String the map value the same way the config parser does, but with
|
||||
// string the map value the same way the config parser does, but with
|
||||
// sorted keys for reproducibility.
|
||||
func (c Simple) String() string {
|
||||
//
|
||||
// If human is set then use fewer quotes.
|
||||
func (c Simple) string(human bool) string {
|
||||
var ks = make([]string, 0, len(c))
|
||||
for k := range c {
|
||||
ks = append(ks, k)
|
||||
@@ -150,20 +152,41 @@ func (c Simple) String() string {
|
||||
out.WriteRune(',')
|
||||
}
|
||||
out.WriteString(k)
|
||||
v := c[k]
|
||||
if human && v == "true" {
|
||||
continue
|
||||
}
|
||||
out.WriteRune('=')
|
||||
out.WriteRune('\'')
|
||||
for _, ch := range c[k] {
|
||||
out.WriteRune(ch)
|
||||
// Escape ' as ''
|
||||
if ch == '\'' {
|
||||
if !human || strings.ContainsAny(v, `'":=,`) {
|
||||
out.WriteRune('\'')
|
||||
for _, ch := range v {
|
||||
out.WriteRune(ch)
|
||||
// Escape ' as ''
|
||||
if ch == '\'' {
|
||||
out.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
out.WriteRune('\'')
|
||||
} else {
|
||||
out.WriteString(v)
|
||||
}
|
||||
out.WriteRune('\'')
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// Human converts the map value the same way the config parser does,
|
||||
// but with sorted keys for reproducibility. This does it in human
|
||||
// readable form with fewer quotes.
|
||||
func (c Simple) Human() string {
|
||||
return c.string(true)
|
||||
}
|
||||
|
||||
// String the map value the same way the config parser does, but with
|
||||
// sorted keys for reproducibility.
|
||||
func (c Simple) String() string {
|
||||
return c.string(false)
|
||||
}
|
||||
|
||||
// Encode from c into a string suitable for putting on the command line
|
||||
func (c Simple) Encode() (string, error) {
|
||||
if len(c) == 0 {
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package configmap_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSimpleString(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
want string
|
||||
in configmap.Simple
|
||||
}{
|
||||
{name: "Nil", want: "", in: configmap.Simple(nil)},
|
||||
{name: "Empty", want: "", in: configmap.Simple{}},
|
||||
{name: "Basic", want: "config1='one'", in: configmap.Simple{
|
||||
"config1": "one",
|
||||
}},
|
||||
{name: "Truthy", want: "config1='true',config2='true'", in: configmap.Simple{
|
||||
"config1": "true",
|
||||
"config2": "true",
|
||||
}},
|
||||
{name: "Quotable", want: `config1='"one"',config2=':two:',config3='''three''',config4='=four=',config5=',five,'`, in: configmap.Simple{
|
||||
"config1": `"one"`,
|
||||
"config2": `:two:`,
|
||||
"config3": `'three'`,
|
||||
"config4": `=four=`,
|
||||
"config5": `,five,`,
|
||||
}},
|
||||
{name: "Order", want: "config1='one',config2='two',config3='three',config4='four',config5='five'", in: configmap.Simple{
|
||||
"config5": "five",
|
||||
"config4": "four",
|
||||
"config3": "three",
|
||||
"config2": "two",
|
||||
"config1": "one",
|
||||
}},
|
||||
{name: "Escaping", want: "apple='',config1='o''n''e'", in: configmap.Simple{
|
||||
"config1": "o'n'e",
|
||||
"apple": "",
|
||||
}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Check forwards
|
||||
params := tt.in.String()
|
||||
assert.Equal(t, tt.want, params)
|
||||
|
||||
// Check config round trips through config parser
|
||||
remote := ":local," + params + ":"
|
||||
if params == "" {
|
||||
remote = ":local:"
|
||||
}
|
||||
what := fmt.Sprintf("remote = %q", remote)
|
||||
parsed, err := fspath.Parse(remote)
|
||||
require.NoError(t, err, what)
|
||||
if len(parsed.Config) != 0 || len(tt.in) != 0 {
|
||||
assert.Equal(t, tt.in, parsed.Config, what)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestSimpleHuman(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
want string
|
||||
in configmap.Simple
|
||||
}{
|
||||
{name: "Nil", want: "", in: configmap.Simple(nil)},
|
||||
{name: "Empty", want: "", in: configmap.Simple{}},
|
||||
{name: "Basic", want: "config1=one", in: configmap.Simple{
|
||||
"config1": "one",
|
||||
}},
|
||||
{name: "Truthy", want: "config1,config2", in: configmap.Simple{
|
||||
"config1": "true",
|
||||
"config2": "true",
|
||||
}},
|
||||
{name: "Quotable", want: `config1='"one"',config2=':two:',config3='''three''',config4='=four=',config5=',five,'`, in: configmap.Simple{
|
||||
"config1": `"one"`,
|
||||
"config2": `:two:`,
|
||||
"config3": `'three'`,
|
||||
"config4": `=four=`,
|
||||
"config5": `,five,`,
|
||||
}},
|
||||
{name: "Order", want: "config1=one,config2=two,config3=three,config4=four,config5=five", in: configmap.Simple{
|
||||
"config5": "five",
|
||||
"config4": "four",
|
||||
"config3": "three",
|
||||
"config2": "two",
|
||||
"config1": "one",
|
||||
}},
|
||||
{name: "Escaping", want: "apple=,config1='o''n''e'", in: configmap.Simple{
|
||||
"config1": "o'n'e",
|
||||
"apple": "",
|
||||
}},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Check forwards
|
||||
params := tt.in.Human()
|
||||
assert.Equal(t, tt.want, params)
|
||||
|
||||
// Check config round trips through config parser
|
||||
remote := ":local," + params + ":"
|
||||
if params == "" {
|
||||
remote = ":local:"
|
||||
}
|
||||
what := fmt.Sprintf("remote = %q", remote)
|
||||
parsed, err := fspath.Parse(remote)
|
||||
require.NoError(t, err, what)
|
||||
if len(parsed.Config) != 0 || len(tt.in) != 0 {
|
||||
assert.Equal(t, tt.in, parsed.Config, what)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -246,30 +246,6 @@ func TestConfigMapClearSetters(t *testing.T) {
|
||||
assert.Equal(t, []Setter(nil), m.setters)
|
||||
}
|
||||
|
||||
func TestSimpleString(t *testing.T) {
|
||||
// Basic
|
||||
assert.Equal(t, "", Simple(nil).String())
|
||||
assert.Equal(t, "", Simple{}.String())
|
||||
assert.Equal(t, "config1='one'", Simple{
|
||||
"config1": "one",
|
||||
}.String())
|
||||
|
||||
// Check ordering
|
||||
assert.Equal(t, "config1='one',config2='two',config3='three',config4='four',config5='five'", Simple{
|
||||
"config5": "five",
|
||||
"config4": "four",
|
||||
"config3": "three",
|
||||
"config2": "two",
|
||||
"config1": "one",
|
||||
}.String())
|
||||
|
||||
// Check escaping
|
||||
assert.Equal(t, "apple='',config1='o''n''e'", Simple{
|
||||
"config1": "o'n'e",
|
||||
"apple": "",
|
||||
}.String())
|
||||
}
|
||||
|
||||
func TestSimpleEncode(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in Simple
|
||||
|
||||
Loaded 100 of 145 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user