Compare commits
140
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d8144a7e84 | ||
|
|
21f5b1076f | ||
|
|
a8cee91929 | ||
|
|
14edaefaa3 | ||
|
|
946b73c5fe | ||
|
|
07f7854a25 | ||
|
|
dfc7393e9c | ||
|
|
7f50143805 | ||
|
|
37308832b1 | ||
|
|
d5d124c9ee | ||
|
|
8b1f737271 | ||
|
|
ed013228ea | ||
|
|
a9b7974196 | ||
|
|
38453bd8ff | ||
|
|
51ba73afe4 | ||
|
|
b782a15f17 | ||
|
|
1d2edf0789 | ||
|
|
8367261a93 | ||
|
|
31dc78905e | ||
|
|
f9493ff1db | ||
|
|
a5cf531b94 | ||
|
|
04d59fe6bd | ||
|
|
cbc448039d | ||
|
|
7a8ade4949 | ||
|
|
8ab6e2bd1f | ||
|
|
d5c5811a54 | ||
|
|
43d21b7e64 | ||
|
|
0501b12e17 | ||
|
|
723c1dfdc5 | ||
|
|
d6894408b9 | ||
|
|
ac0e831090 | ||
|
|
b93ed77495 | ||
|
|
d0a3c40f9a | ||
|
|
6a9a571d87 | ||
|
|
6b68d1f79a | ||
|
|
088756ff8a | ||
|
|
4c1ffcc36d | ||
|
|
e5bd3f1c26 | ||
|
|
e03b4e2ce9 | ||
|
|
6054476c9c | ||
|
|
6cd8d3c4a0 | ||
|
|
78b0329928 | ||
|
|
6882aeff97 | ||
|
|
a0922643e6 | ||
|
|
4d7254f88f | ||
|
|
8499090038 | ||
|
|
ca0c4c7585 | ||
|
|
5aec01100e | ||
|
|
4d6af44045 | ||
|
|
256d17eba8 | ||
|
|
5777f4481f | ||
|
|
6e945fba82 | ||
|
|
ce7047d88a | ||
|
|
4a35100130 | ||
|
|
ef7662d2fa | ||
|
|
16e89706d3 | ||
|
|
41a58d7f6e | ||
|
|
2a7f3eecf4 | ||
|
|
41e12114a8 | ||
|
|
1b12d5d346 | ||
|
|
74b8cbfb84 | ||
|
|
06427371eb | ||
|
|
d08e1616a7 | ||
|
|
79b011305e | ||
|
|
050879f0ca | ||
|
|
0a74f8022e | ||
|
|
0db20c9c56 | ||
|
|
33e9db9745 | ||
|
|
d0145d4359 | ||
|
|
80dab10ec9 | ||
|
|
022cda3109 | ||
|
|
4030672b66 | ||
|
|
348379625c | ||
|
|
ab4a2275eb | ||
|
|
c8683fc916 | ||
|
|
273ee0d696 | ||
|
|
6d19bbba73 | ||
|
|
d6c31b51c6 | ||
|
|
af19f924ff | ||
|
|
80572c544b | ||
|
|
11f8cb32d1 | ||
|
|
62781d0925 | ||
|
|
2bd786a452 | ||
|
|
038648aaca | ||
|
|
0e3ac6b13c | ||
|
|
1e274b01fe | ||
|
|
c4f6439715 | ||
|
|
cdecb44789 | ||
|
|
8af3f61b6e | ||
|
|
48eef2fb3c | ||
|
|
7a5b531bd0 | ||
|
|
78ca08ba8a | ||
|
|
49ba4eeb86 | ||
|
|
c08617c70f | ||
|
|
1bd3365868 | ||
|
|
31f21551bf | ||
|
|
baaec5b126 | ||
|
|
2b72c7f709 | ||
|
|
f204d95a8b | ||
|
|
7a5b814e59 | ||
|
|
aa6c06d751 | ||
|
|
ffb031a71b | ||
|
|
041b201abd | ||
|
|
9db8ecbc32 | ||
|
|
518d39815c | ||
|
|
147f97d1f7 | ||
|
|
7f44735709 | ||
|
|
db9d94f5de | ||
|
|
a36ef8582f | ||
|
|
f34a40a709 | ||
|
|
4aee962233 | ||
|
|
5f71d186b2 | ||
|
|
56c9fdb53c | ||
|
|
47474687eb | ||
|
|
abe753ca86 | ||
|
|
cf5d0f5c1f | ||
|
|
4d431e94b9 | ||
|
|
bdafbad61e | ||
|
|
eb6e9b194a | ||
|
|
ecdfd80459 | ||
|
|
a268eedb1d | ||
|
|
19ff7c9302 | ||
|
|
fb169a8b54 | ||
|
|
bcbfad1482 | ||
|
|
8c37ae8f5c | ||
|
|
610f40f700 | ||
|
|
5cb2a2fa3c | ||
|
|
919a180ad2 | ||
|
|
951099dbed | ||
|
|
0f9267d5fc | ||
|
|
3de9bd9d04 | ||
|
|
57ee25d75a | ||
|
|
633f50cd3e | ||
|
|
d04d4edc40 | ||
|
|
98c34e413d | ||
|
|
c4bc249b66 | ||
|
|
899c8e0697 | ||
|
|
4006345cfb | ||
|
|
1319d7333c | ||
|
|
5f168b3b96 |
No files matched your search
@@ -0,0 +1,4 @@
|
||||
github: [ncw]
|
||||
patreon: njcw
|
||||
liberapay: ncw
|
||||
custom: ["https://rclone.org/donate/"]
|
||||
@@ -0,0 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Rclone Forum Community Support
|
||||
url: https://forum.rclone.org/
|
||||
about: Please ask and answer questions here.
|
||||
@@ -227,7 +227,7 @@ jobs:
|
||||
- name: Cross-compile rclone
|
||||
run: |
|
||||
docker pull billziss/xgo-cgofuse
|
||||
go get -v github.com/karalabe/xgo
|
||||
GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
|
||||
xgo \
|
||||
-image=billziss/xgo-cgofuse \
|
||||
-targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Docker beta build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build image job
|
||||
steps:
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and publish image
|
||||
uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
|
||||
with:
|
||||
tag: beta
|
||||
imageName: rclone/rclone
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
|
||||
publish: true
|
||||
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
|
||||
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
||||
@@ -0,0 +1,33 @@
|
||||
name: Docker release build
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build image job
|
||||
steps:
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Get actual patch version
|
||||
id: actual_patch_version
|
||||
run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
|
||||
- name: Get actual minor version
|
||||
id: actual_minor_version
|
||||
run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
|
||||
- name: Get actual major version
|
||||
id: actual_major_version
|
||||
run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
|
||||
- name: Build and publish image
|
||||
uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
|
||||
with:
|
||||
tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
|
||||
imageName: rclone/rclone
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
|
||||
publish: true
|
||||
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
|
||||
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
||||
+3
-3
@@ -158,7 +158,7 @@ with modules beneath.
|
||||
* fserrors - rclone specific error handling
|
||||
* fshttp - http handling for rclone
|
||||
* fspath - path handling for rclone
|
||||
* hash - defines rclones hash types and functions
|
||||
* hash - defines rclone's hash types and functions
|
||||
* list - list a remote
|
||||
* log - logging facilities
|
||||
* march - iterates directories in lock step
|
||||
@@ -295,7 +295,7 @@ If you need to update a dependency then run
|
||||
GO111MODULE=on go get -u github.com/pkg/errors
|
||||
GO111MODULE=on go mod vendor
|
||||
|
||||
Check in in a single commit as above.
|
||||
Check in a single commit as above.
|
||||
|
||||
## Updating all the dependencies ##
|
||||
|
||||
@@ -383,7 +383,7 @@ alphabetical order of full name of remote (eg `drive` is ordered as
|
||||
* update them with `make backenddocs` - revert any changes in other backends
|
||||
* `docs/content/overview.md` - overview docs
|
||||
* `docs/content/docs.md` - list of remotes in config section
|
||||
* `docs/content/about.md` - front page of rclone.org
|
||||
* `docs/content/_index.md` - front page of rclone.org
|
||||
* `docs/layouts/chrome/navbar.html` - add it to the website navigation
|
||||
* `bin/make_manual.py` - add the page to the `docs` constant
|
||||
|
||||
|
||||
+2
-2
@@ -3,7 +3,6 @@ FROM golang AS builder
|
||||
COPY . /go/src/github.com/rclone/rclone/
|
||||
WORKDIR /go/src/github.com/rclone/rclone/
|
||||
|
||||
RUN make quicktest
|
||||
RUN \
|
||||
CGO_ENABLED=0 \
|
||||
make
|
||||
@@ -12,7 +11,8 @@ RUN ./rclone version
|
||||
# Begin final image
|
||||
FROM alpine:latest
|
||||
|
||||
RUN apk --no-cache add ca-certificates fuse
|
||||
RUN apk --no-cache add ca-certificates fuse tzdata && \
|
||||
echo "user_allow_other" >> /etc/fuse.conf
|
||||
|
||||
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
|
||||
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ Rclone uses the labels like this:
|
||||
* `duplicate` - normally close these and ask the user to subscribe to the original
|
||||
* `enhancement: new remote` - a new rclone backend
|
||||
* `enhancement` - a new feature
|
||||
* `FUSE` - do do with `rclone mount` command
|
||||
* `FUSE` - to do with `rclone mount` command
|
||||
* `good first issue` - mark these if you find a small self contained issue - these get shown to new visitors to the project
|
||||
* `help` wanted - mark these if you find a self contained issue - these get shown to new visitors to the project
|
||||
* `IMPORTANT` - note to maintainers not to forget to fix this for the release
|
||||
|
||||
Generated
+3310
-1197
File diff suppressed because it is too large.
Load diff
Generated
+4126
-803
File diff suppressed because it is too large.
Load diff
@@ -136,11 +136,19 @@ clean:
|
||||
rm -f rclone fs/operations/operations.test fs/sync/sync.test fs/test_all.log test.log
|
||||
|
||||
website:
|
||||
rm -rf docs/public
|
||||
cd docs && hugo
|
||||
@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
|
||||
|
||||
upload_website: website
|
||||
rclone -v sync docs/public memstore:www-rclone-org
|
||||
|
||||
upload_test_website: website
|
||||
rclone -P sync docs/public test-rclone-org:
|
||||
|
||||
validate_website: website
|
||||
find docs/public -type f -name "*.html" | xargs tidy --mute-id yes -errors --gnu-emacs yes --drop-empty-elements no --warn-proprietary-attributes no --mute MISMATCHED_ATTRIBUTE_WARN
|
||||
|
||||
tarball:
|
||||
git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
|
||||
|
||||
@@ -209,7 +217,7 @@ fetch_binaries:
|
||||
rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
|
||||
|
||||
serve: website
|
||||
cd docs && hugo server -v -w
|
||||
cd docs && hugo server -v -w --disableFastRender
|
||||
|
||||
tag: doc
|
||||
@echo "Old tag is $(VERSION)"
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ This file describes how to make the various kinds of releases
|
||||
* make upload_website
|
||||
* make upload_github
|
||||
* make startdev
|
||||
* # announce with forum post, twitter post, G+ post
|
||||
* # announce with forum post, twitter post, patreon post
|
||||
|
||||
Early in the next release cycle update the vendored dependencies
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ func init() {
|
||||
Description: "Amazon Drive",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("amazon cloud drive", name, m, acdConfig)
|
||||
err := oauthutil.Config("amazon cloud drive", name, m, acdConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
@@ -169,7 +169,7 @@ type Fs struct {
|
||||
tokenRenewer *oauthutil.Renew // renew the token on expiry
|
||||
}
|
||||
|
||||
// Object describes a acd object
|
||||
// Object describes an acd object
|
||||
//
|
||||
// Will definitely have info but maybe not meta
|
||||
type Object struct {
|
||||
@@ -229,7 +229,7 @@ func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
}
|
||||
// Work around receiving this error sporadically on authentication
|
||||
//
|
||||
// HTTP code 403: "403 Forbidden", reponse body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
|
||||
// HTTP code 403: "403 Forbidden", response body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
|
||||
if resp.StatusCode == 403 && strings.Contains(err.Error(), "Authorization header requires") {
|
||||
fs.Debugf(f, "403 \"Authorization header requires...\" error received - retry")
|
||||
return true, err
|
||||
|
||||
@@ -201,7 +201,7 @@ type Fs struct {
|
||||
pool *pool.Pool // memory pool
|
||||
}
|
||||
|
||||
// Object describes a azure object
|
||||
// Object describes an azure object
|
||||
type Object struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
@@ -338,7 +338,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
||||
}
|
||||
|
||||
// httpClientFactory creates a Factory object that sends HTTP requests
|
||||
// to a rclone's http.Client.
|
||||
// to an rclone's http.Client.
|
||||
//
|
||||
// copied from azblob.newDefaultHTTPClientFactory
|
||||
func httpClientFactory(client *http.Client) pipeline.Factory {
|
||||
@@ -1455,7 +1455,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
httpHeaders.ContentType = fs.MimeType(ctx, o)
|
||||
// Compute the Content-MD5 of the file, for multiparts uploads it
|
||||
// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
|
||||
// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
|
||||
// Note: If multipart, an MD5 checksum will also be computed for each uploaded block
|
||||
// in order to validate its integrity during transport
|
||||
if !o.fs.opt.DisableCheckSum {
|
||||
if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
|
||||
|
||||
@@ -222,3 +222,23 @@ type AppAuth struct {
|
||||
PrivateKey string `json:"privateKey"`
|
||||
Passphrase string `json:"passphrase"`
|
||||
}
|
||||
|
||||
// User is returned from /users/me
|
||||
type User struct {
|
||||
Type string `json:"type"`
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Login string `json:"login"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ModifiedAt time.Time `json:"modified_at"`
|
||||
Language string `json:"language"`
|
||||
Timezone string `json:"timezone"`
|
||||
SpaceAmount int64 `json:"space_amount"`
|
||||
SpaceUsed int64 `json:"space_used"`
|
||||
MaxUploadSize int64 `json:"max_upload_size"`
|
||||
Status string `json:"status"`
|
||||
JobTitle string `json:"job_title"`
|
||||
Phone string `json:"phone"`
|
||||
Address string `json:"address"`
|
||||
AvatarURL string `json:"avatar_url"`
|
||||
}
|
||||
+27
-2
@@ -93,7 +93,7 @@ func init() {
|
||||
log.Fatalf("Failed to configure token with jwt authentication: %v", err)
|
||||
}
|
||||
} else {
|
||||
err = oauthutil.Config("box", name, m, oauthConfig)
|
||||
err = oauthutil.Config("box", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token with oauth authentication: %v", err)
|
||||
}
|
||||
@@ -296,7 +296,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// parsePath parses an box 'url'
|
||||
// parsePath parses a box 'url'
|
||||
func parsePath(path string) (root string) {
|
||||
root = strings.Trim(path, "/")
|
||||
return
|
||||
@@ -883,6 +883,30 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// About gets quota information
|
||||
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/users/me",
|
||||
}
|
||||
var user api.User
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read user info")
|
||||
}
|
||||
// FIXME max upload size would be useful to use in Update
|
||||
usage = &fs.Usage{
|
||||
Used: fs.NewUsageValue(user.SpaceUsed), // bytes in use
|
||||
Total: fs.NewUsageValue(user.SpaceAmount), // bytes total
|
||||
Free: fs.NewUsageValue(user.SpaceAmount - user.SpaceUsed), // bytes free
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// Move src to this remote using server side move operations.
|
||||
//
|
||||
// This is stored with the remote path given
|
||||
@@ -1274,6 +1298,7 @@ var (
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PutStreamer = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
|
||||
Vendored
+5
-5
@@ -87,7 +87,7 @@ func init() {
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "plex_insecure",
|
||||
Help: "Skip all certificate verifications when connecting to the Plex server",
|
||||
Help: "Skip all certificate verification when connecting to the Plex server",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
@@ -339,7 +339,7 @@ func parseRootPath(path string) (string, error) {
|
||||
return strings.Trim(path, "/"), nil
|
||||
}
|
||||
|
||||
// NewFs constructs a Fs from the path, container:path
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
@@ -520,9 +520,6 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// override only those features that use a temp fs and it doesn't support them
|
||||
//f.features.ChangeNotify = f.ChangeNotify
|
||||
if f.opt.TempWritePath != "" {
|
||||
if f.tempFs.Features().Copy == nil {
|
||||
f.features.Copy = nil
|
||||
}
|
||||
if f.tempFs.Features().Move == nil {
|
||||
f.features.Move = nil
|
||||
}
|
||||
@@ -1533,6 +1530,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
fs.Errorf(src, "source remote (%v) doesn't support Copy", src.Fs())
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
if f.opt.TempWritePath != "" && src.Fs() == f.tempFs {
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
// the source must be a cached object or we abort
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
|
||||
Vendored
+16
-12
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/testy"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfsflags"
|
||||
@@ -299,6 +300,7 @@ func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestInternalCachedWrittenContentMatches(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
@@ -342,6 +344,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestInternalCachedUpdatedContentMatches(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
id := fmt.Sprintf("ticucm%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
@@ -1009,6 +1012,15 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Config to pass to NewFs
|
||||
m := configmap.Simple{}
|
||||
for k, v := range r.runDefaultCfgMap {
|
||||
m.Set(k, v)
|
||||
}
|
||||
for k, v := range flags {
|
||||
m.Set(k, v)
|
||||
}
|
||||
|
||||
// if the remote doesn't exist, create a new one with a local one for it
|
||||
// identify which is the cache remote (it can be wrapped by a crypt too)
|
||||
rootIsCrypt := false
|
||||
@@ -1017,8 +1029,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
||||
localRemote := remote + "-local"
|
||||
config.FileSet(localRemote, "type", "local")
|
||||
config.FileSet(localRemote, "nounc", "true")
|
||||
config.FileSet(remote, "type", "cache")
|
||||
config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote)
|
||||
m.Set("type", "cache")
|
||||
m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
|
||||
} else {
|
||||
remoteType := config.FileGet(remote, "type", "")
|
||||
if remoteType == "" {
|
||||
@@ -1028,8 +1040,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
||||
if remoteType != "cache" {
|
||||
if remoteType == "crypt" {
|
||||
rootIsCrypt = true
|
||||
config.FileSet(remote, "password", cryptPassword1)
|
||||
config.FileSet(remote, "password2", cryptPassword2)
|
||||
m.Set("password", cryptPassword1)
|
||||
m.Set("password2", cryptPassword2)
|
||||
}
|
||||
remoteRemote := config.FileGet(remote, "remote", "")
|
||||
if remoteRemote == "" {
|
||||
@@ -1055,14 +1067,6 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
||||
|
||||
fs.Config.LowLevelRetries = 1
|
||||
|
||||
m := configmap.Simple{}
|
||||
for k, v := range r.runDefaultCfgMap {
|
||||
m.Set(k, v)
|
||||
}
|
||||
for k, v := range flags {
|
||||
m.Set(k, v)
|
||||
}
|
||||
|
||||
// Instantiate root
|
||||
if purge {
|
||||
boltDb.PurgeTempUploads()
|
||||
|
||||
@@ -217,7 +217,7 @@ func decodeFileName(in string) ([]byte, error) {
|
||||
// 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
|
||||
// Rogaway.
|
||||
//
|
||||
// This makes for determinstic encryption which is what we want - the
|
||||
// This makes for deterministic encryption which is what we want - the
|
||||
// same filename must encrypt to the same thing.
|
||||
//
|
||||
// This means that
|
||||
@@ -559,7 +559,7 @@ func (n *nonce) increment() {
|
||||
n.carry(0)
|
||||
}
|
||||
|
||||
// add an uint64 to the nonce
|
||||
// add a uint64 to the nonce
|
||||
func (n *nonce) add(x uint64) {
|
||||
carry := uint16(0)
|
||||
for i := 0; i < 8; i++ {
|
||||
|
||||
@@ -929,7 +929,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
|
||||
assert.Equal(t, 0, n)
|
||||
}
|
||||
|
||||
// Now try decoding it with a open/seek
|
||||
// Now try decoding it with an open/seek
|
||||
for _, offset := range trials {
|
||||
for _, limit := range limits {
|
||||
if offset+limit > len(plaintext) {
|
||||
|
||||
@@ -241,7 +241,7 @@ func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
|
||||
*entries = append(*entries, f.newObject(obj))
|
||||
}
|
||||
|
||||
// Encrypt an directory file name to entries.
|
||||
// Encrypt a directory file name to entries.
|
||||
func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
|
||||
remote := dir.Remote()
|
||||
decryptedRemote, err := f.cipher.DecryptDirName(remote)
|
||||
@@ -943,7 +943,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
|
||||
if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
|
||||
// Prefer direct interface assertion
|
||||
} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
|
||||
// Otherwise likely is a operations.OverrideRemote
|
||||
// Otherwise likely is an operations.OverrideRemote
|
||||
srcObj = do.UnWrap()
|
||||
} else {
|
||||
return "", nil
|
||||
|
||||
@@ -82,7 +82,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
|
||||
|
||||
var oi fs.ObjectInfo = obj
|
||||
if wrap {
|
||||
// wrap the object in a fs.ObjectUnwrapper if required
|
||||
// wrap the object in an fs.ObjectUnwrapper if required
|
||||
oi = testWrapper{oi}
|
||||
}
|
||||
|
||||
|
||||
+115
-41
@@ -24,6 +24,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
@@ -68,6 +69,8 @@ const (
|
||||
minChunkSize = 256 * fs.KibiByte
|
||||
defaultChunkSize = 8 * fs.MebiByte
|
||||
partialFields = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails"
|
||||
listRGrouping = 50 // number of IDs to search at once when using ListR
|
||||
listRInputBuffer = 1000 // size of input buffer when using ListR
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -178,7 +181,7 @@ func init() {
|
||||
}
|
||||
|
||||
if opt.ServiceAccountFile == "" {
|
||||
err = oauthutil.Config("drive", name, m, driveConfig)
|
||||
err = oauthutil.Config("drive", name, m, driveConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
@@ -263,7 +266,7 @@ videos.
|
||||
Setting this flag will cause Google photos and videos to return a
|
||||
blank MD5 checksum.
|
||||
|
||||
Google photos are identifed by being in the "photos" space.
|
||||
Google photos are identified by being in the "photos" space.
|
||||
|
||||
Corrupted checksums are caused by Google modifying the image/video but
|
||||
not updating the checksum.`,
|
||||
@@ -346,9 +349,12 @@ date is used.`,
|
||||
Help: "Size of listing chunk 100-1000. 0 to disable.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "impersonate",
|
||||
Default: "",
|
||||
Help: "Impersonate this user when using a service account.",
|
||||
Name: "impersonate",
|
||||
Default: "",
|
||||
Help: `Impersonate this user when using a service account.
|
||||
|
||||
Note that if this is used then "root_folder_id" will be ignored.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "alternate_export",
|
||||
@@ -558,6 +564,9 @@ type Fs struct {
|
||||
isTeamDrive bool // true if this is a team drive
|
||||
fileFields googleapi.Field // fields to fetch file info with
|
||||
m configmap.Mapper
|
||||
grouping int32 // number of IDs to search at once in ListR - read with atomic
|
||||
listRmu *sync.Mutex // protects listRempties
|
||||
listRempties map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
|
||||
}
|
||||
|
||||
type baseObject struct {
|
||||
@@ -1079,11 +1088,14 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: newPacer(opt),
|
||||
m: m,
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: newPacer(opt),
|
||||
m: m,
|
||||
grouping: listRGrouping,
|
||||
listRmu: new(sync.Mutex),
|
||||
listRempties: make(map[string]struct{}),
|
||||
}
|
||||
f.isTeamDrive = opt.TeamDriveID != ""
|
||||
f.fileFields = f.getFileFields()
|
||||
@@ -1109,9 +1121,20 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// If impersonating warn about root_folder_id if set and unset it
|
||||
//
|
||||
// This is because rclone v1.51 and v1.52 cached root_folder_id when
|
||||
// using impersonate which they shouldn't have done. It is possible
|
||||
// someone is using impersonate and root_folder_id in which case this
|
||||
// breaks their workflow. There isn't an easy way around that.
|
||||
if opt.RootFolderID != "" && opt.Impersonate != "" {
|
||||
fs.Logf(f, "Ignoring cached root_folder_id when using --drive-impersonate")
|
||||
opt.RootFolderID = ""
|
||||
}
|
||||
|
||||
// set root folder for a team drive or query the user root folder
|
||||
if opt.RootFolderID != "" {
|
||||
// override root folder if set or cached in the config
|
||||
// override root folder if set or cached in the config and not impersonating
|
||||
f.rootFolderID = opt.RootFolderID
|
||||
} else if f.isTeamDrive {
|
||||
f.rootFolderID = f.opt.TeamDriveID
|
||||
@@ -1128,7 +1151,10 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
|
||||
}
|
||||
}
|
||||
f.rootFolderID = rootID
|
||||
m.Set("root_folder_id", rootID)
|
||||
// Don't cache the root folder ID if impersonating
|
||||
if opt.Impersonate == "" {
|
||||
m.Set("root_folder_id", rootID)
|
||||
}
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, f.rootFolderID, f)
|
||||
@@ -1220,7 +1246,7 @@ func (f *Fs) getFileFields() (fields googleapi.Field) {
|
||||
return fields
|
||||
}
|
||||
|
||||
// newRegularObject creates a fs.Object for a normal drive.File
|
||||
// newRegularObject creates an fs.Object for a normal drive.File
|
||||
func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||
// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
|
||||
if f.opt.SkipChecksumGphotos {
|
||||
@@ -1239,7 +1265,7 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||
}
|
||||
}
|
||||
|
||||
// newDocumentObject creates a fs.Object for a google docs drive.File
|
||||
// newDocumentObject creates an fs.Object for a google docs drive.File
|
||||
func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(exportMimeType)
|
||||
if err != nil {
|
||||
@@ -1270,7 +1296,7 @@ func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, expor
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newLinkObject creates a fs.Object that represents a link a google docs drive.File
|
||||
// newLinkObject creates an fs.Object that represents a link a google docs drive.File
|
||||
func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
|
||||
t := linkTemplate(exportMimeType)
|
||||
if t == nil {
|
||||
@@ -1296,9 +1322,9 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newObjectWithInfo creates a fs.Object for any drive.File
|
||||
// newObjectWithInfo creates an fs.Object for any drive.File
|
||||
//
|
||||
// When the drive.File cannot be represented as a fs.Object it will return (nil, nil).
|
||||
// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
|
||||
func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
|
||||
// If item has MD5 sum or a length it is a file stored on drive
|
||||
if info.Md5Checksum != "" || info.Size > 0 {
|
||||
@@ -1309,9 +1335,9 @@ func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, erro
|
||||
return f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
|
||||
}
|
||||
|
||||
// newObjectWithExportInfo creates a fs.Object for any drive.File and the result of findExportFormat
|
||||
// newObjectWithExportInfo creates an fs.Object for any drive.File and the result of findExportFormat
|
||||
//
|
||||
// When the drive.File cannot be represented as a fs.Object it will return (nil, nil).
|
||||
// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
|
||||
func (f *Fs) newObjectWithExportInfo(
|
||||
remote string, info *drive.File,
|
||||
extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
|
||||
@@ -1402,6 +1428,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
leaf = f.opt.Enc.FromStandardName(leaf)
|
||||
// fmt.Println("Making", path)
|
||||
// Define the metadata for the directory we are going to create.
|
||||
pathID = actualID(pathID)
|
||||
createInfo := &drive.File{
|
||||
Name: leaf,
|
||||
Description: leaf,
|
||||
@@ -1629,20 +1656,22 @@ func (s listRSlices) Less(i, j int) bool {
|
||||
return s.dirs[i] < s.dirs[j]
|
||||
}
|
||||
|
||||
// listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
|
||||
// listRRunner will read dirIDs from the in channel, perform the file listing and call cb with each DirEntry.
|
||||
//
|
||||
// In each cycle it will read up to grouping entries from the in channel without blocking.
|
||||
// If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
|
||||
// nil is send to the out channel and the function returns.
|
||||
func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan listREntry, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
|
||||
func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listREntry, out chan<- error, cb func(fs.DirEntry) error) {
|
||||
var dirs []string
|
||||
var paths []string
|
||||
var grouping int32
|
||||
|
||||
for dir := range in {
|
||||
dirs = append(dirs[:0], dir.id)
|
||||
paths = append(paths[:0], dir.path)
|
||||
grouping = atomic.LoadInt32(&f.grouping)
|
||||
waitloop:
|
||||
for i := 1; i < grouping; i++ {
|
||||
for i := int32(1); i < grouping; i++ {
|
||||
select {
|
||||
case d, ok := <-in:
|
||||
if !ok {
|
||||
@@ -1655,6 +1684,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
|
||||
}
|
||||
listRSlices{dirs, paths}.Sort()
|
||||
var iErr error
|
||||
foundItems := false
|
||||
_, err := f.list(ctx, dirs, "", false, false, false, func(item *drive.File) bool {
|
||||
// shared with me items have no parents when at the root
|
||||
if f.opt.SharedWithMe && len(item.Parents) == 0 && len(paths) == 1 && paths[0] == "" {
|
||||
@@ -1662,6 +1692,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
|
||||
}
|
||||
for _, parent := range item.Parents {
|
||||
var i int
|
||||
foundItems = true
|
||||
earlyExit := false
|
||||
// If only one item in paths then no need to search for the ID
|
||||
// assuming google drive is doing its job properly.
|
||||
@@ -1702,6 +1733,53 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
|
||||
}
|
||||
return false
|
||||
})
|
||||
// Found no items in more than one directory. Retry these as
|
||||
// individual directories This is to work around a bug in google
|
||||
// drive where (A in parents) or (B in parents) returns nothing
|
||||
// sometimes. See #3114, #4289 and
|
||||
// https://issuetracker.google.com/issues/149522397
|
||||
if len(dirs) > 1 && !foundItems {
|
||||
if atomic.SwapInt32(&f.grouping, 1) != 1 {
|
||||
fs.Logf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
|
||||
}
|
||||
var recycled = make([]listREntry, len(dirs))
|
||||
f.listRmu.Lock()
|
||||
for i := range dirs {
|
||||
recycled[i] = listREntry{id: dirs[i], path: paths[i]}
|
||||
// Make a note of these dirs - if they all turn
|
||||
// out to be empty then we can re-enable grouping
|
||||
f.listRempties[dirs[i]] = struct{}{}
|
||||
}
|
||||
f.listRmu.Unlock()
|
||||
// recycle these in the background so we don't deadlock
|
||||
// the listR runners if they all get here
|
||||
wg.Add(len(recycled))
|
||||
go func() {
|
||||
for _, entry := range recycled {
|
||||
in <- entry
|
||||
}
|
||||
fs.Debugf(f, "Recycled %d entries", len(recycled))
|
||||
}()
|
||||
}
|
||||
// If using a grouping of 1 and dir was empty then check to see if it
|
||||
// is part of the group that caused grouping to be disabled.
|
||||
if grouping == 1 && len(dirs) == 1 && !foundItems {
|
||||
f.listRmu.Lock()
|
||||
if _, found := f.listRempties[dirs[0]]; found {
|
||||
// Remove the ID
|
||||
delete(f.listRempties, dirs[0])
|
||||
// If no empties left => all the directories that
|
||||
// triggered the grouping being set to 1 were actually
|
||||
// empty so must have made a mistake
|
||||
if len(f.listRempties) == 0 {
|
||||
if atomic.SwapInt32(&f.grouping, listRGrouping) != listRGrouping {
|
||||
fs.Logf(f, "Re-enabling ListR as previous detection was in error")
|
||||
}
|
||||
}
|
||||
}
|
||||
f.listRmu.Unlock()
|
||||
}
|
||||
|
||||
for range dirs {
|
||||
wg.Done()
|
||||
}
|
||||
@@ -1736,11 +1814,6 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
|
||||
// Don't implement this unless you have a more efficient way
|
||||
// of listing recursively that doing a directory traversal.
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
|
||||
const (
|
||||
grouping = 50
|
||||
inputBuffer = 1000
|
||||
)
|
||||
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1753,7 +1826,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
|
||||
mu := sync.Mutex{} // protects in and overflow
|
||||
wg := sync.WaitGroup{}
|
||||
in := make(chan listREntry, inputBuffer)
|
||||
in := make(chan listREntry, listRInputBuffer)
|
||||
out := make(chan error, fs.Config.Checkers)
|
||||
list := walk.NewListRHelper(callback)
|
||||
overflow := []listREntry{}
|
||||
@@ -1766,6 +1839,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
job := listREntry{actualID(d.ID()), d.Remote()}
|
||||
select {
|
||||
case in <- job:
|
||||
// Adding the wg after we've entered the item is
|
||||
// safe here because we know when the callback
|
||||
// is called we are holding a waitgroup.
|
||||
wg.Add(1)
|
||||
default:
|
||||
overflow = append(overflow, job)
|
||||
@@ -1779,7 +1855,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
in <- listREntry{directoryID, dir}
|
||||
|
||||
for i := 0; i < fs.Config.Checkers; i++ {
|
||||
go f.listRRunner(ctx, &wg, in, out, cb, grouping)
|
||||
go f.listRRunner(ctx, &wg, in, out, cb)
|
||||
}
|
||||
go func() {
|
||||
// wait until the all directories are processed
|
||||
@@ -1788,9 +1864,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
for len(overflow) > 0 {
|
||||
mu.Lock()
|
||||
l := len(overflow)
|
||||
// only fill half of the channel to prevent entries beeing put into overflow again
|
||||
if l > inputBuffer/2 {
|
||||
l = inputBuffer / 2
|
||||
// only fill half of the channel to prevent entries being put into overflow again
|
||||
if l > listRInputBuffer/2 {
|
||||
l = listRInputBuffer / 2
|
||||
}
|
||||
wg.Add(l)
|
||||
for _, d := range overflow[:l] {
|
||||
@@ -1914,16 +1990,17 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to resolve shortcut")
|
||||
}
|
||||
// make sure we use the Name and Parents from the original item
|
||||
// make sure we use the Name, Parents and Trashed from the original item
|
||||
newItem.Name = item.Name
|
||||
newItem.Parents = item.Parents
|
||||
newItem.Trashed = item.Trashed
|
||||
// the new ID is a composite ID
|
||||
newItem.Id = joinID(newItem.Id, item.Id)
|
||||
return newItem, nil
|
||||
}
|
||||
|
||||
// itemToDirEntry converts a drive.File to a fs.DirEntry.
|
||||
// When the drive.File cannot be represented as a fs.DirEntry
|
||||
// itemToDirEntry converts a drive.File to an fs.DirEntry.
|
||||
// When the drive.File cannot be represented as an fs.DirEntry
|
||||
// (nil, nil) is returned.
|
||||
func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry, err error) {
|
||||
switch {
|
||||
@@ -2074,7 +2151,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
}
|
||||
dstDir := dirs[0]
|
||||
for _, srcDir := range dirs[1:] {
|
||||
// list the the objects
|
||||
// list the objects
|
||||
infos := []*drive.File{}
|
||||
_, err := f.list(ctx, []string{srcDir.ID()}, "", false, false, true, func(info *drive.File) bool {
|
||||
infos = append(infos, info)
|
||||
@@ -2244,10 +2321,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read description for Google Doc")
|
||||
}
|
||||
// set the description if there is one, or use the default if not
|
||||
if info.Description != "" {
|
||||
createInfo.Description = info.Description
|
||||
}
|
||||
createInfo.Description = info.Description
|
||||
} else {
|
||||
// don't overwrite the description on copy for files
|
||||
// this should work for docs but it doesn't - it is probably a bug in Google Drive
|
||||
@@ -3147,7 +3221,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
|
||||
return req, res, nil
|
||||
}
|
||||
|
||||
// openDocumentFile represents an documentObject open for reading.
|
||||
// openDocumentFile represents a documentObject open for reading.
|
||||
// Updates the object size after read successfully.
|
||||
type openDocumentFile struct {
|
||||
o *documentObject // Object we are reading for
|
||||
|
||||
@@ -117,7 +117,10 @@ func init() {
|
||||
Description: "Dropbox",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.ConfigNoOffline("dropbox", name, m, dropboxConfig)
|
||||
opt := oauthutil.Options{
|
||||
NoOffline: true,
|
||||
}
|
||||
err := oauthutil.Config("dropbox", name, m, dropboxConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
|
||||
@@ -298,7 +298,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
|
||||
// return an error or upload it properly (rather than e.g. calling panic).
|
||||
//
|
||||
|
||||
@@ -101,7 +101,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
|
||||
+1
-1
@@ -72,7 +72,7 @@ func init() {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
// The FTP protocal can't handle trailing spaces (for instance
|
||||
// The FTP protocol can't handle trailing spaces (for instance
|
||||
// pureftpd turns them into _)
|
||||
//
|
||||
// proftpd can't handle '*' in file names
|
||||
|
||||
@@ -82,7 +82,7 @@ func init() {
|
||||
if saFile != "" || saCreds != "" {
|
||||
return
|
||||
}
|
||||
err := oauthutil.Config("google cloud storage", name, m, storageConfig)
|
||||
err := oauthutil.Config("google cloud storage", name, m, storageConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ type Error struct {
|
||||
Details ErrorDetails `json:"error"`
|
||||
}
|
||||
|
||||
// Error statisfies error interface
|
||||
// Error satisfies error interface
|
||||
func (e *Error) Error() string {
|
||||
return fmt.Sprintf("%s (%d %s)", e.Details.Message, e.Details.Code, e.Details.Status)
|
||||
}
|
||||
|
||||
@@ -96,7 +96,7 @@ func init() {
|
||||
}
|
||||
|
||||
// Do the oauth
|
||||
err = oauthutil.Config("google photos", name, m, oauthConfig)
|
||||
err = oauthutil.Config("google photos", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
golog.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ var patterns = dirPatterns{
|
||||
fs.NewDir(prefix+"album", f.dirTime()),
|
||||
fs.NewDir(prefix+"shared-album", f.dirTime()),
|
||||
fs.NewDir(prefix+"upload", f.dirTime()),
|
||||
fs.NewDir(prefix+"feature", f.dirTime()),
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
@@ -190,6 +191,28 @@ var patterns = dirPatterns{
|
||||
re: `^shared-album/(.+?)/([^/]+)$`,
|
||||
isFile: true,
|
||||
},
|
||||
{
|
||||
re: `^feature$`,
|
||||
toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
|
||||
return fs.DirEntries{
|
||||
fs.NewDir(prefix+"favorites", f.dirTime()),
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
re: `^feature/favorites$`,
|
||||
toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
|
||||
filter := featureFilter(ctx, f, match)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.listDir(ctx, prefix, filter)
|
||||
},
|
||||
},
|
||||
{
|
||||
re: `^feature/favorites/([^/]+)$`,
|
||||
isFile: true,
|
||||
},
|
||||
}.mustCompile()
|
||||
|
||||
// mustCompile compiles the regexps in the dirPatterns
|
||||
@@ -201,7 +224,7 @@ func (ds dirPatterns) mustCompile() dirPatterns {
|
||||
return ds
|
||||
}
|
||||
|
||||
// match finds the path passed in in the matching structure and
|
||||
// match finds the path passed in the matching structure and
|
||||
// returns the parameters and a pointer to the match, or nil.
|
||||
func (ds dirPatterns) match(root string, itemPath string, isFile bool) (match []string, prefix string, pattern *dirPattern) {
|
||||
itemPath = strings.Trim(itemPath, "/")
|
||||
@@ -290,6 +313,24 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
|
||||
return sf, nil
|
||||
}
|
||||
|
||||
// featureFilter creates a filter for the Feature enum
|
||||
//
|
||||
// The API only supports one feature, FAVORITES, so hardcode that feature
|
||||
//
|
||||
// https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
|
||||
func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {
|
||||
sf = api.SearchFilter{
|
||||
Filters: &api.Filters{
|
||||
FeatureFilter: &api.FeatureFilter{
|
||||
IncludedFeatures: []string{
|
||||
"FAVORITES",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return sf
|
||||
}
|
||||
|
||||
// Turns an albumPath into entries
|
||||
//
|
||||
// These can either be synthetic directory entries if the album path
|
||||
|
||||
@@ -155,6 +155,38 @@ func TestPatternMatch(t *testing.T) {
|
||||
wantPrefix: "file.jpg/",
|
||||
wantPattern: &patterns[5],
|
||||
},
|
||||
{
|
||||
root: "",
|
||||
itemPath: "feature",
|
||||
isFile: false,
|
||||
wantMatch: []string{"feature"},
|
||||
wantPrefix: "feature/",
|
||||
wantPattern: &patterns[23],
|
||||
},
|
||||
{
|
||||
root: "feature/favorites",
|
||||
itemPath: "",
|
||||
isFile: false,
|
||||
wantMatch: []string{"feature/favorites"},
|
||||
wantPrefix: "",
|
||||
wantPattern: &patterns[24],
|
||||
},
|
||||
{
|
||||
root: "feature",
|
||||
itemPath: "favorites",
|
||||
isFile: false,
|
||||
wantMatch: []string{"feature/favorites"},
|
||||
wantPrefix: "favorites/",
|
||||
wantPattern: &patterns[24],
|
||||
},
|
||||
{
|
||||
root: "feature/favorites",
|
||||
itemPath: "file.jpg",
|
||||
isFile: true,
|
||||
wantMatch: []string{"feature/favorites/file.jpg", "file.jpg"},
|
||||
wantPrefix: "file.jpg/",
|
||||
wantPattern: &patterns[25],
|
||||
},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("#%d,root=%q,itemPath=%q,isFile=%v", testNumber, test.root, test.itemPath, test.isFile), func(t *testing.T) {
|
||||
gotMatch, gotPrefix, gotPattern := patterns.match(test.root, test.itemPath, test.isFile)
|
||||
|
||||
@@ -21,7 +21,7 @@ func newAuth(f *Fs) *auth {
|
||||
}
|
||||
}
|
||||
|
||||
// Request constructs a http.Request for authentication
|
||||
// Request constructs an http.Request for authentication
|
||||
//
|
||||
// returns nil for not needed
|
||||
func (a *auth) Request(*swift.Connection) (r *http.Request, err error) {
|
||||
|
||||
@@ -39,7 +39,7 @@ var (
|
||||
// Description of how to auth for this app
|
||||
oauthConfig = &oauth2.Config{
|
||||
Scopes: []string{
|
||||
"credentials.r", // Read Openstack credentials
|
||||
"credentials.r", // Read OpenStack credentials
|
||||
},
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: "https://api.hubic.com/oauth/auth/",
|
||||
@@ -58,7 +58,7 @@ func init() {
|
||||
Description: "Hubic",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("hubic", name, m, oauthConfig)
|
||||
err := oauthutil.Config("hubic", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
@@ -76,8 +76,8 @@ func init() {
|
||||
// credentials is the JSON returned from the Hubic API to read the
|
||||
// OpenStack credentials
|
||||
type credentials struct {
|
||||
Token string `json:"token"` // Openstack token
|
||||
Endpoint string `json:"endpoint"` // Openstack endpoint
|
||||
Token string `json:"token"` // OpenStack token
|
||||
Endpoint string `json:"endpoint"` // OpenStack endpoint
|
||||
Expires string `json:"expires"` // Expires date - eg "2015-11-09T14:24:56+01:00"
|
||||
}
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ func init() {
|
||||
// needs to be done early so we can use oauth during config
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "jottacloud",
|
||||
Description: "JottaCloud",
|
||||
Description: "Jottacloud",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
ctx := context.TODO()
|
||||
@@ -235,7 +235,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// parsePath parses an box 'url'
|
||||
// parsePath parses a box 'url'
|
||||
func parsePath(path string) (root string) {
|
||||
root = strings.Trim(path, "/")
|
||||
return
|
||||
@@ -454,7 +454,7 @@ func errorHandler(resp *http.Response) error {
|
||||
return errResponse
|
||||
}
|
||||
|
||||
// Jottacloud want's '+' to be URL encoded even though the RFC states it's not reserved
|
||||
// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
|
||||
func urlPathEscape(in string) string {
|
||||
return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
|
||||
}
|
||||
@@ -464,7 +464,7 @@ func (f *Fs) filePathRaw(file string) string {
|
||||
return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
|
||||
}
|
||||
|
||||
// filePath returns a escaped file path (f.root, file)
|
||||
// filePath returns an escaped file path (f.root, file)
|
||||
func (f *Fs) filePath(file string) string {
|
||||
return urlPathEscape(f.filePathRaw(file))
|
||||
}
|
||||
@@ -493,7 +493,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
return nil, errors.New("Outdated config - please reconfigure this backend")
|
||||
}
|
||||
|
||||
// if custome endpoints are set use them else stick with defaults
|
||||
// if custom endpoints are set use them else stick with defaults
|
||||
if tokenURL, ok := m.Get(configTokenURL); ok {
|
||||
oauthConfig.Endpoint.TokenURL = tokenURL
|
||||
// jottacloud is weird. we need to use the tokenURL as authURL
|
||||
@@ -1105,7 +1105,7 @@ func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// filePath returns a escaped file path (f.root, remote)
|
||||
// filePath returns an escaped file path (f.root, remote)
|
||||
func (o *Object) filePath() string {
|
||||
return o.fs.filePath(o.remote)
|
||||
}
|
||||
@@ -1210,7 +1210,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// The cleanup function should be called when out is finished with
|
||||
// regardless of whether this function returned an error or not.
|
||||
func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
|
||||
// we need a MD5
|
||||
// we need an MD5
|
||||
md5Hasher := md5.New()
|
||||
// use the teeReader to write to the local file AND calculate the MD5 while doing so
|
||||
teeReader := io.TeeReader(in, md5Hasher)
|
||||
|
||||
@@ -79,7 +79,7 @@ type Options struct {
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// A Fs is a representation of a remote Koofr Fs
|
||||
// An Fs is a representation of a remote Koofr Fs
|
||||
type Fs struct {
|
||||
name string
|
||||
mountID string
|
||||
@@ -250,7 +250,7 @@ func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.MD5)
|
||||
}
|
||||
|
||||
// fullPath constructs a full, absolute path from a Fs root relative path,
|
||||
// fullPath constructs a full, absolute path from an Fs root relative path,
|
||||
func (f *Fs) fullPath(part string) string {
|
||||
return f.opt.Enc.FromStandardPath(path.Join("/", f.root, part))
|
||||
}
|
||||
@@ -421,7 +421,7 @@ func translateErrorsObject(err error) error {
|
||||
}
|
||||
|
||||
// mkdir creates a directory at the given remote path. Creates ancestors if
|
||||
// neccessary
|
||||
// necessary
|
||||
func (f *Fs) mkdir(fullPath string) error {
|
||||
if fullPath == "/" {
|
||||
return nil
|
||||
|
||||
+23
-5
@@ -118,6 +118,16 @@ Windows/macOS and case sensitive for everything else. Use this flag
|
||||
to override the default choice.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "no_sparse",
|
||||
Help: `Disable sparse files for multi-thread downloads
|
||||
|
||||
On Windows platforms rclone will make sparse files when doing
|
||||
multi-thread downloads. This avoids long pauses on large files where
|
||||
the OS zeros the file. However sparse files may be undesirable as they
|
||||
cause disk fragmentation and can be slow to work with.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -139,6 +149,7 @@ type Options struct {
|
||||
OneFileSystem bool `config:"one_file_system"`
|
||||
CaseSensitive bool `config:"case_sensitive"`
|
||||
CaseInsensitive bool `config:"case_insensitive"`
|
||||
NoSparse bool `config:"no_sparse"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
@@ -1095,6 +1106,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return o.lstat()
|
||||
}
|
||||
|
||||
var sparseWarning sync.Once
|
||||
|
||||
// OpenWriterAt opens with a handle for random access writes
|
||||
//
|
||||
// Pass in the remote desired and the size if known.
|
||||
@@ -1122,10 +1135,15 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to pre-allocate: %v", err)
|
||||
}
|
||||
// Set the file to be a sparse file (important on Windows)
|
||||
err = file.SetSparse(out)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to set sparse: %v", err)
|
||||
if !f.opt.NoSparse && file.SetSparseImplemented {
|
||||
sparseWarning.Do(func() {
|
||||
fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable")
|
||||
})
|
||||
// Set the file to be a sparse file (important on Windows)
|
||||
err = file.SetSparse(out)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to set sparse: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return out, nil
|
||||
@@ -1146,7 +1164,7 @@ func (o *Object) setMetadata(info os.FileInfo) {
|
||||
}
|
||||
}
|
||||
|
||||
// Stat a Object into info
|
||||
// Stat an Object into info
|
||||
func (o *Object) lstat() error {
|
||||
info, err := o.fs.lstat(o.path)
|
||||
if err == nil {
|
||||
|
||||
@@ -402,7 +402,7 @@ func (q *quirks) parseQuirks(option string) {
|
||||
// "Accept-Encoding: gzip" header. However, enabling compression
|
||||
// might be good for performance.
|
||||
// Use this quirk to investigate the performance impact.
|
||||
// Remove this quirk if perfomance does not improve.
|
||||
// Remove this quirk if performance does not improve.
|
||||
q.gzip = true
|
||||
case "insecure":
|
||||
// The mailru disk-o protocol is not documented. To compare HTTP
|
||||
|
||||
@@ -150,7 +150,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// parsePath parses an mega 'url'
|
||||
// parsePath parses a mega 'url'
|
||||
func parsePath(path string) (root string) {
|
||||
root = strings.Trim(path, "/")
|
||||
return
|
||||
@@ -871,7 +871,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
return errors.Errorf("MergeDirs failed to find node for: %v", srcDir)
|
||||
}
|
||||
|
||||
// list the the objects
|
||||
// list the objects
|
||||
infos := []*mega.Node{}
|
||||
_, err := f.list(ctx, srcDirNode, func(info *mega.Node) bool {
|
||||
infos = append(infos, info)
|
||||
|
||||
@@ -272,19 +272,19 @@ type CreateShareLinkResponse struct {
|
||||
} `json:"link"`
|
||||
}
|
||||
|
||||
// AsyncOperationStatus provides information on the status of a asynchronous job progress.
|
||||
// AsyncOperationStatus provides information on the status of an asynchronous job progress.
|
||||
//
|
||||
// The following API calls return AsyncOperationStatus resources:
|
||||
//
|
||||
// Copy Item
|
||||
// Upload From URL
|
||||
type AsyncOperationStatus struct {
|
||||
PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete.
|
||||
PercentageComplete float64 `json:"percentageComplete"` // A float value between 0 and 100 that indicates the percentage complete.
|
||||
Status string `json:"status"` // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting"
|
||||
}
|
||||
|
||||
// GetID returns a normalized ID of the item
|
||||
// If DriveID is known it will be prefixed to the ID with # seperator
|
||||
// If DriveID is known it will be prefixed to the ID with # separator
|
||||
// Can be parsed using onedrive.parseNormalizedID(normalizedID)
|
||||
func (i *Item) GetID() string {
|
||||
if i.IsRemote() && i.RemoteItem.ID != "" {
|
||||
|
||||
@@ -79,7 +79,7 @@ func init() {
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
ctx := context.TODO()
|
||||
err := oauthutil.Config("onedrive", name, m, oauthConfig)
|
||||
err := oauthutil.Config("onedrive", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return
|
||||
@@ -396,7 +396,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// parsePath parses an one drive 'url'
|
||||
// parsePath parses a one drive 'url'
|
||||
func parsePath(path string) (root string) {
|
||||
root = strings.Trim(path, "/")
|
||||
return
|
||||
@@ -1310,7 +1310,7 @@ func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(QuickXorHashType)
|
||||
}
|
||||
|
||||
// PublicLink returns a link for downloading without accout.
|
||||
// PublicLink returns a link for downloading without account.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote))
|
||||
if err != nil {
|
||||
|
||||
@@ -63,7 +63,7 @@ func init() {
|
||||
// ? (question mark) -> '?' // FULLWIDTH QUESTION MARK
|
||||
// * (asterisk) -> '*' // FULLWIDTH ASTERISK
|
||||
//
|
||||
// Additionally names can't begin or end with a ASCII whitespace.
|
||||
// Additionally names can't begin or end with an ASCII whitespace.
|
||||
// List of replaced characters:
|
||||
// (space) -> '␠' // SYMBOL FOR SPACE
|
||||
// (horizontal tab) -> '␉' // SYMBOL FOR HORIZONTAL TABULATION
|
||||
@@ -677,7 +677,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
}
|
||||
|
||||
if "" == o.id {
|
||||
// We need to create a ID for this file
|
||||
// We need to create an ID for this file
|
||||
var resp *http.Response
|
||||
response := createFileResponse{}
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
|
||||
@@ -18,13 +18,13 @@ func (e *Error) Error() string {
|
||||
return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
|
||||
}
|
||||
|
||||
// Account describes a OpenDRIVE account
|
||||
// Account describes an OpenDRIVE account
|
||||
type Account struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"passwd"`
|
||||
}
|
||||
|
||||
// UserSessionInfo describes a OpenDRIVE session
|
||||
// UserSessionInfo describes an OpenDRIVE session
|
||||
type UserSessionInfo struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"passwd"`
|
||||
@@ -45,7 +45,7 @@ type UserSessionInfo struct {
|
||||
PartnerUsersDomain string `json:"PartnerUsersDomain"`
|
||||
}
|
||||
|
||||
// FolderList describes a OpenDRIVE listing
|
||||
// FolderList describes an OpenDRIVE listing
|
||||
type FolderList struct {
|
||||
// DirUpdateTime string `json:"DirUpdateTime,string"`
|
||||
Name string `json:"Name"`
|
||||
@@ -56,7 +56,7 @@ type FolderList struct {
|
||||
Files []File `json:"Files"`
|
||||
}
|
||||
|
||||
// Folder describes a OpenDRIVE folder
|
||||
// Folder describes an OpenDRIVE folder
|
||||
type Folder struct {
|
||||
FolderID string `json:"FolderID"`
|
||||
Name string `json:"Name"`
|
||||
@@ -109,7 +109,7 @@ type removeFolder struct {
|
||||
FolderID string `json:"folder_id"`
|
||||
}
|
||||
|
||||
// File describes a OpenDRIVE file
|
||||
// File describes an OpenDRIVE file
|
||||
type File struct {
|
||||
FileID string `json:"FileId"`
|
||||
FileHash string `json:"FileHash"`
|
||||
|
||||
@@ -67,7 +67,10 @@ func init() {
|
||||
Description: "Pcloud",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("pcloud", name, m, oauthConfig)
|
||||
opt := oauthutil.Options{
|
||||
StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
|
||||
}
|
||||
err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
@@ -152,7 +155,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// parsePath parses an pcloud 'url'
|
||||
// parsePath parses a pcloud 'url'
|
||||
func parsePath(path string) (root string) {
|
||||
root = strings.Trim(path, "/")
|
||||
return
|
||||
|
||||
@@ -10,7 +10,7 @@ type Response struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// Error statisfies the error interface
|
||||
// Error satisfies the error interface
|
||||
func (e *Response) Error() string {
|
||||
return fmt.Sprintf("%s: %s", e.Status, e.Message)
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ func init() {
|
||||
Description: "premiumize.me",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("premiumizeme", name, m, oauthConfig)
|
||||
err := oauthutil.Config("premiumizeme", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
|
||||
@@ -248,7 +248,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
_ = resp.Body.Close()
|
||||
return nil, fserrors.NoRetryError(err)
|
||||
}
|
||||
return resp.Body, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Update the already existing object
|
||||
|
||||
@@ -60,7 +60,10 @@ func init() {
|
||||
Description: "Put.io",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.ConfigNoOffline("putio", name, m, putioConfig)
|
||||
opt := oauthutil.Options{
|
||||
NoOffline: true,
|
||||
}
|
||||
err := oauthutil.Config("putio", name, m, putioConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@ func init() {
|
||||
Help: "QingStor Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Enter a endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
|
||||
Help: "Enter an endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
|
||||
}, {
|
||||
Name: "zone",
|
||||
Help: "Zone to connect to.\nDefault is \"pek3a\".",
|
||||
@@ -203,10 +203,10 @@ func (o *Object) split() (bucket, bucketPath string) {
|
||||
return o.fs.split(o.remote)
|
||||
}
|
||||
|
||||
// Split an URL into three parts: protocol host and port
|
||||
// Split a URL into three parts: protocol host and port
|
||||
func qsParseEndpoint(endpoint string) (protocol, host, port string, err error) {
|
||||
/*
|
||||
Pattern to match a endpoint,
|
||||
Pattern to match an endpoint,
|
||||
eg: "http(s)://qingstor.com:443" --> "http(s)", "qingstor.com", 443
|
||||
"http(s)//qingstor.com" --> "http(s)", "qingstor.com", ""
|
||||
"qingstor.com" --> "", "qingstor.com", ""
|
||||
|
||||
@@ -22,7 +22,7 @@ const (
|
||||
// maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor
|
||||
// maxMultiPartSize = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor
|
||||
minMultiPartSize = 1024 * 1024 * 4 // The minimum allowed part size when uploading a part to QingStor
|
||||
maxMultiParts = 10000 // The maximum allowed number of parts in an multi-part upload
|
||||
maxMultiParts = 10000 // The maximum allowed number of parts in a multi-part upload
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -168,7 +168,7 @@ func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload upload a object into QingStor
|
||||
// Upload upload an object into QingStor
|
||||
func (u *uploader) upload() error {
|
||||
u.init()
|
||||
|
||||
@@ -255,7 +255,7 @@ func (mu *multiUploader) readChunk(ch chan chunk) {
|
||||
}
|
||||
}
|
||||
|
||||
// initiate init an Multiple Object and obtain UploadID
|
||||
// initiate init a Multiple Object and obtain UploadID
|
||||
func (mu *multiUploader) initiate() error {
|
||||
bucketInit, _ := mu.bucketInit()
|
||||
req := qs.InitiateMultipartUploadInput{
|
||||
@@ -297,7 +297,7 @@ func (mu *multiUploader) send(c chunk) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// complete complete an multipart upload
|
||||
// complete complete a multipart upload
|
||||
func (mu *multiUploader) complete() error {
|
||||
var err error
|
||||
if err = mu.getErr(); err != nil {
|
||||
@@ -324,7 +324,7 @@ func (mu *multiUploader) complete() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// abort abort an multipart upload
|
||||
// abort abort a multipart upload
|
||||
func (mu *multiUploader) abort() error {
|
||||
var err error
|
||||
bucketInit, _ := mu.bucketInit()
|
||||
@@ -342,7 +342,7 @@ func (mu *multiUploader) abort() error {
|
||||
|
||||
// multiPartUpload upload a multiple object into QingStor
|
||||
func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) (err error) {
|
||||
// Initiate an multi-part upload
|
||||
// Initiate a multi-part upload
|
||||
if err = mu.initiate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+40
-23
@@ -59,6 +59,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/structs"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -677,7 +678,7 @@ isn't set then "acl" is used instead.`,
|
||||
}},
|
||||
}, {
|
||||
Name: "sse_customer_key",
|
||||
Help: "If using SSE-C you must provide the secret encyption key used to encrypt/decrypt your data.",
|
||||
Help: "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
|
||||
Provider: "AWS,Ceph,Minio",
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
@@ -2033,11 +2034,17 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
if err != nil {
|
||||
if awsErr, ok := err.(awserr.RequestFailure); ok {
|
||||
if awsErr.StatusCode() == http.StatusNotFound {
|
||||
// NotFound indicates bucket was OK
|
||||
// NoSuchBucket would be returned if bucket was bad
|
||||
if awsErr.Code() == "NotFound" {
|
||||
o.fs.cache.MarkOK(bucket)
|
||||
}
|
||||
return fs.ErrorObjectNotFound
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
o.fs.cache.MarkOK(bucket)
|
||||
var size int64
|
||||
// Ignore missing Content-Length assuming it is 0
|
||||
// Some versions of ceph do this due their apache proxies
|
||||
@@ -2198,19 +2205,12 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
|
||||
memPool := f.getMemoryPool(int64(partSize))
|
||||
|
||||
var mReq s3.CreateMultipartUploadInput
|
||||
structs.SetFrom(&mReq, req)
|
||||
var cout *s3.CreateMultipartUploadOutput
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
cout, err = f.c.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: req.Bucket,
|
||||
ACL: req.ACL,
|
||||
Key: req.Key,
|
||||
ContentType: req.ContentType,
|
||||
Metadata: req.Metadata,
|
||||
ServerSideEncryption: req.ServerSideEncryption,
|
||||
SSEKMSKeyId: req.SSEKMSKeyId,
|
||||
StorageClass: req.StorageClass,
|
||||
})
|
||||
cout, err = f.c.CreateMultipartUploadWithContext(ctx, &mReq)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2423,6 +2423,35 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if o.fs.opt.StorageClass != "" {
|
||||
req.StorageClass = &o.fs.opt.StorageClass
|
||||
}
|
||||
// Apply upload options
|
||||
for _, option := range options {
|
||||
key, value := option.Header()
|
||||
lowerKey := strings.ToLower(key)
|
||||
switch lowerKey {
|
||||
case "":
|
||||
// ignore
|
||||
case "cache-control":
|
||||
req.CacheControl = aws.String(value)
|
||||
case "content-disposition":
|
||||
req.ContentDisposition = aws.String(value)
|
||||
case "content-encoding":
|
||||
req.ContentEncoding = aws.String(value)
|
||||
case "content-language":
|
||||
req.ContentLanguage = aws.String(value)
|
||||
case "content-type":
|
||||
req.ContentType = aws.String(value)
|
||||
case "x-amz-tagging":
|
||||
req.Tagging = aws.String(value)
|
||||
default:
|
||||
const amzMetaPrefix = "x-amz-meta-"
|
||||
if strings.HasPrefix(lowerKey, amzMetaPrefix) {
|
||||
metaKey := lowerKey[len(amzMetaPrefix):]
|
||||
req.Metadata[metaKey] = aws.String(value)
|
||||
} else {
|
||||
fs.Errorf(o, "Don't know how to set key %q on upload", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if multipart {
|
||||
err = o.uploadMultipart(ctx, &req, size, in)
|
||||
@@ -2463,18 +2492,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
httpReq.Header = headers
|
||||
httpReq.ContentLength = size
|
||||
|
||||
for _, option := range options {
|
||||
switch option.(type) {
|
||||
case *fs.HTTPOption:
|
||||
key, value := option.Header()
|
||||
httpReq.Header.Add(key, value)
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := o.fs.srv.Do(httpReq)
|
||||
if err != nil {
|
||||
|
||||
@@ -85,7 +85,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
|
||||
+131
-39
@@ -32,8 +32,16 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
librariesCacheKey = "all"
|
||||
retryAfterHeader = "Retry-After"
|
||||
librariesCacheKey = "all"
|
||||
retryAfterHeader = "Retry-After"
|
||||
configURL = "url"
|
||||
configUser = "user"
|
||||
configPassword = "pass"
|
||||
config2FA = "2fa"
|
||||
configLibrary = "library"
|
||||
configLibraryKey = "library_key"
|
||||
configCreateLibrary = "create_library"
|
||||
configAuthToken = "auth_token"
|
||||
)
|
||||
|
||||
// This is global to all instances of fs
|
||||
@@ -49,8 +57,9 @@ func init() {
|
||||
Name: "seafile",
|
||||
Description: "seafile",
|
||||
NewFs: NewFs,
|
||||
Config: Config,
|
||||
Options: []fs.Option{{
|
||||
Name: "url",
|
||||
Name: configURL,
|
||||
Help: "URL of seafile host to connect to",
|
||||
Required: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
@@ -58,26 +67,35 @@ func init() {
|
||||
Help: "Connect to cloud.seafile.com",
|
||||
}},
|
||||
}, {
|
||||
Name: "user",
|
||||
Help: "User name",
|
||||
Name: configUser,
|
||||
Help: "User name (usually email address)",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "pass",
|
||||
// Password is not required, it will be left blank for 2FA
|
||||
Name: configPassword,
|
||||
Help: "Password",
|
||||
IsPassword: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "library",
|
||||
Name: config2FA,
|
||||
Help: "Two-factor authentication ('true' if the account has 2FA enabled)",
|
||||
Default: false,
|
||||
}, {
|
||||
Name: configLibrary,
|
||||
Help: "Name of the library. Leave blank to access all non-encrypted libraries.",
|
||||
}, {
|
||||
Name: "library_key",
|
||||
Name: configLibraryKey,
|
||||
Help: "Library password (for encrypted libraries only). Leave blank if you pass it through the command line.",
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "create_library",
|
||||
Help: "Should create library if it doesn't exist",
|
||||
Name: configCreateLibrary,
|
||||
Help: "Should rclone create a library if it doesn't exist",
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}, {
|
||||
// Keep the authentication token after entering the 2FA code
|
||||
Name: configAuthToken,
|
||||
Help: "Authentication token",
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -97,6 +115,8 @@ type Options struct {
|
||||
URL string `config:"url"`
|
||||
User string `config:"user"`
|
||||
Password string `config:"pass"`
|
||||
Is2FA bool `config:"2fa"`
|
||||
AuthToken string `config:"auth_token"`
|
||||
LibraryName string `config:"library"`
|
||||
LibraryKey string `config:"library_key"`
|
||||
CreateLibrary bool `config:"create_library"`
|
||||
@@ -192,7 +212,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
}
|
||||
fs.Debugf(nil, "Seafile server version %s", serverInfo.Version)
|
||||
|
||||
// We don't support bellow seafile v6.0 (version 6.0 is already more than 3 years old)
|
||||
// We don't support lower than seafile v6.0 (version 6.0 is already more than 3 years old)
|
||||
serverVersion := semver.New(serverInfo.Version)
|
||||
if serverVersion.Major < 6 {
|
||||
return nil, errors.New("unsupported Seafile server (version < 6.0)")
|
||||
@@ -205,10 +225,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
f.moveDirNotAvailable = true
|
||||
}
|
||||
|
||||
err = f.authorizeAccount(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// Take the authentication token from the configuration first
|
||||
token := f.opt.AuthToken
|
||||
if token == "" {
|
||||
// If not available, send the user/password instead
|
||||
token, err = f.authorizeAccount(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
f.setAuthorizationToken(token)
|
||||
|
||||
if f.libraryName != "" {
|
||||
// Check if the library exists
|
||||
@@ -270,26 +296,108 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Config callback for 2FA
|
||||
func Config(name string, m configmap.Mapper) {
|
||||
serverURL, ok := m.Get(configURL)
|
||||
if !ok || serverURL == "" {
|
||||
// If there's no server URL, it means we're trying an operation at the backend level, like a "rclone authorize seafile"
|
||||
fmt.Print("\nOperation not supported on this remote.\nIf you need a 2FA code on your account, use the command:\n\nrclone config reconnect <remote name>:\n\n")
|
||||
return
|
||||
}
|
||||
|
||||
// Stop if we are running non-interactive config
|
||||
if fs.Config.AutoConfirm {
|
||||
return
|
||||
}
|
||||
|
||||
u, err := url.Parse(serverURL)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Invalid server URL %s", serverURL)
|
||||
return
|
||||
}
|
||||
|
||||
is2faEnabled, _ := m.Get(config2FA)
|
||||
if is2faEnabled != "true" {
|
||||
fmt.Println("Two-factor authentication is not enabled on this account.")
|
||||
return
|
||||
}
|
||||
|
||||
username, _ := m.Get(configUser)
|
||||
if username == "" {
|
||||
fs.Errorf(nil, "A username is required")
|
||||
return
|
||||
}
|
||||
|
||||
password, _ := m.Get(configPassword)
|
||||
if password != "" {
|
||||
password, _ = obscure.Reveal(password)
|
||||
}
|
||||
// Just make sure we do have a password
|
||||
for password == "" {
|
||||
fmt.Print("Two-factor authentication: please enter your password (it won't be saved in the configuration)\npassword> ")
|
||||
password = config.ReadPassword()
|
||||
}
|
||||
|
||||
// Create rest client for getAuthorizationToken
|
||||
url := u.String()
|
||||
if !strings.HasPrefix(url, "/") {
|
||||
url += "/"
|
||||
}
|
||||
srv := rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(url)
|
||||
|
||||
// We loop asking for a 2FA code
|
||||
for {
|
||||
code := ""
|
||||
for code == "" {
|
||||
fmt.Print("Two-factor authentication: please enter your 2FA code\n2fa code> ")
|
||||
code = config.ReadLine()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
fmt.Println("Authenticating...")
|
||||
token, err := getAuthorizationToken(ctx, srv, username, password, code)
|
||||
if err != nil {
|
||||
fmt.Printf("Authentication failed: %v\n", err)
|
||||
tryAgain := strings.ToLower(config.ReadNonEmptyLine("Do you want to try again (y/n)?"))
|
||||
if tryAgain != "y" && tryAgain != "yes" {
|
||||
// The user is giving up, we're done here
|
||||
break
|
||||
}
|
||||
}
|
||||
if token != "" {
|
||||
fmt.Println("Success!")
|
||||
// Let's save the token into the configuration
|
||||
m.Set(configAuthToken, token)
|
||||
// And delete any previous entry for password
|
||||
m.Set(configPassword, "")
|
||||
config.SaveConfig()
|
||||
// And we're done here
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sets the AuthorizationToken up
|
||||
func (f *Fs) setAuthorizationToken(token string) {
|
||||
f.srv.SetHeader("Authorization", "Token "+token)
|
||||
}
|
||||
|
||||
// authorizeAccount gets the auth token.
|
||||
func (f *Fs) authorizeAccount(ctx context.Context) error {
|
||||
func (f *Fs) authorizeAccount(ctx context.Context) (string, error) {
|
||||
f.authMu.Lock()
|
||||
defer f.authMu.Unlock()
|
||||
|
||||
token, err := f.getAuthorizationToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return "", err
|
||||
}
|
||||
f.setAuthorizationToken(token)
|
||||
return nil
|
||||
return token, nil
|
||||
}
|
||||
|
||||
// retryErrorCodes is a slice of error codes that we will retry
|
||||
var retryErrorCodes = []int{
|
||||
401, // Unauthorized (eg "Token has expired")
|
||||
408, // Request Timeout
|
||||
429, // Rate exceeded.
|
||||
500, // Get occasional 500 Internal Server Error
|
||||
@@ -298,9 +406,9 @@ var retryErrorCodes = []int{
|
||||
520, // Operation failed (We get them sometimes when running tests in parallel)
|
||||
}
|
||||
|
||||
// shouldRetryNoAuth returns a boolean as to whether this resp and err
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
|
||||
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// For 429 errors look at the Retry-After: header and
|
||||
// set the retry appropriately, starting with a minimum of 1
|
||||
// second if it isn't set.
|
||||
@@ -319,22 +427,6 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
// It looks like seafile is using the 403 error code instead of the standard 401.
|
||||
if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 403) {
|
||||
fs.Debugf(f, "Unauthorized: %v", err)
|
||||
// Reauth
|
||||
authErr := f.authorizeAccount(ctx)
|
||||
if authErr != nil {
|
||||
err = authErr
|
||||
}
|
||||
return true, err
|
||||
}
|
||||
return f.shouldRetryNoReauth(resp, err)
|
||||
}
|
||||
|
||||
func (f *Fs) shouldRetryUpload(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if err != nil || (resp != nil && resp.StatusCode > 400) {
|
||||
return true, err
|
||||
@@ -436,7 +528,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
|
||||
// return an error or upload it properly (rather than e.g. calling panic).
|
||||
//
|
||||
|
||||
+76
-63
@@ -32,37 +32,44 @@ var (
|
||||
// ==================== Seafile API ====================
|
||||
|
||||
func (f *Fs) getAuthorizationToken(ctx context.Context) (string, error) {
|
||||
// API Socumentation
|
||||
return getAuthorizationToken(ctx, f.srv, f.opt.User, f.opt.Password, "")
|
||||
}
|
||||
|
||||
// getAuthorizationToken can be called outside of an fs (during configuration of the remote to get the authentication token)
|
||||
// it's doing a single call (no pacer involved)
|
||||
func getAuthorizationToken(ctx context.Context, srv *rest.Client, user, password, oneTimeCode string) (string, error) {
|
||||
// API Documentation
|
||||
// https://download.seafile.com/published/web-api/home.md#user-content-Quick%20Start
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "api2/auth-token/",
|
||||
ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request
|
||||
IgnoreStatus: true, // so we can load the error messages back into result
|
||||
}
|
||||
|
||||
// 2FA
|
||||
if oneTimeCode != "" {
|
||||
opts.ExtraHeaders["X-SEAFILE-OTP"] = oneTimeCode
|
||||
}
|
||||
|
||||
request := api.AuthenticationRequest{
|
||||
Username: f.opt.User,
|
||||
Password: f.opt.Password,
|
||||
Username: user,
|
||||
Password: password,
|
||||
}
|
||||
result := api.AuthenticationResult{}
|
||||
|
||||
var resp *http.Response
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetryNoReauth(resp, err)
|
||||
})
|
||||
_, err := srv.CallJSON(ctx, &opts, &request, &result)
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
return "", fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
// This is only going to be http errors here
|
||||
return "", errors.Wrap(err, "failed to authenticate")
|
||||
}
|
||||
if result.Errors != nil && len(result.Errors) > 1 {
|
||||
if result.Errors != nil && len(result.Errors) > 0 {
|
||||
return "", errors.New(strings.Join(result.Errors, ", "))
|
||||
}
|
||||
if result.Token == "" {
|
||||
// No error in "non_field_errors" field but still empty token
|
||||
return "", errors.New("failed to authenticate")
|
||||
}
|
||||
return result.Token, nil
|
||||
}
|
||||
|
||||
@@ -79,11 +86,11 @@ func (f *Fs) getServerInfo(ctx context.Context) (account *api.ServerInfo, err er
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -105,11 +112,11 @@ func (f *Fs) getUserAccountInfo(ctx context.Context) (account *api.AccountInfo,
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -132,11 +139,11 @@ func (f *Fs) getLibraries(ctx context.Context) ([]api.Library, error) {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -163,11 +170,11 @@ func (f *Fs) createLibrary(ctx context.Context, libraryName, password string) (l
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -190,11 +197,11 @@ func (f *Fs) deleteLibrary(ctx context.Context, libraryID string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -221,7 +228,7 @@ func (f *Fs) decryptLibrary(ctx context.Context, libraryID, password string) err
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -264,10 +271,13 @@ func (f *Fs) getDirectoryEntriesAPIv21(ctx context.Context, libraryID, dirPath s
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
@@ -306,11 +316,11 @@ func (f *Fs) getDirectoryDetails(ctx context.Context, libraryID, dirPath string)
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -348,11 +358,11 @@ func (f *Fs) createDir(ctx context.Context, libraryID, dirPath string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -388,11 +398,11 @@ func (f *Fs) renameDir(ctx context.Context, libraryID, dirPath, newName string)
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -428,11 +438,11 @@ func (f *Fs) moveDir(ctx context.Context, srcLibraryID, srcDir, srcName, dstLibr
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -464,11 +474,11 @@ func (f *Fs) deleteDir(ctx context.Context, libraryID, filePath string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -495,14 +505,14 @@ func (f *Fs) getFileDetails(ctx context.Context, libraryID, filePath string) (*a
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 404 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -529,7 +539,7 @@ func (f *Fs) deleteFile(ctx context.Context, libraryID, filePath string) error {
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to delete file")
|
||||
@@ -555,7 +565,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -604,7 +614,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -649,11 +659,11 @@ func (f *Fs) getUploadLink(ctx context.Context, libraryID string) (string, error
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return "", fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -693,7 +703,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 500 {
|
||||
@@ -729,11 +739,11 @@ func (f *Fs) listShareLinks(ctx context.Context, libraryID, remote string) ([]ap
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -767,11 +777,11 @@ func (f *Fs) createShareLink(ctx context.Context, libraryID, remote string) (*ap
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -808,11 +818,11 @@ func (f *Fs) copyFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -850,11 +860,11 @@ func (f *Fs) moveFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -890,11 +900,11 @@ func (f *Fs) renameFile(ctx context.Context, libraryID, filePath, newname string
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -928,11 +938,11 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
@@ -966,10 +976,13 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
@@ -1017,11 +1030,11 @@ func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibrar
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return nil, fs.ErrorPermissionDenied
|
||||
}
|
||||
}
|
||||
@@ -1045,7 +1058,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
|
||||
// No luck with JSON input with the older api2
|
||||
postParameters := url.Values{
|
||||
"operation": {"rename"},
|
||||
"reloaddir": {"true"}, // This is an undocumented trick to avoid a http code 301 response (found in https://github.com/haiwen/seahub/blob/master/seahub/api2/views.py)
|
||||
"reloaddir": {"true"}, // This is an undocumented trick to avoid an http code 301 response (found in https://github.com/haiwen/seahub/blob/master/seahub/api2/views.py)
|
||||
"newname": {f.opt.Enc.FromStandardName(newname)},
|
||||
}
|
||||
|
||||
@@ -1062,7 +1075,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.Call(ctx, &opts)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
return f.shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
@@ -1070,7 +1083,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
|
||||
// This is the normal response from the server
|
||||
return nil
|
||||
}
|
||||
if resp.StatusCode == 403 {
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return fs.ErrorPermissionDenied
|
||||
}
|
||||
if resp.StatusCode == 404 {
|
||||
|
||||
+24
-9
@@ -69,6 +69,9 @@ func init() {
|
||||
Name: "pass",
|
||||
Help: "SSH password, leave blank to use ssh-agent.",
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "key_pem",
|
||||
Help: "Raw PEM-encoded private key, If specified, will override key_file parameter.",
|
||||
}, {
|
||||
Name: "key_file",
|
||||
Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
|
||||
@@ -91,7 +94,7 @@ when the ssh-agent contains many keys.`,
|
||||
Name: "use_insecure_cipher",
|
||||
Help: `Enable the use of insecure ciphers and key exchange methods.
|
||||
|
||||
This enables the use of the the following insecure ciphers and key exchange methods:
|
||||
This enables the use of the following insecure ciphers and key exchange methods:
|
||||
|
||||
- aes128-cbc
|
||||
- aes192-cbc
|
||||
@@ -172,6 +175,7 @@ type Options struct {
|
||||
User string `config:"user"`
|
||||
Port string `config:"port"`
|
||||
Pass string `config:"pass"`
|
||||
KeyPem string `config:"key_pem"`
|
||||
KeyFile string `config:"key_file"`
|
||||
KeyFilePass string `config:"key_file_pass"`
|
||||
KeyUseAgent bool `config:"key_use_agent"`
|
||||
@@ -390,8 +394,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
keyFile := env.ShellExpand(opt.KeyFile)
|
||||
// Add ssh agent-auth if no password or file specified
|
||||
if (opt.Pass == "" && keyFile == "" && !opt.AskPassword) || opt.KeyUseAgent {
|
||||
//keyPem := env.ShellExpand(opt.KeyPem)
|
||||
// Add ssh agent-auth if no password or file or key PEM specified
|
||||
if (opt.Pass == "" && keyFile == "" && !opt.AskPassword && opt.KeyPem == "") || opt.KeyUseAgent {
|
||||
sshAgentClient, _, err := sshagent.New()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
|
||||
@@ -427,10 +432,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
// Load key file if specified
|
||||
if keyFile != "" {
|
||||
key, err := ioutil.ReadFile(keyFile)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read private key file")
|
||||
if keyFile != "" || opt.KeyPem != "" {
|
||||
var key []byte
|
||||
if opt.KeyPem == "" {
|
||||
key, err = ioutil.ReadFile(keyFile)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read private key file")
|
||||
}
|
||||
} else {
|
||||
// wrap in quotes because the config is a coming as a literal without them.
|
||||
opt.KeyPem, err = strconv.Unquote("\"" + opt.KeyPem + "\"")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "pem key not formatted properly")
|
||||
}
|
||||
key = []byte(opt.KeyPem)
|
||||
}
|
||||
clearpass := ""
|
||||
if opt.KeyFilePass != "" {
|
||||
@@ -470,7 +485,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
return NewFsWithConnection(ctx, name, root, m, opt, sshConfig)
|
||||
}
|
||||
|
||||
// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
|
||||
// NewFsWithConnection creates a new Fs object from the name and root and an ssh.ClientConfig. It connects to
|
||||
// the host specified in the ssh.ClientConfig
|
||||
func NewFsWithConnection(ctx context.Context, name string, root string, m configmap.Mapper, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
@@ -1021,7 +1036,7 @@ func parseHash(bytes []byte) string {
|
||||
|
||||
// Parses the byte array output from the SSH session
|
||||
// returned by an invocation of df into
|
||||
// the disk size, used space, and avaliable space on the disk, in that order.
|
||||
// the disk size, used space, and available space on the disk, in that order.
|
||||
// Only works when `df` has output info on only one disk
|
||||
func parseUsage(bytes []byte) (spaceTotal int64, spaceUsed int64, spaceAvail int64) {
|
||||
spaceTotal, spaceUsed, spaceAvail = -1, -1, -1
|
||||
|
||||
@@ -102,7 +102,7 @@ type UploadSpecification struct {
|
||||
MaxNumberOfThreads int `json:"MaxNumberOfThreads"` // Specifies the max number of chunks that can be sent simultaneously for threaded uploads
|
||||
}
|
||||
|
||||
// UploadFinishResponse is returnes from calling UploadSpecification.FinishURI
|
||||
// UploadFinishResponse is returns from calling UploadSpecification.FinishURI
|
||||
type UploadFinishResponse struct {
|
||||
Error bool `json:"error"`
|
||||
ErrorMessage string `json:"errorMessage"`
|
||||
|
||||
@@ -152,7 +152,10 @@ func init() {
|
||||
oauthConfig.Endpoint.TokenURL = endpoint + tokenPath
|
||||
return nil
|
||||
}
|
||||
err := oauthutil.ConfigWithCallback("sharefile", name, m, oauthConfig, checkAuth)
|
||||
opt := oauthutil.Options{
|
||||
CheckAuth: checkAuth,
|
||||
}
|
||||
err := oauthutil.Config("sharefile", name, m, oauthConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
|
||||
@@ -230,7 +230,7 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// parsePath parses an sugarsync 'url'
|
||||
// parsePath parses a sugarsync 'url'
|
||||
func parsePath(path string) (root string) {
|
||||
root = strings.Trim(path, "/")
|
||||
return
|
||||
|
||||
@@ -73,7 +73,7 @@ copy operations.`,
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "swift",
|
||||
Description: "Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
|
||||
Description: "OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
|
||||
NewFs: NewFs,
|
||||
Options: append([]fs.Option{{
|
||||
Name: "env_auth",
|
||||
@@ -284,7 +284,7 @@ var retryErrorCodes = []int{
|
||||
// shouldRetry returns a boolean as to whether this err deserves to be
|
||||
// retried. It returns the err as a convenience
|
||||
func shouldRetry(err error) (bool, error) {
|
||||
// If this is an swift.Error object extract the HTTP error code
|
||||
// If this is a swift.Error object extract the HTTP error code
|
||||
if swiftError, ok := err.(*swift.Error); ok {
|
||||
for _, e := range retryErrorCodes {
|
||||
if swiftError.StatusCode == e {
|
||||
@@ -1124,6 +1124,9 @@ func (o *Object) removeSegments(except string) error {
|
||||
// remove the segments container if empty, ignore errors
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = o.fs.c.ContainerDelete(segmentsContainer)
|
||||
if err == swift.ContainerNotFound || err == swift.ContainerNotEmpty {
|
||||
return false, err
|
||||
}
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
@@ -1253,7 +1256,7 @@ func deleteChunks(o *Object, segmentsContainer string, segmentInfos []string) {
|
||||
fs.Debugf(o, "Delete segment file %q on %q", v, segmentsContainer)
|
||||
e := o.fs.c.ObjectDelete(segmentsContainer, v)
|
||||
if e != nil {
|
||||
fs.Errorf(o, "Error occured in delete segment file %q on %q , error: %q", v, segmentsContainer, e)
|
||||
fs.Errorf(o, "Error occurred in delete segment file %q on %q, error: %q", v, segmentsContainer, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +103,7 @@ func init() {
|
||||
},
|
||||
{
|
||||
Name: "satellite_address",
|
||||
Help: "Satellite Address. Custom satellite address should match the format: <nodeid>@<address>:<port>.",
|
||||
Help: "Satellite Address. Custom satellite address should match the format: `<nodeid>@<address>:<port>`.",
|
||||
Required: false,
|
||||
Provider: newProvider,
|
||||
Default: "us-central-1.tardigrade.io",
|
||||
@@ -269,7 +269,9 @@ func (f *Fs) connect(ctx context.Context) (project *uplink.Project, err error) {
|
||||
fs.Debugf(f, "connecting...")
|
||||
defer fs.Debugf(f, "connected: %+v", err)
|
||||
|
||||
cfg := uplink.Config{}
|
||||
cfg := uplink.Config{
|
||||
UserAgent: "rclone",
|
||||
}
|
||||
|
||||
project, err = cfg.OpenProject(ctx, f.access)
|
||||
if err != nil {
|
||||
@@ -518,7 +520,7 @@ func (f *Fs) NewObject(ctx context.Context, relative string) (_ fs.Object, err e
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Put should
|
||||
// either return an error or upload it properly (rather than e.g. calling
|
||||
// panic).
|
||||
@@ -669,7 +671,7 @@ func (f *Fs) Rmdir(ctx context.Context, relative string) (err error) {
|
||||
// requirements. In particular, libuplink requires a trailing slash for
|
||||
// listings, but rclone does not always provide one. Further, depending on how
|
||||
// the path was initially path normalization may have removed it (e.g. a
|
||||
// trailing slash from the CLI is removed before it ever get's to the backend
|
||||
// trailing slash from the CLI is removed before it ever gets to the backend
|
||||
// code).
|
||||
func newPrefix(prefix string) string {
|
||||
if prefix == "" {
|
||||
|
||||
@@ -177,7 +177,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (_ io.ReadC
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
|
||||
@@ -55,7 +55,7 @@ func (d *Directory) candidates() []upstream.Entry {
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
|
||||
@@ -33,7 +33,7 @@ func (e Errors) FilterNil() Errors {
|
||||
return ne
|
||||
}
|
||||
|
||||
// Err returns a error interface that filtered nil,
|
||||
// Err returns an error interface that filtered nil,
|
||||
// or nil if no non-nil Error is presented.
|
||||
func (e Errors) Err() error {
|
||||
ne := e.FilterNil()
|
||||
|
||||
@@ -31,7 +31,7 @@ func (p *All) Create(ctx context.Context, upstreams []*upstream.Fs, path string)
|
||||
return upstreams, nil
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *All) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -61,7 +61,7 @@ func (p *EpAll) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return p.epall(ctx, upstreams, path)
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receivng a set of candidate entries
|
||||
func (p *EpAll) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -86,7 +86,7 @@ func (p *EpAll) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return upstreams, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *EpAll) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -61,7 +61,7 @@ func (p *EpFF) Action(ctx context.Context, upstreams []*upstream.Fs, path string
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *EpFF) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -86,7 +86,7 @@ func (p *EpFF) Create(ctx context.Context, upstreams []*upstream.Fs, path string
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *EpFF) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -106,7 +106,7 @@ func (p *EpFF) Search(ctx context.Context, upstreams []*upstream.Fs, path string
|
||||
return p.epff(ctx, upstreams, path)
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
func (p *EpFF) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -65,7 +65,7 @@ func (p *EpLfs) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *EpLfs) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.ActionEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -85,7 +85,7 @@ func (p *EpLfs) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *EpLfs) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.CreateEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -107,7 +107,7 @@ func (p *EpLfs) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return p.lfs(upstreams)
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
func (p *EpLfs) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -65,7 +65,7 @@ func (p *EpLno) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *EpLno) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.ActionEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -85,7 +85,7 @@ func (p *EpLno) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *EpLno) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.CreateEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -107,7 +107,7 @@ func (p *EpLno) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return p.lno(upstreams)
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
func (p *EpLno) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -65,7 +65,7 @@ func (p *EpLus) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *EpLus) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.ActionEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -85,7 +85,7 @@ func (p *EpLus) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *EpLus) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.CreateEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -107,7 +107,7 @@ func (p *EpLus) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return p.lus(upstreams)
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
func (p *EpLus) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -64,7 +64,7 @@ func (p *EpMfs) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *EpMfs) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.ActionEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -84,7 +84,7 @@ func (p *EpMfs) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *EpMfs) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.CreateEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -106,7 +106,7 @@ func (p *EpMfs) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
|
||||
return p.mfs(upstreams)
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receivng a set of candidate entries
|
||||
func (p *EpMfs) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -38,7 +38,7 @@ func (p *EpRand) Action(ctx context.Context, upstreams []*upstream.Fs, path stri
|
||||
return []*upstream.Fs{p.rand(upstreams)}, nil
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *EpRand) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.ActionEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -56,7 +56,7 @@ func (p *EpRand) Create(ctx context.Context, upstreams []*upstream.Fs, path stri
|
||||
return []*upstream.Fs{p.rand(upstreams)}, nil
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *EpRand) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.EpAll.CreateEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -77,7 +77,7 @@ func (p *EpRand) Search(ctx context.Context, upstreams []*upstream.Fs, path stri
|
||||
return p.rand(upstreams), nil
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
func (p *EpRand) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -15,7 +15,7 @@ func init() {
|
||||
}
|
||||
|
||||
// Newest policy picks the file / directory with the largest mtime
|
||||
// It implies the existance of a path
|
||||
// It implies the existence of a path
|
||||
type Newest struct {
|
||||
EpAll
|
||||
}
|
||||
@@ -93,7 +93,7 @@ func (p *Newest) Action(ctx context.Context, upstreams []*upstream.Fs, path stri
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *Newest) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -119,7 +119,7 @@ func (p *Newest) Create(ctx context.Context, upstreams []*upstream.Fs, path stri
|
||||
return []*upstream.Fs{u}, err
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *Newest) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -140,7 +140,7 @@ func (p *Newest) Search(ctx context.Context, upstreams []*upstream.Fs, path stri
|
||||
return p.newest(ctx, upstreams, path)
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
func (p *Newest) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -26,13 +26,13 @@ type Policy interface {
|
||||
// Search category policy, governing the access to files and directories
|
||||
Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error)
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error)
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error)
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
SearchEntries(entries ...upstream.Entry) (upstream.Entry, error)
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,7 @@ func (p *Rand) Action(ctx context.Context, upstreams []*upstream.Fs, path string
|
||||
return []*upstream.Fs{p.rand(upstreams)}, nil
|
||||
}
|
||||
|
||||
// ActionEntries is ACTION category policy but receving a set of candidate entries
|
||||
// ActionEntries is ACTION category policy but receiving a set of candidate entries
|
||||
func (p *Rand) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.All.ActionEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -53,7 +53,7 @@ func (p *Rand) Create(ctx context.Context, upstreams []*upstream.Fs, path string
|
||||
return []*upstream.Fs{p.rand(upstreams)}, nil
|
||||
}
|
||||
|
||||
// CreateEntries is CREATE category policy but receving a set of candidate entries
|
||||
// CreateEntries is CREATE category policy but receiving a set of candidate entries
|
||||
func (p *Rand) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
|
||||
entries, err := p.All.CreateEntries(entries...)
|
||||
if err != nil {
|
||||
@@ -74,7 +74,7 @@ func (p *Rand) Search(ctx context.Context, upstreams []*upstream.Fs, path string
|
||||
return p.rand(upstreams), nil
|
||||
}
|
||||
|
||||
// SearchEntries is SEARCH category policy but receving a set of candidate entries
|
||||
// SearchEntries is SEARCH category policy but receiving a set of candidate entries
|
||||
func (p *Rand) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
|
||||
if len(entries) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -80,7 +80,7 @@ type Fs struct {
|
||||
searchPolicy policy.Policy // policy for SEARCH
|
||||
}
|
||||
|
||||
// Wrap candidate objects in to an union Object
|
||||
// Wrap candidate objects in to a union Object
|
||||
func (f *Fs) wrapEntries(entries ...upstream.Entry) (entry, error) {
|
||||
e, err := f.searchEntries(entries...)
|
||||
if err != nil {
|
||||
|
||||
@@ -100,7 +100,7 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) {
|
||||
return f, err
|
||||
}
|
||||
|
||||
// WrapDirectory wraps a fs.Directory to include the info
|
||||
// WrapDirectory wraps an fs.Directory to include the info
|
||||
// of the upstream Fs
|
||||
func (f *Fs) WrapDirectory(e fs.Directory) *Directory {
|
||||
if e == nil {
|
||||
@@ -112,7 +112,7 @@ func (f *Fs) WrapDirectory(e fs.Directory) *Directory {
|
||||
}
|
||||
}
|
||||
|
||||
// WrapObject wraps a fs.Object to include the info
|
||||
// WrapObject wraps an fs.Object to include the info
|
||||
// of the upstream Fs
|
||||
func (f *Fs) WrapObject(o fs.Object) *Object {
|
||||
if o == nil {
|
||||
@@ -124,7 +124,7 @@ func (f *Fs) WrapObject(o fs.Object) *Object {
|
||||
}
|
||||
}
|
||||
|
||||
// WrapEntry wraps a fs.DirEntry to include the info
|
||||
// WrapEntry wraps an fs.DirEntry to include the info
|
||||
// of the upstream Fs
|
||||
func (f *Fs) WrapEntry(e fs.DirEntry) (Entry, error) {
|
||||
switch e.(type) {
|
||||
@@ -219,7 +219,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
|
||||
@@ -155,7 +155,7 @@ func (e *Error) Error() string {
|
||||
return strings.Join(out, ": ")
|
||||
}
|
||||
|
||||
// Time represents represents date and time information for the
|
||||
// Time represents date and time information for the
|
||||
// webdav API marshalling to and from timeFormat
|
||||
type Time time.Time
|
||||
|
||||
@@ -226,6 +226,6 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
// </d:response>
|
||||
// </d:multistatus>
|
||||
type Quota struct {
|
||||
Available int64 `xml:"DAV: response>propstat>prop>quota-available-bytes"`
|
||||
Used int64 `xml:"DAV: response>propstat>prop>quota-used-bytes"`
|
||||
Available string `xml:"DAV: response>propstat>prop>quota-available-bytes"`
|
||||
Used string `xml:"DAV: response>propstat>prop>quota-used-bytes"`
|
||||
}
|
||||
@@ -48,7 +48,7 @@ type SuccessResponseBody struct {
|
||||
Token string `xml:"RequestSecurityTokenResponse>RequestedSecurityToken>BinarySecurityToken"`
|
||||
}
|
||||
|
||||
// SharepointError holds a error response microsoft login
|
||||
// SharepointError holds an error response microsoft login
|
||||
type SharepointError struct {
|
||||
XMLName xml.Name `xml:"Envelope"`
|
||||
Body ErrorResponseBody `xml:"Body"`
|
||||
@@ -58,7 +58,7 @@ func (e *SharepointError) Error() string {
|
||||
return fmt.Sprintf("%s: %s (%s)", e.Body.FaultCode, e.Body.Reason, e.Body.Detail)
|
||||
}
|
||||
|
||||
// ErrorResponseBody contains the body of a erroneous repsonse
|
||||
// ErrorResponseBody contains the body of an erroneous response
|
||||
type ErrorResponseBody struct {
|
||||
XMLName xml.Name
|
||||
FaultCode string `xml:"Fault>Code>Subcode>Value"`
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -975,10 +976,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
</D:prop>
|
||||
</D:propfind>
|
||||
`))
|
||||
var q = api.Quota{
|
||||
Available: -1,
|
||||
Used: -1,
|
||||
}
|
||||
var q api.Quota
|
||||
var resp *http.Response
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -989,14 +987,14 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
return nil, errors.Wrap(err, "about call failed")
|
||||
}
|
||||
usage := &fs.Usage{}
|
||||
if q.Used >= 0 {
|
||||
usage.Used = fs.NewUsageValue(q.Used)
|
||||
if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 {
|
||||
usage.Used = fs.NewUsageValue(i)
|
||||
}
|
||||
if q.Available >= 0 {
|
||||
usage.Free = fs.NewUsageValue(q.Available)
|
||||
if i, err := strconv.ParseInt(q.Available, 10, 64); err == nil && i >= 0 {
|
||||
usage.Free = fs.NewUsageValue(i)
|
||||
}
|
||||
if q.Available >= 0 && q.Used >= 0 {
|
||||
usage.Total = fs.NewUsageValue(q.Available + q.Used)
|
||||
if usage.Used != nil && usage.Free != nil {
|
||||
usage.Total = fs.NewUsageValue(*usage.Used + *usage.Free)
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func init() {
|
||||
Description: "Yandex Disk",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("yandex", name, m, oauthConfig)
|
||||
err := oauthutil.Config("yandex", name, m, oauthConfig, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
return
|
||||
@@ -200,12 +200,12 @@ func (f *Fs) setRoot(root string) {
|
||||
f.diskRoot = diskRoot
|
||||
}
|
||||
|
||||
// filePath returns a escaped file path (f.root, file)
|
||||
// filePath returns an escaped file path (f.root, file)
|
||||
func (f *Fs) filePath(file string) string {
|
||||
return path.Join(f.diskRoot, file)
|
||||
}
|
||||
|
||||
// dirPath returns a escaped file path (f.root, file) ending with '/'
|
||||
// dirPath returns an escaped file path (f.root, file) ending with '/'
|
||||
func (f *Fs) dirPath(file string) string {
|
||||
return path.Join(f.diskRoot, file) + "/"
|
||||
}
|
||||
@@ -502,7 +502,7 @@ func (f *Fs) mkDirs(ctx context.Context, path string) (err error) {
|
||||
|
||||
if err = f.CreateDir(ctx, dirString); err != nil {
|
||||
if apiErr, ok := err.(*api.ErrorResponse); ok {
|
||||
// allready exists
|
||||
// already exists
|
||||
if apiErr.ErrorName != "DiskPathPointsToExistentDirectoryError" {
|
||||
// 2 if it fails then create all directories in the path from root.
|
||||
dirs := strings.Split(dirString, "/") //path separator
|
||||
|
||||
+1
-1
@@ -55,7 +55,7 @@ func gitDiffDiff(rev1, rev2 string) {
|
||||
if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
|
||||
// OK just different
|
||||
} else {
|
||||
log.Fatalf("git diff diff failed: %#v", err)
|
||||
log.Fatalf("git diff failed: %#v", err)
|
||||
}
|
||||
}
|
||||
_, _ = os.Stdout.Write(out)
|
||||
|
||||
@@ -7,9 +7,10 @@ import os
|
||||
import io
|
||||
import subprocess
|
||||
|
||||
marker = "<!--- autogenerated options"
|
||||
marker = "{{< rem autogenerated options"
|
||||
start = marker + " start"
|
||||
stop = marker + " stop"
|
||||
end = ">}}"
|
||||
|
||||
def find_backends():
|
||||
"""Return a list of all backends"""
|
||||
@@ -38,11 +39,11 @@ def alter_doc(backend):
|
||||
if not in_docs:
|
||||
if start in line:
|
||||
in_docs = True
|
||||
start_full = start + " - DO NOT EDIT, instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs -->\n" % (backend, backend)
|
||||
start_full = (start + "\" - DO NOT EDIT - instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs\" " + end + "\n") % (backend, backend)
|
||||
out_file.write(start_full)
|
||||
output_docs(backend, out_file)
|
||||
output_backend_tool_docs(backend, out_file)
|
||||
out_file.write(stop+" -->\n")
|
||||
out_file.write(stop+" "+end+"\n")
|
||||
altered = True
|
||||
if not in_docs:
|
||||
out_file.write(line)
|
||||
|
||||
+10
-4
@@ -156,17 +156,23 @@ def main():
|
||||
|
||||
# Output combined changelog with new part
|
||||
sys.stdout.write(old_head)
|
||||
today = datetime.date.today()
|
||||
new_features = "\n".join(new_features_lines)
|
||||
bugfixes = "\n".join(bugfix_lines)
|
||||
backend_changes = "\n".join(backend_lines)
|
||||
sys.stdout.write("""
|
||||
|
||||
## %s - %s
|
||||
## %(next_version)s - %(today)s
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/%(version)s...%(next_version)s)
|
||||
|
||||
* New backends
|
||||
* New commands
|
||||
* New Features
|
||||
%s
|
||||
%(new_features)s
|
||||
* Bug Fixes
|
||||
%s
|
||||
%s""" % (next_version, datetime.date.today(), "\n".join(new_features_lines), "\n".join(bugfix_lines), "\n".join(backend_lines)))
|
||||
%(bugfixes)s
|
||||
%(backend_changes)s""" % locals())
|
||||
sys.stdout.write(old_tail)
|
||||
|
||||
|
||||
|
||||
+12
-2
@@ -13,7 +13,7 @@ outfile = "MANUAL.md"
|
||||
|
||||
# Order to add docs segments to make outfile
|
||||
docs = [
|
||||
"about.md",
|
||||
"_index.md",
|
||||
"install.md",
|
||||
"docs.md",
|
||||
"remote_setup.md",
|
||||
@@ -111,11 +111,21 @@ def read_doc(doc):
|
||||
contents = parts[2].strip()+"\n\n"
|
||||
# Remove icons
|
||||
contents = re.sub(r'<i class="fa.*?</i>\s*', "", contents)
|
||||
# Interpret img shortcodes
|
||||
# {{< img ... >}}
|
||||
contents = re.sub(r'\{\{<\s*img\s+(.*?)>\}\}', r"<img \1>", contents)
|
||||
# Make any img tags absolute
|
||||
contents = re.sub(r'(<img.*?src=")/', r"\1https://rclone.org/", contents)
|
||||
# Make [...](/links/) absolute
|
||||
contents = re.sub(r'\]\((\/.*?\/(#.*)?)\)', r"](https://rclone.org\1)", contents)
|
||||
# Add additional links on the front page
|
||||
contents = re.sub(r'\{\{< rem MAINPAGELINK >\}\}', "- [Donate.](https://rclone.org/donate/)", contents)
|
||||
# Interpret provider shortcode
|
||||
# {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
|
||||
contents = re.sub(r'\{\{<\s+provider.*?name="(.*?)".*?>\}\}', r"\1", contents)
|
||||
contents = re.sub(r'\{\{<\s*provider.*?name="(.*?)".*?>\}\}', r"- \1", contents)
|
||||
# Remove remaining shortcodes
|
||||
contents = re.sub(r'\{\{<.*?>\}\}', r"", contents)
|
||||
contents = re.sub(r'\{\{%.*?%\}\}', r"", contents)
|
||||
return contents
|
||||
|
||||
def check_docs(docpath):
|
||||
|
||||
+2
-2
@@ -15,8 +15,8 @@ fusermount -u -z /tmp/rclone/rc_mount > /dev/null 2>&1 || umount /tmp/rclone/rc_
|
||||
|
||||
awk '
|
||||
BEGIN {p=1}
|
||||
/^<!--- autogenerated start/ {print;system("cat /tmp/rclone/z.md");p=0}
|
||||
/^<!--- autogenerated stop/ {p=1}
|
||||
/^\{\{< rem autogenerated start/ {print;system("cat /tmp/rclone/z.md");p=0}
|
||||
/^\{\{< rem autogenerated stop/ {p=1}
|
||||
p' docs/content/rc.md > /tmp/rclone/rc.md
|
||||
|
||||
mv /tmp/rclone/rc.md docs/content/rc.md
|
||||
|
||||
+2
-1
@@ -15,7 +15,8 @@ description: |
|
||||
vendor: "rclone"
|
||||
homepage: "https://rclone.org"
|
||||
license: "MIT"
|
||||
bindir: "/usr/bin"
|
||||
# No longer supported? See https://github.com/goreleaser/nfpm/issues/144
|
||||
# bindir: "/usr/bin"
|
||||
files:
|
||||
./rclone: "/usr/bin/rclone"
|
||||
./README.html: "/usr/share/doc/rclone/README.html"
|
||||
|
||||
@@ -59,7 +59,7 @@ Note to run these commands on a running backend then see
|
||||
[backend/command](/rc/#backend/command) in the rc docs.
|
||||
`,
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
cmd.CheckArgs(2, 1E6, command, args)
|
||||
cmd.CheckArgs(2, 1e6, command, args)
|
||||
name, remote := args[0], args[1]
|
||||
cmd.Run(false, false, command, func() error {
|
||||
// show help if remote is a backend name
|
||||
@@ -140,7 +140,7 @@ func showHelp(fsInfo *fs.RegInfo) error {
|
||||
fmt.Printf("### Backend commands\n\n")
|
||||
fmt.Printf(`Here are the commands specific to the %s backend.
|
||||
|
||||
Run them with with
|
||||
Run them with
|
||||
|
||||
rclone backend COMMAND remote:
|
||||
|
||||
|
||||
+5
-2
@@ -76,7 +76,7 @@ func ShowVersion() {
|
||||
fmt.Printf("- go version: %s\n", runtime.Version())
|
||||
}
|
||||
|
||||
// NewFsFile creates a Fs from a name but may point to a file.
|
||||
// NewFsFile creates an Fs from a name but may point to a file.
|
||||
//
|
||||
// It returns a string with the file name if points to a file
|
||||
// otherwise "".
|
||||
@@ -99,7 +99,7 @@ func NewFsFile(remote string) (fs.Fs, string) {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
// newFsFileAddFilter creates a src Fs from a name
|
||||
// newFsFileAddFilter creates an src Fs from a name
|
||||
//
|
||||
// This works the same as NewFsFile however it adds filters to the Fs
|
||||
// to limit it to a single file if the remote pointed to a file.
|
||||
@@ -486,6 +486,9 @@ func AddBackendFlags() {
|
||||
help = help[:nl]
|
||||
}
|
||||
help = strings.TrimSpace(help)
|
||||
if opt.IsPassword {
|
||||
help += " (obscured)"
|
||||
}
|
||||
flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help)
|
||||
if _, isBool := opt.Default.(bool); isBool {
|
||||
flag.NoOptDefVal = "true"
|
||||
|
||||
+4
-1
@@ -1,4 +1,4 @@
|
||||
// Package cmount implents a FUSE mounting system for rclone remotes.
|
||||
// Package cmount implements a FUSE mounting system for rclone remotes.
|
||||
//
|
||||
// This uses the cgo based cgofuse library
|
||||
|
||||
@@ -39,6 +39,9 @@ func init() {
|
||||
name = "mount"
|
||||
}
|
||||
mountlib.NewMountCommand(name, false, Mount)
|
||||
// Add mount to rc
|
||||
mountlib.AddRc("cmount", mount)
|
||||
|
||||
}
|
||||
|
||||
// mountOptions configures the options from the command line flags
|
||||
|
||||
@@ -11,9 +11,11 @@ package cmount
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/testy"
|
||||
"github.com/rclone/rclone/vfs/vfstest"
|
||||
)
|
||||
|
||||
func TestMount(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
vfstest.RunTests(t, false, mount)
|
||||
}
|
||||
+44
-20
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/cmd"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -69,7 +71,8 @@ var configShowCommand = &cobra.Command{
|
||||
if len(args) == 0 {
|
||||
config.ShowConfig()
|
||||
} else {
|
||||
config.ShowRemote(args[0])
|
||||
name := strings.TrimRight(args[0], ":")
|
||||
config.ShowRemote(name)
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -92,12 +95,32 @@ var configProvidersCommand = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var (
|
||||
configObscure bool
|
||||
configNoObscure bool
|
||||
)
|
||||
|
||||
const configPasswordHelp = `
|
||||
If any of the parameters passed is a password field, then rclone will
|
||||
automatically obscure them if they aren't already obscured before
|
||||
putting them in the config file.
|
||||
|
||||
**NB** If the password parameter is 22 characters or longer and
|
||||
consists only of base64 characters then rclone can get confused about
|
||||
whether the password is already obscured or not and put unobscured
|
||||
passwords into the config file. If you want to be 100% certain that
|
||||
the passwords get obscured then use the "--obscure" flag, or if you
|
||||
are 100% certain you are already passing obscured passwords then use
|
||||
"--no-obscure". You can also set osbscured passwords using the
|
||||
"rclone config password" command.
|
||||
`
|
||||
|
||||
var configCreateCommand = &cobra.Command{
|
||||
Use: "create <name> <type> [<key> <value>]*",
|
||||
Use: "create `name` `type` [`key` `value`]*",
|
||||
Short: `Create a new remote with name, type and options.`,
|
||||
Long: `
|
||||
Create a new remote of <name> with <type> and options. The options
|
||||
should be passed in in pairs of <key> <value>.
|
||||
Create a new remote of ` + "`name`" + ` with ` + "`type`" + ` and options. The options
|
||||
should be passed in pairs of ` + "`key` `value`" + `.
|
||||
|
||||
For example to make a swift remote of name myremote using auto config
|
||||
you would do:
|
||||
@@ -107,10 +130,7 @@ you would do:
|
||||
Note that if the config process would normally ask a question the
|
||||
default is taken. Each time that happens rclone will print a message
|
||||
saying how to affect the value taken.
|
||||
|
||||
If any of the parameters passed is a password field, then rclone will
|
||||
automatically obscure them before putting them in the config file.
|
||||
|
||||
` + configPasswordHelp + `
|
||||
So for example if you wanted to configure a Google Drive remote but
|
||||
using remote authorization you would do this:
|
||||
|
||||
@@ -122,7 +142,7 @@ using remote authorization you would do this:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = config.CreateRemote(args[0], args[1], in)
|
||||
err = config.CreateRemote(args[0], args[1], in, configObscure, configNoObscure)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -131,21 +151,25 @@ using remote authorization you would do this:
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
for _, cmdFlags := range []*pflag.FlagSet{configCreateCommand.Flags(), configUpdateCommand.Flags()} {
|
||||
flags.BoolVarP(cmdFlags, &configObscure, "obscure", "", false, "Force any passwords to be obscured.")
|
||||
flags.BoolVarP(cmdFlags, &configNoObscure, "no-obscure", "", false, "Force any passwords not to be obscured.")
|
||||
}
|
||||
}
|
||||
|
||||
var configUpdateCommand = &cobra.Command{
|
||||
Use: "update <name> [<key> <value>]+",
|
||||
Use: "update `name` [`key` `value`]+",
|
||||
Short: `Update options in an existing remote.`,
|
||||
Long: `
|
||||
Update an existing remote's options. The options should be passed in
|
||||
in pairs of <key> <value>.
|
||||
in pairs of ` + "`key` `value`" + `.
|
||||
|
||||
For example to update the env_auth field of a remote of name myremote
|
||||
you would do:
|
||||
|
||||
rclone config update myremote swift env_auth true
|
||||
|
||||
If any of the parameters passed is a password field, then rclone will
|
||||
automatically obscure them before putting them in the config file.
|
||||
|
||||
` + configPasswordHelp + `
|
||||
If the remote uses OAuth the token will be updated, if you don't
|
||||
require this add an extra parameter thus:
|
||||
|
||||
@@ -157,7 +181,7 @@ require this add an extra parameter thus:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = config.UpdateRemote(args[0], in)
|
||||
err = config.UpdateRemote(args[0], in, configObscure, configNoObscure)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -167,8 +191,8 @@ require this add an extra parameter thus:
|
||||
}
|
||||
|
||||
var configDeleteCommand = &cobra.Command{
|
||||
Use: "delete <name>",
|
||||
Short: `Delete an existing remote <name>.`,
|
||||
Use: "delete `name`",
|
||||
Short: "Delete an existing remote `name`.",
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
config.DeleteRemote(args[0])
|
||||
@@ -176,11 +200,11 @@ var configDeleteCommand = &cobra.Command{
|
||||
}
|
||||
|
||||
var configPasswordCommand = &cobra.Command{
|
||||
Use: "password <name> [<key> <value>]+",
|
||||
Use: "password `name` [`key` `value`]+",
|
||||
Short: `Update password in an existing remote.`,
|
||||
Long: `
|
||||
Update an existing remote's password. The password
|
||||
should be passed in in pairs of <key> <value>.
|
||||
should be passed in pairs of ` + "`key` `value`" + `.
|
||||
|
||||
For example to set password of a remote of name myremote you would do:
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ var commandDefinition = &cobra.Command{
|
||||
Download a URL's content and copy it to the destination without saving
|
||||
it in temporary storage.
|
||||
|
||||
Setting --auto-filename will cause the file name to be retreived from
|
||||
Setting --auto-filename will cause the file name to be retrieved from
|
||||
the from URL (after any redirections) and used in the destination
|
||||
path.
|
||||
|
||||
|
||||
+51
-13
@@ -2,12 +2,14 @@ package gendocs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
@@ -20,14 +22,24 @@ func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
}
|
||||
|
||||
const gendocFrontmatterTemplate = `---
|
||||
date: %s
|
||||
title: "%s"
|
||||
slug: %s
|
||||
url: %s
|
||||
# autogenerated - DO NOT EDIT, instead edit the source code in %s and as part of making a release run "make commanddocs"
|
||||
// define things which go into the frontmatter
|
||||
type frontmatter struct {
|
||||
Date string
|
||||
Title string
|
||||
Description string
|
||||
Slug string
|
||||
URL string
|
||||
Source string
|
||||
}
|
||||
|
||||
var frontmatterTemplate = template.Must(template.New("frontmatter").Parse(`---
|
||||
title: "{{ .Title }}"
|
||||
description: "{{ .Description }}"
|
||||
slug: {{ .Slug }}
|
||||
url: {{ .URL }}
|
||||
# autogenerated - DO NOT EDIT, instead edit the source code in {{ .Source }} and as part of making a release run "make commanddocs"
|
||||
---
|
||||
`
|
||||
`))
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "gendocs output_directory",
|
||||
@@ -57,19 +69,41 @@ rclone.org website.`,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flagsHelp := strings.Replace(buf.String(), "YYYY-MM-DD", now, -1)
|
||||
err = ioutil.WriteFile(filepath.Join(root, "flags.md"), []byte(flagsHelp), 0777)
|
||||
err = ioutil.WriteFile(filepath.Join(root, "flags.md"), buf.Bytes(), 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Look up name => description for prepender
|
||||
var description = map[string]string{}
|
||||
var addDescription func(root *cobra.Command)
|
||||
addDescription = func(root *cobra.Command) {
|
||||
name := strings.Replace(root.CommandPath(), " ", "_", -1) + ".md"
|
||||
description[name] = root.Short
|
||||
for _, c := range root.Commands() {
|
||||
addDescription(c)
|
||||
}
|
||||
}
|
||||
addDescription(cmd.Root)
|
||||
|
||||
// markup for the docs files
|
||||
prepender := func(filename string) string {
|
||||
name := filepath.Base(filename)
|
||||
base := strings.TrimSuffix(name, path.Ext(name))
|
||||
url := "/commands/" + strings.ToLower(base) + "/"
|
||||
source := strings.Replace(strings.Replace(base, "rclone", "cmd", -1), "_", "/", -1) + "/"
|
||||
return fmt.Sprintf(gendocFrontmatterTemplate, now, strings.Replace(base, "_", " ", -1), base, url, source)
|
||||
data := frontmatter{
|
||||
Date: now,
|
||||
Title: strings.Replace(base, "_", " ", -1),
|
||||
Description: description[name],
|
||||
Slug: base,
|
||||
URL: "/commands/" + strings.ToLower(base) + "/",
|
||||
Source: strings.Replace(strings.Replace(base, "rclone", "cmd", -1), "_", "/", -1) + "/",
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err := frontmatterTemplate.Execute(&buf, data)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to render frontmatter template: %v", err)
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
linkHandler := func(name string) string {
|
||||
base := strings.TrimSuffix(name, path.Ext(name))
|
||||
@@ -85,6 +119,8 @@ rclone.org website.`,
|
||||
return err
|
||||
}
|
||||
|
||||
var outdentTitle = regexp.MustCompile(`(?m)^#(#+)`)
|
||||
|
||||
// Munge the files to add a link to the global flags page
|
||||
err = filepath.Walk(out, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
@@ -100,6 +136,8 @@ rclone.org website.`,
|
||||
See the [global flags page](/flags/) for global options not listed here.
|
||||
|
||||
### SEE ALSO`, 1)
|
||||
// outdent all the titles by one
|
||||
doc = outdentTitle.ReplaceAllString(doc, `$1`)
|
||||
err = ioutil.WriteFile(path, []byte(doc), 0777)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -25,7 +25,7 @@ func init() {
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "hashsum <hash> remote:path",
|
||||
Short: `Produces an hashsum file for all the objects in the path.`,
|
||||
Short: `Produces a hashsum file for all the objects in the path.`,
|
||||
Long: `
|
||||
Produces a hash file for all the objects in the path using the hash
|
||||
named. The output is in the same format as the standard
|
||||
|
||||
+3
-1
@@ -245,7 +245,6 @@ Use "rclone help backends" for a list of supported services.
|
||||
var docFlagsTemplate = `---
|
||||
title: "Global Flags"
|
||||
description: "Rclone Global Flags"
|
||||
date: "YYYY-MM-DD"
|
||||
---
|
||||
|
||||
# Global Flags
|
||||
@@ -325,6 +324,9 @@ func showBackend(name string) {
|
||||
}
|
||||
fmt.Printf("#### --%s%s\n\n", opt.FlagName(backend.Prefix), shortOpt)
|
||||
fmt.Printf("%s\n\n", opt.Help)
|
||||
if opt.IsPassword {
|
||||
fmt.Printf("**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).\n\n")
|
||||
}
|
||||
fmt.Printf("- Config: %s\n", opt.Name)
|
||||
fmt.Printf("- Env Var: %s\n", opt.EnvVarName(backend.Prefix))
|
||||
fmt.Printf("- Type: %s\n", opt.Type())
|
||||
|
||||
+1
-1
@@ -108,7 +108,7 @@ func (d *Dir) ReadDirAll(ctx context.Context) (dirents []fuse.Dirent, err error)
|
||||
}
|
||||
for _, node := range items {
|
||||
name := node.Name()
|
||||
if len(name) >= mountlib.MaxLeafSize {
|
||||
if len(name) > mountlib.MaxLeafSize {
|
||||
fs.Errorf(d, "Name too long (%d bytes) for FUSE, skipping: %s", len(name), name)
|
||||
continue
|
||||
}
|
||||
|
||||
+3
-1
@@ -1,4 +1,4 @@
|
||||
// Package mount implents a FUSE mounting system for rclone remotes.
|
||||
// Package mount implements a FUSE mounting system for rclone remotes.
|
||||
|
||||
// +build linux,go1.13 darwin,go1.13 freebsd,go1.13
|
||||
|
||||
@@ -23,6 +23,8 @@ import (
|
||||
|
||||
func init() {
|
||||
mountlib.NewMountCommand("mount", false, Mount)
|
||||
// Add mount to rc
|
||||
mountlib.AddRc("mount", mount)
|
||||
}
|
||||
|
||||
// mountOptions configures the options from the command line flags
|
||||
|
||||
+5
-1
@@ -1,4 +1,4 @@
|
||||
// Package mount implents a FUSE mounting system for rclone remotes.
|
||||
// Package mount implements a FUSE mounting system for rclone remotes.
|
||||
|
||||
// +build linux darwin,amd64
|
||||
|
||||
@@ -24,6 +24,10 @@ import (
|
||||
|
||||
func init() {
|
||||
mountlib.NewMountCommand("mount2", true, Mount)
|
||||
|
||||
// Add mount to rc
|
||||
mountlib.AddRc("mount2", mount)
|
||||
|
||||
}
|
||||
|
||||
// mountOptions configures the options from the command line flags
|
||||
|
||||
Loaded 100 of 631 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user