Compare commits

..
835 changed files with 29994 additions and 92501 deletions

No files matched your search

+2 -18
View File
@@ -1,4 +1,3 @@
---
version: 2
jobs:
@@ -14,10 +13,10 @@ jobs:
- run:
name: Cross-compile rclone
command: |
docker pull rclone/xgo-cgofuse
docker pull billziss/xgo-cgofuse
go get -v github.com/karalabe/xgo
xgo \
--image=rclone/xgo-cgofuse \
--image=billziss/xgo-cgofuse \
--targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
-tags cmount \
.
@@ -30,21 +29,6 @@ jobs:
command: |
mkdir -p /tmp/rclone.dist
cp -R rclone-* /tmp/rclone.dist
mkdir build
cp -R rclone-* build/
- run:
name: Build rclone
command: |
go version
go build
- run:
name: Upload artifacts
command: |
if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
make circleci_upload
fi
- store_artifacts:
path: /tmp/rclone.dist
-30
View File
@@ -1,30 +0,0 @@
# golangci-lint configuration options
run:
build-tags:
- cmount
linters:
enable:
- deadcode
- errcheck
- goimports
- golint
- ineffassign
- structcheck
- varcheck
- govet
- unconvert
#- prealloc
#- maligned
disable-all: true
issues:
# Enable some lints excluded by default
exclude-use-default: false
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
max-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
max-same-issues: 0
+14
View File
@@ -0,0 +1,14 @@
{
"Enable": [
"deadcode",
"errcheck",
"goimports",
"golint",
"ineffassign",
"structcheck",
"varcheck",
"vet"
],
"EnableGC": true,
"Vendor": true
}
+2 -4
View File
@@ -8,7 +8,6 @@ go:
- 1.9.x
- 1.10.x
- 1.11.x
- 1.12rc1
- tip
go_import_path: github.com/ncw/rclone
before_install:
@@ -25,7 +24,6 @@ script:
env:
global:
- GOTAGS=cmount
- GO111MODULE=off
- secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
- secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
- secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
@@ -44,7 +42,7 @@ matrix:
- go: tip
include:
- os: osx
go: 1.12rc1
go: 1.11.x
env: GOTAGS=""
cache:
directories:
@@ -56,5 +54,5 @@ deploy:
on:
repo: ncw/rclone
all_branches: true
go: 1.12rc1
go: 1.11.x
condition: $TRAVIS_PULL_REQUEST == false
+1 -6
View File
@@ -351,12 +351,6 @@ Unit tests
Integration tests
* Add your backend to `fstest/test_all/config.yaml`
* Once you've done that then you can use the integration test framework from the project root:
* go install ./...
* test_all -backend remote
Or if you want to run the integration tests manually:
* Make sure integration tests pass with
* `cd fs/operations`
* `go test -v -remote TestRemote:`
@@ -378,3 +372,4 @@ Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](
* `docs/content/about.md` - front page of rclone.org
* `docs/layouts/chrome/navbar.html` - add it to the website navigation
* `bin/make_manual.py` - add the page to the `docs` constant
* `cmd/cmd.go` - the main help for rclone
+8 -11
View File
@@ -1,17 +1,14 @@
# Maintainers guide for rclone #
Current active maintainers of rclone are:
Current active maintainers of rclone are
| Name | GitHub ID | Specific Responsibilities |
| :--------------- | :---------- | :-------------------------- |
| Nick Craig-Wood | @ncw | overall project health |
| Stefan Breunig | @breunigs | |
| Ishuah Kariuki | @ishuah | |
| Remus Bunduc | @remusb | cache backend |
| Fabian Möller | @B4dM4n | |
| Alex Chen | @Cnly | onedrive backend |
| Sandeep Ummadi | @sandeepkru | azureblob backend |
| Sebastian Bünger | @buengese | jottacloud & yandex backends |
* Nick Craig-Wood @ncw
* Stefan Breunig @breunigs
* Ishuah Kariuki @ishuah
* Remus Bunduc @remusb - cache subsystem maintainer
* Fabian Möller @B4dM4n
* Alex Chen @Cnly
* Sandeep Ummadi @sandeepkru
**This is a work in progress Draft**
+267 -1028
View File
File diff suppressed because it is too large. Load diff
+237 -1179
View File
File diff suppressed because it is too large. Load diff
+256 -1269
View File
File diff suppressed because it is too large. Load diff
+19 -16
View File
@@ -11,7 +11,7 @@ ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
BRANCH_PATH :=
endif
TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
ifneq ($(TAG),$(LAST_TAG))
TAG := $(TAG)-beta
endif
@@ -64,20 +64,30 @@ endif
# Do source code quality checks
check: rclone
ifdef FULL_TESTS
@# we still run go vet for -printfuncs which golangci-lint doesn't do yet
@# see: https://github.com/golangci/golangci-lint/issues/204
@echo "-- START CODE QUALITY REPORT -------------------------------"
@go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
@golangci-lint run ./...
@echo "-- END CODE QUALITY REPORT ---------------------------------"
go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
errcheck $(BUILDTAGS) ./...
find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1
else
@echo Skipping source quality tests as version of go too old
endif
gometalinter_install:
go get -u github.com/alecthomas/gometalinter
gometalinter --install --update
# We aren't using gometalinter as the default linter yet because
# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
# 2. can't get -printfuncs working with the vet linter
gometalinter:
gometalinter ./...
# Get the build dependencies
build_dep:
ifdef FULL_TESTS
go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
go get -u github.com/kisielk/errcheck
go get -u golang.org/x/tools/cmd/goimports
go get -u golang.org/x/lint/golint
endif
# Get the release dependencies
@@ -175,13 +185,6 @@ ifndef BRANCH_PATH
endif
@echo Beta release ready at $(BETA_URL)
circleci_upload:
./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
ifndef BRANCH_PATH
./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
endif
@echo Beta release ready at $(BETA_URL)/testbuilds
BUILD_FLAGS := -exclude "^(windows|darwin)/"
ifeq ($(TRAVIS_OS_NAME),osx)
BUILD_FLAGS := -include "^darwin/" -cgo
@@ -189,7 +192,7 @@ endif
travis_beta:
ifeq ($(TRAVIS_OS_NAME),linux)
go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
endif
git log $(LAST_TAG).. > /tmp/git-log.txt
go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)
+4 -6
View File
@@ -20,7 +20,6 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
## Storage providers
* Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
* Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
* Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
* Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
@@ -44,14 +43,13 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
* Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
* OVH [:page_facing_up:](https://rclone.org/swift/)
* OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
* OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
* Openstack Swift [:page_facing_up:](https://rclone.org/swift/)
* Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
* ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
* pCloud [:page_facing_up:](https://rclone.org/pcloud/)
* put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
* QingStor [:page_facing_up:](https://rclone.org/qingstor/)
* Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
* Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
* SFTP [:page_facing_up:](https://rclone.org/sftp/)
* Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
* WebDAV [:page_facing_up:](https://rclone.org/webdav/)
@@ -62,13 +60,13 @@ Please see [the full list of all storage providers and their features](https://r
## Features
* MD5/SHA-1 hashes checked at all times for file integrity
* MD5/SHA1 hashes checked at all times for file integrity
* Timestamps preserved on files
* Partial syncs supported on a whole file basis
* [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
* [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
* [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
* Can sync to and from network, e.g. two different cloud accounts
* Can sync to and from network, eg two different cloud accounts
* Optional encryption ([Crypt](https://rclone.org/crypt/))
* Optional cache ([Cache](https://rclone.org/cache/))
* Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
@@ -93,4 +91,4 @@ License
-------
This is free software under the terms of MIT the license (check the
[COPYING file](/COPYING) included in this package).
[COPYING file](/rclone/COPYING) included in this package).
-821
View File
@@ -1,821 +0,0 @@
package adb
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config/configmap"
"github.com/ncw/rclone/fs/config/configstruct"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/lib/readers"
"github.com/pkg/errors"
adb "github.com/thinkhy/go-adb"
"github.com/thinkhy/go-adb/wire"
)
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
Name: "adb",
Description: "Android Debug Bridge",
NewFs: NewFs,
Options: []fs.Option{{
Name: "serial",
Help: "The device serial to use. Leave empty for auto selection.",
Advanced: true,
}, {
Name: "host",
Default: "localhost",
Help: "The ADB server host.",
Advanced: true,
}, {
Name: "port",
Default: 5037,
Help: "The ADB server port.",
Advanced: true,
}, {
Name: "executable",
Help: "The ADB executable path.",
Advanced: true,
}, {
Name: "copy_links",
Help: "Follow symlinks and copy the pointed to item.",
Default: false,
Advanced: true,
}},
})
}
// Options defines the configuration for this backend
type Options struct {
Serial string
Host string
Port uint16
Executable string
FollowSymlinks bool `config:"copy_links"`
}
// Fs represents a adb device
type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
features *fs.Features // optional features
client *adb.Adb
device *execDevice
statFunc statFunc
statFuncMu sync.Mutex
touchFunc touchFunc
touchFuncMu sync.Mutex
}
// Object describes a adb file
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
size int64
mode os.FileMode
modTime time.Time
}
// ------------------------------------------------------------
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
// String converts this Fs to a string
func (f *Fs) String() string {
return fmt.Sprintf("ADB root '%s'", f.root)
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// NewFs constructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
}
if root == "" {
root = "/"
}
f := &Fs{
name: name,
root: root,
opt: *opt,
statFunc: (*Object).statTry,
touchFunc: (*Object).touchTry,
}
f.features = (&fs.Features{
CanHaveEmptyDirectories: true,
}).Fill(f)
f.client, err = adb.NewWithConfig(adb.ServerConfig{
Host: opt.Host,
Port: int(opt.Port),
PathToAdb: opt.Executable,
})
if err != nil {
return nil, errors.Wrapf(err, "Could not configure ADB server")
}
err = f.client.StartServer()
if err != nil {
return nil, errors.Wrapf(err, "Could not start ADB server")
}
serverVersion, err := f.client.ServerVersion()
if err != nil {
return nil, errors.Wrapf(err, "Could not get ADB server version")
}
fs.Debugf(f, "ADB server version: 0x%X", serverVersion)
serials, err := f.client.ListDeviceSerials()
if err != nil {
return nil, errors.Wrapf(err, "Could not get ADB devices")
}
descriptor := adb.AnyDevice()
if opt.Serial != "" {
descriptor = adb.DeviceWithSerial(opt.Serial)
}
if len(serials) > 1 && opt.Serial == "" {
return nil, errors.New("Multiple ADB devices found. Use the serial config to select a specific device")
}
f.device = &execDevice{f.client.Device(descriptor)}
// follow symlinks for root pathes
entry, err := f.newEntryFollowSymlinks("")
switch err {
case nil:
case fs.ErrorObjectNotFound:
default:
return nil, err
}
switch entry.(type) {
case fs.Object:
f.root = path.Dir(f.root)
return f, fs.ErrorIsFile
case nil:
return f, nil
case fs.Directory:
return f, nil
default:
return nil, errors.Errorf("Invalid root entry type %t", entry)
}
}
// Precision of the object storage system
func (f *Fs) Precision() time.Duration {
return 1 * time.Second
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.None)
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
p := path.Join(f.root, dir)
dirEntries, err := f.device.ListDirEntries(p)
if err != nil {
return nil, errors.Wrap(err, "ListDirEntries")
}
defer fs.CheckClose(dirEntries, &err)
found := false
for dirEntries.Next() {
found = true
dirEntry := dirEntries.Entry()
switch dirEntry.Name {
case ".", "..":
continue
}
fsEntry, err := f.entryForDirEntry(path.Join(dir, dirEntry.Name), dirEntry, f.opt.FollowSymlinks)
if err != nil {
fs.Errorf(p, "Listing error: %q: %v", dirEntry.Name, err)
return nil, err
} else if fsEntry != nil {
entries = append(entries, fsEntry)
} else {
fs.Debugf(f, "Skipping DirEntry %#v", dirEntry)
}
}
err = dirEntries.Err()
if err != nil {
return nil, errors.Wrap(err, "ListDirEntries")
}
if !found {
return nil, fs.ErrorDirNotFound
}
return
}
func (f *Fs) entryForDirEntry(remote string, e *adb.DirEntry, followSymlinks bool) (fs.DirEntry, error) {
o := f.newObjectWithInfo(remote, e)
// Follow symlinks if required
if followSymlinks && (e.Mode&os.ModeSymlink) != 0 {
err := f.statFunc(&o)
if err != nil {
return nil, err
}
}
if o.mode.IsDir() {
return fs.NewDir(remote, o.modTime), nil
}
return &o, nil
}
func (f *Fs) newEntry(remote string) (fs.DirEntry, error) {
return f.newEntryWithFollow(remote, f.opt.FollowSymlinks)
}
func (f *Fs) newEntryFollowSymlinks(remote string) (fs.DirEntry, error) {
return f.newEntryWithFollow(remote, true)
}
func (f *Fs) newEntryWithFollow(remote string, followSymlinks bool) (fs.DirEntry, error) {
entry, err := f.device.Stat(path.Join(f.root, remote))
if err != nil {
if adb.HasErrCode(err, adb.FileNoExistError) {
return nil, fs.ErrorObjectNotFound
}
return nil, errors.Wrapf(err, "Stat failed")
}
return f.entryForDirEntry(remote, entry, followSymlinks)
}
func (f *Fs) newObjectWithInfo(remote string, e *adb.DirEntry) Object {
return Object{
fs: f,
remote: remote,
size: int64(e.Size),
mode: e.Mode,
modTime: e.ModifiedAt,
}
}
// NewObject finds the Object at remote. If it can't be found
// it returns the error ErrorObjectNotFound.
func (f *Fs) NewObject(remote string) (fs.Object, error) {
entry, err := f.newEntry(remote)
if err != nil {
return nil, err
}
obj, ok := entry.(fs.Object)
if !ok {
return nil, fs.ErrorObjectNotFound
}
return obj, nil
}
// Put in to the remote path with the modTime given of the given size
//
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
// return an error or upload it properly (rather than e.g. calling panic).
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
remote := src.Remote()
// Temporary Object under construction - info filled in by Update()
o := f.newObject(remote)
err := o.Update(in, src, options...)
if err != nil {
return nil, err
}
return o, nil
}
// newObject makes a half completed Object
func (f *Fs) newObject(remote string) *Object {
return &Object{
fs: f,
remote: remote,
}
}
// Mkdir makes the directory (container, bucket)
//
// Shouldn't return an error if it already exists
func (f *Fs) Mkdir(dir string) error {
p := path.Join(f.root, dir)
output, code, err := f.device.execCommandWithExitCode("mkdir -p", p)
switch err := err.(type) {
case nil:
return nil
case adb.ShellExitError:
entry, _ := f.newEntry(p)
if _, ok := entry.(fs.Directory); ok {
return nil
}
return errors.Errorf("mkdir %q failed with %d: %q", dir, code, output)
default:
return errors.Wrap(err, "mkdir")
}
}
// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(dir string) error {
p := path.Join(f.root, dir)
output, code, err := f.device.execCommandWithExitCode("rmdir", p)
switch err := err.(type) {
case nil:
return nil
case adb.ShellExitError:
return errors.Errorf("rmdir %q failed with %d: %q", dir, code, output)
default:
return errors.Wrap(err, "rmdir")
}
}
// Fs returns the parent Fs
func (o *Object) Fs() fs.Info {
return o.fs
}
// Return a string version
func (o *Object) String() string {
if o == nil {
return "<nil>"
}
return o.remote
}
// Remote returns the remote path
func (o *Object) Remote() string {
return o.remote
}
// ModTime returns the modification date of the file
// It should return a best guess if one isn't available
func (o *Object) ModTime() time.Time {
return o.modTime
}
// Size returns the size of the file
func (o *Object) Size() int64 {
return o.size
}
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
func (o *Object) Hash(hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
// Storable says whether this object can be stored
func (o *Object) Storable() bool {
return true
}
// SetModTime sets the metadata on the object to set the modification date
func (o *Object) SetModTime(t time.Time) error {
return o.fs.touchFunc(o, t)
}
func (o *Object) stat() error {
return o.statStatArg(statArgC, path.Join(o.fs.root, o.remote))
}
func (o *Object) setMetadata(entry *adb.DirEntry) {
// Don't overwrite the values if we don't need to
// this avoids upsetting the race detector
if o.size != int64(entry.Size) {
o.size = int64(entry.Size)
}
if !o.modTime.Equal(entry.ModifiedAt) {
o.modTime = entry.ModifiedAt
}
if o.mode != entry.Mode {
o.mode = decodeEntryMode(uint32(entry.Mode))
}
}
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
const blockSize = 1 << 12
var offset, count int64 = 0, -1
for _, option := range options {
switch x := option.(type) {
case *fs.RangeOption:
offset, count = x.Decode(o.size)
case *fs.SeekOption:
offset = x.Offset
default:
if option.Mandatory() {
fs.Logf(o, "Unsupported mandatory option: %v", option)
}
}
}
if offset > o.size {
offset = o.size
}
if count < 0 {
count = o.size - offset
} else if count+offset > o.size {
count = o.size - offset
}
fs.Debugf(o, "Open: remote: %q offset: %d count: %d", o.remote, offset, count)
if count == 0 {
return ioutil.NopCloser(bytes.NewReader(nil)), nil
}
offsetBlocks, offsetRest := offset/blockSize, offset%blockSize
countBlocks := (count-1)/blockSize + 1
conn, err := o.fs.device.execCommand(fmt.Sprintf("sh -c 'dd \"if=$0\" bs=%d skip=%d count=%d 2>/dev/null'", blockSize, offsetBlocks, countBlocks), path.Join(o.fs.root, o.remote))
if err != nil {
return nil, err
}
return &adbReader{
ReadCloser: readers.NewLimitedReadCloser(conn, count+offsetRest),
skip: offsetRest,
expected: count,
}, nil
}
// Update in to the object with the modTime given of the given size
//
// When called from outside a Fs by rclone, src.Size() will always be >= 0.
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
// return an error or update the object properly (rather than e.g. calling panic).
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
for _, option := range options {
if option.Mandatory() {
fs.Logf(option, "Unsupported mandatory option: %v", option)
}
}
written, err := o.writeToFile(path.Join(o.fs.root, o.remote), in, 0666, src.ModTime())
if err != nil {
if removeErr := o.Remove(); removeErr != nil {
fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
}
return err
}
expected := src.Size()
if expected == -1 {
expected = written
}
for _, t := range []int64{100, 250, 500, 1000, 2500, 5000, 10000} {
err = o.stat()
if err != nil {
return err
}
if o.size == expected {
return nil
}
fs.Debugf(o, "Invalid size after update, expected: %d got: %d", expected, o.size)
time.Sleep(time.Duration(t) * time.Millisecond)
}
return o.stat()
}
// Remove this object
func (o *Object) Remove() error {
p := path.Join(o.fs.root, o.remote)
output, code, err := o.fs.device.execCommandWithExitCode("rm", p)
switch err := err.(type) {
case nil:
return nil
case adb.ShellExitError:
return errors.Errorf("rm %q failed with %d: %q", o.remote, code, output)
default:
return errors.Wrap(err, "rm")
}
}
func (o *Object) writeToFile(path string, rd io.Reader, perms os.FileMode, modeTime time.Time) (written int64, err error) {
dst, err := o.fs.device.OpenWrite(path, perms, modeTime)
if err != nil {
return
}
defer fs.CheckClose(dst, &err)
return io.Copy(dst, rd)
}
type statFunc func(*Object) error
func (o *Object) statTry() error {
o.fs.statFuncMu.Lock()
defer o.fs.statFuncMu.Unlock()
for _, f := range []statFunc{
(*Object).statStatL, (*Object).statRealPath, (*Object).statReadLink,
} {
err := f(o)
if err != nil {
fs.Debugf(o, "%s", err)
} else {
o.fs.statFunc = f
return nil
}
}
return errors.Errorf("unable to resolve link target")
}
const (
statArgLc = "-Lc"
statArgC = "-c"
)
func (o *Object) statStatL() error {
return o.statStatArg(statArgLc, path.Join(o.fs.root, o.remote))
}
func (o *Object) statStatArg(arg, path string) error {
output, code, err := o.fs.device.execCommandWithExitCode(fmt.Sprintf("stat %s %s", arg, "%f,%s,%Y"), path)
output = strings.TrimSpace(output)
switch err := err.(type) {
case nil:
case adb.ShellExitError:
return errors.Errorf("stat %q failed with %d: %q", o.remote, code, output)
default:
return errors.Wrap(err, "stat")
}
parts := strings.Split(output, ",")
if len(parts) != 3 {
return errors.Errorf("stat %q invalid output %q", o.remote, output)
}
mode, err := strconv.ParseUint(parts[0], 16, 32)
if err != nil {
return errors.Errorf("stat %q invalid output %q", o.remote, output)
}
size, err := strconv.ParseUint(parts[1], 10, 64)
if err != nil {
return errors.Errorf("stat %q invalid output %q", o.remote, output)
}
modTime, err := strconv.ParseInt(parts[2], 10, 64)
if err != nil {
return errors.Errorf("stat %q invalid output %q", o.remote, output)
}
o.size = int64(size)
o.modTime = time.Unix(modTime, 0)
o.mode = decodeEntryMode(uint32(mode))
return nil
}
func (o *Object) statReadLink() error {
p := path.Join(o.fs.root, o.remote)
output, code, err := o.fs.device.execCommandWithExitCode("readlink -f", p)
output = strings.TrimSuffix(output, "\n")
switch err := err.(type) {
case nil:
case adb.ShellExitError:
return errors.Errorf("readlink %q failed with %d: %q", o.remote, code, output)
default:
return errors.Wrap(err, "readlink")
}
return o.statStatArg(statArgC, output)
}
func (o *Object) statRealPath() error {
p := path.Join(o.fs.root, o.remote)
output, code, err := o.fs.device.execCommandWithExitCode("realpath", p)
output = strings.TrimSuffix(output, "\n")
switch err := err.(type) {
case nil:
case adb.ShellExitError:
return errors.Errorf("realpath %q failed with %d: %q", o.remote, code, output)
default:
return errors.Wrap(err, "realpath")
}
return o.statStatArg(statArgC, output)
}
type touchFunc func(*Object, time.Time) error
func (o *Object) touchTry(t time.Time) error {
o.fs.touchFuncMu.Lock()
defer o.fs.touchFuncMu.Unlock()
for _, f := range []touchFunc{
(*Object).touchCmd, (*Object).touchCd,
} {
err := f(o, t)
if err != nil {
fs.Debugf(o, "%s", err)
} else {
o.fs.touchFunc = f
return nil
}
}
return errors.Errorf("unable to resolve link target")
}
const (
touchArgCmd = "-cmd"
touchArgCd = "-cd"
)
func (o *Object) touchCmd(t time.Time) error {
return o.touchStatArg(touchArgCmd, path.Join(o.fs.root, o.remote), t)
}
func (o *Object) touchCd(t time.Time) error {
return o.touchStatArg(touchArgCd, path.Join(o.fs.root, o.remote), t)
}
func (o *Object) touchStatArg(arg, path string, t time.Time) error {
output, code, err := o.fs.device.execCommandWithExitCode(fmt.Sprintf("touch %s %s", arg, t.Format(time.RFC3339Nano)), path)
output = strings.TrimSpace(output)
switch err := err.(type) {
case nil:
case adb.ShellExitError:
return errors.Errorf("touch %q failed with %d: %q", o.remote, code, output)
default:
return errors.Wrap(err, "touch")
}
err = o.stat()
if err != nil {
return err
}
if diff, ok := checkTimeEqualWithPrecision(t, o.modTime, o.fs.Precision()); !ok {
return errors.Errorf("touch %q to %s was ineffective: %d", o.remote, t, diff)
}
return nil
}
func checkTimeEqualWithPrecision(t0, t1 time.Time, precision time.Duration) (time.Duration, bool) {
dt := t0.Sub(t1)
if dt >= precision || dt <= -precision {
return dt, false
}
return dt, true
}
func decodeEntryMode(entryMode uint32) os.FileMode {
const (
unixIFBLK = 0x6000
unixIFMT = 0xf000
unixIFCHR = 0x2000
unixIFDIR = 0x4000
unixIFIFO = 0x1000
unixIFLNK = 0xa000
unixIFREG = 0x8000
unixIFSOCK = 0xc000
unixISGID = 0x400
unixISUID = 0x800
unixISVTX = 0x200
)
mode := os.FileMode(entryMode & 0777)
switch entryMode & unixIFMT {
case unixIFBLK:
mode |= os.ModeDevice
case unixIFCHR:
mode |= os.ModeDevice | os.ModeCharDevice
case unixIFDIR:
mode |= os.ModeDir
case unixIFIFO:
mode |= os.ModeNamedPipe
case unixIFLNK:
mode |= os.ModeSymlink
case unixIFREG:
// nothing to do
case unixIFSOCK:
mode |= os.ModeSocket
}
if entryMode&unixISGID != 0 {
mode |= os.ModeSetgid
}
if entryMode&unixISUID != 0 {
mode |= os.ModeSetuid
}
if entryMode&unixISVTX != 0 {
mode |= os.ModeSticky
}
return mode
}
type execDevice struct {
*adb.Device
}
func (d *execDevice) execCommandWithExitCode(cmd string, arg string) (string, int, error) {
cmdLine := fmt.Sprintf("sh -c '%s \"$0\"; echo :$?' '%s'", cmd, strings.Replace(arg, "'", "'\\''", -1))
fs.Debugf("adb", "exec: %s", cmdLine)
conn, err := d.execCommand(cmdLine)
if err != nil {
return "", -1, err
}
resp, err := conn.ReadUntilEof()
if err != nil {
return "", -1, errors.Wrap(err, "ExecCommand")
}
outStr := string(resp)
idx := strings.LastIndexByte(outStr, ':')
if idx == -1 {
return outStr, -1, fmt.Errorf("adb shell aborted, can not parse exit code")
}
exitCode, _ := strconv.Atoi(strings.TrimSpace(outStr[idx+1:]))
if exitCode != 0 {
err = adb.ShellExitError{Command: cmdLine, ExitCode: exitCode}
}
return outStr[:idx], exitCode, err
}
func (d *execDevice) execCommand(cmd string, args ...string) (*wire.Conn, error) {
cmd = prepareCommandLineEscaped(cmd, args...)
conn, err := d.Dial()
if err != nil {
return nil, errors.Wrap(err, "ExecCommand")
}
defer func() {
if err != nil && conn != nil {
_ = conn.Close()
}
}()
req := fmt.Sprintf("exec:%s", cmd)
if err = conn.SendMessage([]byte(req)); err != nil {
return nil, errors.Wrap(err, "ExecCommand")
}
if _, err = conn.ReadStatus(req); err != nil {
return nil, errors.Wrap(err, "ExecCommand")
}
return conn, nil
}
func prepareCommandLineEscaped(cmd string, args ...string) string {
for i, arg := range args {
args[i] = fmt.Sprintf("'%s'", strings.Replace(arg, "'", "'\\''", -1))
}
// Prepend the command to the args array.
if len(args) > 0 {
cmd = fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))
}
return cmd
}
type adbReader struct {
io.ReadCloser
skip int64
read int64
expected int64
}
func (r *adbReader) Read(b []byte) (n int, err error) {
n, err = r.ReadCloser.Read(b)
if s := r.skip; n > 0 && s > 0 {
_n := int64(n)
if _n <= s {
r.skip -= _n
return r.Read(b)
}
r.skip = 0
copy(b, b[s:n])
n -= int(s)
}
r.read += int64(n)
if err == io.EOF && r.read < r.expected {
fs.Debugf("adb", "Read: read: %d expected: %d n: %d", r.read, r.expected, n)
return n, io.ErrUnexpectedEOF
}
return n, err
}
-20
View File
@@ -1,20 +0,0 @@
// Test ADB filesystem interface
package adb_test
import (
"testing"
"github.com/ncw/rclone/backend/adb"
"github.com/ncw/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestAdb:/data/local/tmp",
NilObject: (*adb.Object)(nil),
ExtraConfig: []fstests.ExtraConfigItem{
{Name: "TestAdb", Key: "copy_links", Value: "true"},
},
})
}
+1 -1
View File
@@ -30,7 +30,7 @@ type Options struct {
Remote string `config:"remote"`
}
// NewFs constructs an Fs from the path.
// NewFs contstructs an Fs from the path.
//
// The returned Fs is the actual Fs, referenced by remote in the config
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+1 -1
View File
@@ -80,7 +80,7 @@ func TestNewFS(t *testing.T) {
wantEntry := test.entries[i]
require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
require.Equal(t, wantEntry.size, gotEntry.Size(), what)
require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what)
_, isDir := gotEntry.(fs.Directory)
require.Equal(t, wantEntry.isDir, isDir, what)
}
-1
View File
@@ -2,7 +2,6 @@ package all
import (
// Active file systems
_ "github.com/ncw/rclone/backend/adb"
_ "github.com/ncw/rclone/backend/alias"
_ "github.com/ncw/rclone/backend/amazonclouddrive"
_ "github.com/ncw/rclone/backend/azureblob"
+36 -87
View File
@@ -77,7 +77,7 @@ func init() {
}, {
Name: "upload_cutoff",
Help: "Cutoff for switching to chunked upload (<= 256MB).",
Default: defaultUploadCutoff,
Default: fs.SizeSuffix(defaultUploadCutoff),
Advanced: true,
}, {
Name: "chunk_size",
@@ -85,7 +85,7 @@ func init() {
Note that this is stored in memory and there may be up to
"--transfers" chunks stored at once in memory.`,
Default: defaultChunkSize,
Default: fs.SizeSuffix(defaultChunkSize),
Advanced: true,
}, {
Name: "list_chunk",
@@ -307,7 +307,7 @@ func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline
return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
}
// NewFs constructs an Fs from the path, container:path
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
@@ -392,7 +392,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
return nil, errors.New("Container name in SAS URL and container provided in command do not match")
}
f.container = parts.ContainerName
container = parts.ContainerName
containerURL = azblob.NewContainerURL(*u, pipeline)
} else {
serviceURL = azblob.NewServiceURL(*u, pipeline)
@@ -417,8 +417,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
_, err := f.NewObject(remote)
if err != nil {
if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
// File doesn't exist or is a directory so return old f
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
f.root = oldRoot
return f, nil
}
@@ -474,21 +474,6 @@ func (o *Object) updateMetadataWithModTime(modTime time.Time) {
o.meta[modTimeKey] = modTime.Format(timeFormatOut)
}
// Returns whether file is a directory marker or not
func isDirectoryMarker(size int64, metadata azblob.Metadata, remote string) bool {
// Directory markers are 0 length
if size == 0 {
// Note that metadata with hdi_isfolder = true seems to be a
// defacto standard for marking blobs as directories.
endsWithSlash := strings.HasSuffix(remote, "/")
if endsWithSlash || remote == "" || metadata["hdi_isfolder"] == "true" {
return true
}
}
return false
}
// listFn is called from list to handle an object
type listFn func(remote string, object *azblob.BlobItem, isDirectory bool) error
@@ -554,20 +539,26 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
continue
}
remote := file.Name[len(f.root):]
if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) {
if strings.HasSuffix(remote, "/") {
remote = remote[:len(remote)-1]
// is this a directory marker?
if *file.Properties.ContentLength == 0 {
// Note that metadata with hdi_isfolder = true seems to be a
// defacto standard for marking blobs as directories.
endsWithSlash := strings.HasSuffix(remote, "/")
if endsWithSlash || remote == "" || file.Metadata["hdi_isfolder"] == "true" {
if endsWithSlash {
remote = remote[:len(remote)-1]
}
err = fn(remote, file, true)
if err != nil {
return err
}
// Keep track of directory markers. If recursing then
// there will be no Prefixes so no need to keep track
if !recurse {
directoryMarkers[remote] = struct{}{}
}
continue // skip directory marker
}
err = fn(remote, file, true)
if err != nil {
return err
}
// Keep track of directory markers. If recursing then
// there will be no Prefixes so no need to keep track
if !recurse {
directoryMarkers[remote] = struct{}{}
}
continue // skip directory marker
}
// Send object
err = fn(remote, file, false)
@@ -754,35 +745,6 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
return fs, fs.Update(in, src, options...)
}
// Check if the container exists
//
// NB this can return incorrect results if called immediately after container deletion
func (f *Fs) dirExists() (bool, error) {
options := azblob.ListBlobsSegmentOptions{
Details: azblob.BlobListingDetails{
Copy: false,
Metadata: false,
Snapshots: false,
UncommittedBlobs: false,
Deleted: false,
},
MaxResults: 1,
}
err := f.pacer.Call(func() (bool, error) {
ctx := context.Background()
_, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options)
return f.shouldRetry(err)
})
if err == nil {
return true, nil
}
// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
return false, nil
}
return false, err
}
// Mkdir creates the container if it doesn't exist
func (f *Fs) Mkdir(dir string) error {
f.containerOKMu.Lock()
@@ -790,15 +752,6 @@ func (f *Fs) Mkdir(dir string) error {
if f.containerOK {
return nil
}
if !f.containerDeleted {
exists, err := f.dirExists()
if err == nil {
f.containerOK = exists
}
if err != nil || exists {
return err
}
}
// now try to create the container
err := f.pacer.Call(func() (bool, error) {
@@ -1028,37 +981,27 @@ func (o *Object) setMetadata(metadata azblob.Metadata) {
// o.md5
// o.meta
func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
metadata := info.NewMetadata()
size := info.ContentLength()
if isDirectoryMarker(size, metadata, o.remote) {
return fs.ErrorNotAFile
}
// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
// this as base64 encoded string.
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
o.mimeType = info.ContentType()
o.size = size
o.modTime = info.LastModified()
o.size = info.ContentLength()
o.modTime = time.Time(info.LastModified())
o.accessTier = azblob.AccessTierType(info.AccessTier())
o.setMetadata(metadata)
o.setMetadata(info.NewMetadata())
return nil
}
func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItem) (err error) {
metadata := info.Metadata
size := *info.Properties.ContentLength
if isDirectoryMarker(size, metadata, o.remote) {
return fs.ErrorNotAFile
}
// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
// this as base64 encoded string.
o.md5 = base64.StdEncoding.EncodeToString(info.Properties.ContentMD5)
o.mimeType = *info.Properties.ContentType
o.size = size
o.size = *info.Properties.ContentLength
o.modTime = info.Properties.LastModified
o.accessTier = info.Properties.AccessTier
o.setMetadata(metadata)
o.setMetadata(info.Metadata)
return nil
}
@@ -1104,6 +1047,12 @@ func (o *Object) readMetaData() (err error) {
return o.decodeMetaDataFromPropertiesResponse(blobProperties)
}
// timeString returns modTime as the number of milliseconds
// elapsed since January 1, 1970 UTC as a decimal string.
func timeString(modTime time.Time) string {
return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
}
// parseTimeString converts a decimal string number of milliseconds
// elapsed since January 1, 1970 UTC into a time.Time and stores it in
// the modTime variable.
+3 -3
View File
@@ -17,12 +17,12 @@ type Error struct {
Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
}
// Error satisfies the error interface
// Error statisfies the error interface
func (e *Error) Error() string {
return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
}
// Fatal satisfies the Fatal interface
// Fatal statisfies the Fatal interface
//
// It indicates which errors should be treated as fatal
func (e *Error) Fatal() bool {
@@ -100,7 +100,7 @@ func RemoveVersion(remote string) (t Timestamp, newRemote string) {
return Timestamp(newT), base[:versionStart] + ext
}
// IsZero returns true if the timestamp is uninitialized
// IsZero returns true if the timestamp is unitialised
func (t Timestamp) IsZero() bool {
return time.Time(t).IsZero()
}
+19 -37
View File
@@ -108,7 +108,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
Files above this size will be uploaded in chunks of "--b2-chunk-size".
This value should be set no larger than 4.657GiB (== 5GB).`,
Default: defaultUploadCutoff,
Default: fs.SizeSuffix(defaultUploadCutoff),
Advanced: true,
}, {
Name: "chunk_size",
@@ -117,21 +117,8 @@ This value should be set no larger than 4.657GiB (== 5GB).`,
When uploading large files, chunk the file into this size. Note that
these chunks are buffered in memory and there might a maximum of
"--transfers" chunks in progress at once. 5,000,000 Bytes is the
minimum size.`,
Default: defaultChunkSize,
Advanced: true,
}, {
Name: "disable_checksum",
Help: `Disable checksums for large (> upload cutoff) files`,
Default: false,
Advanced: true,
}, {
Name: "download_url",
Help: `Custom endpoint for downloads.
This is usually set to a Cloudflare CDN URL as Backblaze offers
free egress for data downloaded through the Cloudflare network.
Leave blank if you want to use the endpoint provided by Backblaze.`,
minimim size.`,
Default: fs.SizeSuffix(defaultChunkSize),
Advanced: true,
}},
})
@@ -139,16 +126,14 @@ Leave blank if you want to use the endpoint provided by Backblaze.`,
// Options defines the configuration for this backend
type Options struct {
Account string `config:"account"`
Key string `config:"key"`
Endpoint string `config:"endpoint"`
TestMode string `config:"test_mode"`
Versions bool `config:"versions"`
HardDelete bool `config:"hard_delete"`
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
DisableCheckSum bool `config:"disable_checksum"`
DownloadURL string `config:"download_url"`
Account string `config:"account"`
Key string `config:"key"`
Endpoint string `config:"endpoint"`
TestMode string `config:"test_mode"`
Versions bool `config:"versions"`
HardDelete bool `config:"hard_delete"`
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
}
// Fs represents a remote b2 server
@@ -328,7 +313,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
return
}
// NewFs constructs an Fs from the path, bucket:path
// NewFs contstructs an Fs from the path, bucket:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
@@ -1305,17 +1290,9 @@ var _ io.ReadCloser = &openFile{}
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
opts := rest.Opts{
Method: "GET",
RootURL: o.fs.info.DownloadURL,
Options: options,
}
// Use downloadUrl from backblaze if downloadUrl is not set
// otherwise use the custom downloadUrl
if o.fs.opt.DownloadURL == "" {
opts.RootURL = o.fs.info.DownloadURL
} else {
opts.RootURL = o.fs.opt.DownloadURL
}
// Download by id if set otherwise by name
if o.id != "" {
opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
@@ -1476,7 +1453,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
// Content-Type b2/x-auto to automatically set the stored Content-Type
// post upload. In the case where a file extension is absent or the
// lookup fails, the Content-Type is set to application/octet-stream. The
// Content-Type mappings can be pursued here.
// Content-Type mappings can be purused here.
//
// X-Bz-Content-Sha1
// required
@@ -1523,6 +1500,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
},
ContentLength: &size,
}
// for go1.8 (see release notes) we must nil the Body if we want a
// "Content-Length: 0" header which b2 requires for all files.
if size == 0 {
opts.Body = nil
}
var response api.FileInfo
// Don't retry, return a retry error instead
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+2 -4
View File
@@ -116,10 +116,8 @@ func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *lar
},
}
// Set the SHA1 if known
if !o.fs.opt.DisableCheckSum {
if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
request.Info[sha1Key] = calculatedSha1
}
if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
request.Info[sha1Key] = calculatedSha1
}
var response api.StartLargeFileResponse
err = f.pacer.Call(func() (bool, error) {
+2 -2
View File
@@ -45,7 +45,7 @@ type Error struct {
RequestID string `json:"request_id"`
}
// Error returns a string for the error and satisfies the error interface
// Error returns a string for the error and statistifes the error interface
func (e *Error) Error() string {
out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status)
if e.Message != "" {
@@ -57,7 +57,7 @@ func (e *Error) Error() string {
return out
}
// Check Error satisfies the error interface
// Check Error statisfies the error interface
var _ error = (*Error)(nil)
// ItemFields are the fields needed for FileInfo
+5 -5
View File
@@ -171,13 +171,13 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) {
authRetry := false
authRety := false
if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
authRetry = true
authRety = true
fs.Debugf(nil, "Should retry: %v", err)
}
return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
// substitute reserved characters for box
@@ -530,10 +530,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
//
// The new object may have been created if an error is returned
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
switch err {
case nil:
return existingObj, existingObj.Update(in, src, options...)
return exisitingObj, exisitingObj.Update(in, src, options...)
case fs.ErrorObjectNotFound:
// Not found so create it
return f.PutUnchecked(in, src)
+2 -2
View File
@@ -211,8 +211,8 @@ outer:
}
reqSize := remaining
if reqSize >= chunkSize {
reqSize = chunkSize
if reqSize >= int64(chunkSize) {
reqSize = int64(chunkSize)
}
// Make a block of memory
+4 -4
View File
@@ -576,7 +576,7 @@ The slice indices are similar to Python slices: start[:end]
start is the 0 based chunk number from the beginning of the file
to fetch inclusive. end is 0 based chunk number from the beginning
of the file to fetch exclusive.
of the file to fetch exclisive.
Both values can be negative, in which case they count from the back
of the file. The value "-5:" represents the last 5 chunks of a file.
@@ -870,7 +870,7 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
}
}
// ChangeNotify can subscribe multiple callers
// ChangeNotify can subsribe multiple callers
// this is coupled with the wrapped fs ChangeNotify (if it supports it)
// and also notifies other caches (i.e VFS) to clear out whenever something changes
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
@@ -1549,7 +1549,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
}
if srcObj.isTempFile() {
// we check if the feature is still active
// we check if the feature is stil active
if f.opt.TempWritePath == "" {
fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run")
return nil, fs.ErrorCantCopy
@@ -1625,7 +1625,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
// if this is a temp object then we perform the changes locally
if srcObj.isTempFile() {
// we check if the feature is still active
// we check if the feature is stil active
if f.opt.TempWritePath == "" {
fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run")
return nil, fs.ErrorCantMove
+13 -18
View File
@@ -387,10 +387,10 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
// write the object
o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
require.Equal(t, o.Size(), testSize)
require.Equal(t, o.Size(), int64(testSize))
time.Sleep(time.Second * 3)
checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
require.NoError(t, err)
require.Equal(t, int64(len(checkSample)), o.Size())
@@ -726,7 +726,6 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
// Call the rc function
m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
require.NoError(t, err)
require.Contains(t, m, "status")
require.Contains(t, m, "message")
require.Equal(t, "ok", m["status"])
@@ -736,21 +735,18 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
co, err = rootFs.NewObject("data.bin")
require.NoError(t, err)
require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
_, err = runInstance.list(t, rootFs, "")
require.NoError(t, err)
li1, err := runInstance.list(t, rootFs, "")
// create some rand test data
testData2 := randStringBytes(int(chunkSize))
runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
// list should have 1 item only
li1, err := runInstance.list(t, rootFs, "")
require.NoError(t, err)
li1, err = runInstance.list(t, rootFs, "")
require.Len(t, li1, 1)
// Call the rc function
m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
require.NoError(t, err)
require.Contains(t, m, "status")
require.Contains(t, m, "message")
require.Equal(t, "ok", m["status"])
@@ -758,7 +754,6 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
// list should have 2 items now
li2, err := runInstance.list(t, rootFs, "")
require.NoError(t, err)
require.Len(t, li2, 2)
}
@@ -1495,8 +1490,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
var err error
if r.useMount {
var f *os.File
f, err = os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return err
}
@@ -1506,8 +1500,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
}()
_, err = f.WriteString(data + append)
} else {
var obj1 fs.Object
obj1, err = rootFs.NewObject(src)
obj1, err := rootFs.NewObject(src)
if err != nil {
return err
}
@@ -1639,13 +1632,15 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
cfs, ok := f.(*cache.Fs)
if ok {
return cfs, nil
}
if f.Features().UnWrap != nil {
cfs, ok := f.Features().UnWrap().(*cache.Fs)
if ok {
return cfs, nil
} else {
if f.Features().UnWrap != nil {
cfs, ok := f.Features().UnWrap().(*cache.Fs)
if ok {
return cfs, nil
}
}
}
return nil, errors.New("didn't found a cache fs")
}
+3 -3
View File
@@ -398,7 +398,7 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
if err != nil {
return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
}
err = bucket.Put([]byte(cachedObject.Name), encoded)
err = bucket.Put([]byte(cachedObject.Name), []byte(encoded))
if err != nil {
return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
}
@@ -809,7 +809,7 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
if err != nil {
return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
}
err = bucket.Put([]byte(destPath), encoded)
err = bucket.Put([]byte(destPath), []byte(encoded))
if err != nil {
return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
}
@@ -1049,7 +1049,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
if err != nil {
return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
}
err = bucket.Put([]byte(destPath), encoded)
err = bucket.Put([]byte(destPath), []byte(encoded))
if err != nil {
return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
}
+2 -2
View File
@@ -463,7 +463,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
if int(newRune) < base {
newRune += 256
}
_, _ = result.WriteRune(newRune)
_, _ = result.WriteRune(rune(newRune))
default:
_, _ = result.WriteRune(runeValue)
@@ -748,7 +748,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
}
// retrieve the nonce
// retreive the nonce
fh.nonce.fromBuf(readBuf[fileMagicSize:])
fh.initialNonce = fh.nonce
return fh, nil
+2 -2
View File
@@ -122,7 +122,7 @@ func NewCipher(m configmap.Mapper) (Cipher, error) {
return newCipherForConfig(opt)
}
// NewFs constructs an Fs from the path, container:path
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
@@ -555,7 +555,7 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
}
// ComputeHash takes the nonce from o, and encrypts the contents of
// src with it, and calculates the hash given by HashType on the fly
// src with it, and calcuates the hash given by HashType on the fly
//
// Note that we break lots of encapsulation in this function.
func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {
+62 -169
View File
@@ -21,7 +21,6 @@ import (
"net/url"
"os"
"path"
"sort"
"strconv"
"strings"
"sync"
@@ -40,7 +39,6 @@ import (
"github.com/ncw/rclone/lib/dircache"
"github.com/ncw/rclone/lib/oauthutil"
"github.com/ncw/rclone/lib/pacer"
"github.com/ncw/rclone/lib/readers"
"github.com/pkg/errors"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
@@ -56,8 +54,7 @@ const (
driveFolderType = "application/vnd.google-apps.folder"
timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
defaultMinSleep = fs.Duration(100 * time.Millisecond)
defaultBurst = 100
minSleep = 10 * time.Millisecond
defaultExportExtensions = "docx,xlsx,pptx,svg"
scopePrefix = "https://www.googleapis.com/auth/"
defaultScope = "drive"
@@ -128,29 +125,6 @@ var (
_linkTemplates map[string]*template.Template // available link types
)
// Parse the scopes option returning a slice of scopes
func driveScopes(scopesString string) (scopes []string) {
if scopesString == "" {
scopesString = defaultScope
}
for _, scope := range strings.Split(scopesString, ",") {
scope = strings.TrimSpace(scope)
scopes = append(scopes, scopePrefix+scope)
}
return scopes
}
// Returns true if one of the scopes was "drive.appfolder"
func driveScopesContainsAppFolder(scopes []string) bool {
for _, scope := range scopes {
if scope == scopePrefix+"drive.appfolder" {
return true
}
}
return false
}
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
@@ -165,14 +139,18 @@ func init() {
fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
return
}
// Fill in the scopes
driveConfig.Scopes = driveScopes(opt.Scope)
// Set the root_folder_id if using drive.appfolder
if driveScopesContainsAppFolder(driveConfig.Scopes) {
m.Set("root_folder_id", "appDataFolder")
if opt.Scope == "" {
opt.Scope = defaultScope
}
driveConfig.Scopes = nil
for _, scope := range strings.Split(opt.Scope, ",") {
driveConfig.Scopes = append(driveConfig.Scopes, scopePrefix+strings.TrimSpace(scope))
// Set the root_folder_id if using drive.appfolder
if scope == "drive.appfolder" {
m.Set("root_folder_id", "appDataFolder")
}
}
if opt.ServiceAccountFile == "" {
err = oauthutil.Config("drive", name, m, driveConfig)
if err != nil {
@@ -359,16 +337,6 @@ will download it anyway.`,
Default: fs.SizeSuffix(-1),
Help: "If Object's are greater, use drive v2 API to download.",
Advanced: true,
}, {
Name: "pacer_min_sleep",
Default: defaultMinSleep,
Help: "Minimum time to sleep between API calls.",
Advanced: true,
}, {
Name: "pacer_burst",
Default: defaultBurst,
Help: "Number of API calls to allow without sleeping.",
Advanced: true,
}},
})
@@ -411,8 +379,6 @@ type Options struct {
AcknowledgeAbuse bool `config:"acknowledge_abuse"`
KeepRevisionForever bool `config:"keep_revision_forever"`
V2DownloadMinSize fs.SizeSuffix `config:"v2_download_min_size"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
PacerBurst int `config:"pacer_burst"`
}
// Fs represents a remote drive server
@@ -482,7 +448,7 @@ func (f *Fs) Features() *fs.Features {
return f.features
}
// shouldRetry determines whether a given err rates being retried
// shouldRetry determines whehter a given err rates being retried
func shouldRetry(err error) (bool, error) {
if err == nil {
return false, nil
@@ -733,16 +699,12 @@ func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, er
// Figure out if the user wants to use a team drive
func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
// Stop if we are running non-interactive config
if fs.Config.AutoConfirm {
return nil
}
if opt.TeamDriveID == "" {
fmt.Printf("Configure this as a team drive?\n")
} else {
fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID)
}
if !config.Confirm() {
if !config.ConfirmWithDefault(false) {
return nil
}
client, err := createOAuthClient(opt, name, m)
@@ -759,7 +721,7 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
listFailed := false
for {
var teamDrives *drive.TeamDriveList
err = newPacer(opt).Call(func() (bool, error) {
err = newPacer().Call(func() (bool, error) {
teamDrives, err = listTeamDrives.Do()
return shouldRetry(err)
})
@@ -789,13 +751,12 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
}
// newPacer makes a pacer configured for drive
func newPacer(opt *Options) *pacer.Pacer {
return pacer.New().SetMinSleep(time.Duration(opt.PacerMinSleep)).SetBurst(opt.PacerBurst).SetPacer(pacer.GoogleDrivePacer)
func newPacer() *pacer.Pacer {
return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
}
func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
scopes := driveScopes(opt.Scope)
conf, err := google.JWTConfigFromJSON(credentialsData, scopes...)
conf, err := google.JWTConfigFromJSON(credentialsData, driveConfig.Scopes...)
if err != nil {
return nil, errors.Wrap(err, "error processing credentials")
}
@@ -863,7 +824,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
return
}
// NewFs constructs an Fs from the path, container:path
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
@@ -894,7 +855,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
name: name,
root: root,
opt: *opt,
pacer: newPacer(opt),
pacer: newPacer(),
}
f.isTeamDrive = opt.TeamDriveID != ""
f.features = (&fs.Features{
@@ -1340,46 +1301,17 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
return entries, nil
}
// listREntry is a task to be executed by a litRRunner
type listREntry struct {
id, path string
}
// listRSlices is a helper struct to sort two slices at once
type listRSlices struct {
dirs []string
paths []string
}
func (s listRSlices) Sort() {
sort.Sort(s)
}
func (s listRSlices) Len() int {
return len(s.dirs)
}
func (s listRSlices) Swap(i, j int) {
s.dirs[i], s.dirs[j] = s.dirs[j], s.dirs[i]
s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
}
func (s listRSlices) Less(i, j int) bool {
return s.dirs[i] < s.dirs[j]
}
// listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
//
// In each cycle it will read up to grouping entries from the in channel without blocking.
// In each cycle, will wait up to 10ms to read up to grouping entries from the in channel.
// If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
// nil is send to the out channel and the function returns.
func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
var dirs []string
var paths []string
for dir := range in {
dirs = append(dirs[:0], dir.id)
paths = append(paths[:0], dir.path)
dirs = append(dirs[:0], dir)
wait := time.After(10 * time.Millisecond)
waitloop:
for i := 1; i < grouping; i++ {
select {
@@ -1387,32 +1319,31 @@ func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- er
if !ok {
break waitloop
}
dirs = append(dirs, d.id)
paths = append(paths, d.path)
default:
dirs = append(dirs, d)
case <-wait:
break waitloop
}
}
listRSlices{dirs, paths}.Sort()
var iErr error
_, err := f.list(dirs, "", false, false, false, func(item *drive.File) bool {
for _, parent := range item.Parents {
// only handle parents that are in the requested dirs list
i := sort.SearchStrings(dirs, parent)
if i == len(dirs) || dirs[i] != parent {
continue
}
remote := path.Join(paths[i], item.Name)
entry, err := f.itemToDirEntry(remote, item)
if err != nil {
iErr = err
return true
parentPath := ""
if len(item.Parents) > 0 {
p, ok := f.dirCache.GetInv(item.Parents[0])
if ok {
parentPath = p
}
}
remote := path.Join(parentPath, item.Name)
entry, err := f.itemToDirEntry(remote, item)
if err != nil {
iErr = err
return true
}
err = cb(entry)
if err != nil {
iErr = err
return true
}
err = cb(entry)
if err != nil {
iErr = err
return true
}
return false
})
@@ -1463,44 +1394,30 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
if err != nil {
return err
}
if directoryID == "root" {
var info *drive.File
err = f.pacer.CallNoRetry(func() (bool, error) {
info, err = f.svc.Files.Get("root").
Fields("id").
SupportsTeamDrives(f.isTeamDrive).
Do()
return shouldRetry(err)
})
if err != nil {
return err
}
directoryID = info.Id
}
mu := sync.Mutex{} // protects in and overflow
wg := sync.WaitGroup{}
in := make(chan listREntry, inputBuffer)
in := make(chan string, inputBuffer)
out := make(chan error, fs.Config.Checkers)
list := walk.NewListRHelper(callback)
overflow := []listREntry{}
overfflow := []string{}
cb := func(entry fs.DirEntry) error {
mu.Lock()
defer mu.Unlock()
if d, isDir := entry.(*fs.Dir); isDir && in != nil {
select {
case in <- listREntry{d.ID(), d.Remote()}:
case in <- d.ID():
wg.Add(1)
default:
overflow = append(overflow, listREntry{d.ID(), d.Remote()})
overfflow = append(overfflow, d.ID())
}
}
return list.Add(entry)
}
wg.Add(1)
in <- listREntry{directoryID, dir}
in <- directoryID
for i := 0; i < fs.Config.Checkers; i++ {
go f.listRRunner(&wg, in, out, cb, grouping)
@@ -1509,18 +1426,18 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
// wait until the all directories are processed
wg.Wait()
// if the input channel overflowed add the collected entries to the channel now
for len(overflow) > 0 {
for len(overfflow) > 0 {
mu.Lock()
l := len(overflow)
// only fill half of the channel to prevent entries beeing put into overflow again
l := len(overfflow)
// only fill half of the channel to prevent entries beeing put into overfflow again
if l > inputBuffer/2 {
l = inputBuffer / 2
}
wg.Add(l)
for _, d := range overflow[:l] {
for _, d := range overfflow[:l] {
in <- d
}
overflow = overflow[l:]
overfflow = overfflow[l:]
mu.Unlock()
// wait again for the completion of all directories
@@ -1711,14 +1628,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
return shouldRetry(err)
})
if err != nil {
return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Name, srcDir)
}
}
// rmdir (into trash) the now empty source directory
fs.Infof(srcDir, "removing empty directory")
err = f.rmdir(srcDir.ID(), true)
if err != nil {
return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
}
}
return nil
@@ -2137,7 +2054,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
// ChangeNotify calls the passed function with a path that has had changes.
// If the implementation uses polling, it should adhere to the given interval.
//
// Automatically restarts itself in case of unexpected behavior of the remote.
// Automatically restarts itself in case of unexpected behaviour of the remote.
//
// Close the returned channel to stop being notified.
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
@@ -2244,13 +2161,11 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), startPage
// translate the parent dir of this object
if len(change.File.Parents) > 0 {
for _, parent := range change.File.Parents {
if parentPath, ok := f.dirCache.GetInv(parent); ok {
// and append the drive file name to compute the full file name
newPath := path.Join(parentPath, change.File.Name)
// this will now clear the actual file too
pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
}
if parentPath, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
// and append the drive file name to compute the full file name
newPath := path.Join(parentPath, change.File.Name)
// this will now clear the actual file too
pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
}
} else { // a true root object that is changed
pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
@@ -2542,32 +2457,16 @@ func (o *documentObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err e
// Update the size with what we are reading as it can change from
// the HEAD in the listing to this GET. This stops rclone marking
// the transfer as corrupted.
var offset, end int64 = 0, -1
var newOptions = options[:0]
for _, o := range options {
// Note that Range requests don't work on Google docs:
// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
// So do a subset of them manually
switch x := o.(type) {
case *fs.RangeOption:
offset, end = x.Start, x.End
case *fs.SeekOption:
offset, end = x.Offset, -1
default:
newOptions = append(newOptions, o)
if _, ok := o.(*fs.RangeOption); ok {
return nil, errors.New("partial downloads are not supported while exporting Google Documents")
}
}
options = newOptions
if offset != 0 {
return nil, errors.New("partial downloads are not supported while exporting Google Documents")
}
in, err = o.baseObject.open(o.url, options...)
if in != nil {
in = &openDocumentFile{o: o, in: in}
}
if end >= 0 {
in = readers.NewLimitedReadCloser(in, end-offset+1)
}
return
}
func (o *linkObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
@@ -2633,9 +2532,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
return err
}
newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
if err != nil {
return err
}
switch newO := newO.(type) {
case *Object:
*o = *newO
@@ -2674,9 +2570,6 @@ func (o *documentObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.O
remote = remote[:len(remote)-o.extLen]
newO, err := o.fs.newObjectWithInfo(remote, info)
if err != nil {
return err
}
switch newO := newO.(type) {
case *documentObject:
*o = *newO
-25
View File
@@ -22,31 +22,6 @@ import (
"google.golang.org/api/drive/v3"
)
func TestDriveScopes(t *testing.T) {
for _, test := range []struct {
in string
want []string
wantFlag bool
}{
{"", []string{
"https://www.googleapis.com/auth/drive",
}, false},
{" drive.file , drive.readonly", []string{
"https://www.googleapis.com/auth/drive.file",
"https://www.googleapis.com/auth/drive.readonly",
}, false},
{" drive.file , drive.appfolder", []string{
"https://www.googleapis.com/auth/drive.file",
"https://www.googleapis.com/auth/drive.appfolder",
}, true},
} {
got := driveScopes(test.in)
assert.Equal(t, test.want, got, test.in)
gotFlag := driveScopesContainsAppFolder(got)
assert.Equal(t, test.wantFlag, gotFlag, test.in)
}
}
/*
var additionalMimeTypes = map[string]string{
"application/vnd.ms-excel.sheet.macroenabled.12": ".xlsm",
+1 -1
View File
@@ -185,7 +185,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
// been 200 OK.
//
// So parse the response out of the body. We aren't expecting
// any other 2xx codes, so we parse it unconditionally on
// any other 2xx codes, so we parse it unconditionaly on
// StatusCode
if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
return 598, err
+5 -5
View File
@@ -130,8 +130,8 @@ Any files larger than this will be uploaded in chunks of this size.
Note that chunks are buffered in memory (one at a time) so rclone can
deal with retries. Setting this larger will increase the speed
slightly (at most 10%% for 128MB in tests) at the cost of using more
memory. It can be set smaller if you are tight on memory.`, maxChunkSize),
Default: defaultChunkSize,
memory. It can be set smaller if you are tight on memory.`, fs.SizeSuffix(maxChunkSize)),
Default: fs.SizeSuffix(defaultChunkSize),
Advanced: true,
}, {
Name: "impersonate",
@@ -213,8 +213,8 @@ func shouldRetry(err error) (bool, error) {
}
return true, err
}
// Keep old behavior for backward compatibility
if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") || baseErrString == "" {
// Keep old behaviour for backward compatibility
if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") {
return true, err
}
return fserrors.ShouldRetry(err), err
@@ -239,7 +239,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
return
}
// NewFs constructs an Fs from the path, container:path
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
+2 -16
View File
@@ -166,7 +166,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
f.poolMu.Unlock()
}
// NewFs constructs an Fs from the path, container:path
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
// Parse config into Options struct
@@ -646,21 +646,7 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
// Close the FTP reader and return the connection to the pool
func (f *ftpReadCloser) Close() error {
var err error
errchan := make(chan error, 1)
go func() {
errchan <- f.rc.Close()
}()
// Wait for Close for up to 60 seconds
timer := time.NewTimer(60 * time.Second)
select {
case err = <-errchan:
timer.Stop()
case <-timer.C:
// if timer fired assume no error but connection dead
fs.Errorf(f.f, "Timeout when waiting for connection Close")
return nil
}
err := f.rc.Close()
// if errors while reading or closing, dump the connection
if err != nil || f.err != nil {
_ = f.c.Quit()
@@ -162,36 +162,21 @@ func init() {
}, {
Value: "asia-east1",
Help: "Taiwan.",
}, {
Value: "asia-east2",
Help: "Hong Kong.",
}, {
Value: "asia-northeast1",
Help: "Tokyo.",
}, {
Value: "asia-south1",
Help: "Mumbai.",
}, {
Value: "asia-southeast1",
Help: "Singapore.",
}, {
Value: "australia-southeast1",
Help: "Sydney.",
}, {
Value: "europe-north1",
Help: "Finland.",
}, {
Value: "europe-west1",
Help: "Belgium.",
}, {
Value: "europe-west2",
Help: "London.",
}, {
Value: "europe-west3",
Help: "Frankfurt.",
}, {
Value: "europe-west4",
Help: "Netherlands.",
}, {
Value: "us-central1",
Help: "Iowa.",
@@ -204,9 +189,6 @@ func init() {
}, {
Value: "us-west1",
Help: "Oregon.",
}, {
Value: "us-west2",
Help: "California.",
}},
}, {
Name: "storage_class",
@@ -300,7 +282,7 @@ func (f *Fs) Features() *fs.Features {
return f.features
}
// shouldRetry determines whether a given err rates being retried
// shouldRetry determines whehter a given err rates being retried
func shouldRetry(err error) (again bool, errOut error) {
again = false
if err != nil {
@@ -348,7 +330,7 @@ func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
}
// NewFs constructs an Fs from the path, bucket:path
// NewFs contstructs an Fs from the path, bucket:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
var oAuthClient *http.Client
+1 -4
View File
@@ -40,9 +40,6 @@ func init() {
Examples: []fs.OptionExample{{
Value: "https://example.com",
Help: "Connect to example.com",
}, {
Value: "https://user:pass@example.com",
Help: "Connect to example.com using a username and password",
}},
}},
}
@@ -251,7 +248,7 @@ func parseName(base *url.URL, name string) (string, error) {
}
// calculate the name relative to the base
name = u.Path[len(base.Path):]
// mustn't be empty
// musn't be empty
if name == "" {
return "", errNameIsEmpty
}
+1 -5
View File
@@ -9,10 +9,8 @@ package hubic
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
"github.com/ncw/rclone/backend/swift"
@@ -126,9 +124,7 @@ func (f *Fs) getCredentials() (err error) {
}
defer fs.CheckClose(resp.Body, &err)
if resp.StatusCode < 200 || resp.StatusCode > 299 {
body, _ := ioutil.ReadAll(resp.Body)
bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
return errors.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
return errors.Errorf("failed to get credentials: %s", resp.Status)
}
decoder := json.NewDecoder(resp.Body)
var result credentials
-49
View File
@@ -9,10 +9,7 @@ import (
)
const (
// default time format for almost all request and responses
timeFormat = "2006-01-02-T15:04:05Z0700"
// the API server seems to use a different format
apiTimeFormat = "2006-01-02T15:04:05Z07:00"
)
// Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
@@ -43,9 +40,6 @@ func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
// Return Time string in Jottacloud format
func (t Time) String() string { return time.Time(t).Format(timeFormat) }
// APIString returns Time string in Jottacloud API format
func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
// Flag is a hacky type for checking if an attribute is present
type Flag bool
@@ -64,15 +58,6 @@ func (f *Flag) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
return attr, errors.New("unimplemented")
}
// TokenJSON is the struct representing the HTTP response from OAuth2
// providers returning a token in JSON form.
type TokenJSON struct {
AccessToken string `json:"access_token"`
TokenType string `json:"token_type"`
RefreshToken string `json:"refresh_token"`
ExpiresIn int32 `json:"expires_in"` // at least PayPal returns string, while most return number
}
/*
GET http://www.jottacloud.com/JFS/<account>
@@ -280,37 +265,3 @@ func (e *Error) Error() string {
}
return out
}
// AllocateFileRequest to prepare an upload to Jottacloud
type AllocateFileRequest struct {
Bytes int64 `json:"bytes"`
Created string `json:"created"`
Md5 string `json:"md5"`
Modified string `json:"modified"`
Path string `json:"path"`
}
// AllocateFileResponse for upload requests
type AllocateFileResponse struct {
Name string `json:"name"`
Path string `json:"path"`
State string `json:"state"`
UploadID string `json:"upload_id"`
UploadURL string `json:"upload_url"`
Bytes int64 `json:"bytes"`
ResumePos int64 `json:"resume_pos"`
}
// UploadResponse after an upload
type UploadResponse struct {
Name string `json:"name"`
Path string `json:"path"`
Kind string `json:"kind"`
ContentID string `json:"content_id"`
Bytes int64 `json:"bytes"`
Md5 string `json:"md5"`
Created int64 `json:"created"`
Modified int64 `json:"modified"`
Deleted interface{} `json:"deleted"`
Mime string `json:"mime"`
}
+84 -228
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
@@ -27,41 +26,22 @@ import (
"github.com/ncw/rclone/fs/fshttp"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/fs/walk"
"github.com/ncw/rclone/lib/oauthutil"
"github.com/ncw/rclone/lib/pacer"
"github.com/ncw/rclone/lib/rest"
"github.com/pkg/errors"
"golang.org/x/oauth2"
)
// Globals
const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
defaultDevice = "Jotta"
defaultMountpoint = "Sync" // nolint
rootURL = "https://www.jottacloud.com/jfs/"
apiURL = "https://api.jottacloud.com/files/v1/"
baseURL = "https://www.jottacloud.com/"
tokenURL = "https://api.jottacloud.com/auth/v1/token"
cachePrefix = "rclone-jcmd5-"
rcloneClientID = "nibfk8biu12ju7hpqomr8b1e40"
rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
configUsername = "user"
)
var (
// Description of how to auth for this app for a personal account
oauthConfig = &oauth2.Config{
Endpoint: oauth2.Endpoint{
AuthURL: tokenURL,
TokenURL: tokenURL,
},
ClientID: rcloneClientID,
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
RedirectURL: oauthutil.RedirectLocalhostURL,
}
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
defaultDevice = "Jotta"
defaultMountpoint = "Sync"
rootURL = "https://www.jottacloud.com/jfs/"
apiURL = "https://api.jottacloud.com"
shareURL = "https://www.jottacloud.com/"
cachePrefix = "rclone-jcmd5-"
)
// Register with Fs
@@ -70,71 +50,13 @@ func init() {
Name: "jottacloud",
Description: "JottaCloud",
NewFs: NewFs,
Config: func(name string, m configmap.Mapper) {
tokenString, ok := m.Get("token")
if ok && tokenString != "" {
fmt.Printf("Already have a token - refresh?\n")
if !config.Confirm() {
return
}
}
username, ok := m.Get(configUsername)
if !ok {
log.Fatalf("No username defined")
}
password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
// prepare out token request with username and password
srv := rest.NewClient(fshttp.NewClient(fs.Config))
values := url.Values{}
values.Set("grant_type", "PASSWORD")
values.Set("password", password)
values.Set("username", username)
values.Set("client_id", oauthConfig.ClientID)
values.Set("client_secret", oauthConfig.ClientSecret)
opts := rest.Opts{
Method: "POST",
RootURL: oauthConfig.Endpoint.AuthURL,
ContentType: "application/x-www-form-urlencoded",
Parameters: values,
}
var jsonToken api.TokenJSON
resp, err := srv.CallJSON(&opts, nil, &jsonToken)
if err != nil {
// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
if resp != nil {
if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
fmt.Printf("Enter verification code> ")
authCode := config.ReadLine()
authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
opts.ExtraHeaders = make(map[string]string)
opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
resp, err = srv.CallJSON(&opts, nil, &jsonToken)
}
}
if err != nil {
log.Fatalf("Failed to get resource token: %v", err)
}
}
var token oauth2.Token
token.AccessToken = jsonToken.AccessToken
token.RefreshToken = jsonToken.RefreshToken
token.TokenType = jsonToken.TokenType
token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
// finally save them in the config
err = oauthutil.PutToken(name, m, &token, true)
if err != nil {
log.Fatalf("Error while setting token: %s", err)
}
},
Options: []fs.Option{{
Name: configUsername,
Help: "User Name:",
Name: "user",
Help: "User Name",
}, {
Name: "pass",
Help: "Password.",
IsPassword: true,
}, {
Name: "mountpoint",
Help: "The mountpoint to use.",
@@ -161,11 +83,6 @@ func init() {
Help: "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
Default: false,
Advanced: true,
}, {
Name: "upload_resume_limit",
Help: "Files bigger than this can be resumed if the upload fail's.",
Default: fs.SizeSuffix(10 * 1024 * 1024),
Advanced: true,
}},
})
}
@@ -173,25 +90,23 @@ func init() {
// Options defines the configuration for this backend
type Options struct {
User string `config:"user"`
Pass string `config:"pass"`
Mountpoint string `config:"mountpoint"`
MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
HardDelete bool `config:"hard_delete"`
Unlink bool `config:"unlink"`
UploadThreshold fs.SizeSuffix `config:"upload_resume_limit"`
}
// Fs represents a remote jottacloud
type Fs struct {
name string
root string
user string
opt Options
features *fs.Features
endpointURL string
srv *rest.Client
apiSrv *rest.Client
pacer *pacer.Pacer
tokenRenewer *oauthutil.Renew // renew the token on expiry
name string
root string
user string
opt Options
features *fs.Features
endpointURL string
srv *rest.Client
pacer *pacer.Pacer
}
// Object describes a jottacloud object
@@ -346,29 +261,6 @@ func (o *Object) filePath() string {
return o.fs.filePath(o.remote)
}
// Jottacloud requires the grant_type 'refresh_token' string
// to be uppercase and throws a 400 Bad Request if we use the
// lower case used by the oauth2 module
//
// This filter catches all refresh requests, reads the body,
// changes the case and then sends it on
func grantTypeFilter(req *http.Request) {
if tokenURL == req.URL.String() {
// read the entire body
refreshBody, err := ioutil.ReadAll(req.Body)
if err != nil {
return
}
_ = req.Body.Close()
// make the refresh token upper case
refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
// set the new ReadCloser (with a dummy Close())
req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
}
}
// NewFs constructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -381,29 +273,25 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
rootIsDir := strings.HasSuffix(root, "/")
root = parsePath(root)
// the oauth client for the api servers needs
// a filter to fix the grant_type issues (see above)
baseClient := fshttp.NewClient(fs.Config)
if do, ok := baseClient.Transport.(interface {
SetRequestFilter(f func(req *http.Request))
}); ok {
do.SetRequestFilter(grantTypeFilter)
} else {
fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
}
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
if err != nil {
return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
user := config.FileGet(name, "user")
pass := config.FileGet(name, "pass")
if opt.Pass != "" {
var err error
opt.Pass, err = obscure.Reveal(opt.Pass)
if err != nil {
return nil, errors.Wrap(err, "couldn't decrypt password")
}
}
f := &Fs{
name: name,
root: root,
user: opt.User,
opt: *opt,
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
name: name,
root: root,
user: opt.User,
opt: *opt,
//endpointURL: rest.URLPathEscape(path.Join(user, defaultDevice, opt.Mountpoint)),
srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL),
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
}
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -411,13 +299,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
ReadMimeType: true,
WriteMimeType: true,
}).Fill(f)
f.srv.SetErrorHandler(errorHandler)
// Renew the token in the background
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
_, err := f.readMetaDataForPath("")
return err
})
if user == "" || pass == "" {
return nil, errors.New("jottacloud needs user and password")
}
f.srv.SetUserPass(opt.User, opt.Pass)
f.srv.SetErrorHandler(errorHandler)
err = f.setEndpointURL(opt.Mountpoint)
if err != nil {
@@ -443,6 +331,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// return an error with an fs which points to the parent
return f, fs.ErrorIsFile
}
return f, nil
}
@@ -459,7 +348,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
// Set info
err = o.setMetaData(info)
} else {
err = o.readMetaData(false) // reads info and meta, returning an error
err = o.readMetaData() // reads info and meta, returning an error
}
if err != nil {
return nil, err
@@ -507,7 +396,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) {
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
//fmt.Printf("List: %s\n", f.filePath(dir))
//fmt.Printf("List: %s\n", dir)
opts := rest.Opts{
Method: "GET",
Path: f.filePath(dir),
@@ -769,7 +658,7 @@ func (f *Fs) Purge() error {
return f.purgeCheck("", false)
}
// copyOrMoves copies or moves directories or files depending on the method parameter
// copyOrMoves copys or moves directories or files depending on the mthod parameter
func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) {
opts := rest.Opts{
Method: "POST",
@@ -787,6 +676,7 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro
if err != nil {
return nil, err
}
return info, nil
}
@@ -934,7 +824,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
if result.PublicSharePath == "" {
return "", errors.New("couldn't create public link - no link path received")
}
link = path.Join(baseURL, result.PublicSharePath)
link = path.Join(shareURL, result.PublicSharePath)
return link, nil
}
@@ -990,7 +880,7 @@ func (o *Object) Hash(t hash.Type) (string, error) {
// Size returns the size of an object in bytes
func (o *Object) Size() int64 {
err := o.readMetaData(false)
err := o.readMetaData()
if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err)
return 0
@@ -1006,24 +896,21 @@ func (o *Object) MimeType() string {
// setMetaData sets the metadata from info
func (o *Object) setMetaData(info *api.JottaFile) (err error) {
o.hasMetaData = true
o.size = info.Size
o.size = int64(info.Size)
o.md5 = info.MD5
o.mimeType = info.MimeType
o.modTime = time.Time(info.ModifiedAt)
return nil
}
func (o *Object) readMetaData(force bool) (err error) {
if o.hasMetaData && !force {
func (o *Object) readMetaData() (err error) {
if o.hasMetaData {
return nil
}
info, err := o.fs.readMetaDataForPath(o.remote)
if err != nil {
return err
}
if info.Deleted {
return fs.ErrorObjectNotFound
}
return o.setMetaData(info)
}
@@ -1032,7 +919,7 @@ func (o *Object) readMetaData(force bool) (err error) {
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime() time.Time {
err := o.readMetaData(false)
err := o.readMetaData()
if err != nil {
fs.Logf(o, "Failed to read metadata: %v", err)
return time.Now()
@@ -1080,7 +967,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
// we need a MD5
md5Hasher := md5.New()
// use the teeReader to write to the local file AND calculate the MD5 while doing so
// use the teeReader to write to the local file AND caclulate the MD5 while doing so
teeReader := io.TeeReader(in, md5Hasher)
// nothing to clean up by default
@@ -1153,74 +1040,43 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
in = wrap(in)
}
// use the api to allocate the file first and get resume / deduplication info
var resp *http.Response
var result api.JottaFile
opts := rest.Opts{
Method: "POST",
Path: "allocate",
ExtraHeaders: make(map[string]string),
}
fileDate := api.Time(src.ModTime()).APIString()
// the allocate request
var request = api.AllocateFileRequest{
Bytes: size,
Created: fileDate,
Modified: fileDate,
Md5: md5String,
Path: path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
Method: "POST",
Path: o.filePath(),
Body: in,
ContentType: fs.MimeType(src),
ContentLength: &size,
ExtraHeaders: make(map[string]string),
Parameters: url.Values{},
}
// send it
var response api.AllocateFileResponse
opts.ExtraHeaders["JMd5"] = md5String
opts.Parameters.Set("cphash", md5String)
opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
// opts.ExtraHeaders["JCreated"] = api.Time(src.ModTime()).String()
opts.ExtraHeaders["JModified"] = api.Time(src.ModTime()).String()
// Parameters observed in other implementations
//opts.ExtraHeaders["X-Jfs-DeviceName"] = "Jotta"
//opts.ExtraHeaders["X-Jfs-Devicename-Base64"] = ""
//opts.ExtraHeaders["X-Jftp-Version"] = "2.4" this appears to be the current version
//opts.ExtraHeaders["jx_csid"] = ""
//opts.ExtraHeaders["jx_lisence"] = ""
opts.Parameters.Set("umode", "nomultipart")
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response)
resp, err = o.fs.srv.CallXML(&opts, nil, &result)
return shouldRetry(resp, err)
})
if err != nil {
return err
}
// If the file state is INCOMPLETE and CORRPUT, try to upload a then
if response.State != "COMPLETED" {
// how much do we still have to upload?
remainingBytes := size - response.ResumePos
opts = rest.Opts{
Method: "POST",
RootURL: response.UploadURL,
ContentLength: &remainingBytes,
ContentType: "application/octet-stream",
Body: in,
ExtraHeaders: make(map[string]string),
}
if response.ResumePos != 0 {
opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
}
// copy the already uploaded bytes into the trash :)
var result api.UploadResponse
_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
if err != nil {
return err
}
// send the remaining bytes
resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result)
if err != nil {
return err
}
// finally update the meta data
o.hasMetaData = true
o.size = result.Bytes
o.md5 = result.Md5
o.modTime = time.Unix(result.Modified/1000, 0)
} else {
// If the file state is COMPLETE we don't need to upload it because the file was allready found but we still ned to update our metadata
return o.readMetaData(true)
}
return nil
// TODO: Check returned Metadata? Timeout on big uploads?
return o.setMetaData(&result)
}
// Remove an object
+1 -1
View File
@@ -2,7 +2,7 @@
Translate file names for JottaCloud adapted from OneDrive
The following characters are JottaCloud reserved characters, and can't
The following characters are JottaClous reserved characters, and can't
be used in JottaCloud folder and file names.
jottacloud = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"
+1 -1
View File
@@ -16,7 +16,7 @@ func (f *Fs) About() (*fs.Usage, error) {
if err != nil {
return nil, errors.Wrap(err, "failed to read disk usage")
}
bs := int64(s.Bsize) // nolint: unconvert
bs := int64(s.Bsize)
usage := &fs.Usage{
Total: fs.NewUsageValue(bs * int64(s.Blocks)), // quota of bytes that can be used
Used: fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use
-20
View File
@@ -1,20 +0,0 @@
// +build windows plan9
package local
import (
"time"
)
const haveLChtimes = false
// lChtimes changes the access and modification times of the named
// link, similar to the Unix utime() or utimes() functions.
//
// The underlying filesystem may truncate or round the values to a
// less precise time unit.
// If there is an error, it will be of type *PathError.
func lChtimes(name string, atime time.Time, mtime time.Time) error {
// Does nothing
return nil
}
-28
View File
@@ -1,28 +0,0 @@
// +build !windows,!plan9
package local
import (
"os"
"time"
"golang.org/x/sys/unix"
)
const haveLChtimes = true
// lChtimes changes the access and modification times of the named
// link, similar to the Unix utime() or utimes() functions.
//
// The underlying filesystem may truncate or round the values to a
// less precise time unit.
// If there is an error, it will be of type *PathError.
func lChtimes(name string, atime time.Time, mtime time.Time) error {
var utimes [2]unix.Timespec
utimes[0] = unix.NsecToTimespec(atime.UnixNano())
utimes[1] = unix.NsecToTimespec(mtime.UnixNano())
if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); e != nil {
return &os.PathError{Op: "lchtimes", Path: name, Err: e}
}
return nil
}
+33 -159
View File
@@ -2,7 +2,6 @@
package local
import (
"bytes"
"fmt"
"io"
"io/ioutil"
@@ -22,14 +21,12 @@ import (
"github.com/ncw/rclone/fs/config/configstruct"
"github.com/ncw/rclone/fs/fserrors"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/lib/file"
"github.com/ncw/rclone/lib/readers"
"github.com/pkg/errors"
)
// Constants
const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
const linkSuffix = ".rclonelink" // The suffix added to a translated symbolic link
// Register with Fs
func init() {
@@ -51,13 +48,6 @@ func init() {
NoPrefix: true,
ShortOpt: "L",
Advanced: true,
}, {
Name: "links",
Help: "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension",
Default: false,
NoPrefix: true,
ShortOpt: "l",
Advanced: true,
}, {
Name: "skip_links",
Help: `Don't warn about skipped symlinks.
@@ -102,13 +92,12 @@ check can be disabled with this flag.`,
// Options defines the configuration for this backend
type Options struct {
FollowSymlinks bool `config:"copy_links"`
TranslateSymlinks bool `config:"links"`
SkipSymlinks bool `config:"skip_links"`
NoUTFNorm bool `config:"no_unicode_normalization"`
NoCheckUpdated bool `config:"no_check_updated"`
NoUNC bool `config:"nounc"`
OneFileSystem bool `config:"one_file_system"`
FollowSymlinks bool `config:"copy_links"`
SkipSymlinks bool `config:"skip_links"`
NoUTFNorm bool `config:"no_unicode_normalization"`
NoCheckUpdated bool `config:"no_check_updated"`
NoUNC bool `config:"nounc"`
OneFileSystem bool `config:"one_file_system"`
}
// Fs represents a local filesystem rooted at root
@@ -130,20 +119,17 @@ type Fs struct {
// Object represents a local filesystem object
type Object struct {
fs *Fs // The Fs this object is part of
remote string // The remote path - properly UTF-8 encoded - for rclone
path string // The local path - may not be properly UTF-8 encoded - for OS
size int64 // file metadata - always present
mode os.FileMode
modTime time.Time
hashes map[hash.Type]string // Hashes
translatedLink bool // Is this object a translated link
fs *Fs // The Fs this object is part of
remote string // The remote path - properly UTF-8 encoded - for rclone
path string // The local path - may not be properly UTF-8 encoded - for OS
size int64 // file metadata - always present
mode os.FileMode
modTime time.Time
hashes map[hash.Type]string // Hashes
}
// ------------------------------------------------------------
var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
// NewFs constructs an Fs from the path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -152,9 +138,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
if opt.TranslateSymlinks && opt.FollowSymlinks {
return nil, errLinksAndCopyLinks
}
if opt.NoUTFNorm {
fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
@@ -182,7 +165,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err == nil {
f.dev = readDevice(fi, f.opt.OneFileSystem)
}
if err == nil && f.isRegular(fi.Mode()) {
if err == nil && fi.Mode().IsRegular() {
// It is a file, so use the parent as the root
f.root = filepath.Dir(f.root)
// return an error with an fs which points to the parent
@@ -191,20 +174,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
return f, nil
}
// Determine whether a file is a 'regular' file,
// Symlinks are regular files, only if the TranslateSymlink
// option is in-effect
func (f *Fs) isRegular(mode os.FileMode) bool {
if !f.opt.TranslateSymlinks {
return mode.IsRegular()
}
// fi.Mode().IsRegular() tests that all mode bits are zero
// Since symlinks are accepted, test that all other bits are zero,
// except the symlink bit
return mode&os.ModeType&^os.ModeSymlink == 0
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
@@ -225,48 +194,28 @@ func (f *Fs) Features() *fs.Features {
return f.features
}
// caseInsensitive returns whether the remote is case insensitive or not
// caseInsenstive returns whether the remote is case insensitive or not
func (f *Fs) caseInsensitive() bool {
// FIXME not entirely accurate since you can have case
// sensitive Fses on darwin and case insensitive Fses on linux.
// sensitive Fses on darwin and case insenstive Fses on linux.
// Should probably check but that would involve creating a
// file in the remote to be most accurate which probably isn't
// desirable.
return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
}
// translateLink checks whether the remote is a translated link
// and returns a new path, removing the suffix as needed,
// It also returns whether this is a translated link at all
//
// for regular files, dstPath is returned unchanged
func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
return newDstPath, isTranslatedLink
}
// newObject makes a half completed Object
//
// if dstPath is empty then it is made from remote
func (f *Fs) newObject(remote, dstPath string) *Object {
translatedLink := false
if dstPath == "" {
dstPath = f.cleanPath(filepath.Join(f.root, remote))
}
remote = f.cleanRemote(remote)
if f.opt.TranslateSymlinks {
// Possibly receive a new name for dstPath
dstPath, translatedLink = translateLink(remote, dstPath)
}
return &Object{
fs: f,
remote: remote,
path: dstPath,
translatedLink: translatedLink,
fs: f,
remote: remote,
path: dstPath,
}
}
@@ -288,11 +237,6 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
}
return nil, err
}
// Handle the odd case, that a symlink was specified by name without the link suffix
if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
return nil, fs.ErrorObjectNotFound
}
}
if o.mode.IsDir() {
return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote)
@@ -316,7 +260,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
dir = f.dirNames.Load(dir)
fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
remote := f.cleanRemote(dir)
@@ -373,10 +316,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
entries = append(entries, d)
}
} else {
// Check whether this link should be translated
if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
newRemote += linkSuffix
}
fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
if err != nil {
return nil, err
@@ -590,7 +529,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
// OK
} else if err != nil {
return nil, err
} else if !dstObj.fs.isRegular(dstObj.mode) {
} else if !dstObj.mode.IsRegular() {
// It isn't a file
return nil, errors.New("can't move file onto non-file")
}
@@ -712,13 +651,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
o.fs.objectHashesMu.Unlock()
if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
var in io.ReadCloser
if !o.translatedLink {
in, err = file.Open(o.path)
} else {
in, err = o.openTranslatedLink(0, -1)
}
in, err := os.Open(o.path)
if err != nil {
return "", errors.Wrap(err, "hash: failed to open")
}
@@ -749,12 +682,7 @@ func (o *Object) ModTime() time.Time {
// SetModTime sets the modification time of the local fs object
func (o *Object) SetModTime(modTime time.Time) error {
var err error
if o.translatedLink {
err = lChtimes(o.path, modTime, modTime)
} else {
err = os.Chtimes(o.path, modTime, modTime)
}
err := os.Chtimes(o.path, modTime, modTime)
if err != nil {
return err
}
@@ -772,7 +700,7 @@ func (o *Object) Storable() bool {
}
}
mode := o.mode
if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
if mode&os.ModeSymlink != 0 {
if !o.fs.opt.SkipSymlinks {
fs.Logf(o, "Can't follow symlink without -L/--copy-links")
}
@@ -833,16 +761,6 @@ func (file *localOpenFile) Close() (err error) {
return err
}
// Returns a ReadCloser() object that contains the contents of a symbolic link
func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
// Read the link and return the destination it as the contents of the object
linkdst, err := os.Readlink(o.path)
if err != nil {
return nil, err
}
return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
}
// Open an object for read
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
var offset, limit int64 = 0, -1
@@ -862,12 +780,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
}
}
// Handle a translated link
if o.translatedLink {
return o.openTranslatedLink(offset, limit)
}
fd, err := file.Open(o.path)
fd, err := os.Open(o.path)
if err != nil {
return
}
@@ -898,19 +811,8 @@ func (o *Object) mkdirAll() error {
return os.MkdirAll(dir, 0777)
}
type nopWriterCloser struct {
*bytes.Buffer
}
func (nwc nopWriterCloser) Close() error {
// noop
return nil
}
// Update the object from in with modTime and size
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
var out io.WriteCloser
hashes := hash.Supported
for _, option := range options {
switch x := option.(type) {
@@ -924,23 +826,15 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
return err
}
var symlinkData bytes.Buffer
// If the object is a regular file, create it.
// If it is a translated link, just read in the contents, and
// then create a symlink
if !o.translatedLink {
f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return err
}
// Pre-allocate the file for performance reasons
err = preAllocate(src.Size(), f)
if err != nil {
fs.Debugf(o, "Failed to pre-allocate: %v", err)
}
out = f
} else {
out = nopWriterCloser{&symlinkData}
out, err := os.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return err
}
// Pre-allocate the file for performance reasons
err = preAllocate(src.Size(), out)
if err != nil {
fs.Debugf(o, "Failed to pre-allocate: %v", err)
}
// Calculate the hash of the object we are reading as we go along
@@ -955,26 +849,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
if err == nil {
err = closeErr
}
if o.translatedLink {
if err == nil {
// Remove any current symlink or file, if one exists
if _, err := os.Lstat(o.path); err == nil {
if removeErr := os.Remove(o.path); removeErr != nil {
fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
return removeErr
}
}
// Use the contents for the copied object to create a symlink
err = os.Symlink(symlinkData.String(), o.path)
}
// only continue if symlink creation succeeded
if err != nil {
return err
}
}
if err != nil {
fs.Logf(o, "Removing partially written file on error: %v", err)
if removeErr := os.Remove(o.path); removeErr != nil {
+1 -115
View File
@@ -1,19 +1,13 @@
package local
import (
"io/ioutil"
"os"
"path"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config/configmap"
"github.com/ncw/rclone/fs/hash"
"github.com/ncw/rclone/fstest"
"github.com/ncw/rclone/lib/file"
"github.com/ncw/rclone/lib/readers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -44,13 +38,10 @@ func TestUpdatingCheck(t *testing.T) {
filePath := "sub dir/local test"
r.WriteFile(filePath, "content", time.Now())
fd, err := file.Open(path.Join(r.LocalName, filePath))
fd, err := os.Open(path.Join(r.LocalName, filePath))
if err != nil {
t.Fatalf("failed opening file %q: %v", filePath, err)
}
defer func() {
require.NoError(t, fd.Close())
}()
fi, err := fd.Stat()
require.NoError(t, err)
@@ -81,108 +72,3 @@ func TestUpdatingCheck(t *testing.T) {
require.NoError(t, err)
}
func TestSymlink(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
f := r.Flocal.(*Fs)
dir := f.root
// Write a file
modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
file1 := r.WriteFile("file.txt", "hello", modTime1)
// Write a symlink
modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
symlinkPath := filepath.Join(dir, "symlink.txt")
require.NoError(t, os.Symlink("file.txt", symlinkPath))
require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
// Object viewed as symlink
file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
if runtime.GOOS == "windows" {
file2.Size = 0 // symlinks are 0 length under Windows
}
// Object viewed as destination
file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
// Check with no symlink flags
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote)
// Set fs into "-L" mode
f.opt.FollowSymlinks = true
f.opt.TranslateSymlinks = false
f.lstat = os.Stat
fstest.CheckItems(t, r.Flocal, file1, file2d)
fstest.CheckItems(t, r.Fremote)
// Set fs into "-l" mode
f.opt.FollowSymlinks = false
f.opt.TranslateSymlinks = true
f.lstat = os.Lstat
fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
if haveLChtimes {
fstest.CheckItems(t, r.Flocal, file1, file2)
}
// Create a symlink
modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
if runtime.GOOS == "windows" {
file3.Size = 0 // symlinks are 0 length under Windows
}
fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
if haveLChtimes {
fstest.CheckItems(t, r.Flocal, file1, file2, file3)
}
// Check it got the correct contents
symlinkPath = filepath.Join(dir, "symlink2.txt")
fi, err := os.Lstat(symlinkPath)
require.NoError(t, err)
assert.False(t, fi.Mode().IsRegular())
linkText, err := os.Readlink(symlinkPath)
require.NoError(t, err)
assert.Equal(t, "file.txt", linkText)
// Check that NewObject gets the correct object
o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
require.NoError(t, err)
assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
if runtime.GOOS != "windows" {
assert.Equal(t, int64(8), o.Size())
}
// Check that NewObject doesn't see the non suffixed version
_, err = r.Flocal.NewObject("symlink2.txt")
require.Equal(t, fs.ErrorObjectNotFound, err)
// Check reading the object
in, err := o.Open()
require.NoError(t, err)
contents, err := ioutil.ReadAll(in)
require.NoError(t, err)
require.Equal(t, "file.txt", string(contents))
require.NoError(t, in.Close())
// Check reading the object with range
in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
require.NoError(t, err)
contents, err = ioutil.ReadAll(in)
require.NoError(t, err)
require.Equal(t, "file.txt"[2:5+1], string(contents))
require.NoError(t, in.Close())
}
func TestSymlinkError(t *testing.T) {
m := configmap.Simple{
"links": "true",
"copy_links": "true",
}
_, err := NewFs("local", "/", m)
assert.Equal(t, errLinksAndCopyLinks, err)
}
+1 -1
View File
@@ -22,5 +22,5 @@ func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
fs.Debugf(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys())
return devUnset
}
return uint64(statT.Dev) // nolint: unconvert
return uint64(statT.Dev)
}
+6 -9
View File
@@ -497,7 +497,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
// Creates from the parameters passed in a half finished Object which
// must have setMetaData called on it
//
// Returns the dirNode, object, leaf and error
// Returns the dirNode, obect, leaf and error
//
// Used to create new objects
func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
@@ -523,10 +523,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
// This will create a duplicate if we upload a new file without
// checking to see if there is one already - use Put() for that.
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
switch err {
case nil:
return existingObj, existingObj.Update(in, src, options...)
return exisitingObj, exisitingObj.Update(in, src, options...)
case fs.ErrorObjectNotFound:
// Not found so create it
return f.PutUnchecked(in, src)
@@ -847,14 +847,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
return shouldRetry(err)
})
if err != nil {
return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.GetName(), srcDir)
}
}
// rmdir (into trash) the now empty source directory
fs.Infof(srcDir, "removing empty directory")
err = f.deleteNode(srcDirNode)
if err != nil {
return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
}
}
return nil
@@ -1076,9 +1076,6 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
// The new object may have been created if an error is returned
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
size := src.Size()
if size < 0 {
return errors.New("mega backend can't upload a file of unknown length")
}
//modTime := src.ModTime()
remote := o.Remote()
@@ -1129,7 +1126,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
return errors.Wrap(err, "failed to finish upload")
}
// If the upload succeeded and the original object existed, then delete it
// If the upload succeded and the original object existed, then delete it
if o.info != nil {
err = o.fs.deleteNode(o.info)
if err != nil {
+6 -6
View File
@@ -25,7 +25,7 @@ type Error struct {
} `json:"error"`
}
// Error returns a string for the error and satisfies the error interface
// Error returns a string for the error and statistifes the error interface
func (e *Error) Error() string {
out := e.ErrorInfo.Code
if e.ErrorInfo.InnerError.Code != "" {
@@ -35,7 +35,7 @@ func (e *Error) Error() string {
return out
}
// Check Error satisfies the error interface
// Check Error statisfies the error interface
var _ error = (*Error)(nil)
// Identity represents an identity of an actor. For example, and actor
@@ -295,9 +295,9 @@ func (i *Item) GetID() string {
return i.ID
}
// GetDriveID returns a normalized ParentReference of the item
// GetDriveID returns a normalized ParentReferance of the item
func (i *Item) GetDriveID() string {
return i.GetParentReference().DriveID
return i.GetParentReferance().DriveID
}
// GetName returns a normalized Name of the item
@@ -398,8 +398,8 @@ func (i *Item) GetLastModifiedDateTime() Timestamp {
return i.LastModifiedDateTime
}
// GetParentReference returns a normalized ParentReference of the item
func (i *Item) GetParentReference() *ItemReference {
// GetParentReferance returns a normalized ParentReferance of the item
func (i *Item) GetParentReferance() *ItemReference {
if i.IsRemote() && i.ParentReference == nil {
return i.RemoteItem.ParentReference
}
+17 -12
View File
@@ -75,8 +75,9 @@ func init() {
return
}
// Stop if we are running non-interactive config
if fs.Config.AutoConfirm {
// Are we running headless?
if automatic, _ := m.Get(config.ConfigAutomatic); automatic != "" {
// Yes, okay we are done
return
}
@@ -198,7 +199,7 @@ func init() {
fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
// This does not work, YET :)
if !config.ConfirmWithConfig(m, "config_drive_ok", true) {
if !config.Confirm() {
log.Fatalf("Cancelled by user")
}
@@ -227,7 +228,7 @@ that the chunks will be buffered into memory.`,
Advanced: true,
}, {
Name: "drive_type",
Help: "The type of the drive ( " + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + " )",
Help: "The type of the drive ( personal | business | documentLibrary )",
Default: "",
Advanced: true,
}, {
@@ -324,13 +325,13 @@ var retryErrorCodes = []int{
// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried. It returns the err as a convenience
func shouldRetry(resp *http.Response, err error) (bool, error) {
authRetry := false
authRety := false
if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
authRetry = true
authRety = true
fs.Debugf(nil, "Should retry: %v", err)
}
return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
// readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
@@ -492,11 +493,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Get rootID
rootInfo, _, err := f.readMetaDataForPath("")
if err != nil || rootInfo.GetID() == "" {
if err != nil || rootInfo.ID == "" {
return nil, errors.Wrap(err, "failed to get root")
}
f.dirCache = dircache.New(root, rootInfo.GetID(), f)
f.dirCache = dircache.New(root, rootInfo.ID, f)
// Find the current root
err = f.dirCache.FindRoot(false)
@@ -1488,7 +1489,7 @@ func (o *Object) cancelUploadSession(url string) (err error) {
// uploadMultipart uploads a file using multipart upload
func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
if size <= 0 {
return nil, errors.New("unknown-sized upload not supported")
panic("size passed into uploadMultipart must be > 0")
}
// Create upload session
@@ -1535,7 +1536,7 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
// This function will set modtime after uploading, which will create a new version for the remote file
func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
panic("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
}
fs.Debugf(o, "Starting singlepart upload")
@@ -1560,6 +1561,10 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
}
}
if size == 0 {
opts.Body = nil
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(&opts, nil, &info)
if apiErr, ok := err.(*api.Error); ok {
@@ -1602,7 +1607,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
} else if size == 0 {
info, err = o.uploadSinglepart(in, size, modTime)
} else {
return errors.New("unknown-sized upload not supported")
panic("src file size must be >= 0")
}
if err != nil {
return err
+2 -2
View File
@@ -119,7 +119,7 @@ func (f *Fs) DirCacheFlush() {
f.dirCache.ResetRoot()
}
// NewFs constructs an Fs from the path, bucket:path
// NewFs contstructs an Fs from the path, bucket:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
@@ -785,7 +785,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
remote := path.Join(dir, folder.Name)
// cache the directory ID for later lookups
f.dirCache.Put(remote, folder.FolderID)
d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
d := fs.NewDir(remote, time.Unix(int64(folder.DateModified), 0)).SetID(folder.FolderID)
d.SetItems(int64(folder.ChildFolders))
entries = append(entries, d)
}
+1 -1
View File
@@ -13,7 +13,7 @@ type Error struct {
} `json:"error"`
}
// Error satisfies the error interface
// Error statisfies the error interface
func (e *Error) Error() string {
return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
}
+3 -2
View File
@@ -41,7 +41,7 @@ type Error struct {
ErrorString string `json:"error"`
}
// Error returns a string for the error and satisfies the error interface
// Error returns a string for the error and statistifes the error interface
func (e *Error) Error() string {
return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result)
}
@@ -58,7 +58,7 @@ func (e *Error) Update(err error) error {
return e
}
// Check Error satisfies the error interface
// Check Error statisfies the error interface
var _ error = (*Error)(nil)
// Item describes a folder or a file as returned by Get Folder Items and others
@@ -161,6 +161,7 @@ type UserInfo struct {
PublicLinkQuota int64 `json:"publiclinkquota"`
Email string `json:"email"`
UserID int `json:"userid"`
Result int `json:"result"`
Quota int64 `json:"quota"`
TrashRevretentionDays int `json:"trashrevretentiondays"`
Premium bool `json:"premium"`
+1 -1
View File
@@ -385,7 +385,7 @@ func fileIDtoNumber(fileID string) string {
if len(fileID) > 0 && fileID[0] == 'f' {
return fileID[1:]
}
fs.Debugf(nil, "Invalid file id %q", fileID)
fs.Debugf(nil, "Invalid filee id %q", fileID)
return fileID
}
+18 -107
View File
@@ -72,54 +72,14 @@ func init() {
Help: "Number of connection retries.",
Default: 3,
Advanced: true,
}, {
Name: "upload_cutoff",
Help: `Cutoff for switching to chunked upload
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5GB.`,
Default: defaultUploadCutoff,
Advanced: true,
}, {
Name: "chunk_size",
Help: `Chunk size to use for uploading.
When uploading files larger than upload_cutoff they will be uploaded
as multipart uploads using this chunk size.
Note that "--qingstor-upload-concurrency" chunks of this size are buffered
in memory per transfer.
If you are transferring large files over high speed links and you have
enough memory, then increasing this will speed up the transfers.`,
Default: minChunkSize,
Advanced: true,
}, {
Name: "upload_concurrency",
Help: `Concurrency for multipart uploads.
This is the number of chunks of the same file that are uploaded
concurrently.
NB if you set this to > 1 then the checksums of multpart uploads
become corrupted (the uploads themselves are not corrupted though).
If you are uploading small numbers of large file over high speed link
and these uploads do not fully utilize your bandwidth, then increasing
this may help to speed up the transfers.`,
Default: 1,
Advanced: true,
}},
})
}
// Constants
const (
listLimitSize = 1000 // Number of items to read at once
maxSizeForCopy = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
minChunkSize = fs.SizeSuffix(minMultiPartSize)
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
listLimitSize = 1000 // Number of items to read at once
maxSizeForCopy = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
)
// Globals
@@ -132,15 +92,12 @@ func timestampToTime(tp int64) time.Time {
// Options defines the configuration for this backend
type Options struct {
EnvAuth bool `config:"env_auth"`
AccessKeyID string `config:"access_key_id"`
SecretAccessKey string `config:"secret_access_key"`
Endpoint string `config:"endpoint"`
Zone string `config:"zone"`
ConnectionRetries int `config:"connection_retries"`
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
UploadConcurrency int `config:"upload_concurrency"`
EnvAuth bool `config:"env_auth"`
AccessKeyID string `config:"access_key_id"`
SecretAccessKey string `config:"secret_access_key"`
Endpoint string `config:"endpoint"`
Zone string `config:"zone"`
ConnectionRetries int `config:"connection_retries"`
}
// Fs represents a remote qingstor server
@@ -270,36 +227,6 @@ func qsServiceConnection(opt *Options) (*qs.Service, error) {
return qs.Init(cf)
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
if cs < minChunkSize {
return errors.Errorf("%s is less than %s", cs, minChunkSize)
}
return nil
}
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
}
return
}
func checkUploadCutoff(cs fs.SizeSuffix) error {
if cs > maxUploadCutoff {
return errors.Errorf("%s is greater than %s", cs, maxUploadCutoff)
}
return nil
}
func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkUploadCutoff(cs)
if err == nil {
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
}
return
}
// NewFs constructs an Fs from the path, bucket:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -308,14 +235,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
err = checkUploadChunkSize(opt.ChunkSize)
if err != nil {
return nil, errors.Wrap(err, "qingstor: chunk size")
}
err = checkUploadCutoff(opt.UploadCutoff)
if err != nil {
return nil, errors.Wrap(err, "qingstor: upload cutoff")
}
bucket, key, err := qsParsePath(root)
if err != nil {
return nil, err
@@ -449,7 +368,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
}
_, err = bucketInit.PutObject(key, &req)
if err != nil {
fs.Debugf(f, "Copy Failed, API Error: %v", err)
fs.Debugf(f, "Copied Faild, API Error: %v", err)
return nil, err
}
return f.NewObject(remote)
@@ -756,7 +675,7 @@ func (f *Fs) Mkdir(dir string) error {
}
switch *statistics.Status {
case "deleted":
fs.Debugf(f, "Wait for qingstor sync bucket status, retries: %d", retries)
fs.Debugf(f, "Wiat for qingstor sync bucket status, retries: %d", retries)
time.Sleep(time.Second * 1)
retries++
continue
@@ -875,7 +794,7 @@ func (o *Object) readMetaData() (err error) {
fs.Debugf(o, "Read metadata of key: %s", key)
resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{})
if err != nil {
fs.Debugf(o, "Read metadata failed, API Error: %v", err)
fs.Debugf(o, "Read metadata faild, API Error: %v", err)
if e, ok := err.(*qsErr.QingStorError); ok {
if e.StatusCode == http.StatusNotFound {
return fs.ErrorObjectNotFound
@@ -994,24 +913,16 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
mimeType := fs.MimeType(src)
req := uploadInput{
body: in,
qsSvc: o.fs.svc,
bucket: o.fs.bucket,
zone: o.fs.zone,
key: key,
mimeType: mimeType,
partSize: int64(o.fs.opt.ChunkSize),
concurrency: o.fs.opt.UploadConcurrency,
body: in,
qsSvc: o.fs.svc,
bucket: o.fs.bucket,
zone: o.fs.zone,
key: key,
mimeType: mimeType,
}
uploader := newUploader(&req)
size := src.Size()
multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
if multipart {
err = uploader.upload()
} else {
err = uploader.singlePartUpload(in, size)
}
err = uploader.upload()
if err != nil {
return err
}
+3 -16
View File
@@ -2,12 +2,12 @@
// +build !plan9
package qingstor
package qingstor_test
import (
"testing"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/backend/qingstor"
"github.com/ncw/rclone/fstest/fstests"
)
@@ -15,19 +15,6 @@ import (
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestQingStor:",
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: minChunkSize,
},
NilObject: (*qingstor.Object)(nil),
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadChunkSize(cs)
}
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setUploadCutoff(cs)
}
var _ fstests.SetUploadChunkSizer = (*Fs)(nil)
+13 -21
View File
@@ -143,7 +143,7 @@ func (u *uploader) init() {
// Try to adjust partSize if it is too small and account for
// integer division truncation.
if u.totalSize/u.cfg.partSize >= u.cfg.partSize {
if u.totalSize/u.cfg.partSize >= int64(u.cfg.partSize) {
// Add one to the part size to account for remainders
// during the size calculation. e.g odd number of bytes.
u.cfg.partSize = (u.totalSize / int64(u.cfg.maxUploadParts)) + 1
@@ -152,18 +152,18 @@ func (u *uploader) init() {
}
// singlePartUpload upload a single object that contentLength less than "defaultUploadPartSize"
func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
func (u *uploader) singlePartUpload(buf io.ReadSeeker) error {
bucketInit, _ := u.bucketInit()
req := qs.PutObjectInput{
ContentLength: &size,
ContentLength: &u.readerPos,
ContentType: &u.cfg.mimeType,
Body: buf,
}
_, err := bucketInit.PutObject(u.cfg.key, &req)
if err == nil {
fs.Debugf(u, "Upload single object finished")
fs.Debugf(u, "Upload single objcet finished")
}
return err
}
@@ -179,13 +179,13 @@ func (u *uploader) upload() error {
// Do one read to determine if we have more than one part
reader, _, err := u.nextReader()
if err == io.EOF { // single part
fs.Debugf(u, "Uploading as single part object to QingStor")
return u.singlePartUpload(reader, u.readerPos)
fs.Debugf(u, "Tried to upload a singile object to QingStor")
return u.singlePartUpload(reader)
} else if err != nil {
return errors.Errorf("read upload data failed: %s", err)
}
fs.Debugf(u, "Uploading as multi-part object to QingStor")
fs.Debugf(u, "Treied to upload a multi-part object to QingStor")
mu := multiUploader{uploader: u}
return mu.multiPartUpload(reader)
}
@@ -261,7 +261,7 @@ func (mu *multiUploader) initiate() error {
req := qs.InitiateMultipartUploadInput{
ContentType: &mu.cfg.mimeType,
}
fs.Debugf(mu, "Initiating a multi-part upload")
fs.Debugf(mu, "Tried to initiate a multi-part upload")
rsp, err := bucketInit.InitiateMultipartUpload(mu.cfg.key, &req)
if err == nil {
mu.uploadID = rsp.UploadID
@@ -279,12 +279,12 @@ func (mu *multiUploader) send(c chunk) error {
ContentLength: &c.size,
Body: c.buffer,
}
fs.Debugf(mu, "Uploading a part to QingStor with partNumber %d and partSize %d", c.partNumber, c.size)
fs.Debugf(mu, "Tried to upload a part to QingStor that partNumber %d and partSize %d", c.partNumber, c.size)
_, err := bucketInit.UploadMultipart(mu.cfg.key, &req)
if err != nil {
return err
}
fs.Debugf(mu, "Done uploading part partNumber %d and partSize %d", c.partNumber, c.size)
fs.Debugf(mu, "Upload part finished that partNumber %d and partSize %d", c.partNumber, c.size)
mu.mtx.Lock()
defer mu.mtx.Unlock()
@@ -304,7 +304,7 @@ func (mu *multiUploader) list() error {
req := qs.ListMultipartInput{
UploadID: mu.uploadID,
}
fs.Debugf(mu, "Reading multi-part details")
fs.Debugf(mu, "Tried to list a multi-part")
rsp, err := bucketInit.ListMultipart(mu.cfg.key, &req)
if err == nil {
mu.objectParts = rsp.ObjectParts
@@ -331,7 +331,7 @@ func (mu *multiUploader) complete() error {
ObjectParts: mu.objectParts,
ETag: &md5String,
}
fs.Debugf(mu, "Completing multi-part object")
fs.Debugf(mu, "Tried to complete a multi-part")
_, err = bucketInit.CompleteMultipartUpload(mu.cfg.key, &req)
if err == nil {
fs.Debugf(mu, "Complete multi-part finished")
@@ -348,7 +348,7 @@ func (mu *multiUploader) abort() error {
req := qs.AbortMultipartUploadInput{
UploadID: uploadID,
}
fs.Debugf(mu, "Aborting multi-part object %q", *uploadID)
fs.Debugf(mu, "Tried to abort a multi-part")
_, err = bucketInit.AbortMultipartUpload(mu.cfg.key, &req)
}
@@ -392,14 +392,6 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) error {
var nextChunkLen int
reader, nextChunkLen, err = mu.nextReader()
if err != nil && err != io.EOF {
// empty ch
go func() {
for range ch {
}
}()
// Wait for all goroutines finish
close(ch)
mu.wg.Wait()
return err
}
if nextChunkLen == 0 && partNumber > 0 {
+35 -222
View File
@@ -53,7 +53,7 @@ import (
func init() {
fs.Register(&fs.RegInfo{
Name: "s3",
Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)",
NewFs: NewFs,
Options: []fs.Option{{
Name: fs.ConfigProvider,
@@ -61,9 +61,6 @@ func init() {
Examples: []fs.OptionExample{{
Value: "AWS",
Help: "Amazon Web Services (AWS) S3",
}, {
Value: "Alibaba",
Help: "Alibaba Cloud Object Storage System (OSS) formerly Aliyun",
}, {
Value: "Ceph",
Help: "Ceph Object Storage",
@@ -79,9 +76,6 @@ func init() {
}, {
Value: "Minio",
Help: "Minio Object Storage",
}, {
Value: "Netease",
Help: "Netease Object Storage (NOS)",
}, {
Value: "Wasabi",
Help: "Wasabi Object Storage",
@@ -131,9 +125,6 @@ func init() {
}, {
Value: "eu-west-2",
Help: "EU (London) Region\nNeeds location constraint eu-west-2.",
}, {
Value: "eu-north-1",
Help: "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
}, {
Value: "eu-central-1",
Help: "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -159,7 +150,7 @@ func init() {
}, {
Name: "region",
Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
Provider: "!AWS,Alibaba",
Provider: "!AWS",
Examples: []fs.OptionExample{{
Value: "",
Help: "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -237,10 +228,10 @@ func init() {
Help: "EU Cross Region Amsterdam Private Endpoint",
}, {
Value: "s3.eu-gb.objectstorage.softlayer.net",
Help: "Great Britain Endpoint",
Help: "Great Britan Endpoint",
}, {
Value: "s3.eu-gb.objectstorage.service.networklayer.com",
Help: "Great Britain Private Endpoint",
Help: "Great Britan Private Endpoint",
}, {
Value: "s3.ap-geo.objectstorage.softlayer.net",
Help: "APAC Cross Regional Endpoint",
@@ -278,75 +269,12 @@ func init() {
Value: "s3.tor01.objectstorage.service.networklayer.com",
Help: "Toronto Single Site Private Endpoint",
}},
}, {
// oss endpoints: https://help.aliyun.com/document_detail/31837.html
Name: "endpoint",
Help: "Endpoint for OSS API.",
Provider: "Alibaba",
Examples: []fs.OptionExample{{
Value: "oss-cn-hangzhou.aliyuncs.com",
Help: "East China 1 (Hangzhou)",
}, {
Value: "oss-cn-shanghai.aliyuncs.com",
Help: "East China 2 (Shanghai)",
}, {
Value: "oss-cn-qingdao.aliyuncs.com",
Help: "North China 1 (Qingdao)",
}, {
Value: "oss-cn-beijing.aliyuncs.com",
Help: "North China 2 (Beijing)",
}, {
Value: "oss-cn-zhangjiakou.aliyuncs.com",
Help: "North China 3 (Zhangjiakou)",
}, {
Value: "oss-cn-huhehaote.aliyuncs.com",
Help: "North China 5 (Huhehaote)",
}, {
Value: "oss-cn-shenzhen.aliyuncs.com",
Help: "South China 1 (Shenzhen)",
}, {
Value: "oss-cn-hongkong.aliyuncs.com",
Help: "Hong Kong (Hong Kong)",
}, {
Value: "oss-us-west-1.aliyuncs.com",
Help: "US West 1 (Silicon Valley)",
}, {
Value: "oss-us-east-1.aliyuncs.com",
Help: "US East 1 (Virginia)",
}, {
Value: "oss-ap-southeast-1.aliyuncs.com",
Help: "Southeast Asia Southeast 1 (Singapore)",
}, {
Value: "oss-ap-southeast-2.aliyuncs.com",
Help: "Asia Pacific Southeast 2 (Sydney)",
}, {
Value: "oss-ap-southeast-3.aliyuncs.com",
Help: "Southeast Asia Southeast 3 (Kuala Lumpur)",
}, {
Value: "oss-ap-southeast-5.aliyuncs.com",
Help: "Asia Pacific Southeast 5 (Jakarta)",
}, {
Value: "oss-ap-northeast-1.aliyuncs.com",
Help: "Asia Pacific Northeast 1 (Japan)",
}, {
Value: "oss-ap-south-1.aliyuncs.com",
Help: "Asia Pacific South 1 (Mumbai)",
}, {
Value: "oss-eu-central-1.aliyuncs.com",
Help: "Central Europe 1 (Frankfurt)",
}, {
Value: "oss-eu-west-1.aliyuncs.com",
Help: "West Europe (London)",
}, {
Value: "oss-me-east-1.aliyuncs.com",
Help: "Middle East 1 (Dubai)",
}},
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.\nRequired when using an S3 clone.",
Provider: "!AWS,IBMCOS,Alibaba",
Provider: "!AWS,IBMCOS",
Examples: []fs.OptionExample{{
Value: "objects-us-east-1.dream.io",
Value: "objects-us-west-1.dream.io",
Help: "Dream Objects endpoint",
Provider: "Dreamhost",
}, {
@@ -395,9 +323,6 @@ func init() {
}, {
Value: "eu-west-2",
Help: "EU (London) Region.",
}, {
Value: "eu-north-1",
Help: "EU (Stockholm) Region.",
}, {
Value: "EU",
Help: "EU Region.",
@@ -450,7 +375,7 @@ func init() {
Help: "US East Region Flex",
}, {
Value: "us-south-standard",
Help: "US South Region Standard",
Help: "US Sout hRegion Standard",
}, {
Value: "us-south-vault",
Help: "US South Region Vault",
@@ -474,16 +399,16 @@ func init() {
Help: "EU Cross Region Flex",
}, {
Value: "eu-gb-standard",
Help: "Great Britain Standard",
Help: "Great Britan Standard",
}, {
Value: "eu-gb-vault",
Help: "Great Britain Vault",
Help: "Great Britan Vault",
}, {
Value: "eu-gb-cold",
Help: "Great Britain Cold",
Help: "Great Britan Cold",
}, {
Value: "eu-gb-flex",
Help: "Great Britain Flex",
Help: "Great Britan Flex",
}, {
Value: "ap-standard",
Help: "APAC Standard",
@@ -524,13 +449,11 @@ func init() {
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
Provider: "!AWS,IBMCOS,Alibaba",
Provider: "!AWS,IBMCOS",
}, {
Name: "acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Note that this ACL is applied when server side copying objects as S3
@@ -576,28 +499,6 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
Help: "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS",
Provider: "IBMCOS",
}},
}, {
Name: "bucket_acl",
Help: `Canned ACL used when creating buckets.
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Note that this ACL is applied when only when creating buckets. If it
isn't set then "acl" is used instead.`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "private",
Help: "Owner gets FULL_CONTROL. No one else has access rights (default).",
}, {
Value: "public-read",
Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
}, {
Value: "public-read-write",
Help: "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.",
}, {
Value: "authenticated-read",
Help: "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.",
}},
}, {
Name: "server_side_encryption",
Help: "The server-side encryption algorithm used when storing this object in S3.",
@@ -646,24 +547,6 @@ isn't set then "acl" is used instead.`,
Value: "GLACIER",
Help: "Glacier storage class",
}},
}, {
// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
Name: "storage_class",
Help: "The storage class to use when storing new objects in OSS.",
Provider: "Alibaba",
Examples: []fs.OptionExample{{
Value: "",
Help: "Default",
}, {
Value: "STANDARD",
Help: "Standard storage class",
}, {
Value: "GLACIER",
Help: "Archive storage mode.",
}, {
Value: "STANDARD_IA",
Help: "Infrequent access storage mode.",
}},
}, {
Name: "upload_cutoff",
Help: `Cutoff for switching to chunked upload
@@ -757,7 +640,6 @@ type Options struct {
Endpoint string `config:"endpoint"`
LocationConstraint string `config:"location_constraint"`
ACL string `config:"acl"`
BucketACL string `config:"bucket_acl"`
ServerSideEncryption string `config:"server_side_encryption"`
SSEKMSKeyID string `config:"sse_kms_key_id"`
StorageClass string `config:"storage_class"`
@@ -832,31 +714,23 @@ func (f *Fs) Features() *fs.Features {
// retryErrorCodes is a slice of error codes that we will retry
// See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
var retryErrorCodes = []int{
// 409, // Conflict - various states that could be resolved on a retry
409, // Conflict - various states that could be resolved on a retry
503, // Service Unavailable/Slow Down - "Reduce your request rate"
}
//S3 is pretty resilient, and the built in retry handling is probably sufficient
// as it should notice closed connections and timeouts which are the most likely
// sort of failure modes
func (f *Fs) shouldRetry(err error) (bool, error) {
func shouldRetry(err error) (bool, error) {
// If this is an awserr object, try and extract more useful information to determine if we should retry
if awsError, ok := err.(awserr.Error); ok {
// Simple case, check the original embedded error in case it's generically retryable
// Simple case, check the original embedded error in case it's generically retriable
if fserrors.ShouldRetry(awsError.OrigErr()) {
return true, err
}
// Failing that, if it's a RequestFailure it's probably got an http status code we can check
//Failing that, if it's a RequestFailure it's probably got an http status code we can check
if reqErr, ok := err.(awserr.RequestFailure); ok {
// 301 if wrong region for bucket
if reqErr.StatusCode() == http.StatusMovedPermanently {
urfbErr := f.updateRegionForBucket()
if urfbErr != nil {
fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
return false, err
}
return true, err
}
for _, e := range retryErrorCodes {
if reqErr.StatusCode() == e {
return true, err
@@ -864,7 +738,7 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
}
}
}
// Ok, not an awserr, check for generic failure conditions
//Ok, not an awserr, check for generic failure conditions
return fserrors.ShouldRetry(err), err
}
@@ -941,21 +815,13 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
if opt.Region == "" {
opt.Region = "us-east-1"
}
if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
opt.ForcePathStyle = false
}
awsConfig := aws.NewConfig().
WithRegion(opt.Region).
WithMaxRetries(maxRetries).
WithCredentials(cred).
WithEndpoint(opt.Endpoint).
WithHTTPClient(fshttp.NewClient(fs.Config)).
WithS3ForcePathStyle(opt.ForcePathStyle)
if opt.Region != "" {
awsConfig.WithRegion(opt.Region)
}
if opt.Endpoint != "" {
awsConfig.WithEndpoint(opt.Endpoint)
}
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
awsSessionOpts := session.Options{
Config: *awsConfig,
@@ -1038,12 +904,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, err
}
if opt.ACL == "" {
opt.ACL = "private"
}
if opt.BucketACL == "" {
opt.BucketACL = opt.ACL
}
c, ses, err := s3Connection(opt)
if err != nil {
return nil, err
@@ -1072,7 +932,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.c.HeadObject(&req)
return f.shouldRetry(err)
return shouldRetry(err)
})
if err == nil {
f.root = path.Dir(directory)
@@ -1122,51 +982,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
// Gets the bucket location
func (f *Fs) getBucketLocation() (string, error) {
req := s3.GetBucketLocationInput{
Bucket: &f.bucket,
}
var resp *s3.GetBucketLocationOutput
var err error
err = f.pacer.Call(func() (bool, error) {
resp, err = f.c.GetBucketLocation(&req)
return f.shouldRetry(err)
})
if err != nil {
return "", err
}
return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
}
// Updates the region for the bucket by reading the region from the
// bucket then updating the session.
func (f *Fs) updateRegionForBucket() error {
region, err := f.getBucketLocation()
if err != nil {
return errors.Wrap(err, "reading bucket location failed")
}
if aws.StringValue(f.c.Config.Endpoint) != "" {
return errors.Errorf("can't set region to %q as endpoint is set", region)
}
if aws.StringValue(f.c.Config.Region) == region {
return errors.Errorf("region is already %q - not updating", region)
}
// Make a new session with the new region
oldRegion := f.opt.Region
f.opt.Region = region
c, ses, err := s3Connection(&f.opt)
if err != nil {
return errors.Wrap(err, "creating new session failed")
}
f.c = c
f.ses = ses
fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
return nil
}
// listFn is called from list to handle an object.
type listFn func(remote string, object *s3.Object, isDirectory bool) error
@@ -1199,7 +1014,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
var err error
err = f.pacer.Call(func() (bool, error) {
resp, err = f.c.ListObjects(&req)
return f.shouldRetry(err)
return shouldRetry(err)
})
if err != nil {
if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1328,7 +1143,7 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
var resp *s3.ListBucketsOutput
err = f.pacer.Call(func() (bool, error) {
resp, err = f.c.ListBuckets(&req)
return f.shouldRetry(err)
return shouldRetry(err)
})
if err != nil {
return nil, err
@@ -1416,7 +1231,7 @@ func (f *Fs) dirExists() (bool, error) {
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.c.HeadBucket(&req)
return f.shouldRetry(err)
return shouldRetry(err)
})
if err == nil {
return true, nil
@@ -1447,7 +1262,7 @@ func (f *Fs) Mkdir(dir string) error {
}
req := s3.CreateBucketInput{
Bucket: &f.bucket,
ACL: &f.opt.BucketACL,
ACL: &f.opt.ACL,
}
if f.opt.LocationConstraint != "" {
req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
@@ -1456,7 +1271,7 @@ func (f *Fs) Mkdir(dir string) error {
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.c.CreateBucket(&req)
return f.shouldRetry(err)
return shouldRetry(err)
})
if err, ok := err.(awserr.Error); ok {
if err.Code() == "BucketAlreadyOwnedByYou" {
@@ -1466,7 +1281,6 @@ func (f *Fs) Mkdir(dir string) error {
if err == nil {
f.bucketOK = true
f.bucketDeleted = false
fs.Infof(f, "Bucket created with ACL %q", *req.ACL)
}
return err
}
@@ -1485,12 +1299,11 @@ func (f *Fs) Rmdir(dir string) error {
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.c.DeleteBucket(&req)
return f.shouldRetry(err)
return shouldRetry(err)
})
if err == nil {
f.bucketOK = false
f.bucketDeleted = true
fs.Infof(f, "Bucket deleted")
}
return err
}
@@ -1546,7 +1359,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.c.CopyObject(&req)
return f.shouldRetry(err)
return shouldRetry(err)
})
if err != nil {
return nil, err
@@ -1628,7 +1441,7 @@ func (o *Object) readMetaData() (err error) {
err = o.fs.pacer.Call(func() (bool, error) {
var err error
resp, err = o.fs.c.HeadObject(&req)
return o.fs.shouldRetry(err)
return shouldRetry(err)
})
if err != nil {
if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1724,7 +1537,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
}
err = o.fs.pacer.Call(func() (bool, error) {
_, err := o.fs.c.CopyObject(&req)
return o.fs.shouldRetry(err)
return shouldRetry(err)
})
return err
}
@@ -1756,7 +1569,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
err = o.fs.pacer.Call(func() (bool, error) {
var err error
resp, err = o.fs.c.GetObject(&req)
return o.fs.shouldRetry(err)
return shouldRetry(err)
})
if err, ok := err.(awserr.RequestFailure); ok {
if err.Code() == "InvalidObjectState" {
@@ -1847,7 +1660,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
}
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
_, err = uploader.Upload(&req)
return o.fs.shouldRetry(err)
return shouldRetry(err)
})
if err != nil {
return err
@@ -1903,11 +1716,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err := o.fs.srv.Do(httpReq)
if err != nil {
return o.fs.shouldRetry(err)
return shouldRetry(err)
}
body, err := rest.ReadBody(resp)
if err != nil {
return o.fs.shouldRetry(err)
return shouldRetry(err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
return false, nil
@@ -1935,7 +1748,7 @@ func (o *Object) Remove() error {
}
err := o.fs.pacer.Call(func() (bool, error) {
_, err := o.fs.c.DeleteObject(&req)
return o.fs.shouldRetry(err)
return shouldRetry(err)
})
return err
}
+7 -71
View File
@@ -66,22 +66,7 @@ func init() {
IsPassword: true,
}, {
Name: "key_file",
Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
}, {
Name: "key_file_pass",
Help: `The passphrase to decrypt the PEM-encoded private key file.
Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
in the new OpenSSH format can't be used.`,
IsPassword: true,
}, {
Name: "key_use_agent",
Help: `When set forces the usage of the ssh-agent.
When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
requested from the ssh-agent. This allows to avoid ` + "`Too many authentication failures for *username*`" + ` errors
when the ssh-agent contains many keys.`,
Default: false,
Help: "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
}, {
Name: "use_insecure_cipher",
Help: "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
@@ -137,8 +122,6 @@ type Options struct {
Port string `config:"port"`
Pass string `config:"pass"`
KeyFile string `config:"key_file"`
KeyFilePass string `config:"key_file_pass"`
KeyUseAgent bool `config:"key_use_agent"`
UseInsecureCipher bool `config:"use_insecure_cipher"`
DisableHashCheck bool `config:"disable_hashcheck"`
AskPassword bool `config:"ask_password"`
@@ -315,18 +298,6 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
f.poolMu.Unlock()
}
// shellExpand replaces a leading "~" with "${HOME}" and expands all environment
// variables afterwards.
func shellExpand(s string) string {
if s != "" {
if s[0] == '~' {
s = "${HOME}" + s[1:]
}
s = os.ExpandEnv(s)
}
return s
}
// NewFs creates a new Fs object from the name and root. It connects to
// the host specified in the config file.
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -354,9 +325,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
}
keyFile := shellExpand(opt.KeyFile)
// Add ssh agent-auth if no password or file specified
if (opt.Pass == "" && keyFile == "") || opt.KeyUseAgent {
if opt.Pass == "" && opt.KeyFile == "" {
sshAgentClient, _, err := sshagent.New()
if err != nil {
return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
@@ -365,46 +335,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
if err != nil {
return nil, errors.Wrap(err, "couldn't read ssh agent signers")
}
if keyFile != "" {
pubBytes, err := ioutil.ReadFile(keyFile + ".pub")
if err != nil {
return nil, errors.Wrap(err, "failed to read public key file")
}
pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
if err != nil {
return nil, errors.Wrap(err, "failed to parse public key file")
}
pubM := pub.Marshal()
found := false
for _, s := range signers {
if bytes.Equal(pubM, s.PublicKey().Marshal()) {
sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(s))
found = true
break
}
}
if !found {
return nil, errors.New("private key not found in the ssh-agent")
}
} else {
sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
}
sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
}
// Load key file if specified
if keyFile != "" {
key, err := ioutil.ReadFile(keyFile)
if opt.KeyFile != "" {
key, err := ioutil.ReadFile(opt.KeyFile)
if err != nil {
return nil, errors.Wrap(err, "failed to read private key file")
}
clearpass := ""
if opt.KeyFilePass != "" {
clearpass, err = obscure.Reveal(opt.KeyFilePass)
if err != nil {
return nil, err
}
}
signer, err := ssh.ParsePrivateKeyWithPassphrase(key, []byte(clearpass))
signer, err := ssh.ParsePrivateKey(key)
if err != nil {
return nil, errors.Wrap(err, "failed to parse private key file")
}
@@ -565,13 +505,9 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
// pick up the size and type of the destination, instead of the size and type of the symlink.
if !info.Mode().IsRegular() {
oldInfo := info
info, err = f.stat(remote)
if err != nil {
if !os.IsNotExist(err) {
fs.Errorf(remote, "stat of non-regular file/dir failed: %v", err)
}
info = oldInfo
return nil, errors.Wrap(err, "stat of non-regular file/dir failed")
}
}
if info.IsDir() {
+39 -56
View File
@@ -130,15 +130,6 @@ func init() {
}, {
Name: "auth_token",
Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)",
}, {
Name: "application_credential_id",
Help: "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)",
}, {
Name: "application_credential_name",
Help: "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)",
}, {
Name: "application_credential_secret",
Help: "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)",
}, {
Name: "auth_version",
Help: "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)",
@@ -182,26 +173,23 @@ provider.`,
// Options defines the configuration for this backend
type Options struct {
EnvAuth bool `config:"env_auth"`
User string `config:"user"`
Key string `config:"key"`
Auth string `config:"auth"`
UserID string `config:"user_id"`
Domain string `config:"domain"`
Tenant string `config:"tenant"`
TenantID string `config:"tenant_id"`
TenantDomain string `config:"tenant_domain"`
Region string `config:"region"`
StorageURL string `config:"storage_url"`
AuthToken string `config:"auth_token"`
AuthVersion int `config:"auth_version"`
ApplicationCredentialID string `config:"application_credential_id"`
ApplicationCredentialName string `config:"application_credential_name"`
ApplicationCredentialSecret string `config:"application_credential_secret"`
StoragePolicy string `config:"storage_policy"`
EndpointType string `config:"endpoint_type"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
NoChunk bool `config:"no_chunk"`
EnvAuth bool `config:"env_auth"`
User string `config:"user"`
Key string `config:"key"`
Auth string `config:"auth"`
UserID string `config:"user_id"`
Domain string `config:"domain"`
Tenant string `config:"tenant"`
TenantID string `config:"tenant_id"`
TenantDomain string `config:"tenant_domain"`
Region string `config:"region"`
StorageURL string `config:"storage_url"`
AuthToken string `config:"auth_token"`
AuthVersion int `config:"auth_version"`
StoragePolicy string `config:"storage_policy"`
EndpointType string `config:"endpoint_type"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
NoChunk bool `config:"no_chunk"`
}
// Fs represents a remote swift server
@@ -305,25 +293,22 @@ func parsePath(path string) (container, directory string, err error) {
func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
c := &swift.Connection{
// Keep these in the same order as the Config for ease of checking
UserName: opt.User,
ApiKey: opt.Key,
AuthUrl: opt.Auth,
UserId: opt.UserID,
Domain: opt.Domain,
Tenant: opt.Tenant,
TenantId: opt.TenantID,
TenantDomain: opt.TenantDomain,
Region: opt.Region,
StorageUrl: opt.StorageURL,
AuthToken: opt.AuthToken,
AuthVersion: opt.AuthVersion,
ApplicationCredentialId: opt.ApplicationCredentialID,
ApplicationCredentialName: opt.ApplicationCredentialName,
ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
EndpointType: swift.EndpointType(opt.EndpointType),
ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport
Transport: fshttp.NewTransport(fs.Config),
UserName: opt.User,
ApiKey: opt.Key,
AuthUrl: opt.Auth,
UserId: opt.UserID,
Domain: opt.Domain,
Tenant: opt.Tenant,
TenantId: opt.TenantID,
TenantDomain: opt.TenantDomain,
Region: opt.Region,
StorageUrl: opt.StorageURL,
AuthToken: opt.AuthToken,
AuthVersion: opt.AuthVersion,
EndpointType: swift.EndpointType(opt.EndpointType),
ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport
Transport: fshttp.NewTransport(fs.Config),
}
if opt.EnvAuth {
err := c.ApplyEnvironment()
@@ -333,13 +318,11 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
}
StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
if !c.Authenticated() {
if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret == "" {
if c.UserName == "" && c.UserId == "" {
return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
}
if c.ApiKey == "" {
return nil, errors.New("key not found")
}
if c.UserName == "" && c.UserId == "" {
return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
}
if c.ApiKey == "" {
return nil, errors.New("key not found")
}
if c.AuthUrl == "" {
return nil, errors.New("auth not found")
@@ -430,7 +413,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
return f, nil
}
// NewFs constructs an Fs from the path, container:path
// NewFs contstructs an Fs from the path, container:path
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := new(Options)
+2 -7
View File
@@ -177,8 +177,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
// At least one value will be written to the channel,
// specifying the initial value and updated values might
// follow. A 0 Duration should pause the polling.
// The ChangeNotify implementation must empty the channel
// regularly. When the channel gets closed, the implementation
// The ChangeNotify implemantion must empty the channel
// regulary. When the channel gets closed, the implemantion
// should stop polling and release resources.
func (f *Fs) ChangeNotify(fn func(string, fs.EntryType), ch <-chan time.Duration) {
var remoteChans []chan time.Duration
@@ -376,11 +376,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
}).Fill(f)
features = features.Mask(f.wr) // mask the features just on the writable fs
// Really need the union of all remotes for these, so
// re-instate and calculate separately.
features.ChangeNotify = f.ChangeNotify
features.DirCacheFlush = f.DirCacheFlush
// FIXME maybe should be masking the bools here?
// Clear ChangeNotify and DirCacheFlush if all are nil
+7 -49
View File
@@ -10,7 +10,6 @@ import (
"time"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/hash"
)
const (
@@ -66,13 +65,11 @@ type Response struct {
// Note that status collects all the status values for which we just
// check the first is OK.
type Prop struct {
Status []string `xml:"DAV: status"`
Name string `xml:"DAV: prop>displayname,omitempty"`
Type *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
IsCollection *string `xml:"DAV: prop>iscollection,omitempty"` // this is a Microsoft extension see #2716
Size int64 `xml:"DAV: prop>getcontentlength,omitempty"`
Modified Time `xml:"DAV: prop>getlastmodified,omitempty"`
Checksums []string `xml:"prop>checksums>checksum,omitempty"`
Status []string `xml:"DAV: status"`
Name string `xml:"DAV: prop>displayname,omitempty"`
Type *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
Size int64 `xml:"DAV: prop>getcontentlength,omitempty"`
Modified Time `xml:"DAV: prop>getlastmodified,omitempty"`
}
// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -98,33 +95,13 @@ func (p *Prop) StatusOK() bool {
return false
}
// Hashes returns a map of all checksums - may be nil
func (p *Prop) Hashes() (hashes map[hash.Type]string) {
if len(p.Checksums) == 0 {
return nil
}
hashes = make(map[hash.Type]string)
for _, checksums := range p.Checksums {
checksums = strings.ToLower(checksums)
for _, checksum := range strings.Split(checksums, " ") {
switch {
case strings.HasPrefix(checksum, "sha1:"):
hashes[hash.SHA1] = checksum[5:]
case strings.HasPrefix(checksum, "md5:"):
hashes[hash.MD5] = checksum[4:]
}
}
}
return hashes
}
// PropValue is a tagged name and value
type PropValue struct {
XMLName xml.Name `xml:""`
Value string `xml:",chardata"`
}
// Error is used to describe webdav errors
// Error is used to desribe webdav errors
//
// <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
// <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
@@ -137,7 +114,7 @@ type Error struct {
StatusCode int
}
// Error returns a string for the error and satisfies the error interface
// Error returns a string for the error and statistifes the error interface
func (e *Error) Error() string {
var out []string
if e.Message != "" {
@@ -210,22 +187,3 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
}
return err
}
// Quota is used to read the bytes used and available
//
// <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
// <d:response>
// <d:href>/remote.php/webdav/</d:href>
// <d:propstat>
// <d:prop>
// <d:quota-available-bytes>-3</d:quota-available-bytes>
// <d:quota-used-bytes>376461895</d:quota-used-bytes>
// </d:prop>
// <d:status>HTTP/1.1 200 OK</d:status>
// </d:propstat>
// </d:response>
// </d:multistatus>
type Quota struct {
Available int64 `xml:"DAV: response>propstat>prop>quota-available-bytes"`
Used int64 `xml:"DAV: response>propstat>prop>quota-used-bytes"`
}
+2 -2
View File
@@ -102,7 +102,7 @@ func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
spRoot, err := url.Parse(ca.endpoint)
if err != nil {
return nil, errors.Wrap(err, "Error while constructing endpoint URL")
return nil, errors.Wrap(err, "Error while contructing endpoint URL")
}
u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
@@ -121,7 +121,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
Jar: jar,
}
// Send the previously acquired Token as a Post parameter
// Send the previously aquired Token as a Post parameter
if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
}
+3
View File
@@ -2,10 +2,13 @@ package odrvcookie
import (
"time"
"github.com/ncw/rclone/lib/rest"
)
// CookieRenew holds information for the renew
type CookieRenew struct {
srv *rest.Client
timer *time.Ticker
renewFn func()
}
+21 -116
View File
@@ -2,13 +2,23 @@
// object storage system.
package webdav
// Owncloud: Getting Oc-Checksum:
// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on
// nextcloud?
// docs for file webdav
// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html
// indicates checksums can be set as metadata here
// https://github.com/nextcloud/server/issues/6129
// owncloud seems to have checksums as metadata though - can read them
// SetModTime might be possible
// https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
// ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
// For example the ownCloud WebDAV server does it that way.
import (
"bytes"
"encoding/xml"
"fmt"
"io"
@@ -106,7 +116,6 @@ type Fs struct {
canStream bool // set if can stream
useOCMtime bool // set if can use X-OC-Mtime
retryWithZeroDepth bool // some vendors (sharepoint) won't list files when Depth is 1 (our default)
hasChecksums bool // set if can use owncloud style checksums
}
// Object describes a webdav object
@@ -118,8 +127,7 @@ type Object struct {
hasMetaData bool // whether info below has been set
size int64 // size of the object
modTime time.Time // modification time of the object
sha1 string // SHA-1 of the object content if known
md5 string // MD5 of the object content if known
sha1 string // SHA-1 of the object content
}
// ------------------------------------------------------------
@@ -172,18 +180,6 @@ func itemIsDir(item *api.Response) bool {
}
fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
}
// the iscollection prop is a Microsoft extension, but if present it is a reliable indicator
// if the above check failed - see #2716. This can be an integer or a boolean - see #2964
if t := item.Props.IsCollection; t != nil {
switch x := strings.ToLower(*t); x {
case "0", "false":
return false
case "1", "true":
return true
default:
fs.Debugf(nil, "Unknown value %q for IsCollection", x)
}
}
return false
}
@@ -198,9 +194,6 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
},
NoRedirect: true,
}
if f.hasChecksums {
opts.Body = bytes.NewBuffer(owncloudProps)
}
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
@@ -256,7 +249,7 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
// addSlash makes sure s is terminated with a / if non empty
// addShlash makes sure s is terminated with a / if non empty
func addSlash(s string) string {
if s != "" && !strings.HasSuffix(s, "/") {
s += "/"
@@ -364,11 +357,9 @@ func (f *Fs) setQuirks(vendor string) error {
f.canStream = true
f.precision = time.Second
f.useOCMtime = true
f.hasChecksums = true
case "nextcloud":
f.precision = time.Second
f.useOCMtime = true
f.hasChecksums = true
case "sharepoint":
// To mount sharepoint, two Cookies are required
// They have to be set instead of BasicAuth
@@ -435,22 +426,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
return f.newObjectWithInfo(remote, nil)
}
// Read the normal props, plus the checksums
//
// <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
var owncloudProps = []byte(`<?xml version="1.0"?>
<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
<d:prop>
<d:displayname />
<d:getlastmodified />
<d:getcontentlength />
<d:resourcetype />
<d:getcontenttype />
<oc:checksums />
</d:prop>
</d:propfind>
`)
// list the objects into the function supplied
//
// If directories is set it only sends directories
@@ -470,9 +445,6 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
"Depth": depth,
},
}
if f.hasChecksums {
opts.Body = bytes.NewBuffer(owncloudProps)
}
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
@@ -656,8 +628,7 @@ func (f *Fs) mkdir(dirPath string) error {
err := f._mkdir(dirPath)
if apiErr, ok := err.(*api.Error); ok {
// already exists
// owncloud returns 423/StatusLocked if the create is already in progress
if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable {
return nil
}
// parent does not exist
@@ -876,52 +847,9 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
if f.hasChecksums {
return hash.NewHashSet(hash.MD5, hash.SHA1)
}
return hash.Set(hash.None)
}
// About gets quota information
func (f *Fs) About() (*fs.Usage, error) {
opts := rest.Opts{
Method: "PROPFIND",
Path: "",
ExtraHeaders: map[string]string{
"Depth": "0",
},
}
opts.Body = bytes.NewBuffer([]byte(`<?xml version="1.0" ?>
<D:propfind xmlns:D="DAV:">
<D:prop>
<D:quota-available-bytes/>
<D:quota-used-bytes/>
</D:prop>
</D:propfind>
`))
var q = api.Quota{
Available: -1,
Used: -1,
}
var resp *http.Response
var err error
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(&opts, nil, &q)
return shouldRetry(resp, err)
})
if err != nil {
return nil, errors.Wrap(err, "about call failed")
}
usage := &fs.Usage{}
if q.Available >= 0 && q.Used >= 0 {
usage.Total = fs.NewUsageValue(q.Available + q.Used)
}
if q.Used >= 0 {
usage.Used = fs.NewUsageValue(q.Used)
}
return usage, nil
}
// ------------------------------------------------------------
// Fs returns the parent Fs
@@ -942,17 +870,12 @@ func (o *Object) Remote() string {
return o.remote
}
// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
// Hash returns the SHA-1 of an object returning a lowercase hex string
func (o *Object) Hash(t hash.Type) (string, error) {
if o.fs.hasChecksums {
switch t {
case hash.SHA1:
return o.sha1, nil
case hash.MD5:
return o.md5, nil
}
if t != hash.SHA1 {
return "", hash.ErrUnsupported
}
return "", hash.ErrUnsupported
return o.sha1, nil
}
// Size returns the size of an object in bytes
@@ -970,11 +893,6 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
o.hasMetaData = true
o.size = info.Size
o.modTime = time.Time(info.Modified)
if o.fs.hasChecksums {
hashes := info.Hashes()
o.sha1 = hashes[hash.SHA1]
o.md5 = hashes[hash.MD5]
}
return nil
}
@@ -1054,21 +972,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
ContentType: fs.MimeType(src),
}
if o.fs.useOCMtime || o.fs.hasChecksums {
opts.ExtraHeaders = map[string]string{}
if o.fs.useOCMtime {
opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9)
}
if o.fs.hasChecksums {
// Set an upload checksum - prefer SHA1
//
// This is used as an upload integrity test. If we set
// only SHA1 here, owncloud will calculate the MD5 too.
if sha1, _ := src.Hash(hash.SHA1); sha1 != "" {
opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
} else if md5, _ := src.Hash(hash.MD5); md5 != "" {
opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
}
if o.fs.useOCMtime {
opts.ExtraHeaders = map[string]string{
"X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9),
}
}
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1112,6 +1018,5 @@ var (
_ fs.Copier = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
)
+1 -1
View File
@@ -56,7 +56,7 @@ type AsyncInfo struct {
Templated bool `json:"templated"`
}
// AsyncStatus is returned when requesting the status of an async operations. Possible values in-progress, success, failure
// AsyncStatus is returned when requesting the status of an async operations. Possble values in-progress, success, failure
type AsyncStatus struct {
Status string `json:"status"`
}
+3 -3
View File
@@ -307,7 +307,7 @@ func (f *Fs) itemToDirEntry(remote string, object *api.ResourceInfoResponse) (fs
if err != nil {
return nil, errors.Wrap(err, "error parsing time in directory item")
}
d := fs.NewDir(remote, t).SetSize(object.Size)
d := fs.NewDir(remote, t).SetSize(int64(object.Size))
return d, nil
case "file":
o, err := f.newObjectWithInfo(remote, object)
@@ -634,7 +634,7 @@ func (f *Fs) Purge() error {
return f.purgeCheck("", false)
}
// copyOrMoves copies or moves directories or files depending on the method parameter
// copyOrMoves copys or moves directories or files depending on the mthod parameter
func (f *Fs) copyOrMove(method, src, dst string, overwrite bool) (err error) {
opts := rest.Opts{
Method: "POST",
@@ -1107,7 +1107,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
return err
}
//if file uploaded successfully then return metadata
//if file uploaded sucessfully then return metadata
o.modTime = modTime
o.md5sum = "" // according to unit tests after put the md5 is empty.
o.size = int64(in1.BytesRead()) // better solution o.readMetaData() ?
-5
View File
@@ -1,5 +0,0 @@
#!/bin/bash
set -e
docker build -t rclone/xgo-cgofuse https://github.com/billziss-gh/cgofuse.git
docker images
docker push rclone/xgo-cgofuse
+13 -113
View File
@@ -8,8 +8,6 @@
package main
import (
"archive/tar"
"compress/gzip"
"encoding/json"
"flag"
"fmt"
@@ -21,7 +19,6 @@ import (
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
"time"
@@ -34,13 +31,7 @@ var (
extract = flag.String("extract", "", "Extract the named executable from the .tar.gz and install into bindir.")
bindir = flag.String("bindir", defaultBinDir(), "Directory to install files downloaded with -extract.")
// Globals
matchProject = regexp.MustCompile(`^([\w-]+)/([\w-]+)$`)
osAliases = map[string][]string{
"darwin": []string{"macos", "osx"},
}
archAliases = map[string][]string{
"amd64": []string{"x86_64"},
}
matchProject = regexp.MustCompile(`^(\w+)/(\w+)$`)
)
// A github release
@@ -122,41 +113,25 @@ func writable(path string) bool {
// Directory to install releases in by default
//
// Find writable directories on $PATH. Use $GOPATH/bin if that is on
// the path and writable or use the first writable directory which is
// in $HOME or failing that the first writable directory.
// Find writable directories on $PATH. Use the first writable
// directory which is in $HOME or failing that the first writable
// directory.
//
// Returns "" if none of the above were found
func defaultBinDir() string {
home := os.Getenv("HOME")
var (
bin string
homeBin string
goHomeBin string
gopath = os.Getenv("GOPATH")
)
var binDir string
for _, dir := range strings.Split(os.Getenv("PATH"), ":") {
if writable(dir) {
if strings.HasPrefix(dir, home) {
if homeBin != "" {
homeBin = dir
}
if gopath != "" && strings.HasPrefix(dir, gopath) && goHomeBin == "" {
goHomeBin = dir
}
return dir
}
if bin == "" {
bin = dir
if binDir != "" {
binDir = dir
}
}
}
if goHomeBin != "" {
return goHomeBin
}
if homeBin != "" {
return homeBin
}
return bin
return binDir
}
// read the body or an error message
@@ -200,8 +175,7 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
}
for _, asset := range release.Assets {
//log.Printf("Finding %s", asset.Name)
if matchName.MatchString(asset.Name) && isOurOsArch(asset.Name) {
if matchName.MatchString(asset.Name) {
return asset.BrowserDownloadURL, asset.Name
}
}
@@ -209,22 +183,6 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
return "", ""
}
// isOurOsArch returns true if s contains our OS and our Arch
func isOurOsArch(s string) bool {
s = strings.ToLower(s)
check := func(base string, aliases map[string][]string) bool {
names := []string{base}
names = append(names, aliases[base]...)
for _, name := range names {
if strings.Contains(s, name) {
return true
}
}
return false
}
return check(runtime.GOARCH, archAliases) && check(runtime.GOOS, osAliases)
}
// get a file for download
func getFile(url, fileName string) {
log.Printf("Downloading %q from %q", fileName, url)
@@ -271,66 +229,6 @@ func run(args ...string) {
}
}
// Untars fileName from srcFile
func untar(srcFile, fileName, extractDir string) {
f, err := os.Open(srcFile)
if err != nil {
log.Fatalf("Couldn't open tar: %v", err)
}
defer func() {
err := f.Close()
if err != nil {
log.Fatalf("Couldn't close tar: %v", err)
}
}()
var in io.Reader = f
srcExt := filepath.Ext(srcFile)
if srcExt == ".gz" || srcExt == ".tgz" {
gzf, err := gzip.NewReader(f)
if err != nil {
log.Fatalf("Couldn't open gzip: %v", err)
}
in = gzf
}
tarReader := tar.NewReader(in)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalf("Trouble reading tar file: %v", err)
}
name := header.Name
switch header.Typeflag {
case tar.TypeReg:
baseName := filepath.Base(name)
if baseName == fileName {
outPath := filepath.Join(extractDir, fileName)
out, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
if err != nil {
log.Fatalf("Couldn't open output file: %v", err)
}
defer func() {
err := out.Close()
if err != nil {
log.Fatalf("Couldn't close output: %v", err)
}
}()
n, err := io.Copy(out, tarReader)
if err != nil {
log.Fatalf("Couldn't write output file: %v", err)
}
log.Printf("Wrote %s (%d bytes) as %q", fileName, n, outPath)
}
}
}
}
func main() {
flag.Parse()
args := flag.Args()
@@ -359,6 +257,8 @@ func main() {
log.Fatalf("Need to set -bindir")
}
log.Printf("Unpacking %s from %s and installing into %s", *extract, fileName, *bindir)
untar(fileName, *extract, *bindir+"/")
run("tar", "xf", fileName, *extract)
run("chmod", "a+x", *extract)
run("mv", "-f", *extract, *bindir+"/")
}
}
+2 -2
View File
@@ -3,7 +3,7 @@
version="$1"
if [ "$version" = "" ]; then
echo "Syntax: $0 <version, eg v1.42> [delete]"
echo "Syntax: $0 <version> [delete]"
exit 1
fi
dry_run="--dry-run"
@@ -14,4 +14,4 @@ else
echo "Use '$0 $version delete' to actually delete files"
fi
rclone ${dry_run} -P --fast-list --checkers 16 --transfers 16 delete --include "**${version}**" memstore:beta-rclone-org
rclone ${dry_run} --fast-list -P --checkers 16 --transfers 16 delete --include "**/${version}**" memstore:beta-rclone-org
+1 -1
View File
@@ -29,7 +29,7 @@ github-release release \
--name "rclone" \
--description "Rclone - rsync for cloud storage. Sync files to and from many cloud storage providers."
for build in `ls build | grep -v current | grep -v testbuilds`; do
for build in `ls build | grep -v current`; do
echo "Uploading ${build}"
base="${build%.*}"
parts=(${base//-/ })
+7 -5
View File
@@ -51,7 +51,7 @@ var (
errorCommandNotFound = errors.New("command not found")
errorUncategorized = errors.New("uncategorized error")
errorNotEnoughArguments = errors.New("not enough arguments")
errorTooManyArguments = errors.New("too many arguments")
errorTooManyArguents = errors.New("too many arguments")
)
const (
@@ -294,12 +294,14 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) {
if len(args) < MinArgs {
_ = cmd.Usage()
_, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum: you provided %d non flag arguments: %q\n", cmd.Name(), MinArgs, len(args), args)
_, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum\n", cmd.Name(), MinArgs)
// os.Exit(1)
resolveExitCode(errorNotEnoughArguments)
} else if len(args) > MaxArgs {
_ = cmd.Usage()
_, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum: you provided %d non flag arguments: %q\n", cmd.Name(), MaxArgs, len(args), args)
resolveExitCode(errorTooManyArguments)
_, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name(), MaxArgs)
// os.Exit(1)
resolveExitCode(errorTooManyArguents)
}
}
@@ -456,7 +458,7 @@ func AddBackendFlags() {
help = help[:nl]
}
help = strings.TrimSpace(help)
flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help)
flag := pflag.CommandLine.VarPF(opt, name, string(opt.ShortOpt), help)
if _, isBool := opt.Default.(bool); isBool {
flag.NoOptDefVal = "true"
}
-14
View File
@@ -93,15 +93,6 @@ For example to make a swift remote of name myremote using auto config
you would do:
rclone config create myremote swift env_auth true
Note that if the config process would normally ask a question the
default is taken. Each time that happens rclone will print a message
saying how to affect the value taken.
So for example if you wanted to configure a Google Drive remote but
using remote authorization you would do this:
rclone config create mydrive drive config_is_local false
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(2, 256, command, args)
@@ -128,11 +119,6 @@ in pairs of <key> <value>.
For example to update the env_auth field of a remote of name myremote you would do:
rclone config update myremote swift env_auth true
If the remote uses oauth the token will be updated, if you don't
require this add an extra parameter thus:
rclone config update myremote swift env_auth true config_refresh_token false
`,
RunE: func(command *cobra.Command, args []string) error {
cmd.CheckArgs(3, 256, command, args)
-39
View File
@@ -32,47 +32,8 @@ documentation, changelog and configuration walkthroughs.
fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
atexit.Run()
},
BashCompletionFunction: bashCompletionFunc,
}
const (
bashCompletionFunc = `
__rclone_custom_func() {
if [[ ${#COMPREPLY[@]} -eq 0 ]]; then
local cur cword prev words
if declare -F _init_completion > /dev/null; then
_init_completion -n : || return
else
__rclone_init_completion -n : || return
fi
if [[ $cur =~ ^[[:alnum:]]*$ ]]; then
local remote
while IFS= read -r remote; do
[[ $remote != $cur* ]] || COMPREPLY+=("$remote")
done < <(command rclone listremotes)
if [[ ${COMPREPLY[@]} ]]; then
local paths=("$cur"*)
[[ ! -f ${paths[0]} ]] || COMPREPLY+=("${paths[@]}")
fi
elif [[ $cur =~ ^[[:alnum:]]+: ]]; then
local path=${cur#*:}
if [[ $path == */* ]]; then
local prefix=${path%/*}
else
local prefix=
fi
local line
while IFS= read -r line; do
local reply=${prefix:+$prefix/}$line
[[ $reply != $path* ]] || COMPREPLY+=("$reply")
done < <(rclone lsf "${cur%%:*}:$prefix" 2>/dev/null)
fi
[[ ! ${COMPREPLY[@]} ]] || compopt -o nospace
fi
}
`
)
// root help command
var helpCommand = &cobra.Command{
Use: "help",
+23 -100
View File
@@ -21,22 +21,11 @@ import (
"github.com/spf13/cobra"
)
type position int
const (
positionMiddle position = 1 << iota
positionLeft
positionRight
positionNone position = 0
positionAll position = positionRight<<1 - 1
)
var (
checkNormalization bool
checkControl bool
checkLength bool
checkStreaming bool
positionList = []position{positionMiddle, positionLeft, positionRight}
)
func init() {
@@ -70,7 +59,7 @@ a bit of go code for each one.
type results struct {
f fs.Fs
mu sync.Mutex
stringNeedsEscaping map[string]position
charNeedsEscaping map[rune]bool
maxFileLength int
canWriteUnnormalized bool
canReadUnnormalized bool
@@ -80,8 +69,8 @@ type results struct {
func newResults(f fs.Fs) *results {
return &results{
f: f,
stringNeedsEscaping: make(map[string]position),
f: f,
charNeedsEscaping: make(map[rune]bool),
}
}
@@ -90,13 +79,13 @@ func (r *results) Print() {
fmt.Printf("// %s\n", r.f.Name())
if checkControl {
escape := []string{}
for c, needsEscape := range r.stringNeedsEscaping {
if needsEscape != positionNone {
for c, needsEscape := range r.charNeedsEscaping {
if needsEscape {
escape = append(escape, fmt.Sprintf("0x%02X", c))
}
}
sort.Strings(escape)
fmt.Printf("stringNeedsEscaping = []byte{\n")
fmt.Printf("charNeedsEscaping = []byte{\n")
fmt.Printf("\t%s\n", strings.Join(escape, ", "))
fmt.Printf("}\n")
}
@@ -141,45 +130,20 @@ func (r *results) checkUTF8Normalization() {
}
}
func (r *results) checkStringPositions(s string) {
fs.Infof(r.f, "Writing position file 0x%0X", s)
positionError := positionNone
for _, pos := range positionList {
path := ""
switch pos {
case positionMiddle:
path = fmt.Sprintf("position-middle-%0X-%s-", s, s)
case positionLeft:
path = fmt.Sprintf("%s-position-left-%0X", s, s)
case positionRight:
path = fmt.Sprintf("position-right-%0X-%s", s, s)
default:
panic("invalid position: " + pos.String())
}
_, writeErr := r.writeFile(path)
if writeErr != nil {
fs.Infof(r.f, "Writing %s position file 0x%0X Error: %s", pos.String(), s, writeErr)
} else {
fs.Infof(r.f, "Writing %s position file 0x%0X OK", pos.String(), s)
}
obj, getErr := r.f.NewObject(path)
if getErr != nil {
fs.Infof(r.f, "Getting %s position file 0x%0X Error: %s", pos.String(), s, getErr)
} else {
if obj.Size() != 50 {
fs.Infof(r.f, "Getting %s position file 0x%0X Invalid Size: %d", pos.String(), s, obj.Size())
} else {
fs.Infof(r.f, "Getting %s position file 0x%0X OK", pos.String(), s)
}
}
if writeErr != nil || getErr != nil {
positionError += pos
}
// check we can write file with the rune passed in
func (r *results) checkChar(c rune) {
fs.Infof(r.f, "Writing file 0x%02X", c)
path := fmt.Sprintf("0x%02X-%c-", c, c)
_, err := r.writeFile(path)
escape := false
if err != nil {
fs.Infof(r.f, "Couldn't write file 0x%02X", c)
escape = true
} else {
fs.Infof(r.f, "OK writing file 0x%02X", c)
}
r.mu.Lock()
r.stringNeedsEscaping[s] = positionError
r.charNeedsEscaping[c] = escape
r.mu.Unlock()
}
@@ -193,28 +157,19 @@ func (r *results) checkControls() {
}
var wg sync.WaitGroup
for i := rune(0); i < 128; i++ {
s := string(i)
if i == 0 || i == '/' {
// We're not even going to check NULL or /
r.stringNeedsEscaping[s] = positionAll
r.charNeedsEscaping[i] = true
continue
}
wg.Add(1)
go func(s string) {
c := i
go func() {
defer wg.Done()
token := <-tokens
r.checkStringPositions(s)
r.checkChar(c)
tokens <- token
}(s)
}
for _, s := range []string{"\", "\xBF", "\xFE"} {
wg.Add(1)
go func(s string) {
defer wg.Done()
token := <-tokens
r.checkStringPositions(s)
tokens <- token
}(s)
}()
}
wg.Wait()
fs.Infof(r.f, "Done trying to create control character file names")
@@ -313,35 +268,3 @@ func readInfo(f fs.Fs) error {
r.Print()
return nil
}
func (e position) String() string {
switch e {
case positionNone:
return "none"
case positionAll:
return "all"
}
var buf bytes.Buffer
if e&positionMiddle != 0 {
buf.WriteString("middle")
e &= ^positionMiddle
}
if e&positionLeft != 0 {
if buf.Len() != 0 {
buf.WriteRune(',')
}
buf.WriteString("left")
e &= ^positionLeft
}
if e&positionRight != 0 {
if buf.Len() != 0 {
buf.WriteRune(',')
}
buf.WriteString("right")
e &= ^positionRight
}
if e != positionNone {
panic("invalid position")
}
return buf.String()
}
-40
View File
@@ -1,40 +0,0 @@
set -euo pipefail
for f in info-*.log; do
for pos in middle left right; do
egrep -oe " Writing $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.write_$pos
egrep -oe " Getting $pos position file [^ ]* \w+" $f | sort | cut -d' ' -f 7 > $f.get_$pos
done
{
echo "${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}\t${${f%.log}#info-}"
echo "Write\tWrite\tWrite\tGet\tGet\tGet"
echo "Mid\tLeft\tRight\tMid\tLeft\tRight"
paste $f.write_{middle,left,right} $f.get_{middle,left,right}
} > $f.csv
done
for f in info-*.list; do
for pos in middle left right; do
cat $f | perl -lne 'print $1 if /^\s+[0-9]+\s+(.*)/' | grep -a "position-$pos-" | sort > $f.$pos
done
{
echo "${${f%.list}#info-}\t${${f%.list}#info-}\t${${f%.list}#info-}"
echo "List\tList\tList"
echo "Mid\tLeft\tRight"
for e in 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 10 11 12 13 14 15 16 17 18 19 1A 1B 1C 1D 1E 1F 20 21 22 23 24 25 26 27 28 29 2A 2B 2C 2D 2E 30 31 32 33 34 35 36 37 38 39 3A 3B 3C 3D 3E 3F 40 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F 50 51 52 53 54 55 56 57 58 59 5A 5B 5C 5D 5E 5F 60 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6E 6F 70 71 72 73 74 75 76 77 78 79 7A 7B 7C 7D 7E 7F BF EFBCBC FE; do
echo -n $(perl -lne 'print "'$e'-$1" if /^position-middle-'$e'-(.*)-/' $f.middle | tr -d "\t\r" | grep -a . || echo Miss)
echo -n "\t"
echo -n $(perl -lne 'print "'$e'-$1" if /^(.*)-position-left-'$e'/' $f.left | tr -d "\t\r" | grep -a . || echo Miss)
echo -n "\t"
echo $(perl -lne 'print "'$e'-$1" if /^position-right-'$e'-(.*)/' $f.right | tr -d "\t\r" | grep -a . || echo Miss)
# echo -n $(grep -a "position-middle-$e-" $f.middle | tr -d "\t\r" || echo Miss)"\t"
# echo -n $(grep -a "position-left-$e" $f.left | tr -d "\t\r" || echo Miss)"\t"
# echo $(grep -a "position-right-$e-" $f.right | tr -d "\t\r" || echo Miss)
done
} > $f.csv
done
for f in info-*.list; do
paste ${f%.list}.log.csv $f.csv > ${f%.list}.full.csv
done
paste *.full.csv > info-complete.csv
-3
View File
@@ -1,3 +0,0 @@
rclone.exe purge info
rclone.exe info -vv info > info-LocalWindows.log 2>&1
rclone.exe ls -vv info > info-LocalWindows.list 2>&1
-43
View File
@@ -1,43 +0,0 @@
#!/usr/bin/env zsh
#
# example usage:
# $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh --list | \
# parallel -P20 $GOPATH/src/github.com/ncw/rclone/cmd/info/test.sh
export PATH=$GOPATH/src/github.com/ncw/rclone:$PATH
typeset -A allRemotes
allRemotes=(
TestAmazonCloudDrive '--low-level-retries=2 --checkers=5'
TestB2 ''
TestBox ''
TestDrive '--tpslimit=5'
TestCrypt ''
TestDropbox '--checkers=1'
TestJottacloud ''
TestMega ''
TestOneDrive ''
TestOpenDrive '--low-level-retries=2 --checkers=5'
TestPcloud '--low-level-retries=2 --timeout=15s'
TestS3 ''
Local ''
)
set -euo pipefail
if [[ $# -eq 0 ]]; then
set -- ${(k)allRemotes[@]}
elif [[ $1 = --list ]]; then
printf '%s\n' ${(k)allRemotes[@]}
exit 0
fi
for remote; do
dir=$remote:infotest
if [[ $remote = Local ]]; then
dir=infotest
fi
rclone purge $dir || :
rclone info -vv $dir ${=allRemotes[$remote]} &> info-$remote.log
rclone ls -vv $dir &> info-$remote.list
done
+1 -1
View File
@@ -16,7 +16,7 @@ var (
func init() {
cmd.Root.AddCommand(commandDefintion)
commandDefintion.Flags().BoolVarP(&listLong, "long", "", listLong, "Show the type as well as names.")
commandDefintion.Flags().BoolVarP(&listLong, "long", "l", listLong, "Show the type as well as names.")
}
var commandDefintion = &cobra.Command{
+1 -7
View File
@@ -60,13 +60,7 @@ If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt"
will be "subfolder/file.txt", not "remote:path/subfolder/file.txt".
When used without --recursive the Path will always be the same as Name.
The time is in RFC3339 format with up to nanosecond precision. The
number of decimal digits in the seconds will depend on the precision
that the remote can hold the times, so if times are accurate to the
nearest millisecond (eg Google Drive) then 3 digits will always be
shown ("2017-05-31T16:15:57.034+01:00") whereas if the times are
accurate to the nearest second (Dropbox, Box, WebDav etc) no digits
will be shown ("2017-05-31T16:15:57+01:00").
The time is in RFC3339 format with nanosecond precision.
The whole output can be processed as a JSON blob, or alternatively it
can be processed line by line as each item is written one to a line.
+1 -1
View File
@@ -45,7 +45,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
if err != nil {
return translateError(err)
}
resp.Size = n
resp.Size = int(n)
return nil
}
+1 -5
View File
@@ -20,12 +20,12 @@ var (
)
func randomSeekTest(size int64, in *os.File, name string) {
startTime := time.Now()
start := rand.Int63n(size)
blockSize := rand.Intn(*maxBlockSize)
if int64(blockSize) > size-start {
blockSize = int(size - start)
}
log.Printf("Reading %d from %d", blockSize, start)
_, err := in.Seek(start, io.SeekStart)
if err != nil {
@@ -37,8 +37,6 @@ func randomSeekTest(size int64, in *os.File, name string) {
if err != nil {
log.Fatalf("Read failed on %q: %v", name, err)
}
log.Printf("Reading %d from %d took %v ", blockSize, start, time.Since(startTime))
}
func main() {
@@ -50,12 +48,10 @@ func main() {
rand.Seed(*randSeed)
name := args[0]
openStart := time.Now()
in, err := os.Open(name)
if err != nil {
log.Fatalf("Couldn't open %q: %v", name, err)
}
log.Printf("File Open took %v", time.Since(openStart))
fi, err := in.Stat()
if err != nil {
+3 -37
View File
@@ -4,7 +4,6 @@ import (
"io"
"log"
"os"
"path/filepath"
"runtime"
"strings"
"time"
@@ -63,28 +62,6 @@ func checkMountEmpty(mountpoint string) error {
return nil
}
// Check the root doesn't overlap the mountpoint
func checkMountpointOverlap(root, mountpoint string) error {
abs := func(x string) string {
if absX, err := filepath.EvalSymlinks(x); err == nil {
x = absX
}
if absX, err := filepath.Abs(x); err == nil {
x = absX
}
x = filepath.ToSlash(x)
if !strings.HasSuffix(x, "/") {
x += "/"
}
return x
}
rootAbs, mountpointAbs := abs(root), abs(mountpoint)
if strings.HasPrefix(rootAbs, mountpointAbs) || strings.HasPrefix(mountpointAbs, rootAbs) {
return errors.Errorf("mount point %q and directory to be mounted %q mustn't overlap", mountpoint, root)
}
return nil
}
// NewMountCommand makes a mount command with the given name and Mount function
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
var commandDefintion = &cobra.Command{
@@ -243,14 +220,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
config.PassConfigKeyForDaemonization = true
}
mountpoint := args[1]
fdst := cmd.NewFsDir(args)
if fdst.Name() == "" || fdst.Name() == "local" {
err := checkMountpointOverlap(fdst.Root(), mountpoint)
if err != nil {
log.Fatalf("Fatal error: %v", err)
}
}
// Show stats if the user has specifically requested them
if cmd.ShowStats() {
@@ -260,7 +230,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
// Skip checkMountEmpty if --allow-non-empty flag is used or if
// the Operating System is Windows
if !AllowNonEmpty && runtime.GOOS != "windows" {
err := checkMountEmpty(mountpoint)
err := checkMountEmpty(args[1])
if err != nil {
log.Fatalf("Fatal error: %v", err)
}
@@ -283,7 +253,7 @@ be copied to the vfs cache before opening with --vfs-cache-mode full.
}
}
err := Mount(fdst, mountpoint)
err := Mount(fdst, args[1])
if err != nil {
log.Fatalf("Fatal error: %v", err)
}
@@ -326,11 +296,7 @@ func ClipBlocks(b *uint64) {
var max uint64
switch runtime.GOOS {
case "windows":
if runtime.GOARCH == "386" {
max = (1 << 32) - 1
} else {
max = (1 << 43) - 1
}
max = (1 << 43) - 1
case "darwin":
// OSX FUSE only supports 32 bit number of blocks
// https://github.com/osxfuse/osxfuse/issues/396
+1 -1
View File
@@ -17,7 +17,7 @@ var commandDefintion = &cobra.Command{
Use: "rcd <path to files to serve>*",
Short: `Run rclone listening to remote control commands only.`,
Long: `
This runs rclone so that it only listens to remote control commands.
This runs rclone so that it only listents to remote control commands.
This is useful if you are controlling rclone via the rc API.
-451
View File
@@ -1,451 +0,0 @@
package dlna
const contentDirectoryServiceDescription = `<?xml version="1.0"?>
<scpd xmlns="urn:schemas-upnp-org:service-1-0">
<specVersion>
<major>1</major>
<minor>0</minor>
</specVersion>
<actionList>
<action>
<name>GetSearchCapabilities</name>
<argumentList>
<argument>
<name>SearchCaps</name>
<direction>out</direction>
<relatedStateVariable>SearchCapabilities</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>GetSortCapabilities</name>
<argumentList>
<argument>
<name>SortCaps</name>
<direction>out</direction>
<relatedStateVariable>SortCapabilities</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>GetSortExtensionCapabilities</name>
<argumentList>
<argument>
<name>SortExtensionCaps</name>
<direction>out</direction>
<relatedStateVariable>SortExtensionCapabilities</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>GetFeatureList</name>
<argumentList>
<argument>
<name>FeatureList</name>
<direction>out</direction>
<relatedStateVariable>FeatureList</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>GetSystemUpdateID</name>
<argumentList>
<argument>
<name>Id</name>
<direction>out</direction>
<relatedStateVariable>SystemUpdateID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>Browse</name>
<argumentList>
<argument>
<name>ObjectID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>BrowseFlag</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_BrowseFlag</relatedStateVariable>
</argument>
<argument>
<name>Filter</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
</argument>
<argument>
<name>StartingIndex</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
</argument>
<argument>
<name>RequestedCount</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
</argument>
<argument>
<name>SortCriteria</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
</argument>
<argument>
<name>Result</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
</argument>
<argument>
<name>NumberReturned</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
</argument>
<argument>
<name>TotalMatches</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
</argument>
<argument>
<name>UpdateID</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>Search</name>
<argumentList>
<argument>
<name>ContainerID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>SearchCriteria</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_SearchCriteria</relatedStateVariable>
</argument>
<argument>
<name>Filter</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
</argument>
<argument>
<name>StartingIndex</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
</argument>
<argument>
<name>RequestedCount</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
</argument>
<argument>
<name>SortCriteria</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
</argument>
<argument>
<name>Result</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
</argument>
<argument>
<name>NumberReturned</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
</argument>
<argument>
<name>TotalMatches</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
</argument>
<argument>
<name>UpdateID</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>CreateObject</name>
<argumentList>
<argument>
<name>ContainerID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>Elements</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
</argument>
<argument>
<name>ObjectID</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>Result</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>DestroyObject</name>
<argumentList>
<argument>
<name>ObjectID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>UpdateObject</name>
<argumentList>
<argument>
<name>ObjectID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>CurrentTagValue</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
</argument>
<argument>
<name>NewTagValue</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>MoveObject</name>
<argumentList>
<argument>
<name>ObjectID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>NewParentID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>NewObjectID</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>ImportResource</name>
<argumentList>
<argument>
<name>SourceURI</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
</argument>
<argument>
<name>DestinationURI</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
</argument>
<argument>
<name>TransferID</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>ExportResource</name>
<argumentList>
<argument>
<name>SourceURI</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
</argument>
<argument>
<name>DestinationURI</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
</argument>
<argument>
<name>TransferID</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>StopTransferResource</name>
<argumentList>
<argument>
<name>TransferID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>DeleteResource</name>
<argumentList>
<argument>
<name>ResourceURI</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>GetTransferProgress</name>
<argumentList>
<argument>
<name>TransferID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
</argument>
<argument>
<name>TransferStatus</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_TransferStatus</relatedStateVariable>
</argument>
<argument>
<name>TransferLength</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_TransferLength</relatedStateVariable>
</argument>
<argument>
<name>TransferTotal</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_TransferTotal</relatedStateVariable>
</argument>
</argumentList>
</action>
<action>
<name>CreateReference</name>
<argumentList>
<argument>
<name>ContainerID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>ObjectID</name>
<direction>in</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
<argument>
<name>NewID</name>
<direction>out</direction>
<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
</argument>
</argumentList>
</action>
</actionList>
<serviceStateTable>
<stateVariable sendEvents="no">
<name>SearchCapabilities</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>SortCapabilities</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>SortExtensionCapabilities</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="yes">
<name>SystemUpdateID</name>
<dataType>ui4</dataType>
</stateVariable>
<stateVariable sendEvents="yes">
<name>ContainerUpdateIDs</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="yes">
<name>TransferIDs</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>FeatureList</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_ObjectID</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_Result</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_SearchCriteria</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_BrowseFlag</name>
<dataType>string</dataType>
<allowedValueList>
<allowedValue>BrowseMetadata</allowedValue>
<allowedValue>BrowseDirectChildren</allowedValue>
</allowedValueList>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_Filter</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_SortCriteria</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_Index</name>
<dataType>ui4</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_Count</name>
<dataType>ui4</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_UpdateID</name>
<dataType>ui4</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_TransferID</name>
<dataType>ui4</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_TransferStatus</name>
<dataType>string</dataType>
<allowedValueList>
<allowedValue>COMPLETED</allowedValue>
<allowedValue>ERROR</allowedValue>
<allowedValue>IN_PROGRESS</allowedValue>
<allowedValue>STOPPED</allowedValue>
</allowedValueList>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_TransferLength</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_TransferTotal</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_TagValueList</name>
<dataType>string</dataType>
</stateVariable>
<stateVariable sendEvents="no">
<name>A_ARG_TYPE_URI</name>
<dataType>uri</dataType>
</stateVariable>
</serviceStateTable>
</scpd>`
-240
View File
@@ -1,240 +0,0 @@
package dlna
import (
"encoding/xml"
"fmt"
"log"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"github.com/anacrolix/dms/dlna"
"github.com/anacrolix/dms/upnp"
"github.com/anacrolix/dms/upnpav"
"github.com/ncw/rclone/vfs"
"github.com/pkg/errors"
)
type contentDirectoryService struct {
*server
upnp.Eventing
}
func (cds *contentDirectoryService) updateIDString() string {
return fmt.Sprintf("%d", uint32(os.Getpid()))
}
// Turns the given entry and DMS host into a UPnP object. A nil object is
// returned if the entry is not of interest.
func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fileInfo os.FileInfo, host string) (ret interface{}, err error) {
obj := upnpav.Object{
ID: cdsObject.ID(),
Restricted: 1,
ParentID: cdsObject.ParentID(),
}
if fileInfo.IsDir() {
obj.Class = "object.container.storageFolder"
obj.Title = fileInfo.Name()
ret = upnpav.Container{Object: obj}
return
}
if !fileInfo.Mode().IsRegular() {
return
}
// Hardcode "videoItem" so that files show up in VLC.
obj.Class = "object.item.videoItem"
obj.Title = fileInfo.Name()
item := upnpav.Item{
Object: obj,
Res: make([]upnpav.Resource, 0, 1),
}
item.Res = append(item.Res, upnpav.Resource{
URL: (&url.URL{
Scheme: "http",
Host: host,
Path: resPath,
RawQuery: url.Values{
"path": {cdsObject.Path},
}.Encode(),
}).String(),
// Hardcode "video/x-matroska" so that files show up in VLC.
ProtocolInfo: fmt.Sprintf("http-get:*:video/x-matroska:%s", dlna.ContentFeatures{
SupportRange: true,
}.String()),
Bitrate: 0,
Duration: "",
Size: uint64(fileInfo.Size()),
Resolution: "",
})
ret = item
return
}
// Returns all the upnpav objects in a directory.
func (cds *contentDirectoryService) readContainer(o object, host string) (ret []interface{}, err error) {
node, err := cds.vfs.Stat(o.Path)
if err != nil {
return
}
if !node.IsDir() {
err = errors.New("not a directory")
return
}
dir := node.(*vfs.Dir)
dirEntries, err := dir.ReadDirAll()
if err != nil {
err = errors.New("failed to list directory")
return
}
sort.Sort(dirEntries)
for _, de := range dirEntries {
child := object{
path.Join(o.Path, de.Name()),
}
obj, err := cds.cdsObjectToUpnpavObject(child, de, host)
if err != nil {
log.Printf("error with %s: %s", child.FilePath(), err)
continue
}
if obj != nil {
ret = append(ret, obj)
} else {
log.Printf("bad %s", de)
}
}
return
}
type browse struct {
ObjectID string
BrowseFlag string
Filter string
StartingIndex int
RequestedCount int
}
// ContentDirectory object from ObjectID.
func (cds *contentDirectoryService) objectFromID(id string) (o object, err error) {
o.Path, err = url.QueryUnescape(id)
if err != nil {
return
}
if o.Path == "0" {
o.Path = "/"
}
o.Path = path.Clean(o.Path)
if !path.IsAbs(o.Path) {
err = fmt.Errorf("bad ObjectID %v", o.Path)
return
}
return
}
func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *http.Request) (map[string]string, error) {
host := r.Host
switch action {
case "GetSystemUpdateID":
return map[string]string{
"Id": cds.updateIDString(),
}, nil
case "GetSortCapabilities":
return map[string]string{
"SortCaps": "dc:title",
}, nil
case "Browse":
var browse browse
if err := xml.Unmarshal(argsXML, &browse); err != nil {
return nil, err
}
obj, err := cds.objectFromID(browse.ObjectID)
if err != nil {
return nil, upnp.Errorf(upnpav.NoSuchObjectErrorCode, err.Error())
}
switch browse.BrowseFlag {
case "BrowseDirectChildren":
objs, err := cds.readContainer(obj, host)
if err != nil {
return nil, upnp.Errorf(upnpav.NoSuchObjectErrorCode, err.Error())
}
totalMatches := len(objs)
objs = objs[func() (low int) {
low = browse.StartingIndex
if low > len(objs) {
low = len(objs)
}
return
}():]
if browse.RequestedCount != 0 && browse.RequestedCount < len(objs) {
objs = objs[:browse.RequestedCount]
}
result, err := xml.Marshal(objs)
if err != nil {
return nil, err
}
return map[string]string{
"TotalMatches": fmt.Sprint(totalMatches),
"NumberReturned": fmt.Sprint(len(objs)),
"Result": didlLite(string(result)),
"UpdateID": cds.updateIDString(),
}, nil
default:
return nil, upnp.Errorf(upnp.ArgumentValueInvalidErrorCode, "unhandled browse flag: %v", browse.BrowseFlag)
}
case "GetSearchCapabilities":
return map[string]string{
"SearchCaps": "",
}, nil
default:
return nil, upnp.InvalidActionError
}
}
// Represents a ContentDirectory object.
type object struct {
Path string // The cleaned, absolute path for the object relative to the server.
}
// Returns the actual local filesystem path for the object.
func (o *object) FilePath() string {
return filepath.FromSlash(o.Path)
}
// Returns the ObjectID for the object. This is used in various ContentDirectory actions.
func (o object) ID() string {
if !path.IsAbs(o.Path) {
log.Panicf("Relative object path: %s", o.Path)
}
if len(o.Path) == 1 {
return "0"
}
return url.QueryEscape(o.Path)
}
func (o *object) IsRoot() bool {
return o.Path == "/"
}
// Returns the object's parent ObjectID. Fortunately it can be deduced from the
// ObjectID (for now).
func (o object) ParentID() string {
if o.IsRoot() {
return "-1"
}
o.Path = path.Dir(o.Path)
return o.ID()
}
-440
View File
@@ -1,440 +0,0 @@
package dlna
import (
"bytes"
"encoding/xml"
"fmt"
"log"
"net"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/anacrolix/dms/soap"
"github.com/anacrolix/dms/ssdp"
"github.com/anacrolix/dms/upnp"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/dlna/dlnaflags"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/vfs"
"github.com/ncw/rclone/vfs/vfsflags"
"github.com/spf13/cobra"
)
func init() {
dlnaflags.AddFlags(Command.Flags())
vfsflags.AddFlags(Command.Flags())
}
// Command definition for cobra.
var Command = &cobra.Command{
Use: "dlna remote:path",
Short: `Serve remote:path over DLNA`,
Long: `rclone serve dlna is a DLNA media server for media stored in a rclone remote. Many
devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN
and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast
packets (SSDP) and will thus only work on LANs.
Rclone will list all files present in the remote, without filtering based on media formats or
file extensions. Additionally, there is no media transcoding support. This means that some
players might show files that they are not able to play back correctly.
` + dlnaflags.Help + vfs.Help,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
s := newServer(f, &dlnaflags.Opt)
if err := s.Serve(); err != nil {
log.Fatal(err)
}
s.Wait()
return nil
})
},
}
const (
serverField = "Linux/3.4 DLNADOC/1.50 UPnP/1.0 DMS/1.0"
rootDeviceType = "urn:schemas-upnp-org:device:MediaServer:1"
rootDeviceModelName = "rclone"
resPath = "/res"
rootDescPath = "/rootDesc.xml"
serviceControlURL = "/ctl"
)
// Groups the service definition with its XML description.
type service struct {
upnp.Service
SCPD string
}
// Exposed UPnP AV services.
var services = []*service{
{
Service: upnp.Service{
ServiceType: "urn:schemas-upnp-org:service:ContentDirectory:1",
ServiceId: "urn:upnp-org:serviceId:ContentDirectory",
ControlURL: serviceControlURL,
},
SCPD: contentDirectoryServiceDescription,
},
}
func devices() []string {
return []string{
"urn:schemas-upnp-org:device:MediaServer:1",
}
}
func serviceTypes() (ret []string) {
for _, s := range services {
ret = append(ret, s.ServiceType)
}
return
}
type server struct {
// The service SOAP handler keyed by service URN.
services map[string]UPnPService
Interfaces []net.Interface
HTTPConn net.Listener
httpListenAddr string
httpServeMux *http.ServeMux
rootDeviceUUID string
rootDescXML []byte
FriendlyName string
// For waiting on the listener to close
waitChan chan struct{}
// Time interval between SSPD announces
AnnounceInterval time.Duration
f fs.Fs
vfs *vfs.VFS
}
func newServer(f fs.Fs, opt *dlnaflags.Options) *server {
hostName, err := os.Hostname()
if err != nil {
hostName = ""
} else {
hostName = " (" + hostName + ")"
}
s := &server{
AnnounceInterval: 10 * time.Second,
FriendlyName: "rclone" + hostName,
httpListenAddr: opt.ListenAddr,
f: f,
vfs: vfs.New(f, &vfsflags.Opt),
}
s.initServicesMap()
s.listInterfaces()
s.httpServeMux = http.NewServeMux()
s.rootDeviceUUID = makeDeviceUUID(s.FriendlyName)
s.rootDescXML, err = xml.MarshalIndent(
upnp.DeviceDesc{
SpecVersion: upnp.SpecVersion{Major: 1, Minor: 0},
Device: upnp.Device{
DeviceType: rootDeviceType,
FriendlyName: s.FriendlyName,
Manufacturer: "rclone (rclone.org)",
ModelName: rootDeviceModelName,
UDN: s.rootDeviceUUID,
ServiceList: func() (ss []upnp.Service) {
for _, s := range services {
ss = append(ss, s.Service)
}
return
}(),
},
},
" ", " ")
if err != nil {
// Contents are hardcoded, so this will never happen in production.
log.Panicf("Marshal root descriptor XML: %v", err)
}
s.rootDescXML = append([]byte(`<?xml version="1.0"?>`), s.rootDescXML...)
s.initMux(s.httpServeMux)
return s
}
// UPnPService is the interface for the SOAP service.
type UPnPService interface {
Handle(action string, argsXML []byte, r *http.Request) (respArgs map[string]string, err error)
Subscribe(callback []*url.URL, timeoutSeconds int) (sid string, actualTimeout int, err error)
Unsubscribe(sid string) error
}
// initServicesMap is called during initialization of the server to prepare some internal datastructures.
func (s *server) initServicesMap() {
urn, err := upnp.ParseServiceType(services[0].ServiceType)
if err != nil {
// The service type is hardcoded, so this error should never happen.
log.Panicf("ParseServiceType: %v", err)
}
s.services = map[string]UPnPService{
urn.Type: &contentDirectoryService{
server: s,
},
}
return
}
// listInterfaces is called during initialization of the server to list the network interfaces
// on the machine.
func (s *server) listInterfaces() {
ifs, err := net.Interfaces()
if err != nil {
fs.Errorf(s.f, "list network interfaces: %v", err)
return
}
var tmp []net.Interface
for _, intf := range ifs {
if intf.Flags&net.FlagUp == 0 || intf.MTU <= 0 {
continue
}
s.Interfaces = append(s.Interfaces, intf)
tmp = append(tmp, intf)
}
}
func (s *server) initMux(mux *http.ServeMux) {
mux.HandleFunc(resPath, func(w http.ResponseWriter, r *http.Request) {
remotePath := r.URL.Query().Get("path")
node, err := s.vfs.Stat(remotePath)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
file := node.(*vfs.File)
in, err := file.Open(os.O_RDONLY)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
defer fs.CheckClose(in, &err)
http.ServeContent(w, r, remotePath, node.ModTime(), in)
return
})
mux.HandleFunc(rootDescPath, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("content-type", `text/xml; charset="utf-8"`)
w.Header().Set("content-length", fmt.Sprint(len(s.rootDescXML)))
w.Header().Set("server", serverField)
_, err := w.Write(s.rootDescXML)
if err != nil {
fs.Errorf(s, "Failed to serve root descriptor XML: %v", err)
}
})
// Install handlers to serve SCPD for each UPnP service.
for _, s := range services {
p := path.Join("/scpd", s.ServiceId)
s.SCPDURL = p
mux.HandleFunc(s.SCPDURL, func(serviceDesc string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("content-type", `text/xml; charset="utf-8"`)
http.ServeContent(w, r, ".xml", time.Time{}, bytes.NewReader([]byte(serviceDesc)))
}
}(s.SCPD))
}
mux.HandleFunc(serviceControlURL, s.serviceControlHandler)
}
// Handle a service control HTTP request.
func (s *server) serviceControlHandler(w http.ResponseWriter, r *http.Request) {
soapActionString := r.Header.Get("SOAPACTION")
soapAction, err := upnp.ParseActionHTTPHeader(soapActionString)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var env soap.Envelope
if err := xml.NewDecoder(r.Body).Decode(&env); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", `text/xml; charset="utf-8"`)
w.Header().Set("Ext", "")
w.Header().Set("server", serverField)
soapRespXML, code := func() ([]byte, int) {
respArgs, err := s.soapActionResponse(soapAction, env.Body.Action, r)
if err != nil {
upnpErr := upnp.ConvertError(err)
return mustMarshalXML(soap.NewFault("UPnPError", upnpErr)), 500
}
return marshalSOAPResponse(soapAction, respArgs), 200
}()
bodyStr := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" standalone="yes"?><s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"><s:Body>%s</s:Body></s:Envelope>`, soapRespXML)
w.WriteHeader(code)
if _, err := w.Write([]byte(bodyStr)); err != nil {
log.Print(err)
}
}
// Handle a SOAP request and return the response arguments or UPnP error.
func (s *server) soapActionResponse(sa upnp.SoapAction, actionRequestXML []byte, r *http.Request) (map[string]string, error) {
service, ok := s.services[sa.Type]
if !ok {
// TODO: What's the invalid service error?
return nil, upnp.Errorf(upnp.InvalidActionErrorCode, "Invalid service: %s", sa.Type)
}
return service.Handle(sa.Action, actionRequestXML, r)
}
// Serve runs the server - returns the error only if
// the listener was not started; does not block, so
// use s.Wait() to block on the listener indefinitely.
func (s *server) Serve() (err error) {
if s.HTTPConn == nil {
s.HTTPConn, err = net.Listen("tcp", s.httpListenAddr)
if err != nil {
return
}
}
go func() {
s.startSSDP()
}()
go func() {
fs.Logf(s.f, "Serving HTTP on %s", s.HTTPConn.Addr().String())
err = s.serveHTTP()
if err != nil {
fs.Logf(s.f, "Error on serving HTTP server: %v", err)
}
}()
return nil
}
// Wait blocks while the listener is open.
func (s *server) Wait() {
<-s.waitChan
}
func (s *server) Close() {
err := s.HTTPConn.Close()
if err != nil {
fs.Errorf(s.f, "Error closing HTTP server: %v", err)
return
}
close(s.waitChan)
}
// Run SSDP (multicast for server discovery) on all interfaces.
func (s *server) startSSDP() {
active := 0
stopped := make(chan struct{})
for _, intf := range s.Interfaces {
active++
go func(intf2 net.Interface) {
defer func() {
stopped <- struct{}{}
}()
s.ssdpInterface(intf2)
}(intf)
}
for active > 0 {
<-stopped
active--
}
}
// Run SSDP server on an interface.
func (s *server) ssdpInterface(intf net.Interface) {
// Figure out which HTTP location to advertise based on the interface IP.
advertiseLocationFn := func(ip net.IP) string {
url := url.URL{
Scheme: "http",
Host: (&net.TCPAddr{
IP: ip,
Port: s.HTTPConn.Addr().(*net.TCPAddr).Port,
}).String(),
Path: rootDescPath,
}
return url.String()
}
ssdpServer := ssdp.Server{
Interface: intf,
Devices: devices(),
Services: serviceTypes(),
Location: advertiseLocationFn,
Server: serverField,
UUID: s.rootDeviceUUID,
NotifyInterval: s.AnnounceInterval,
}
// An interface with these flags should be valid for SSDP.
const ssdpInterfaceFlags = net.FlagUp | net.FlagMulticast
if err := ssdpServer.Init(); err != nil {
if intf.Flags&ssdpInterfaceFlags != ssdpInterfaceFlags {
// Didn't expect it to work anyway.
return
}
if strings.Contains(err.Error(), "listen") {
// OSX has a lot of dud interfaces. Failure to create a socket on
// the interface are what we're expecting if the interface is no
// good.
return
}
log.Printf("Error creating ssdp server on %s: %s", intf.Name, err)
return
}
defer ssdpServer.Close()
log.Println("Started SSDP on", intf.Name)
stopped := make(chan struct{})
go func() {
defer close(stopped)
if err := ssdpServer.Serve(); err != nil {
log.Printf("%q: %q\n", intf.Name, err)
}
}()
select {
case <-s.waitChan:
// Returning will close the server.
case <-stopped:
}
}
func (s *server) serveHTTP() error {
srv := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.httpServeMux.ServeHTTP(w, r)
}),
}
err := srv.Serve(s.HTTPConn)
select {
case <-s.waitChan:
return nil
default:
return err
}
}
-88
View File
@@ -1,88 +0,0 @@
// +build go1.8
package dlna
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"testing"
"github.com/ncw/rclone/vfs"
_ "github.com/ncw/rclone/backend/local"
"github.com/ncw/rclone/cmd/serve/dlna/dlnaflags"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var (
dlnaServer *server
)
const (
testBindAddress = "localhost:51777"
testURL = "http://" + testBindAddress + "/"
)
func startServer(t *testing.T, f fs.Fs) {
opt := dlnaflags.DefaultOpt
opt.ListenAddr = testBindAddress
dlnaServer = newServer(f, &opt)
assert.NoError(t, dlnaServer.Serve())
}
func TestInit(t *testing.T) {
config.LoadConfig()
f, err := fs.NewFs("testdata/files")
l, _ := f.List("")
fmt.Println(l)
require.NoError(t, err)
startServer(t, f)
}
// Make sure that it serves rootDesc.xml (SCPD in uPnP parlance).
func TestRootSCPD(t *testing.T) {
req, err := http.NewRequest("GET", testURL+"rootDesc.xml", nil)
require.NoError(t, err)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
// Make sure that the SCPD contains a CDS service.
require.Contains(t, string(body),
"<serviceType>urn:schemas-upnp-org:service:ContentDirectory:1</serviceType>")
}
// Make sure that it serves content from the remote.
func TestServeContent(t *testing.T) {
itemPath := "/small_jpeg.jpg"
pathQuery := url.QueryEscape(itemPath)
req, err := http.NewRequest("GET", testURL+"res?path="+pathQuery, nil)
require.NoError(t, err)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer fs.CheckClose(resp.Body, &err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
actualContents, err := ioutil.ReadAll(resp.Body)
assert.NoError(t, err)
// Now compare the contents with the golden file.
node, err := dlnaServer.vfs.Stat(itemPath)
assert.NoError(t, err)
goldenFile := node.(*vfs.File)
goldenReader, err := goldenFile.Open(os.O_RDONLY)
assert.NoError(t, err)
defer fs.CheckClose(goldenReader, &err)
goldenContents, err := ioutil.ReadAll(goldenReader)
assert.NoError(t, err)
require.Equal(t, goldenContents, actualContents)
}
-52
View File
@@ -1,52 +0,0 @@
package dlna
import (
"crypto/md5"
"encoding/xml"
"fmt"
"io"
"log"
"github.com/anacrolix/dms/soap"
"github.com/anacrolix/dms/upnp"
)
func makeDeviceUUID(unique string) string {
h := md5.New()
if _, err := io.WriteString(h, unique); err != nil {
log.Panicf("makeDeviceUUID write failed: %s", err)
}
buf := h.Sum(nil)
return upnp.FormatUUID(buf)
}
func didlLite(chardata string) string {
return `<DIDL-Lite` +
` xmlns:dc="http://purl.org/dc/elements/1.1/"` +
` xmlns:upnp="urn:schemas-upnp-org:metadata-1-0/upnp/"` +
` xmlns="urn:schemas-upnp-org:metadata-1-0/DIDL-Lite/"` +
` xmlns:dlna="urn:schemas-dlna-org:metadata-1-0/">` +
chardata +
`</DIDL-Lite>`
}
func mustMarshalXML(value interface{}) []byte {
ret, err := xml.MarshalIndent(value, "", " ")
if err != nil {
log.Panicf("mustMarshalXML failed to marshal %v: %s", value, err)
}
return ret
}
// Marshal SOAP response arguments into a response XML snippet.
func marshalSOAPResponse(sa upnp.SoapAction, args map[string]string) []byte {
soapArgs := make([]soap.Arg, 0, len(args))
for argName, value := range args {
soapArgs = append(soapArgs, soap.Arg{
XMLName: xml.Name{Local: argName},
Value: value,
})
}
return []byte(fmt.Sprintf(`<u:%[1]sResponse xmlns:u="%[2]s">%[3]s</u:%[1]sResponse>`,
sa.Action, sa.ServiceURN.String(), mustMarshalXML(soapArgs)))
}
-42
View File
@@ -1,42 +0,0 @@
package dlnaflags
import (
"github.com/ncw/rclone/fs/config/flags"
"github.com/ncw/rclone/fs/rc"
"github.com/spf13/pflag"
)
// Help contains the text for the command line help and manual.
var Help = `
### Server options
Use --addr to specify which IP address and port the server should
listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
IPs.
`
// Options is the type for DLNA serving options.
type Options struct {
ListenAddr string
}
// DefaultOpt contains the defaults options for DLNA serving.
var DefaultOpt = Options{
ListenAddr: ":7879",
}
// Opt contains the options for DLNA serving.
var (
Opt = DefaultOpt
)
func addFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) {
rc.AddOption("dlna", &Opt)
flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "ip:port or :port to bind the DLNA http server to.")
}
// AddFlags add the command line flags for DLNA serving.
func AddFlags(flagSet *pflag.FlagSet) {
addFlagsPrefix(flagSet, "", &Opt)
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 107 B

+1 -1
View File
@@ -126,7 +126,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
}
// Make the entries for display
directory := serve.NewDirectory(dirRemote, s.HTMLTemplate)
directory := serve.NewDirectory(dirRemote)
for _, node := range dirEntries {
directory.AddEntry(node.Path(), node.IsDir())
}
+2 -11
View File
@@ -6,7 +6,6 @@ import (
"crypto/x509"
"encoding/base64"
"fmt"
"html/template"
"io/ioutil"
"log"
"net"
@@ -15,7 +14,6 @@ import (
"time"
auth "github.com/abbot/go-http-auth"
"github.com/ncw/rclone/cmd/serve/httplib/serve/data"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
@@ -109,9 +107,8 @@ type Server struct {
waitChan chan struct{} // for waiting on the listener to close
httpServer *http.Server
basicPassHashed string
useSSL bool // if server is configured for SSL/TLS
usingAuth bool // set if authentication is configured
HTMLTemplate *template.Template // HTML template for web interface
useSSL bool // if server is configured for SSL/TLS
usingAuth bool // set if authentication is configured
}
// singleUserProvider provides the encrypted password for a single user
@@ -208,12 +205,6 @@ func NewServer(handler http.Handler, opt *Options) *Server {
s.httpServer.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert
}
htmlTemplate, templateErr := data.GetTemplate()
if templateErr != nil {
log.Fatalf(templateErr.Error())
}
s.HTMLTemplate = htmlTemplate
return s
}
@@ -1,22 +0,0 @@
// +build ignore
package main
import (
"log"
"net/http"
"github.com/shurcooL/vfsgen"
)
func main() {
var AssetDir http.FileSystem = http.Dir("./templates")
err := vfsgen.Generate(AssetDir, vfsgen.Options{
PackageName: "data",
BuildTags: "!dev",
VariableName: "Assets",
})
if err != nil {
log.Fatalln(err)
}
}
@@ -1,186 +0,0 @@
// Code generated by vfsgen; DO NOT EDIT.
// +build !dev
package data
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
pathpkg "path"
"time"
)
// Assets statically implements the virtual filesystem provided to vfsgen.
var Assets = func() http.FileSystem {
fs := vfsgen۰FS{
"/": &vfsgen۰DirInfo{
name: "/",
modTime: time.Date(2018, 12, 16, 6, 54, 42, 894445775, time.UTC),
},
"/index.html": &vfsgen۰CompressedFileInfo{
name: "index.html",
modTime: time.Date(2018, 12, 16, 6, 54, 42, 790442328, time.UTC),
uncompressedSize: 226,
compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x5c\x8f\x31\xcf\x83\x20\x10\x86\x77\x7e\xc5\x7d\xc4\xf5\x93\xb8\x35\x0d\xb0\xb4\x6e\x26\x6d\x1a\x3b\x74\x3c\xeb\x29\x24\x4a\x13\xa4\x43\x43\xf8\xef\x0d\xea\xd4\x09\xee\x79\xef\x9e\xcb\xc9\xbf\xf3\xe5\xd4\x3e\xae\x35\x98\x30\x4f\x9a\xc9\xfc\xc0\x84\x6e\x54\x9c\x1c\xcf\x80\xb0\xd7\x4c\xce\x14\x10\x9e\x06\xfd\x42\x41\xf1\x77\x18\xfe\x0f\x39\x0d\x36\x4c\xa4\x63\x84\xb2\xcd\x3f\x48\x49\x8a\x8d\x31\x29\xf6\xd1\xee\xd5\x7f\xb2\xa8\xfa\xe9\x33\x95\x66\x31\x82\x47\x37\x12\x14\x16\x8e\x0a\xca\xda\x05\x6f\x69\xc9\x39\x82\xf1\x34\x28\x1e\x23\x14\xb6\xbc\xdf\x1a\x48\x89\xeb\xad\x6a\x08\x87\xd5\x81\x5a\x76\x1e\xc4\x2a\x22\xd7\xaf\x6c\xdf\x27\xb6\x8b\xbe\x01\x00\x00\xff\xff\x92\x2e\x35\x75\xe2\x00\x00\x00"),
},
}
fs["/"].(*vfsgen۰DirInfo).entries = []os.FileInfo{
fs["/index.html"].(os.FileInfo),
}
return fs
}()
type vfsgen۰FS map[string]interface{}
func (fs vfsgen۰FS) Open(path string) (http.File, error) {
path = pathpkg.Clean("/" + path)
f, ok := fs[path]
if !ok {
return nil, &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist}
}
switch f := f.(type) {
case *vfsgen۰CompressedFileInfo:
gr, err := gzip.NewReader(bytes.NewReader(f.compressedContent))
if err != nil {
// This should never happen because we generate the gzip bytes such that they are always valid.
panic("unexpected error reading own gzip compressed bytes: " + err.Error())
}
return &vfsgen۰CompressedFile{
vfsgen۰CompressedFileInfo: f,
gr: gr,
}, nil
case *vfsgen۰DirInfo:
return &vfsgen۰Dir{
vfsgen۰DirInfo: f,
}, nil
default:
// This should never happen because we generate only the above types.
panic(fmt.Sprintf("unexpected type %T", f))
}
}
// vfsgen۰CompressedFileInfo is a static definition of a gzip compressed file.
type vfsgen۰CompressedFileInfo struct {
name string
modTime time.Time
compressedContent []byte
uncompressedSize int64
}
func (f *vfsgen۰CompressedFileInfo) Readdir(count int) ([]os.FileInfo, error) {
return nil, fmt.Errorf("cannot Readdir from file %s", f.name)
}
func (f *vfsgen۰CompressedFileInfo) Stat() (os.FileInfo, error) { return f, nil }
func (f *vfsgen۰CompressedFileInfo) GzipBytes() []byte {
return f.compressedContent
}
func (f *vfsgen۰CompressedFileInfo) Name() string { return f.name }
func (f *vfsgen۰CompressedFileInfo) Size() int64 { return f.uncompressedSize }
func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0444 }
func (f *vfsgen۰CompressedFileInfo) ModTime() time.Time { return f.modTime }
func (f *vfsgen۰CompressedFileInfo) IsDir() bool { return false }
func (f *vfsgen۰CompressedFileInfo) Sys() interface{} { return nil }
// vfsgen۰CompressedFile is an opened compressedFile instance.
type vfsgen۰CompressedFile struct {
*vfsgen۰CompressedFileInfo
gr *gzip.Reader
grPos int64 // Actual gr uncompressed position.
seekPos int64 // Seek uncompressed position.
}
func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
if f.grPos > f.seekPos {
// Rewind to beginning.
err = f.gr.Reset(bytes.NewReader(f.compressedContent))
if err != nil {
return 0, err
}
f.grPos = 0
}
if f.grPos < f.seekPos {
// Fast-forward.
_, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos)
if err != nil {
return 0, err
}
f.grPos = f.seekPos
}
n, err = f.gr.Read(p)
f.grPos += int64(n)
f.seekPos = f.grPos
return n, err
}
func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
f.seekPos = 0 + offset
case io.SeekCurrent:
f.seekPos += offset
case io.SeekEnd:
f.seekPos = f.uncompressedSize + offset
default:
panic(fmt.Errorf("invalid whence value: %v", whence))
}
return f.seekPos, nil
}
func (f *vfsgen۰CompressedFile) Close() error {
return f.gr.Close()
}
// vfsgen۰DirInfo is a static definition of a directory.
type vfsgen۰DirInfo struct {
name string
modTime time.Time
entries []os.FileInfo
}
func (d *vfsgen۰DirInfo) Read([]byte) (int, error) {
return 0, fmt.Errorf("cannot Read from directory %s", d.name)
}
func (d *vfsgen۰DirInfo) Close() error { return nil }
func (d *vfsgen۰DirInfo) Stat() (os.FileInfo, error) { return d, nil }
func (d *vfsgen۰DirInfo) Name() string { return d.name }
func (d *vfsgen۰DirInfo) Size() int64 { return 0 }
func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0755 | os.ModeDir }
func (d *vfsgen۰DirInfo) ModTime() time.Time { return d.modTime }
func (d *vfsgen۰DirInfo) IsDir() bool { return true }
func (d *vfsgen۰DirInfo) Sys() interface{} { return nil }
// vfsgen۰Dir is an opened dir instance.
type vfsgen۰Dir struct {
*vfsgen۰DirInfo
pos int // Position within entries for Seek and Readdir.
}
func (d *vfsgen۰Dir) Seek(offset int64, whence int) (int64, error) {
if offset == 0 && whence == io.SeekStart {
d.pos = 0
return 0, nil
}
return 0, fmt.Errorf("unsupported Seek in directory %s", d.name)
}
func (d *vfsgen۰Dir) Readdir(count int) ([]os.FileInfo, error) {
if d.pos >= len(d.entries) && count > 0 {
return nil, io.EOF
}
if count <= 0 || count > len(d.entries)-d.pos {
count = len(d.entries) - d.pos
}
e := d.entries[d.pos : d.pos+count]
d.pos += count
return e, nil
}
-36
View File
@@ -1,36 +0,0 @@
//go:generate go run assets_generate.go
// The "go:generate" directive compiles static assets by running assets_generate.go
package data
import (
"html/template"
"io/ioutil"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
// GetTemplate eturns the HTML template for serving directories via HTTP
func GetTemplate() (tpl *template.Template, err error) {
templateFile, err := Assets.Open("index.html")
if err != nil {
return nil, errors.Wrap(err, "get template open")
}
defer fs.CheckClose(templateFile, &err)
templateBytes, err := ioutil.ReadAll(templateFile)
if err != nil {
return nil, errors.Wrap(err, "get template read")
}
var templateString = string(templateBytes)
tpl, err = template.New("index").Parse(templateString)
if err != nil {
return nil, errors.Wrap(err, "get template parse")
}
return
}
@@ -1,11 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
</head>
<body>
<h1>{{ .Title }}</h1>
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
{{ end }}</body>
</html>
+25 -11
View File
@@ -21,19 +21,17 @@ type DirEntry struct {
// Directory represents a directory
type Directory struct {
DirRemote string
Title string
Entries []DirEntry
Query string
HTMLTemplate *template.Template
DirRemote string
Title string
Entries []DirEntry
Query string
}
// NewDirectory makes an empty Directory
func NewDirectory(dirRemote string, htmlTemplate *template.Template) *Directory {
func NewDirectory(dirRemote string) *Directory {
d := &Directory{
DirRemote: dirRemote,
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
HTMLTemplate: htmlTemplate,
DirRemote: dirRemote,
Title: fmt.Sprintf("Directory listing of /%s", dirRemote),
}
return d
}
@@ -79,10 +77,26 @@ func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
defer accounting.Stats.DoneTransferring(d.DirRemote, true)
fs.Infof(d.DirRemote, "%s: Serving directory", r.RemoteAddr)
err := d.HTMLTemplate.Execute(w, d)
err := indexTemplate.Execute(w, d)
if err != nil {
Error(d.DirRemote, w, "Failed to render template", err)
return
}
}
// indexPage is a directory listing template
var indexPage = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>{{ .Title }}</title>
</head>
<body>
<h1>{{ .Title }}</h1>
{{ range $i := .Entries }}<a href="{{ $i.URL }}">{{ $i.Leaf }}</a><br />
{{ end }}</body>
</html>
`
// indexTemplate is the instantiated indexPage
var indexTemplate = template.Must(template.New("index").Parse(indexPage))
+5 -14
View File
@@ -2,32 +2,23 @@ package serve
import (
"errors"
"html/template"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/ncw/rclone/cmd/serve/httplib/serve/data"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func GetTemplate(t *testing.T) *template.Template {
htmlTemplate, err := data.GetTemplate()
require.NoError(t, err)
return htmlTemplate
}
func TestNewDirectory(t *testing.T) {
d := NewDirectory("z", GetTemplate(t))
d := NewDirectory("z")
assert.Equal(t, "z", d.DirRemote)
assert.Equal(t, "Directory listing of /z", d.Title)
}
func TestSetQuery(t *testing.T) {
d := NewDirectory("z", GetTemplate(t))
d := NewDirectory("z")
assert.Equal(t, "", d.Query)
d.SetQuery(url.Values{"potato": []string{"42"}})
assert.Equal(t, "?potato=42", d.Query)
@@ -36,7 +27,7 @@ func TestSetQuery(t *testing.T) {
}
func TestAddEntry(t *testing.T) {
var d = NewDirectory("z", GetTemplate(t))
var d = NewDirectory("z")
d.AddEntry("", true)
d.AddEntry("dir", true)
d.AddEntry("a/b/c/d.txt", false)
@@ -51,7 +42,7 @@ func TestAddEntry(t *testing.T) {
}, d.Entries)
// Now test with a query parameter
d = NewDirectory("z", GetTemplate(t)).SetQuery(url.Values{"potato": []string{"42"}})
d = NewDirectory("z").SetQuery(url.Values{"potato": []string{"42"}})
d.AddEntry("file", false)
d.AddEntry("dir", true)
assert.Equal(t, []DirEntry{
@@ -71,7 +62,7 @@ func TestError(t *testing.T) {
}
func TestServe(t *testing.T) {
d := NewDirectory("aDirectory", GetTemplate(t))
d := NewDirectory("aDirectory")
d.AddEntry("file", false)
d.AddEntry("dir", true)
-5
View File
@@ -3,8 +3,6 @@ package serve
import (
"errors"
"github.com/ncw/rclone/cmd/serve/dlna"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/ftp"
"github.com/ncw/rclone/cmd/serve/http"
@@ -21,9 +19,6 @@ func init() {
if restic.Command != nil {
Command.AddCommand(restic.Command)
}
if dlna.Command != nil {
Command.AddCommand(dlna.Command)
}
if ftp.Command != nil {
Command.AddCommand(ftp.Command)
}
-2
View File
@@ -13,7 +13,6 @@ Rclone
Rclone is a command line program to sync files and directories to and from:
* {{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
* {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status))
* {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
* {{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}}
@@ -44,7 +43,6 @@ Rclone is a command line program to sync files and directories to and from:
* {{< provider name="put.io" home="https://put.io/" config="/webdav/#put-io" >}}
* {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
* {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}
* {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}}
* {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SFTP" config="/sftp/" >}}
* {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
* {{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
+1 -17
View File
@@ -154,7 +154,7 @@ Contributors
* Michael P. Dubner <pywebmail@list.ru>
* Antoine GIRARD <sapk@users.noreply.github.com>
* Mateusz Piotrowski <mpp302@gmail.com>
* Animosity022 <animosity22@users.noreply.github.com> <earl.texter@gmail.com>
* Animosity022 <animosity22@users.noreply.github.com>
* Peter Baumgartner <pete@lincolnloop.com>
* Craig Rachel <craig@craigrachel.com>
* Michael G. Noll <miguno@users.noreply.github.com>
@@ -221,19 +221,3 @@ Contributors
* Mathieu Carbou <mathieu.carbou@gmail.com>
* Mark Otway <mark@otway.com>
* William Cocker <37018962+WilliamCocker@users.noreply.github.com>
* François Leurent <131.js@cloudyks.org>
* Arkadius Stefanski <arkste@gmail.com>
* Jay <dev@jaygoel.com>
* andrea rota <a@xelera.eu>
* nicolov <nicolov@users.noreply.github.com>
* Dario Guzik <dario@guzik.com.ar>
* qip <qip@users.noreply.github.com>
* yair@unicorn <yair@unicorn>
* Matt Robinson <brimstone@the.narro.ws>
* kayrus <kay.diam@gmail.com>
* Rémy Léone <remy.leone@gmail.com>
* Wojciech Smigielski <wojciech.hieronim.smigielski@gmail.com>
* weetmuts <oehrstroem@gmail.com>
* Jonathan <vanillajonathan@users.noreply.github.com>
* James Carpenter <orbsmiv@users.noreply.github.com>
* Vince <vince0villamora@gmail.com>
Loaded 100 of 835 files, more files were not shown because too many files have changed in this diff. Show more