Compare commits
23
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
77b1eaeffe | ||
|
|
ab78eb13e4 | ||
|
|
b1f31c2acf | ||
|
|
dcc74fa404 | ||
|
|
6759d36e2f | ||
|
|
a4797014c9 | ||
|
|
4d7d240c12 | ||
|
|
d046402d80 | ||
|
|
9bdf465c10 | ||
|
|
f3f48d7d49 | ||
|
|
3c89406886 | ||
|
|
85d09729f2 | ||
|
|
b3bd2d1c9e | ||
|
|
4c586a9264 | ||
|
|
1c80e84f8a | ||
|
|
028f8a69d3 | ||
|
|
b0d1fa1d6b | ||
|
|
dbb4b2c900 | ||
|
|
99201f8ba4 | ||
|
|
5ad8bcb43a | ||
|
|
6efedc4043 | ||
|
|
a3d9a38f51 | ||
|
|
b1bd17a220 |
No files matched your search
Generated
+9
-36
@@ -18,24 +18,16 @@
|
||||
version = "v0.23.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/Azure/azure-sdk-for-go"
|
||||
packages = [
|
||||
"storage",
|
||||
"version"
|
||||
]
|
||||
revision = "fbe7db0e3f9793ba3e5704efbab84f51436c136e"
|
||||
version = "v18.0.0"
|
||||
name = "github.com/Azure/azure-pipeline-go"
|
||||
packages = ["pipeline"]
|
||||
revision = "7571e8eb0876932ab505918ff7ed5107773e5ee2"
|
||||
version = "0.1.7"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/Azure/go-autorest"
|
||||
packages = [
|
||||
"autorest",
|
||||
"autorest/adal",
|
||||
"autorest/azure",
|
||||
"autorest/date"
|
||||
]
|
||||
revision = "1f7cd6cfe0adea687ad44a512dfe76140f804318"
|
||||
version = "v10.12.0"
|
||||
branch = "master"
|
||||
name = "github.com/Azure/azure-storage-blob-go"
|
||||
packages = ["2018-03-28/azblob"]
|
||||
revision = "eaae161d9d5e07363f04ddb19d84d57efc66d1a1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
@@ -124,12 +116,6 @@
|
||||
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/dgrijalva/jwt-go"
|
||||
packages = ["."]
|
||||
revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
|
||||
version = "v3.2.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/djherbis/times"
|
||||
packages = ["."]
|
||||
@@ -201,12 +187,6 @@
|
||||
revision = "1455def202f6e05b95cc7bfc7e8ae67ae5141eba"
|
||||
version = "v0.1.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/marstr/guid"
|
||||
packages = ["."]
|
||||
revision = "8bd9a64bf37eb297b492a4101fb28e80ac0b290f"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/mattn/go-runewidth"
|
||||
packages = ["."]
|
||||
@@ -285,12 +265,6 @@
|
||||
revision = "55d61fa8aa702f59229e6cff85793c22e580eaf5"
|
||||
version = "v1.5.1"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/satori/go.uuid"
|
||||
packages = ["."]
|
||||
revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/sevlyar/go-daemon"
|
||||
@@ -470,7 +444,6 @@
|
||||
"internal/modules",
|
||||
"internal/remote_api",
|
||||
"internal/urlfetch",
|
||||
"log",
|
||||
"urlfetch"
|
||||
]
|
||||
revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
|
||||
@@ -485,6 +458,6 @@
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "c1378c5fc821e27711155958ff64b3c74b56818ba4733dbfe0c86d518c32880e"
|
||||
inputs-digest = "670cdb55138aa1394b4c8f87345e9be9c8105248edda4be7176dddee2a4f5d26"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
@@ -9,3 +9,7 @@
|
||||
[[override]]
|
||||
branch = "master"
|
||||
name = "github.com/coreos/bbolt"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/Azure/azure-storage-blob-go"
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
Rclone is a command line program to sync files and directories to and from
|
||||
|
||||
* Amazon Drive
|
||||
* Amazon Drive ([See note](https://rclone.org/amazonclouddrive/#status))
|
||||
* Amazon S3 / Dreamhost / Ceph / Minio / Wasabi
|
||||
* Backblaze B2
|
||||
* Box
|
||||
|
||||
+29
-15
@@ -7,7 +7,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -17,29 +18,42 @@ func init() {
|
||||
Description: "Alias for a existing remote",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "remote",
|
||||
Help: "Remote or path to alias.\nCan be \"myremote:path/to/dir\", \"myremote:bucket\", \"myremote:\" or \"/local/path\".",
|
||||
Name: "remote",
|
||||
Help: "Remote or path to alias.\nCan be \"myremote:path/to/dir\", \"myremote:bucket\", \"myremote:\" or \"/local/path\".",
|
||||
Required: true,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Remote string `config:"remote"`
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path.
|
||||
//
|
||||
// The returned Fs is the actual Fs, referenced by remote in the config
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
remote := config.FileGet(name, "remote")
|
||||
if remote == "" {
|
||||
return nil, errors.New("alias can't point to an empty remote - check the value of the remote setting")
|
||||
}
|
||||
if strings.HasPrefix(remote, name+":") {
|
||||
return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
|
||||
}
|
||||
fsInfo, configName, fsPath, err := fs.ParseRemote(remote)
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = filepath.ToSlash(root)
|
||||
return fsInfo.NewFs(configName, path.Join(fsPath, root))
|
||||
if opt.Remote == "" {
|
||||
return nil, errors.New("alias can't point to an empty remote - check the value of the remote setting")
|
||||
}
|
||||
if strings.HasPrefix(opt.Remote, name+":") {
|
||||
return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
|
||||
}
|
||||
_, configName, fsPath, err := fs.ParseRemote(opt.Remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root = path.Join(fsPath, filepath.ToSlash(root))
|
||||
if configName == "local" {
|
||||
return fs.NewFs(root)
|
||||
}
|
||||
return fs.NewFs(configName + ":" + root)
|
||||
}
|
||||
@@ -24,7 +24,8 @@ import (
|
||||
"github.com/ncw/go-acd"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -37,19 +38,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
folderKind = "FOLDER"
|
||||
fileKind = "FILE"
|
||||
statusAvailable = "AVAILABLE"
|
||||
timeFormat = time.RFC3339 // 2014-03-07T22:31:12.173Z
|
||||
minSleep = 20 * time.Millisecond
|
||||
warnFileSize = 50000 << 20 // Display warning for files larger than this size
|
||||
folderKind = "FOLDER"
|
||||
fileKind = "FILE"
|
||||
statusAvailable = "AVAILABLE"
|
||||
timeFormat = time.RFC3339 // 2014-03-07T22:31:12.173Z
|
||||
minSleep = 20 * time.Millisecond
|
||||
warnFileSize = 50000 << 20 // Display warning for files larger than this size
|
||||
defaultTempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
tempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink
|
||||
uploadWaitPerGB = flags.DurationP("acd-upload-wait-per-gb", "", 180*time.Second, "Additional time per GB to wait after a failed complete upload to see if it appears.")
|
||||
// Description of how to auth for this app
|
||||
acdConfig = &oauth2.Config{
|
||||
Scopes: []string{"clouddrive:read_all", "clouddrive:write"},
|
||||
@@ -67,35 +66,62 @@ var (
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "amazon cloud drive",
|
||||
Prefix: "acd",
|
||||
Description: "Amazon Drive",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
err := oauthutil.Config("amazon cloud drive", name, acdConfig)
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("amazon cloud drive", name, m, acdConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Amazon Application Client Id - required.",
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Amazon Application Client ID.",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Amazon Application Client Secret - required.",
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Amazon Application Client Secret.",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: config.ConfigAuthURL,
|
||||
Help: "Auth server URL - leave blank to use Amazon's.",
|
||||
Name: config.ConfigAuthURL,
|
||||
Help: "Auth server URL.\nLeave blank to use Amazon's.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigTokenURL,
|
||||
Help: "Token server url - leave blank to use Amazon's.",
|
||||
Name: config.ConfigTokenURL,
|
||||
Help: "Token server url.\nleave blank to use Amazon's.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "checkpoint",
|
||||
Help: "Checkpoint for internal polling (debug).",
|
||||
Hide: fs.OptionHideBoth,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_wait_per_gb",
|
||||
Help: "Additional time per GB to wait after a failed complete upload to see if it appears.",
|
||||
Default: fs.Duration(180 * time.Second),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "templink_threshold",
|
||||
Help: "Files >= this size will be downloaded via their tempLink.",
|
||||
Default: defaultTempLinkThreshold,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&tempLinkThreshold, "acd-templink-threshold", "", "Files >= this size will be downloaded via their tempLink.")
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Checkpoint string `config:"checkpoint"`
|
||||
UploadWaitPerGB fs.Duration `config:"upload_wait_per_gb"`
|
||||
TempLinkThreshold fs.SizeSuffix `config:"templink_threshold"`
|
||||
}
|
||||
|
||||
// Fs represents a remote acd server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
features *fs.Features // optional features
|
||||
opt Options // options for this Fs
|
||||
c *acd.Client // the connection to the acd server
|
||||
noAuthClient *http.Client // unauthenticated http client
|
||||
root string // the path we are working on
|
||||
@@ -191,7 +217,13 @@ func filterRequest(req *http.Request) {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root = parsePath(root)
|
||||
baseClient := fshttp.NewClient(fs.Config)
|
||||
if do, ok := baseClient.Transport.(interface {
|
||||
@@ -201,7 +233,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
} else {
|
||||
fs.Debugf(name+":", "Couldn't add request filter - large file downloads will fail")
|
||||
}
|
||||
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, acdConfig, baseClient)
|
||||
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, acdConfig, baseClient)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure Amazon Drive: %v", err)
|
||||
}
|
||||
@@ -210,6 +242,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
c: c,
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetPacer(pacer.AmazonCloudDrivePacer),
|
||||
noAuthClient: fshttp.NewClient(fs.Config),
|
||||
@@ -527,13 +560,13 @@ func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, i
|
||||
}
|
||||
|
||||
// Don't wait for uploads - assume they will appear later
|
||||
if *uploadWaitPerGB <= 0 {
|
||||
if f.opt.UploadWaitPerGB <= 0 {
|
||||
fs.Debugf(src, "Upload error detected but waiting disabled: %v (%q)", inErr, httpStatus)
|
||||
return false, inInfo, inErr
|
||||
}
|
||||
|
||||
// Time we should wait for the upload
|
||||
uploadWaitPerByte := float64(*uploadWaitPerGB) / 1024 / 1024 / 1024
|
||||
uploadWaitPerByte := float64(f.opt.UploadWaitPerGB) / 1024 / 1024 / 1024
|
||||
timeToWait := time.Duration(uploadWaitPerByte * float64(src.Size()))
|
||||
|
||||
const sleepTime = 5 * time.Second // sleep between tries
|
||||
@@ -1015,7 +1048,7 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// Open an object for read
|
||||
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
bigObject := o.Size() >= int64(tempLinkThreshold)
|
||||
bigObject := o.Size() >= int64(o.fs.opt.TempLinkThreshold)
|
||||
if bigObject {
|
||||
fs.Debugf(o, "Downloading large object via tempLink")
|
||||
}
|
||||
@@ -1208,7 +1241,7 @@ func (o *Object) MimeType() string {
|
||||
//
|
||||
// Close the returned channel to stop being notified.
|
||||
func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval time.Duration) chan bool {
|
||||
checkpoint := config.FileGet(f.name, "checkpoint")
|
||||
checkpoint := f.opt.Checkpoint
|
||||
|
||||
quit := make(chan bool)
|
||||
go func() {
|
||||
|
||||
+341
-262
@@ -1,9 +1,12 @@
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
|
||||
// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
@@ -18,13 +21,12 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
"github.com/Azure/azure-storage-blob-go/2018-03-28/azblob"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/accounting"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/ncw/rclone/fs/walk"
|
||||
"github.com/ncw/rclone/lib/pacer"
|
||||
@@ -32,24 +34,20 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
apiVersion = "2017-04-17"
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 10 * time.Second
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
listChunkSize = 5000 // number of items to read at once
|
||||
modTimeKey = "mtime"
|
||||
timeFormatIn = time.RFC3339
|
||||
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
maxTotalParts = 50000 // in multipart upload
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 10 * time.Second
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
listChunkSize = 5000 // number of items to read at once
|
||||
modTimeKey = "mtime"
|
||||
timeFormatIn = time.RFC3339
|
||||
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
maxTotalParts = 50000 // in multipart upload
|
||||
storageDefaultBaseURL = "blob.core.windows.net"
|
||||
// maxUncommittedSize = 9 << 30 // can't upload bigger than this
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
maxChunkSize = fs.SizeSuffix(100 * 1024 * 1024)
|
||||
chunkSize = fs.SizeSuffix(4 * 1024 * 1024)
|
||||
uploadCutoff = fs.SizeSuffix(256 * 1024 * 1024)
|
||||
maxUploadCutoff = fs.SizeSuffix(256 * 1024 * 1024)
|
||||
defaultChunkSize = 4 * 1024 * 1024
|
||||
maxChunkSize = 100 * 1024 * 1024
|
||||
defaultUploadCutoff = 256 * 1024 * 1024
|
||||
maxUploadCutoff = 256 * 1024 * 1024
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -64,31 +62,45 @@ func init() {
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: "Storage Account Key (leave blank to use connection string or SAS URL)",
|
||||
}, {
|
||||
Name: "connection_string",
|
||||
Help: "Connection string (leave blank if using account/key or SAS URL)",
|
||||
}, {
|
||||
Name: "sas_url",
|
||||
Help: "SAS URL for container level access only\n(leave blank if using account/key or connection string)",
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service - leave blank normally.",
|
||||
},
|
||||
},
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service\nLeave blank normally.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: "Cutoff for switching to chunked upload.",
|
||||
Default: fs.SizeSuffix(defaultUploadCutoff),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "Upload chunk size. Must fit in memory.",
|
||||
Default: fs.SizeSuffix(defaultChunkSize),
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&uploadCutoff, "azureblob-upload-cutoff", "", "Cutoff for switching to chunked upload")
|
||||
flags.VarP(&chunkSize, "azureblob-chunk-size", "", "Upload chunk size. Must fit in memory.")
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Account string `config:"account"`
|
||||
Key string `config:"key"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
SASURL string `config:"sas_url"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
}
|
||||
|
||||
// Fs represents a remote azure server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
features *fs.Features // optional features
|
||||
account string // account name
|
||||
endpoint string // name of the starting api endpoint
|
||||
bc *storage.BlobStorageClient
|
||||
cc *storage.Container
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed config options
|
||||
features *fs.Features // optional features
|
||||
svcURL *azblob.ServiceURL // reference to serviceURL
|
||||
cntURL *azblob.ContainerURL // reference to containerURL
|
||||
container string // the container we are working on
|
||||
containerOKMu sync.Mutex // mutex to protect container OK
|
||||
containerOK bool // true if we have created the container
|
||||
@@ -99,13 +111,14 @@ type Fs struct {
|
||||
|
||||
// Object describes a azure object
|
||||
type Object struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
meta map[string]string // blob metadata
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
accessTier azblob.AccessTierType // Blob Access Tier
|
||||
meta map[string]string // blob metadata
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -165,8 +178,8 @@ var retryErrorCodes = []int{
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (f *Fs) shouldRetry(err error) (bool, error) {
|
||||
// FIXME interpret special errors - more to do here
|
||||
if storageErr, ok := err.(storage.AzureStorageServiceError); ok {
|
||||
statusCode := storageErr.StatusCode
|
||||
if storageErr, ok := err.(azblob.StorageError); ok {
|
||||
statusCode := storageErr.Response().StatusCode
|
||||
for _, e := range retryErrorCodes {
|
||||
if statusCode == e {
|
||||
return true, err
|
||||
@@ -177,66 +190,74 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
if uploadCutoff > maxUploadCutoff {
|
||||
return nil, errors.Errorf("azure: upload cutoff (%v) must be less than or equal to %v", uploadCutoff, maxUploadCutoff)
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if chunkSize > maxChunkSize {
|
||||
return nil, errors.Errorf("azure: chunk size can't be greater than %v - was %v", maxChunkSize, chunkSize)
|
||||
|
||||
if opt.UploadCutoff > maxUploadCutoff {
|
||||
return nil, errors.Errorf("azure: upload cutoff (%v) must be less than or equal to %v", opt.UploadCutoff, maxUploadCutoff)
|
||||
}
|
||||
if opt.ChunkSize > maxChunkSize {
|
||||
return nil, errors.Errorf("azure: chunk size can't be greater than %v - was %v", maxChunkSize, opt.ChunkSize)
|
||||
}
|
||||
container, directory, err := parsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
account := config.FileGet(name, "account")
|
||||
key := config.FileGet(name, "key")
|
||||
connectionString := config.FileGet(name, "connection_string")
|
||||
sasURL := config.FileGet(name, "sas_url")
|
||||
endpoint := config.FileGet(name, "endpoint", storage.DefaultBaseURL)
|
||||
if opt.Endpoint == "" {
|
||||
opt.Endpoint = storageDefaultBaseURL
|
||||
}
|
||||
|
||||
var (
|
||||
oclient storage.Client
|
||||
client = &oclient
|
||||
cc *storage.Container
|
||||
u *url.URL
|
||||
serviceURL azblob.ServiceURL
|
||||
containerURL azblob.ContainerURL
|
||||
)
|
||||
switch {
|
||||
case account != "" && key != "":
|
||||
oclient, err = storage.NewClient(account, key, endpoint, apiVersion, true)
|
||||
case opt.Account != "" && opt.Key != "":
|
||||
credential := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
|
||||
u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to make azure storage client from account/key")
|
||||
return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint")
|
||||
}
|
||||
case connectionString != "":
|
||||
oclient, err = storage.NewClientFromConnectionString(connectionString)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to make azure storage client from connection string")
|
||||
}
|
||||
case sasURL != "":
|
||||
URL, err := url.Parse(sasURL)
|
||||
pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
|
||||
serviceURL = azblob.NewServiceURL(*u, pipeline)
|
||||
containerURL = serviceURL.NewContainerURL(container)
|
||||
case opt.SASURL != "":
|
||||
u, err = url.Parse(opt.SASURL)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse SAS URL")
|
||||
}
|
||||
cc, err = storage.GetContainerReferenceFromSASURI(*URL)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to make azure storage client from SAS URL")
|
||||
// use anonymous credentials in case of sas url
|
||||
pipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{})
|
||||
// Check if we have container level SAS or account level sas
|
||||
parts := azblob.NewBlobURLParts(*u)
|
||||
if parts.ContainerName != "" {
|
||||
if container != "" && parts.ContainerName != container {
|
||||
return nil, errors.New("Container name in SAS URL and container provided in command do not match")
|
||||
}
|
||||
|
||||
container = parts.ContainerName
|
||||
containerURL = azblob.NewContainerURL(*u, pipeline)
|
||||
} else {
|
||||
serviceURL = azblob.NewServiceURL(*u, pipeline)
|
||||
containerURL = serviceURL.NewContainerURL(container)
|
||||
}
|
||||
client = cc.Client()
|
||||
default:
|
||||
return nil, errors.New("Need account+key or connectionString or sasURL")
|
||||
}
|
||||
client.HTTPClient = fshttp.NewClient(fs.Config)
|
||||
bc := client.GetBlobService()
|
||||
if cc == nil {
|
||||
cc = bc.GetContainerReference(container)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
container: container,
|
||||
root: directory,
|
||||
account: account,
|
||||
endpoint: endpoint,
|
||||
bc: &bc,
|
||||
cc: cc,
|
||||
svcURL: &serviceURL,
|
||||
cntURL: &containerURL,
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
|
||||
}
|
||||
@@ -274,13 +295,13 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
// Return an Object from a path
|
||||
//
|
||||
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) newObjectWithInfo(remote string, info *storage.Blob) (fs.Object, error) {
|
||||
func (f *Fs) newObjectWithInfo(remote string, info *azblob.BlobItem) (fs.Object, error) {
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
if info != nil {
|
||||
err := o.decodeMetaData(info)
|
||||
err := o.decodeMetaDataFromBlob(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -300,13 +321,12 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
||||
}
|
||||
|
||||
// getBlobReference creates an empty blob reference with no metadata
|
||||
func (f *Fs) getBlobReference(remote string) *storage.Blob {
|
||||
return f.cc.GetBlobReference(f.root + remote)
|
||||
func (f *Fs) getBlobReference(remote string) azblob.BlobURL {
|
||||
return f.cntURL.NewBlobURL(f.root + remote)
|
||||
}
|
||||
|
||||
// getBlobWithModTime adds the modTime passed in to o.meta and creates
|
||||
// a Blob from it.
|
||||
func (o *Object) getBlobWithModTime(modTime time.Time) *storage.Blob {
|
||||
// updateMetadataWithModTime adds the modTime passed in to o.meta.
|
||||
func (o *Object) updateMetadataWithModTime(modTime time.Time) {
|
||||
// Make sure o.meta is not nil
|
||||
if o.meta == nil {
|
||||
o.meta = make(map[string]string, 1)
|
||||
@@ -314,14 +334,10 @@ func (o *Object) getBlobWithModTime(modTime time.Time) *storage.Blob {
|
||||
|
||||
// Set modTimeKey in it
|
||||
o.meta[modTimeKey] = modTime.Format(timeFormatOut)
|
||||
|
||||
blob := o.getBlobReference()
|
||||
blob.Metadata = o.meta
|
||||
return blob
|
||||
}
|
||||
|
||||
// listFn is called from list to handle an object
|
||||
type listFn func(remote string, object *storage.Blob, isDirectory bool) error
|
||||
type listFn func(remote string, object *azblob.BlobItem, isDirectory bool) error
|
||||
|
||||
// list lists the objects into the function supplied from
|
||||
// the container and root supplied
|
||||
@@ -342,32 +358,39 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
|
||||
if !recurse {
|
||||
delimiter = "/"
|
||||
}
|
||||
params := storage.ListBlobsParameters{
|
||||
MaxResults: maxResults,
|
||||
Prefix: root,
|
||||
Delimiter: delimiter,
|
||||
Include: &storage.IncludeBlobDataset{
|
||||
Snapshots: false,
|
||||
Metadata: true,
|
||||
UncommittedBlobs: false,
|
||||
|
||||
options := azblob.ListBlobsSegmentOptions{
|
||||
Details: azblob.BlobListingDetails{
|
||||
Copy: false,
|
||||
Metadata: true,
|
||||
Snapshots: false,
|
||||
UncommittedBlobs: false,
|
||||
Deleted: false,
|
||||
},
|
||||
Prefix: root,
|
||||
MaxResults: int32(maxResults),
|
||||
}
|
||||
for {
|
||||
var response storage.BlobListResponse
|
||||
ctx := context.Background()
|
||||
for marker := (azblob.Marker{}); marker.NotDone(); {
|
||||
var response *azblob.ListBlobsHierarchySegmentResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
response, err = f.cc.ListBlobs(params)
|
||||
response, err = f.cntURL.ListBlobsHierarchySegment(ctx, marker, delimiter, options)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if storageErr, ok := err.(storage.AzureStorageServiceError); ok && storageErr.StatusCode == http.StatusNotFound {
|
||||
// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
|
||||
if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
for i := range response.Blobs {
|
||||
file := &response.Blobs[i]
|
||||
// Advance marker to next
|
||||
marker = response.NextMarker
|
||||
|
||||
for i := range response.Segment.BlobItems {
|
||||
file := &response.Segment.BlobItems[i]
|
||||
// Finish if file name no longer has prefix
|
||||
// if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
|
||||
// return nil
|
||||
@@ -389,8 +412,8 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
|
||||
}
|
||||
}
|
||||
// Send the subdirectories
|
||||
for _, remote := range response.BlobPrefixes {
|
||||
remote := strings.TrimRight(remote, "/")
|
||||
for _, remote := range response.Segment.BlobPrefixes {
|
||||
remote := strings.TrimRight(remote.Name, "/")
|
||||
if !strings.HasPrefix(remote, f.root) {
|
||||
fs.Debugf(f, "Odd directory name received %q", remote)
|
||||
continue
|
||||
@@ -402,17 +425,12 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// end if no NextFileName
|
||||
if response.NextMarker == "" {
|
||||
break
|
||||
}
|
||||
params.Marker = response.NextMarker
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert a list item into a DirEntry
|
||||
func (f *Fs) itemToDirEntry(remote string, object *storage.Blob, isDirectory bool) (fs.DirEntry, error) {
|
||||
func (f *Fs) itemToDirEntry(remote string, object *azblob.BlobItem, isDirectory bool) (fs.DirEntry, error) {
|
||||
if isDirectory {
|
||||
d := fs.NewDir(remote, time.Time{})
|
||||
return d, nil
|
||||
@@ -436,7 +454,7 @@ func (f *Fs) markContainerOK() {
|
||||
|
||||
// listDir lists a single directory
|
||||
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.list(dir, false, listChunkSize, func(remote string, object *storage.Blob, isDirectory bool) error {
|
||||
err = f.list(dir, false, listChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(remote, object, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -459,13 +477,8 @@ func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) {
|
||||
if dir != "" {
|
||||
return nil, fs.ErrorListBucketRequired
|
||||
}
|
||||
err = f.listContainersToFn(func(container *storage.Container) error {
|
||||
t, err := time.Parse(time.RFC1123, container.Properties.LastModified)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to parse LastModified %q: %v", container.Properties.LastModified, err)
|
||||
t = time.Time{}
|
||||
}
|
||||
d := fs.NewDir(container.Name, t)
|
||||
err = f.listContainersToFn(func(container *azblob.ContainerItem) error {
|
||||
d := fs.NewDir(container.Name, container.Properties.LastModified)
|
||||
entries = append(entries, d)
|
||||
return nil
|
||||
})
|
||||
@@ -512,7 +525,7 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
|
||||
return fs.ErrorListBucketRequired
|
||||
}
|
||||
list := walk.NewListRHelper(callback)
|
||||
err = f.list(dir, true, listChunkSize, func(remote string, object *storage.Blob, isDirectory bool) error {
|
||||
err = f.list(dir, true, listChunkSize, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(remote, object, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -528,27 +541,34 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
|
||||
}
|
||||
|
||||
// listContainerFn is called from listContainersToFn to handle a container
|
||||
type listContainerFn func(*storage.Container) error
|
||||
type listContainerFn func(*azblob.ContainerItem) error
|
||||
|
||||
// listContainersToFn lists the containers to the function supplied
|
||||
func (f *Fs) listContainersToFn(fn listContainerFn) error {
|
||||
// FIXME page the containers if necessary?
|
||||
params := storage.ListContainersParameters{}
|
||||
var response *storage.ContainerListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
response, err = f.bc.ListContainers(params)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
params := azblob.ListContainersSegmentOptions{
|
||||
MaxResults: int32(listChunkSize),
|
||||
}
|
||||
for i := range response.Containers {
|
||||
err = fn(&response.Containers[i])
|
||||
ctx := context.Background()
|
||||
for marker := (azblob.Marker{}); marker.NotDone(); {
|
||||
var response *azblob.ListContainersResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := range response.ContainerItems {
|
||||
err = fn(&response.ContainerItems[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
marker = response.NextMarker
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -573,32 +593,20 @@ func (f *Fs) Mkdir(dir string) error {
|
||||
if f.containerOK {
|
||||
return nil
|
||||
}
|
||||
// List the container to see if it exists
|
||||
err := f.list("", false, 1, func(remote string, object *storage.Blob, isDirectory bool) error {
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
f.markContainerOK()
|
||||
return nil
|
||||
}
|
||||
|
||||
// now try to create the container
|
||||
options := storage.CreateContainerOptions{
|
||||
Access: storage.ContainerAccessTypePrivate,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err := f.cc.Create(&options)
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
ctx := context.Background()
|
||||
_, err := f.cntURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
|
||||
if err != nil {
|
||||
if storageErr, ok := err.(storage.AzureStorageServiceError); ok {
|
||||
switch storageErr.StatusCode {
|
||||
case http.StatusConflict:
|
||||
switch storageErr.Code {
|
||||
case "ContainerAlreadyExists":
|
||||
f.containerOK = true
|
||||
return false, nil
|
||||
case "ContainerBeingDeleted":
|
||||
f.containerDeleted = true
|
||||
return true, err
|
||||
}
|
||||
if storageErr, ok := err.(azblob.StorageError); ok {
|
||||
switch storageErr.ServiceCode() {
|
||||
case azblob.ServiceCodeContainerAlreadyExists:
|
||||
f.containerOK = true
|
||||
return false, nil
|
||||
case azblob.ServiceCodeContainerBeingDeleted:
|
||||
f.containerDeleted = true
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -614,7 +622,7 @@ func (f *Fs) Mkdir(dir string) error {
|
||||
// isEmpty checks to see if a given directory is empty and returns an error if not
|
||||
func (f *Fs) isEmpty(dir string) (err error) {
|
||||
empty := true
|
||||
err = f.list("", true, 1, func(remote string, object *storage.Blob, isDirectory bool) error {
|
||||
err = f.list("", true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
|
||||
empty = false
|
||||
return nil
|
||||
})
|
||||
@@ -632,16 +640,23 @@ func (f *Fs) isEmpty(dir string) (err error) {
|
||||
func (f *Fs) deleteContainer() error {
|
||||
f.containerOKMu.Lock()
|
||||
defer f.containerOKMu.Unlock()
|
||||
options := storage.DeleteContainerOptions{}
|
||||
options := azblob.ContainerAccessConditions{}
|
||||
ctx := context.Background()
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
exists, err := f.cc.Exists()
|
||||
_, err := f.cntURL.GetProperties(ctx, azblob.LeaseAccessConditions{})
|
||||
if err == nil {
|
||||
_, err = f.cntURL.Delete(ctx, options)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
|
||||
if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
|
||||
return false, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
return f.shouldRetry(err)
|
||||
}
|
||||
if !exists {
|
||||
return false, fs.ErrorDirNotFound
|
||||
}
|
||||
err = f.cc.Delete(&options)
|
||||
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
@@ -704,17 +719,36 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
dstBlob := f.getBlobReference(remote)
|
||||
srcBlob := srcObj.getBlobReference()
|
||||
options := storage.CopyOptions{}
|
||||
sourceBlobURL := srcBlob.GetURL()
|
||||
dstBlobURL := f.getBlobReference(remote)
|
||||
srcBlobURL := srcObj.getBlobReference()
|
||||
|
||||
source, err := url.Parse(srcBlobURL.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
options := azblob.BlobAccessConditions{}
|
||||
ctx := context.Background()
|
||||
var startCopy *azblob.BlobStartCopyFromURLResponse
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err = dstBlob.Copy(sourceBlobURL, &options)
|
||||
startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, options, options)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
copyStatus := startCopy.CopyStatus()
|
||||
for copyStatus == azblob.CopyStatusPending {
|
||||
time.Sleep(1 * time.Second)
|
||||
getMetadata, err := dstBlobURL.GetProperties(ctx, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copyStatus = getMetadata.CopyStatus()
|
||||
}
|
||||
|
||||
return f.NewObject(remote)
|
||||
}
|
||||
|
||||
@@ -759,22 +793,10 @@ func (o *Object) Size() int64 {
|
||||
return o.size
|
||||
}
|
||||
|
||||
// decodeMetaData sets the metadata from the data passed in
|
||||
//
|
||||
// Sets
|
||||
// o.id
|
||||
// o.modTime
|
||||
// o.size
|
||||
// o.md5
|
||||
// o.meta
|
||||
func (o *Object) decodeMetaData(info *storage.Blob) (err error) {
|
||||
o.md5 = info.Properties.ContentMD5
|
||||
o.mimeType = info.Properties.ContentType
|
||||
o.size = info.Properties.ContentLength
|
||||
o.modTime = time.Time(info.Properties.LastModified)
|
||||
if len(info.Metadata) > 0 {
|
||||
o.meta = info.Metadata
|
||||
if modTime, ok := info.Metadata[modTimeKey]; ok {
|
||||
func (o *Object) setMetadata(metadata azblob.Metadata) {
|
||||
if len(metadata) > 0 {
|
||||
o.meta = metadata
|
||||
if modTime, ok := metadata[modTimeKey]; ok {
|
||||
when, err := time.Parse(timeFormatIn, modTime)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Couldn't parse %v = %q: %v", modTimeKey, modTime, err)
|
||||
@@ -784,11 +806,42 @@ func (o *Object) decodeMetaData(info *storage.Blob) (err error) {
|
||||
} else {
|
||||
o.meta = nil
|
||||
}
|
||||
}
|
||||
|
||||
// decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
|
||||
//
|
||||
// Sets
|
||||
// o.id
|
||||
// o.modTime
|
||||
// o.size
|
||||
// o.md5
|
||||
// o.meta
|
||||
func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
|
||||
// NOTE - In BlobGetPropertiesResponse, Client library returns MD5 as base64 decoded string
|
||||
// unlike BlobProperties in BlobItem (used in decodeMetadataFromBlob) which returns base64
|
||||
// encoded bytes. Object needs to maintain this as base64 encoded string.
|
||||
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
|
||||
o.mimeType = info.ContentType()
|
||||
o.size = info.ContentLength()
|
||||
o.modTime = time.Time(info.LastModified())
|
||||
o.accessTier = azblob.AccessTierType(info.AccessTier())
|
||||
o.setMetadata(info.NewMetadata())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItem) (err error) {
|
||||
o.md5 = string(info.Properties.ContentMD5[:])
|
||||
o.mimeType = *info.Properties.ContentType
|
||||
o.size = *info.Properties.ContentLength
|
||||
o.modTime = info.Properties.LastModified
|
||||
o.accessTier = info.Properties.AccessTier
|
||||
o.setMetadata(info.Metadata)
|
||||
return nil
|
||||
}
|
||||
|
||||
// getBlobReference creates an empty blob reference with no metadata
|
||||
func (o *Object) getBlobReference() *storage.Blob {
|
||||
func (o *Object) getBlobReference() azblob.BlobURL {
|
||||
return o.fs.getBlobReference(o.remote)
|
||||
}
|
||||
|
||||
@@ -811,19 +864,22 @@ func (o *Object) readMetaData() (err error) {
|
||||
blob := o.getBlobReference()
|
||||
|
||||
// Read metadata (this includes metadata)
|
||||
getPropertiesOptions := storage.GetBlobPropertiesOptions{}
|
||||
options := azblob.BlobAccessConditions{}
|
||||
ctx := context.Background()
|
||||
var blobProperties *azblob.BlobGetPropertiesResponse
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = blob.GetProperties(&getPropertiesOptions)
|
||||
blobProperties, err = blob.GetProperties(ctx, options)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
if storageErr, ok := err.(storage.AzureStorageServiceError); ok && storageErr.StatusCode == http.StatusNotFound {
|
||||
// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
|
||||
if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeBlobNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
|
||||
return fs.ErrorObjectNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return o.decodeMetaData(blob)
|
||||
return o.decodeMetaDataFromPropertiesResponse(blobProperties)
|
||||
}
|
||||
|
||||
// timeString returns modTime as the number of milliseconds
|
||||
@@ -860,10 +916,17 @@ func (o *Object) ModTime() (result time.Time) {
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(modTime time.Time) error {
|
||||
blob := o.getBlobWithModTime(modTime)
|
||||
options := storage.SetBlobMetadataOptions{}
|
||||
// Make sure o.meta is not nil
|
||||
if o.meta == nil {
|
||||
o.meta = make(map[string]string, 1)
|
||||
}
|
||||
// Set modTimeKey in it
|
||||
o.meta[modTimeKey] = modTime.Format(timeFormatOut)
|
||||
|
||||
blob := o.getBlobReference()
|
||||
ctx := context.Background()
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
err := blob.SetMetadata(&options)
|
||||
_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{})
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -880,29 +943,18 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// Open an object for read
|
||||
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
getBlobOptions := storage.GetBlobOptions{}
|
||||
getBlobRangeOptions := storage.GetBlobRangeOptions{
|
||||
GetBlobOptions: &getBlobOptions,
|
||||
}
|
||||
// Offset and Count for range download
|
||||
var offset int64
|
||||
var count int64
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.RangeOption:
|
||||
start, end := x.Start, x.End
|
||||
if end < 0 {
|
||||
end = 0
|
||||
}
|
||||
if start < 0 {
|
||||
start = o.size - end
|
||||
end = 0
|
||||
}
|
||||
getBlobRangeOptions.Range = &storage.BlobRange{
|
||||
Start: uint64(start),
|
||||
End: uint64(end),
|
||||
offset, count = x.Decode(o.size)
|
||||
if count < 0 {
|
||||
count = o.size - offset
|
||||
}
|
||||
case *fs.SeekOption:
|
||||
getBlobRangeOptions.Range = &storage.BlobRange{
|
||||
Start: uint64(x.Offset),
|
||||
}
|
||||
offset = x.Offset
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||||
@@ -910,17 +962,17 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
}
|
||||
}
|
||||
blob := o.getBlobReference()
|
||||
ctx := context.Background()
|
||||
ac := azblob.BlobAccessConditions{}
|
||||
var dowloadResponse *azblob.DownloadResponse
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
if getBlobRangeOptions.Range == nil {
|
||||
in, err = blob.Get(&getBlobOptions)
|
||||
} else {
|
||||
in, err = blob.GetRange(&getBlobRangeOptions)
|
||||
}
|
||||
dowloadResponse, err = blob.Download(ctx, offset, count, ac, false)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to open for download")
|
||||
}
|
||||
in = dowloadResponse.Body(azblob.RetryReaderOptions{})
|
||||
return in, nil
|
||||
}
|
||||
|
||||
@@ -945,12 +997,18 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// readSeeker joins an io.Reader and an io.Seeker
|
||||
type readSeeker struct {
|
||||
io.Reader
|
||||
io.Seeker
|
||||
}
|
||||
|
||||
// uploadMultipart uploads a file using multipart upload
|
||||
//
|
||||
// Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.
|
||||
func (o *Object) uploadMultipart(in io.Reader, size int64, blob *storage.Blob, putBlobOptions *storage.PutBlobOptions) (err error) {
|
||||
func (o *Object) uploadMultipart(in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) {
|
||||
// Calculate correct chunkSize
|
||||
chunkSize := int64(chunkSize)
|
||||
chunkSize := int64(o.fs.opt.ChunkSize)
|
||||
var totalParts int64
|
||||
for {
|
||||
// Calculate number of parts
|
||||
@@ -970,31 +1028,37 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, blob *storage.Blob, p
|
||||
}
|
||||
fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, fs.SizeSuffix(chunkSize))
|
||||
|
||||
// Create an empty blob
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err := blob.CreateBlockBlob(putBlobOptions)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
// https://godoc.org/github.com/Azure/azure-storage-blob-go/2017-07-29/azblob#example-BlockBlobURL
|
||||
// Utilities are cloned from above example
|
||||
// These helper functions convert a binary block ID to a base-64 string and vice versa
|
||||
// NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length
|
||||
blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) }
|
||||
// These helper functions convert an int block ID to a base-64 string and vice versa
|
||||
blockIDIntToBase64 := func(blockID uint64) string {
|
||||
binaryBlockID := (&[8]byte{})[:] // All block IDs are 8 bytes long
|
||||
binary.LittleEndian.PutUint64(binaryBlockID, blockID)
|
||||
return blockIDBinaryToBase64(binaryBlockID)
|
||||
}
|
||||
|
||||
// block ID variables
|
||||
var (
|
||||
rawID uint64
|
||||
bytesID = make([]byte, 8)
|
||||
blockID = "" // id in base64 encoded form
|
||||
blocks = make([]storage.Block, 0, totalParts)
|
||||
blocks = make([]string, totalParts)
|
||||
)
|
||||
|
||||
// increment the blockID
|
||||
nextID := func() {
|
||||
rawID++
|
||||
binary.LittleEndian.PutUint64(bytesID, rawID)
|
||||
blockID = base64.StdEncoding.EncodeToString(bytesID)
|
||||
blocks = append(blocks, storage.Block{
|
||||
ID: blockID,
|
||||
Status: storage.BlockStatusLatest,
|
||||
})
|
||||
blockID = blockIDIntToBase64(rawID)
|
||||
blocks = append(blocks, blockID)
|
||||
}
|
||||
|
||||
// Get BlockBlobURL, we will use default pipeline here
|
||||
blockBlobURL := blob.ToBlockBlobURL()
|
||||
ctx := context.Background()
|
||||
ac := azblob.LeaseAccessConditions{} // Use default lease access conditions
|
||||
|
||||
// unwrap the accounting from the input, we use wrap to put it
|
||||
// back on after the buffering
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
@@ -1037,13 +1101,11 @@ outer:
|
||||
defer o.fs.uploadToken.Put()
|
||||
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
|
||||
|
||||
// Upload the block, with MD5 for check
|
||||
md5sum := md5.Sum(buf)
|
||||
putBlockOptions := storage.PutBlockOptions{
|
||||
ContentMD5: base64.StdEncoding.EncodeToString(md5sum[:]),
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = blob.PutBlockWithLength(blockID, uint64(len(buf)), wrap(bytes.NewBuffer(buf)), &putBlockOptions)
|
||||
bufferReader := bytes.NewReader(buf)
|
||||
wrappedReader := wrap(bufferReader)
|
||||
rs := readSeeker{wrappedReader, bufferReader}
|
||||
_, err = blockBlobURL.StageBlock(ctx, blockID, rs, ac)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
|
||||
@@ -1073,9 +1135,8 @@ outer:
|
||||
}
|
||||
|
||||
// Finalise the upload session
|
||||
putBlockListOptions := storage.PutBlockListOptions{}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err := blob.PutBlockList(blocks, &putBlockListOptions)
|
||||
_, err := blockBlobURL.CommitBlockList(ctx, blocks, *httpHeaders, o.meta, azblob.BlobAccessConditions{})
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1093,29 +1154,45 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
return err
|
||||
}
|
||||
size := src.Size()
|
||||
blob := o.getBlobWithModTime(src.ModTime())
|
||||
blob.Properties.ContentType = fs.MimeType(o)
|
||||
if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
|
||||
sourceMD5bytes, err := hex.DecodeString(sourceMD5)
|
||||
if err == nil {
|
||||
blob.Properties.ContentMD5 = base64.StdEncoding.EncodeToString(sourceMD5bytes)
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
|
||||
// Update Mod time
|
||||
o.updateMetadataWithModTime(src.ModTime())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blob := o.getBlobReference()
|
||||
httpHeaders := azblob.BlobHTTPHeaders{}
|
||||
httpHeaders.ContentType = fs.MimeType(o)
|
||||
// Multipart upload doesn't support MD5 checksums at put block calls, hence calculate
|
||||
// MD5 only for PutBlob requests
|
||||
if size < int64(o.fs.opt.UploadCutoff) {
|
||||
if sourceMD5, _ := src.Hash(hash.MD5); sourceMD5 != "" {
|
||||
sourceMD5bytes, err := hex.DecodeString(sourceMD5)
|
||||
if err == nil {
|
||||
httpHeaders.ContentMD5 = sourceMD5bytes
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
putBlobOptions := storage.PutBlobOptions{}
|
||||
|
||||
putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
|
||||
BufferSize: int(o.fs.opt.ChunkSize),
|
||||
MaxBuffers: 4,
|
||||
Metadata: o.meta,
|
||||
BlobHTTPHeaders: httpHeaders,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// Don't retry, return a retry error instead
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
if size >= int64(uploadCutoff) {
|
||||
if size >= int64(o.fs.opt.UploadCutoff) {
|
||||
// If a large file upload in chunks
|
||||
err = o.uploadMultipart(in, size, blob, &putBlobOptions)
|
||||
err = o.uploadMultipart(in, size, &blob, &httpHeaders)
|
||||
} else {
|
||||
// Write a small blob in one transaction
|
||||
if size == 0 {
|
||||
in = nil
|
||||
}
|
||||
err = blob.CreateBlockBlobFromReader(in, &putBlobOptions)
|
||||
blockBlobURL := blob.ToBlockBlobURL()
|
||||
_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
|
||||
}
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
@@ -1129,9 +1206,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
// Remove an object
|
||||
func (o *Object) Remove() error {
|
||||
blob := o.getBlobReference()
|
||||
options := storage.DeleteBlobOptions{}
|
||||
snapShotOptions := azblob.DeleteSnapshotsOptionNone
|
||||
ac := azblob.BlobAccessConditions{}
|
||||
ctx := context.Background()
|
||||
return o.fs.pacer.Call(func() (bool, error) {
|
||||
err := blob.Delete(&options)
|
||||
_, err := blob.Delete(ctx, snapShotOptions, ac)
|
||||
return o.fs.shouldRetry(err)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
|
||||
|
||||
package azureblob_test
|
||||
|
||||
import (
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
// +build freebsd netbsd openbsd plan9 solaris !go1.8
|
||||
|
||||
package azureblob
|
||||
+99
-63
@@ -22,8 +22,8 @@ import (
|
||||
"github.com/ncw/rclone/backend/b2/api"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/accounting"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -34,30 +34,27 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEndpoint = "https://api.backblazeb2.com"
|
||||
headerPrefix = "x-bz-info-" // lower case as that is what the server returns
|
||||
timeKey = "src_last_modified_millis"
|
||||
timeHeader = headerPrefix + timeKey
|
||||
sha1Key = "large_file_sha1"
|
||||
sha1Header = "X-Bz-Content-Sha1"
|
||||
sha1InfoHeader = headerPrefix + sha1Key
|
||||
testModeHeader = "X-Bz-Test-Mode"
|
||||
retryAfterHeader = "Retry-After"
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 5 * time.Minute
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
maxParts = 10000
|
||||
maxVersions = 100 // maximum number of versions we search in --b2-versions mode
|
||||
defaultEndpoint = "https://api.backblazeb2.com"
|
||||
headerPrefix = "x-bz-info-" // lower case as that is what the server returns
|
||||
timeKey = "src_last_modified_millis"
|
||||
timeHeader = headerPrefix + timeKey
|
||||
sha1Key = "large_file_sha1"
|
||||
sha1Header = "X-Bz-Content-Sha1"
|
||||
sha1InfoHeader = headerPrefix + sha1Key
|
||||
testModeHeader = "X-Bz-Test-Mode"
|
||||
retryAfterHeader = "Retry-After"
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 5 * time.Minute
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
maxParts = 10000
|
||||
maxVersions = 100 // maximum number of versions we search in --b2-versions mode
|
||||
minChunkSize = 5E6
|
||||
defaultChunkSize = 96 * 1024 * 1024
|
||||
defaultUploadCutoff = 200E6
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
minChunkSize = fs.SizeSuffix(5E6)
|
||||
chunkSize = fs.SizeSuffix(96 * 1024 * 1024)
|
||||
uploadCutoff = fs.SizeSuffix(200E6)
|
||||
b2TestMode = flags.StringP("b2-test-mode", "", "", "A flag string for X-Bz-Test-Mode header.")
|
||||
b2Versions = flags.BoolP("b2-versions", "", false, "Include old versions in directory listings.")
|
||||
b2HardDelete = flags.BoolP("b2-hard-delete", "", false, "Permanently delete files on remote removal, otherwise hide files.")
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
)
|
||||
|
||||
@@ -68,29 +65,64 @@ func init() {
|
||||
Description: "Backblaze B2",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "account",
|
||||
Help: "Account ID",
|
||||
Name: "account",
|
||||
Help: "Account ID",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: "Application Key",
|
||||
Name: "key",
|
||||
Help: "Application Key",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service - leave blank normally.",
|
||||
},
|
||||
},
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service.\nLeave blank normally.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "test_mode",
|
||||
Help: "A flag string for X-Bz-Test-Mode header for debugging.",
|
||||
Default: "",
|
||||
Hide: fs.OptionHideConfigurator,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "versions",
|
||||
Help: "Include old versions in directory listings.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "hard_delete",
|
||||
Help: "Permanently delete files on remote removal, otherwise hide files.",
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: "Cutoff for switching to chunked upload.",
|
||||
Default: fs.SizeSuffix(defaultUploadCutoff),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "Upload chunk size. Must fit in memory.",
|
||||
Default: fs.SizeSuffix(defaultChunkSize),
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&uploadCutoff, "b2-upload-cutoff", "", "Cutoff for switching to chunked upload")
|
||||
flags.VarP(&chunkSize, "b2-chunk-size", "", "Upload chunk size. Must fit in memory.")
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Account string `config:"account"`
|
||||
Key string `config:"key"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
TestMode string `config:"test_mode"`
|
||||
Versions bool `config:"versions"`
|
||||
HardDelete bool `config:"hard_delete"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
}
|
||||
|
||||
// Fs represents a remote b2 server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed config options
|
||||
features *fs.Features // optional features
|
||||
account string // account name
|
||||
key string // auth key
|
||||
endpoint string // name of the starting api endpoint
|
||||
srv *rest.Client // the connection to the b2 server
|
||||
bucket string // the bucket we are working on
|
||||
bucketOKMu sync.Mutex // mutex to protect bucket OK
|
||||
@@ -232,33 +264,37 @@ func errorHandler(resp *http.Response) error {
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, bucket:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
if uploadCutoff < chunkSize {
|
||||
return nil, errors.Errorf("b2: upload cutoff (%v) must be greater than or equal to chunk size (%v)", uploadCutoff, chunkSize)
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if chunkSize < minChunkSize {
|
||||
return nil, errors.Errorf("b2: chunk size can't be less than %v - was %v", minChunkSize, chunkSize)
|
||||
if opt.UploadCutoff < opt.ChunkSize {
|
||||
return nil, errors.Errorf("b2: upload cutoff (%v) must be greater than or equal to chunk size (%v)", opt.UploadCutoff, opt.ChunkSize)
|
||||
}
|
||||
if opt.ChunkSize < minChunkSize {
|
||||
return nil, errors.Errorf("b2: chunk size can't be less than %v - was %v", minChunkSize, opt.ChunkSize)
|
||||
}
|
||||
bucket, directory, err := parsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
account := config.FileGet(name, "account")
|
||||
if account == "" {
|
||||
if opt.Account == "" {
|
||||
return nil, errors.New("account not found")
|
||||
}
|
||||
key := config.FileGet(name, "key")
|
||||
if key == "" {
|
||||
if opt.Key == "" {
|
||||
return nil, errors.New("key not found")
|
||||
}
|
||||
endpoint := config.FileGet(name, "endpoint", defaultEndpoint)
|
||||
if opt.Endpoint == "" {
|
||||
opt.Endpoint = defaultEndpoint
|
||||
}
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
bucket: bucket,
|
||||
root: directory,
|
||||
account: account,
|
||||
key: key,
|
||||
endpoint: endpoint,
|
||||
srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
bufferTokens: make(chan []byte, fs.Config.Transfers),
|
||||
@@ -269,8 +305,8 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
BucketBased: true,
|
||||
}).Fill(f)
|
||||
// Set the test flag if required
|
||||
if *b2TestMode != "" {
|
||||
testMode := strings.TrimSpace(*b2TestMode)
|
||||
if opt.TestMode != "" {
|
||||
testMode := strings.TrimSpace(opt.TestMode)
|
||||
f.srv.SetHeader(testModeHeader, testMode)
|
||||
fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
|
||||
}
|
||||
@@ -316,9 +352,9 @@ func (f *Fs) authorizeAccount() error {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/b2api/v1/b2_authorize_account",
|
||||
RootURL: f.endpoint,
|
||||
UserName: f.account,
|
||||
Password: f.key,
|
||||
RootURL: f.opt.Endpoint,
|
||||
UserName: f.opt.Account,
|
||||
Password: f.opt.Key,
|
||||
ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -384,7 +420,7 @@ func (f *Fs) clearUploadURL() {
|
||||
func (f *Fs) getUploadBlock() []byte {
|
||||
buf := <-f.bufferTokens
|
||||
if buf == nil {
|
||||
buf = make([]byte, chunkSize)
|
||||
buf = make([]byte, f.opt.ChunkSize)
|
||||
}
|
||||
// fs.Debugf(f, "Getting upload block %p", buf)
|
||||
return buf
|
||||
@@ -393,7 +429,7 @@ func (f *Fs) getUploadBlock() []byte {
|
||||
// putUploadBlock returns a block to the pool of size chunkSize
|
||||
func (f *Fs) putUploadBlock(buf []byte) {
|
||||
buf = buf[:cap(buf)]
|
||||
if len(buf) != int(chunkSize) {
|
||||
if len(buf) != int(f.opt.ChunkSize) {
|
||||
panic("bad blocksize returned to pool")
|
||||
}
|
||||
// fs.Debugf(f, "Returning upload block %p", buf)
|
||||
@@ -563,7 +599,7 @@ func (f *Fs) markBucketOK() {
|
||||
// listDir lists a single directory
|
||||
func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
|
||||
last := ""
|
||||
err = f.list(dir, false, "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
|
||||
err = f.list(dir, false, "", 0, f.opt.Versions, func(remote string, object *api.File, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(remote, object, isDirectory, &last)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -635,7 +671,7 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
|
||||
}
|
||||
list := walk.NewListRHelper(callback)
|
||||
last := ""
|
||||
err = f.list(dir, true, "", 0, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
|
||||
err = f.list(dir, true, "", 0, f.opt.Versions, func(remote string, object *api.File, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(remote, object, isDirectory, &last)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1035,12 +1071,12 @@ func (o *Object) readMetaData() (err error) {
|
||||
maxSearched := 1
|
||||
var timestamp api.Timestamp
|
||||
baseRemote := o.remote
|
||||
if *b2Versions {
|
||||
if o.fs.opt.Versions {
|
||||
timestamp, baseRemote = api.RemoveVersion(baseRemote)
|
||||
maxSearched = maxVersions
|
||||
}
|
||||
var info *api.File
|
||||
err = o.fs.list("", true, baseRemote, maxSearched, *b2Versions, func(remote string, object *api.File, isDirectory bool) error {
|
||||
err = o.fs.list("", true, baseRemote, maxSearched, o.fs.opt.Versions, func(remote string, object *api.File, isDirectory bool) error {
|
||||
if isDirectory {
|
||||
return nil
|
||||
}
|
||||
@@ -1254,7 +1290,7 @@ func urlEncode(in string) string {
|
||||
//
|
||||
// The new object may have been created if an error is returned
|
||||
func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
if *b2Versions {
|
||||
if o.fs.opt.Versions {
|
||||
return errNotWithVersions
|
||||
}
|
||||
err = o.fs.Mkdir("")
|
||||
@@ -1289,7 +1325,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else if size > int64(uploadCutoff) {
|
||||
} else if size > int64(o.fs.opt.UploadCutoff) {
|
||||
up, err := o.fs.newLargeUpload(o, in, src)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1408,10 +1444,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove() error {
|
||||
if *b2Versions {
|
||||
if o.fs.opt.Versions {
|
||||
return errNotWithVersions
|
||||
}
|
||||
if *b2HardDelete {
|
||||
if o.fs.opt.HardDelete {
|
||||
return o.fs.deleteByID(o.id, o.fs.root+o.remote)
|
||||
}
|
||||
return o.fs.hide(o.fs.root + o.remote)
|
||||
|
||||
@@ -86,10 +86,10 @@ func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *lar
|
||||
parts := int64(0)
|
||||
sha1SliceSize := int64(maxParts)
|
||||
if size == -1 {
|
||||
fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", fs.SizeSuffix(chunkSize), fs.SizeSuffix(maxParts*chunkSize))
|
||||
fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
|
||||
} else {
|
||||
parts = size / int64(chunkSize)
|
||||
if size%int64(chunkSize) != 0 {
|
||||
parts = size / int64(o.fs.opt.ChunkSize)
|
||||
if size%int64(o.fs.opt.ChunkSize) != 0 {
|
||||
parts++
|
||||
}
|
||||
if parts > maxParts {
|
||||
@@ -409,8 +409,8 @@ outer:
|
||||
}
|
||||
|
||||
reqSize := remaining
|
||||
if reqSize >= int64(chunkSize) {
|
||||
reqSize = int64(chunkSize)
|
||||
if reqSize >= int64(up.f.opt.ChunkSize) {
|
||||
reqSize = int64(up.f.opt.ChunkSize)
|
||||
}
|
||||
|
||||
// Get a block of memory
|
||||
|
||||
+31
-12
@@ -23,7 +23,8 @@ import (
|
||||
"github.com/ncw/rclone/backend/box/api"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -46,6 +47,7 @@ const (
|
||||
uploadURL = "https://upload.box.com/api/2.0"
|
||||
listChunks = 1000 // chunk size to read directory listings
|
||||
minUploadCutoff = 50000000 // upload cutoff can be no lower than this
|
||||
defaultUploadCutoff = 50 * 1024 * 1024
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -61,7 +63,6 @@ var (
|
||||
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
||||
RedirectURL: oauthutil.RedirectURL,
|
||||
}
|
||||
uploadCutoff = fs.SizeSuffix(50 * 1024 * 1024)
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -70,27 +71,37 @@ func init() {
|
||||
Name: "box",
|
||||
Description: "Box",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
err := oauthutil.Config("box", name, oauthConfig)
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("box", name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Box App Client Id - leave blank normally.",
|
||||
Help: "Box App Client Id.\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Box App Client Secret - leave blank normally.",
|
||||
Help: "Box App Client Secret\nLeave blank normally.",
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: "Cutoff for switching to multipart upload.",
|
||||
Default: fs.SizeSuffix(defaultUploadCutoff),
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&uploadCutoff, "box-upload-cutoff", "", "Cutoff for switching to multipart upload")
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
}
|
||||
|
||||
// Fs represents a remote box
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
srv *rest.Client // the connection to the one drive server
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
@@ -219,13 +230,20 @@ func errorHandler(resp *http.Response) error {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
if uploadCutoff < minUploadCutoff {
|
||||
return nil, errors.Errorf("box: upload cutoff (%v) must be greater than equal to %v", uploadCutoff, fs.SizeSuffix(minUploadCutoff))
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if opt.UploadCutoff < minUploadCutoff {
|
||||
return nil, errors.Errorf("box: upload cutoff (%v) must be greater than equal to %v", opt.UploadCutoff, fs.SizeSuffix(minUploadCutoff))
|
||||
}
|
||||
|
||||
root = parsePath(root)
|
||||
oAuthClient, ts, err := oauthutil.NewClient(name, oauthConfig)
|
||||
oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure Box: %v", err)
|
||||
}
|
||||
@@ -233,6 +251,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
|
||||
@@ -1035,7 +1054,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
}
|
||||
|
||||
// Upload with simple or multipart
|
||||
if size <= int64(uploadCutoff) {
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err = o.upload(in, leaf, directoryID, modTime)
|
||||
} else {
|
||||
err = o.uploadMultipart(in, leaf, directoryID, size, modTime)
|
||||
|
||||
Vendored
+225
-213
@@ -18,7 +18,8 @@ import (
|
||||
"github.com/ncw/rclone/backend/crypt"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/ncw/rclone/fs/rc"
|
||||
@@ -30,13 +31,13 @@ import (
|
||||
|
||||
const (
|
||||
// DefCacheChunkSize is the default value for chunk size
|
||||
DefCacheChunkSize = "5M"
|
||||
DefCacheChunkSize = fs.SizeSuffix(5 * 1024 * 1024)
|
||||
// DefCacheTotalChunkSize is the default value for the maximum size of stored chunks
|
||||
DefCacheTotalChunkSize = "10G"
|
||||
DefCacheTotalChunkSize = fs.SizeSuffix(10 * 1024 * 1024 * 1024)
|
||||
// DefCacheChunkCleanInterval is the interval at which chunks are cleaned
|
||||
DefCacheChunkCleanInterval = "1m"
|
||||
DefCacheChunkCleanInterval = fs.Duration(time.Minute)
|
||||
// DefCacheInfoAge is the default value for object info age
|
||||
DefCacheInfoAge = "6h"
|
||||
DefCacheInfoAge = fs.Duration(6 * time.Hour)
|
||||
// DefCacheReadRetries is the default value for read retries
|
||||
DefCacheReadRetries = 10
|
||||
// DefCacheTotalWorkers is how many workers run in parallel to download chunks
|
||||
@@ -48,29 +49,9 @@ const (
|
||||
// DefCacheWrites will cache file data on writes through the cache
|
||||
DefCacheWrites = false
|
||||
// DefCacheTmpWaitTime says how long should files be stored in local cache before being uploaded
|
||||
DefCacheTmpWaitTime = "15m"
|
||||
DefCacheTmpWaitTime = fs.Duration(15 * time.Second)
|
||||
// DefCacheDbWaitTime defines how long the cache backend should wait for the DB to be available
|
||||
DefCacheDbWaitTime = 1 * time.Second
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
cacheDbPath = flags.StringP("cache-db-path", "", filepath.Join(config.CacheDir, "cache-backend"), "Directory to cache DB")
|
||||
cacheChunkPath = flags.StringP("cache-chunk-path", "", filepath.Join(config.CacheDir, "cache-backend"), "Directory to cached chunk files")
|
||||
cacheDbPurge = flags.BoolP("cache-db-purge", "", false, "Purge the cache DB before")
|
||||
cacheChunkSize = flags.StringP("cache-chunk-size", "", DefCacheChunkSize, "The size of a chunk")
|
||||
cacheTotalChunkSize = flags.StringP("cache-total-chunk-size", "", DefCacheTotalChunkSize, "The total size which the chunks can take up from the disk")
|
||||
cacheChunkCleanInterval = flags.StringP("cache-chunk-clean-interval", "", DefCacheChunkCleanInterval, "Interval at which chunk cleanup runs")
|
||||
cacheInfoAge = flags.StringP("cache-info-age", "", DefCacheInfoAge, "How much time should object info be stored in cache")
|
||||
cacheReadRetries = flags.IntP("cache-read-retries", "", DefCacheReadRetries, "How many times to retry a read from a cache storage")
|
||||
cacheTotalWorkers = flags.IntP("cache-workers", "", DefCacheTotalWorkers, "How many workers should run in parallel to download chunks")
|
||||
cacheChunkNoMemory = flags.BoolP("cache-chunk-no-memory", "", DefCacheChunkNoMemory, "Disable the in-memory cache for storing chunks during streaming")
|
||||
cacheRps = flags.IntP("cache-rps", "", int(DefCacheRps), "Limits the number of requests per second to the source FS. -1 disables the rate limiter")
|
||||
cacheStoreWrites = flags.BoolP("cache-writes", "", DefCacheWrites, "Will cache file data on writes through the FS")
|
||||
cacheTempWritePath = flags.StringP("cache-tmp-upload-path", "", "", "Directory to keep temporary files until they are uploaded to the cloud storage")
|
||||
cacheTempWaitTime = flags.StringP("cache-tmp-wait-time", "", DefCacheTmpWaitTime, "How long should files be stored in local cache before being uploaded")
|
||||
cacheDbWaitTime = flags.DurationP("cache-db-wait-time", "", DefCacheDbWaitTime, "How long to wait for the DB to be available - 0 is unlimited")
|
||||
DefCacheDbWaitTime = fs.Duration(1 * time.Second)
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -80,73 +61,155 @@ func init() {
|
||||
Description: "Cache a remote",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "remote",
|
||||
Help: "Remote to cache.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
|
||||
Name: "remote",
|
||||
Help: "Remote to cache.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "plex_url",
|
||||
Help: "Optional: The URL of the Plex server",
|
||||
Optional: true,
|
||||
Name: "plex_url",
|
||||
Help: "The URL of the Plex server",
|
||||
}, {
|
||||
Name: "plex_username",
|
||||
Help: "Optional: The username of the Plex user",
|
||||
Optional: true,
|
||||
Name: "plex_username",
|
||||
Help: "The username of the Plex user",
|
||||
}, {
|
||||
Name: "plex_password",
|
||||
Help: "Optional: The password of the Plex user",
|
||||
Help: "The password of the Plex user",
|
||||
IsPassword: true,
|
||||
Optional: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "The size of a chunk. Lower value good for slow connections but can affect seamless reading. \nDefault: " + DefCacheChunkSize,
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "1m",
|
||||
Help: "1MB",
|
||||
}, {
|
||||
Value: "5M",
|
||||
Help: "5 MB",
|
||||
}, {
|
||||
Value: "10M",
|
||||
Help: "10 MB",
|
||||
},
|
||||
},
|
||||
Optional: true,
|
||||
Name: "plex_token",
|
||||
Help: "The plex token for authentication - auto set normally",
|
||||
Hide: fs.OptionHideBoth,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "info_age",
|
||||
Help: "How much time should object info (file size, file hashes etc) be stored in cache. Use a very high value if you don't plan on changing the source FS from outside the cache. \nAccepted units are: \"s\", \"m\", \"h\".\nDefault: " + DefCacheInfoAge,
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "1h",
|
||||
Help: "1 hour",
|
||||
}, {
|
||||
Value: "24h",
|
||||
Help: "24 hours",
|
||||
}, {
|
||||
Value: "48h",
|
||||
Help: "48 hours",
|
||||
},
|
||||
},
|
||||
Optional: true,
|
||||
Name: "chunk_size",
|
||||
Help: "The size of a chunk. Lower value good for slow connections but can affect seamless reading.",
|
||||
Default: DefCacheChunkSize,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "1m",
|
||||
Help: "1MB",
|
||||
}, {
|
||||
Value: "5M",
|
||||
Help: "5 MB",
|
||||
}, {
|
||||
Value: "10M",
|
||||
Help: "10 MB",
|
||||
}},
|
||||
}, {
|
||||
Name: "chunk_total_size",
|
||||
Help: "The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted. \nDefault: " + DefCacheTotalChunkSize,
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "500M",
|
||||
Help: "500 MB",
|
||||
}, {
|
||||
Value: "1G",
|
||||
Help: "1 GB",
|
||||
}, {
|
||||
Value: "10G",
|
||||
Help: "10 GB",
|
||||
},
|
||||
},
|
||||
Optional: true,
|
||||
Name: "info_age",
|
||||
Help: "How much time should object info (file size, file hashes etc) be stored in cache.\nUse a very high value if you don't plan on changing the source FS from outside the cache.\nAccepted units are: \"s\", \"m\", \"h\".",
|
||||
Default: DefCacheInfoAge,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "1h",
|
||||
Help: "1 hour",
|
||||
}, {
|
||||
Value: "24h",
|
||||
Help: "24 hours",
|
||||
}, {
|
||||
Value: "48h",
|
||||
Help: "48 hours",
|
||||
}},
|
||||
}, {
|
||||
Name: "chunk_total_size",
|
||||
Help: "The maximum size of stored chunks. When the storage grows beyond this size, the oldest chunks will be deleted.",
|
||||
Default: DefCacheTotalChunkSize,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "500M",
|
||||
Help: "500 MB",
|
||||
}, {
|
||||
Value: "1G",
|
||||
Help: "1 GB",
|
||||
}, {
|
||||
Value: "10G",
|
||||
Help: "10 GB",
|
||||
}},
|
||||
}, {
|
||||
Name: "db_path",
|
||||
Default: filepath.Join(config.CacheDir, "cache-backend"),
|
||||
Help: "Directory to cache DB",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_path",
|
||||
Default: filepath.Join(config.CacheDir, "cache-backend"),
|
||||
Help: "Directory to cache chunk files",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "db_purge",
|
||||
Default: false,
|
||||
Help: "Purge the cache DB before",
|
||||
Hide: fs.OptionHideConfigurator,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_clean_interval",
|
||||
Default: DefCacheChunkCleanInterval,
|
||||
Help: "Interval at which chunk cleanup runs",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "read_retries",
|
||||
Default: DefCacheReadRetries,
|
||||
Help: "How many times to retry a read from a cache storage",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "workers",
|
||||
Default: DefCacheTotalWorkers,
|
||||
Help: "How many workers should run in parallel to download chunks",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_no_memory",
|
||||
Default: DefCacheChunkNoMemory,
|
||||
Help: "Disable the in-memory cache for storing chunks during streaming",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "rps",
|
||||
Default: int(DefCacheRps),
|
||||
Help: "Limits the number of requests per second to the source FS. -1 disables the rate limiter",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "writes",
|
||||
Default: DefCacheWrites,
|
||||
Help: "Will cache file data on writes through the FS",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "tmp_upload_path",
|
||||
Default: "",
|
||||
Help: "Directory to keep temporary files until they are uploaded to the cloud storage",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "tmp_wait_time",
|
||||
Default: DefCacheTmpWaitTime,
|
||||
Help: "How long should files be stored in local cache before being uploaded",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "db_wait_time",
|
||||
Default: DefCacheDbWaitTime,
|
||||
Help: "How long to wait for the DB to be available - 0 is unlimited",
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Remote string `config:"remote"`
|
||||
PlexURL string `config:"plex_url"`
|
||||
PlexUsername string `config:"plex_username"`
|
||||
PlexPassword string `config:"plex_password"`
|
||||
PlexToken string `config:"plex_token"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
InfoAge fs.Duration `config:"info_age"`
|
||||
ChunkTotalSize fs.SizeSuffix `config:"chunk_total_size"`
|
||||
DbPath string `config:"db_path"`
|
||||
ChunkPath string `config:"chunk_path"`
|
||||
DbPurge bool `config:"db_purge"`
|
||||
ChunkCleanInterval fs.Duration `config:"chunk_clean_interval"`
|
||||
ReadRetries int `config:"read_retries"`
|
||||
TotalWorkers int `config:"workers"`
|
||||
ChunkNoMemory bool `config:"chunk_no_memory"`
|
||||
Rps int `config:"rps"`
|
||||
StoreWrites bool `config:"writes"`
|
||||
TempWritePath string `config:"tmp_upload_path"`
|
||||
TempWaitTime fs.Duration `config:"tmp_wait_time"`
|
||||
DbWaitTime fs.Duration `config:"db_wait_time"`
|
||||
}
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
fs.Fs
|
||||
@@ -154,21 +217,10 @@ type Fs struct {
|
||||
|
||||
name string
|
||||
root string
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
cache *Persistent
|
||||
|
||||
fileAge time.Duration
|
||||
chunkSize int64
|
||||
chunkTotalSize int64
|
||||
chunkCleanInterval time.Duration
|
||||
readRetries int
|
||||
totalWorkers int
|
||||
totalMaxWorkers int
|
||||
chunkMemory bool
|
||||
cacheWrites bool
|
||||
tempWritePath string
|
||||
tempWriteWait time.Duration
|
||||
tempFs fs.Fs
|
||||
tempFs fs.Fs
|
||||
|
||||
lastChunkCleanup time.Time
|
||||
cleanupMu sync.Mutex
|
||||
@@ -188,9 +240,19 @@ func parseRootPath(path string) (string, error) {
|
||||
}
|
||||
|
||||
// NewFs constructs a Fs from the path, container:path
|
||||
func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
remote := config.FileGet(name, "remote")
|
||||
if strings.HasPrefix(remote, name+":") {
|
||||
func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.ChunkTotalSize < opt.ChunkSize*fs.SizeSuffix(opt.TotalWorkers) {
|
||||
return nil, errors.Errorf("don't set cache-total-chunk-size(%v) less than cache-chunk-size(%v) * cache-workers(%v)",
|
||||
opt.ChunkTotalSize, opt.ChunkSize, opt.TotalWorkers)
|
||||
}
|
||||
|
||||
if strings.HasPrefix(opt.Remote, name+":") {
|
||||
return nil, errors.New("can't point cache remote at itself - check the value of the remote setting")
|
||||
}
|
||||
|
||||
@@ -199,7 +261,7 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
return nil, errors.Wrapf(err, "failed to clean root path %q", rootPath)
|
||||
}
|
||||
|
||||
remotePath := path.Join(remote, rpath)
|
||||
remotePath := path.Join(opt.Remote, rpath)
|
||||
wrappedFs, wrapErr := fs.NewFs(remotePath)
|
||||
if wrapErr != nil && wrapErr != fs.ErrorIsFile {
|
||||
return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath)
|
||||
@@ -210,97 +272,46 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
fsErr = fs.ErrorIsFile
|
||||
rpath = cleanPath(path.Dir(rpath))
|
||||
}
|
||||
plexURL := config.FileGet(name, "plex_url")
|
||||
plexToken := config.FileGet(name, "plex_token")
|
||||
var chunkSize fs.SizeSuffix
|
||||
chunkSizeString := config.FileGet(name, "chunk_size", DefCacheChunkSize)
|
||||
if *cacheChunkSize != DefCacheChunkSize {
|
||||
chunkSizeString = *cacheChunkSize
|
||||
}
|
||||
err = chunkSize.Set(chunkSizeString)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to understand chunk size %v", chunkSizeString)
|
||||
}
|
||||
var chunkTotalSize fs.SizeSuffix
|
||||
chunkTotalSizeString := config.FileGet(name, "chunk_total_size", DefCacheTotalChunkSize)
|
||||
if *cacheTotalChunkSize != DefCacheTotalChunkSize {
|
||||
chunkTotalSizeString = *cacheTotalChunkSize
|
||||
}
|
||||
err = chunkTotalSize.Set(chunkTotalSizeString)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to understand chunk total size %v", chunkTotalSizeString)
|
||||
}
|
||||
chunkCleanIntervalStr := *cacheChunkCleanInterval
|
||||
chunkCleanInterval, err := time.ParseDuration(chunkCleanIntervalStr)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to understand duration %v", chunkCleanIntervalStr)
|
||||
}
|
||||
infoAge := config.FileGet(name, "info_age", DefCacheInfoAge)
|
||||
if *cacheInfoAge != DefCacheInfoAge {
|
||||
infoAge = *cacheInfoAge
|
||||
}
|
||||
infoDuration, err := time.ParseDuration(infoAge)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to understand duration %v", infoAge)
|
||||
}
|
||||
waitTime, err := time.ParseDuration(*cacheTempWaitTime)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to understand duration %v", *cacheTempWaitTime)
|
||||
}
|
||||
// configure cache backend
|
||||
if *cacheDbPurge {
|
||||
if opt.DbPurge {
|
||||
fs.Debugf(name, "Purging the DB")
|
||||
}
|
||||
f := &Fs{
|
||||
Fs: wrappedFs,
|
||||
name: name,
|
||||
root: rpath,
|
||||
fileAge: infoDuration,
|
||||
chunkSize: int64(chunkSize),
|
||||
chunkTotalSize: int64(chunkTotalSize),
|
||||
chunkCleanInterval: chunkCleanInterval,
|
||||
readRetries: *cacheReadRetries,
|
||||
totalWorkers: *cacheTotalWorkers,
|
||||
totalMaxWorkers: *cacheTotalWorkers,
|
||||
chunkMemory: !*cacheChunkNoMemory,
|
||||
cacheWrites: *cacheStoreWrites,
|
||||
lastChunkCleanup: time.Now().Truncate(time.Hour * 24 * 30),
|
||||
tempWritePath: *cacheTempWritePath,
|
||||
tempWriteWait: waitTime,
|
||||
cleanupChan: make(chan bool, 1),
|
||||
notifiedRemotes: make(map[string]bool),
|
||||
Fs: wrappedFs,
|
||||
name: name,
|
||||
root: rpath,
|
||||
opt: *opt,
|
||||
lastChunkCleanup: time.Now().Truncate(time.Hour * 24 * 30),
|
||||
cleanupChan: make(chan bool, 1),
|
||||
notifiedRemotes: make(map[string]bool),
|
||||
}
|
||||
if f.chunkTotalSize < (f.chunkSize * int64(f.totalWorkers)) {
|
||||
return nil, errors.Errorf("don't set cache-total-chunk-size(%v) less than cache-chunk-size(%v) * cache-workers(%v)",
|
||||
f.chunkTotalSize, f.chunkSize, f.totalWorkers)
|
||||
}
|
||||
f.rateLimiter = rate.NewLimiter(rate.Limit(float64(*cacheRps)), f.totalWorkers)
|
||||
f.rateLimiter = rate.NewLimiter(rate.Limit(float64(opt.Rps)), opt.TotalWorkers)
|
||||
|
||||
f.plexConnector = &plexConnector{}
|
||||
if plexURL != "" {
|
||||
if plexToken != "" {
|
||||
f.plexConnector, err = newPlexConnectorWithToken(f, plexURL, plexToken)
|
||||
if opt.PlexURL != "" {
|
||||
if opt.PlexToken != "" {
|
||||
f.plexConnector, err = newPlexConnectorWithToken(f, opt.PlexURL, opt.PlexToken)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to connect to the Plex API %v", plexURL)
|
||||
return nil, errors.Wrapf(err, "failed to connect to the Plex API %v", opt.PlexURL)
|
||||
}
|
||||
} else {
|
||||
plexUsername := config.FileGet(name, "plex_username")
|
||||
plexPassword := config.FileGet(name, "plex_password")
|
||||
if plexPassword != "" && plexUsername != "" {
|
||||
decPass, err := obscure.Reveal(plexPassword)
|
||||
if opt.PlexPassword != "" && opt.PlexUsername != "" {
|
||||
decPass, err := obscure.Reveal(opt.PlexPassword)
|
||||
if err != nil {
|
||||
decPass = plexPassword
|
||||
decPass = opt.PlexPassword
|
||||
}
|
||||
f.plexConnector, err = newPlexConnector(f, plexURL, plexUsername, decPass)
|
||||
f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, func(token string) {
|
||||
m.Set("plex_token", token)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to connect to the Plex API %v", plexURL)
|
||||
return nil, errors.Wrapf(err, "failed to connect to the Plex API %v", opt.PlexURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dbPath := *cacheDbPath
|
||||
chunkPath := *cacheChunkPath
|
||||
dbPath := f.opt.DbPath
|
||||
chunkPath := f.opt.ChunkPath
|
||||
// if the dbPath is non default but the chunk path is default, we overwrite the last to follow the same one as dbPath
|
||||
if dbPath != filepath.Join(config.CacheDir, "cache-backend") &&
|
||||
chunkPath == filepath.Join(config.CacheDir, "cache-backend") {
|
||||
@@ -326,7 +337,8 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
fs.Infof(name, "Cache DB path: %v", dbPath)
|
||||
fs.Infof(name, "Cache chunk path: %v", chunkPath)
|
||||
f.cache, err = GetPersistent(dbPath, chunkPath, &Features{
|
||||
PurgeDb: *cacheDbPurge,
|
||||
PurgeDb: opt.DbPurge,
|
||||
DbWaitTime: time.Duration(opt.DbWaitTime),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to start cache db")
|
||||
@@ -335,7 +347,7 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
c := make(chan os.Signal, 1)
|
||||
signal.Notify(c, syscall.SIGHUP)
|
||||
atexit.Register(func() {
|
||||
if plexURL != "" {
|
||||
if opt.PlexURL != "" {
|
||||
f.plexConnector.closeWebsocket()
|
||||
}
|
||||
f.StopBackgroundRunners()
|
||||
@@ -350,35 +362,35 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
}
|
||||
}()
|
||||
|
||||
fs.Infof(name, "Chunk Memory: %v", f.chunkMemory)
|
||||
fs.Infof(name, "Chunk Size: %v", fs.SizeSuffix(f.chunkSize))
|
||||
fs.Infof(name, "Chunk Total Size: %v", fs.SizeSuffix(f.chunkTotalSize))
|
||||
fs.Infof(name, "Chunk Clean Interval: %v", f.chunkCleanInterval.String())
|
||||
fs.Infof(name, "Workers: %v", f.totalWorkers)
|
||||
fs.Infof(name, "File Age: %v", f.fileAge.String())
|
||||
if f.cacheWrites {
|
||||
fs.Infof(name, "Chunk Memory: %v", !f.opt.ChunkNoMemory)
|
||||
fs.Infof(name, "Chunk Size: %v", f.opt.ChunkSize)
|
||||
fs.Infof(name, "Chunk Total Size: %v", f.opt.ChunkTotalSize)
|
||||
fs.Infof(name, "Chunk Clean Interval: %v", f.opt.ChunkCleanInterval)
|
||||
fs.Infof(name, "Workers: %v", f.opt.TotalWorkers)
|
||||
fs.Infof(name, "File Age: %v", f.opt.InfoAge)
|
||||
if !f.opt.StoreWrites {
|
||||
fs.Infof(name, "Cache Writes: enabled")
|
||||
}
|
||||
|
||||
if f.tempWritePath != "" {
|
||||
err = os.MkdirAll(f.tempWritePath, os.ModePerm)
|
||||
if f.opt.TempWritePath != "" {
|
||||
err = os.MkdirAll(f.opt.TempWritePath, os.ModePerm)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create cache directory %v", f.tempWritePath)
|
||||
return nil, errors.Wrapf(err, "failed to create cache directory %v", f.opt.TempWritePath)
|
||||
}
|
||||
f.tempWritePath = filepath.ToSlash(f.tempWritePath)
|
||||
f.tempFs, err = fs.NewFs(f.tempWritePath)
|
||||
f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath)
|
||||
f.tempFs, err = fs.NewFs(f.opt.TempWritePath)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create temp fs: %v", err)
|
||||
}
|
||||
fs.Infof(name, "Upload Temp Rest Time: %v", f.tempWriteWait.String())
|
||||
fs.Infof(name, "Upload Temp FS: %v", f.tempWritePath)
|
||||
fs.Infof(name, "Upload Temp Rest Time: %v", f.opt.TempWaitTime)
|
||||
fs.Infof(name, "Upload Temp FS: %v", f.opt.TempWritePath)
|
||||
f.backgroundRunner, _ = initBackgroundUploader(f)
|
||||
go f.backgroundRunner.run()
|
||||
}
|
||||
|
||||
go func() {
|
||||
for {
|
||||
time.Sleep(f.chunkCleanInterval)
|
||||
time.Sleep(time.Duration(f.opt.ChunkCleanInterval))
|
||||
select {
|
||||
case <-f.cleanupChan:
|
||||
fs.Infof(f, "stopping cleanup")
|
||||
@@ -391,7 +403,7 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
}()
|
||||
|
||||
if doChangeNotify := wrappedFs.Features().ChangeNotify; doChangeNotify != nil {
|
||||
doChangeNotify(f.receiveChangeNotify, f.chunkCleanInterval)
|
||||
doChangeNotify(f.receiveChangeNotify, time.Duration(f.opt.ChunkCleanInterval))
|
||||
}
|
||||
|
||||
f.features = (&fs.Features{
|
||||
@@ -400,7 +412,7 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
|
||||
}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
|
||||
// override only those features that use a temp fs and it doesn't support them
|
||||
//f.features.ChangeNotify = f.ChangeNotify
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
if f.tempFs.Features().Copy == nil {
|
||||
f.features.Copy = nil
|
||||
}
|
||||
@@ -563,7 +575,7 @@ func (f *Fs) receiveChangeNotify(forgetPath string, entryType fs.EntryType) {
|
||||
// notifyChangeUpstreamIfNeeded will check if the wrapped remote doesn't notify on changes
|
||||
// or if we use a temp fs
|
||||
func (f *Fs) notifyChangeUpstreamIfNeeded(remote string, entryType fs.EntryType) {
|
||||
if f.Fs.Features().ChangeNotify == nil || f.tempWritePath != "" {
|
||||
if f.Fs.Features().ChangeNotify == nil || f.opt.TempWritePath != "" {
|
||||
f.notifyChangeUpstream(remote, entryType)
|
||||
}
|
||||
}
|
||||
@@ -613,17 +625,17 @@ func (f *Fs) String() string {
|
||||
|
||||
// ChunkSize returns the configured chunk size
|
||||
func (f *Fs) ChunkSize() int64 {
|
||||
return f.chunkSize
|
||||
return int64(f.opt.ChunkSize)
|
||||
}
|
||||
|
||||
// InfoAge returns the configured file age
|
||||
func (f *Fs) InfoAge() time.Duration {
|
||||
return f.fileAge
|
||||
return time.Duration(f.opt.InfoAge)
|
||||
}
|
||||
|
||||
// TempUploadWaitTime returns the configured temp file upload wait time
|
||||
func (f *Fs) TempUploadWaitTime() time.Duration {
|
||||
return f.tempWriteWait
|
||||
return time.Duration(f.opt.TempWaitTime)
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
@@ -636,16 +648,16 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
|
||||
err = f.cache.GetObject(co)
|
||||
if err != nil {
|
||||
fs.Debugf(remote, "find: error: %v", err)
|
||||
} else if time.Now().After(co.CacheTs.Add(f.fileAge)) {
|
||||
} else if time.Now().After(co.CacheTs.Add(time.Duration(f.opt.InfoAge))) {
|
||||
fs.Debugf(co, "find: cold object: %+v", co)
|
||||
} else {
|
||||
fs.Debugf(co, "find: warm object: %v, expiring on: %v", co, co.CacheTs.Add(f.fileAge))
|
||||
fs.Debugf(co, "find: warm object: %v, expiring on: %v", co, co.CacheTs.Add(time.Duration(f.opt.InfoAge)))
|
||||
return co, nil
|
||||
}
|
||||
|
||||
// search for entry in source or temp fs
|
||||
var obj fs.Object
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
obj, err = f.tempFs.NewObject(remote)
|
||||
// not found in temp fs
|
||||
if err != nil {
|
||||
@@ -679,13 +691,13 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
entries, err = f.cache.GetDirEntries(cd)
|
||||
if err != nil {
|
||||
fs.Debugf(dir, "list: error: %v", err)
|
||||
} else if time.Now().After(cd.CacheTs.Add(f.fileAge)) {
|
||||
} else if time.Now().After(cd.CacheTs.Add(time.Duration(f.opt.InfoAge))) {
|
||||
fs.Debugf(dir, "list: cold listing: %v", cd.CacheTs)
|
||||
} else if len(entries) == 0 {
|
||||
// TODO: read empty dirs from source?
|
||||
fs.Debugf(dir, "list: empty listing")
|
||||
} else {
|
||||
fs.Debugf(dir, "list: warm %v from cache for: %v, expiring on: %v", len(entries), cd.abs(), cd.CacheTs.Add(f.fileAge))
|
||||
fs.Debugf(dir, "list: warm %v from cache for: %v, expiring on: %v", len(entries), cd.abs(), cd.CacheTs.Add(time.Duration(f.opt.InfoAge)))
|
||||
fs.Debugf(dir, "list: cached entries: %v", entries)
|
||||
return entries, nil
|
||||
}
|
||||
@@ -693,7 +705,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
|
||||
// we first search any temporary files stored locally
|
||||
var cachedEntries fs.DirEntries
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
queuedEntries, err := f.cache.searchPendingUploadFromDir(cd.abs())
|
||||
if err != nil {
|
||||
fs.Errorf(dir, "list: error getting pending uploads: %v", err)
|
||||
@@ -744,7 +756,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
case fs.Directory:
|
||||
cdd := DirectoryFromOriginal(f, o)
|
||||
// check if the dir isn't expired and add it in cache if it isn't
|
||||
if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(f.fileAge)) {
|
||||
if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(time.Duration(f.opt.InfoAge))) {
|
||||
batchDirectories = append(batchDirectories, cdd)
|
||||
}
|
||||
cachedEntries = append(cachedEntries, cdd)
|
||||
@@ -867,7 +879,7 @@ func (f *Fs) Mkdir(dir string) error {
|
||||
func (f *Fs) Rmdir(dir string) error {
|
||||
fs.Debugf(f, "rmdir '%s'", dir)
|
||||
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
// pause background uploads
|
||||
f.backgroundRunner.pause()
|
||||
defer f.backgroundRunner.play()
|
||||
@@ -952,7 +964,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
// pause background uploads
|
||||
f.backgroundRunner.pause()
|
||||
defer f.backgroundRunner.play()
|
||||
@@ -1079,7 +1091,7 @@ func (f *Fs) cacheReader(u io.Reader, src fs.ObjectInfo, originalRead func(inn i
|
||||
go func() {
|
||||
var offset int64
|
||||
for {
|
||||
chunk := make([]byte, f.chunkSize)
|
||||
chunk := make([]byte, f.opt.ChunkSize)
|
||||
readSize, err := io.ReadFull(pr, chunk)
|
||||
// we ignore 3 failures which are ok:
|
||||
// 1. EOF - original reading finished and we got a full buffer too
|
||||
@@ -1127,7 +1139,7 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
|
||||
var obj fs.Object
|
||||
|
||||
// queue for upload and store in temp fs if configured
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
// we need to clear the caches before a put through temp fs
|
||||
parentCd := NewDirectory(f, cleanPath(path.Dir(src.Remote())))
|
||||
_ = f.cache.ExpireDir(parentCd)
|
||||
@@ -1146,7 +1158,7 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
|
||||
}
|
||||
fs.Infof(obj, "put: queued for upload")
|
||||
// if cache writes is enabled write it first through cache
|
||||
} else if f.cacheWrites {
|
||||
} else if f.opt.StoreWrites {
|
||||
f.cacheReader(in, src, func(inn io.Reader) {
|
||||
obj, err = put(inn, src, options...)
|
||||
})
|
||||
@@ -1243,7 +1255,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||
|
||||
if srcObj.isTempFile() {
|
||||
// we check if the feature is stil active
|
||||
if f.tempWritePath == "" {
|
||||
if f.opt.TempWritePath == "" {
|
||||
fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
@@ -1319,7 +1331,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
||||
// if this is a temp object then we perform the changes locally
|
||||
if srcObj.isTempFile() {
|
||||
// we check if the feature is stil active
|
||||
if f.tempWritePath == "" {
|
||||
if f.opt.TempWritePath == "" {
|
||||
fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run")
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
@@ -1460,8 +1472,8 @@ func (f *Fs) CleanUpCache(ignoreLastTs bool) {
|
||||
f.cleanupMu.Lock()
|
||||
defer f.cleanupMu.Unlock()
|
||||
|
||||
if ignoreLastTs || time.Now().After(f.lastChunkCleanup.Add(f.chunkCleanInterval)) {
|
||||
f.cache.CleanChunksBySize(f.chunkTotalSize)
|
||||
if ignoreLastTs || time.Now().After(f.lastChunkCleanup.Add(time.Duration(f.opt.ChunkCleanInterval))) {
|
||||
f.cache.CleanChunksBySize(int64(f.opt.ChunkTotalSize))
|
||||
f.lastChunkCleanup = time.Now()
|
||||
}
|
||||
}
|
||||
@@ -1470,7 +1482,7 @@ func (f *Fs) CleanUpCache(ignoreLastTs bool) {
|
||||
// can be triggered from a terminate signal or from testing between runs
|
||||
func (f *Fs) StopBackgroundRunners() {
|
||||
f.cleanupChan <- false
|
||||
if f.tempWritePath != "" && f.backgroundRunner != nil && f.backgroundRunner.isRunning() {
|
||||
if f.opt.TempWritePath != "" && f.backgroundRunner != nil && f.backgroundRunner.isRunning() {
|
||||
f.backgroundRunner.close()
|
||||
}
|
||||
f.cache.Close()
|
||||
@@ -1528,7 +1540,7 @@ func (f *Fs) DirCacheFlush() {
|
||||
// GetBackgroundUploadChannel returns a channel that can be listened to for remote activities that happen
|
||||
// in the background
|
||||
func (f *Fs) GetBackgroundUploadChannel() chan BackgroundUploadState {
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
return f.backgroundRunner.notifyCh
|
||||
}
|
||||
return nil
|
||||
|
||||
Vendored
+24
-58
@@ -33,13 +33,13 @@ import (
|
||||
"github.com/ncw/rclone/backend/local"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/object"
|
||||
"github.com/ncw/rclone/fs/rc"
|
||||
"github.com/ncw/rclone/fs/rc/rcflags"
|
||||
"github.com/ncw/rclone/fstest"
|
||||
"github.com/ncw/rclone/vfs"
|
||||
"github.com/ncw/rclone/vfs/vfsflags"
|
||||
flag "github.com/spf13/pflag"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -140,7 +140,7 @@ func TestInternalVfsCache(t *testing.T) {
|
||||
|
||||
vfsflags.Opt.CacheMode = vfs.CacheModeWrites
|
||||
id := "tiuufo"
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"cache-writes": "true", "cache-info-age": "1h"})
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
err := rootFs.Mkdir("test")
|
||||
@@ -699,7 +699,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
|
||||
rc.Start(&rcflags.Opt)
|
||||
|
||||
id := fmt.Sprintf("ticsarc%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"rc": "true"})
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
if !runInstance.useMount {
|
||||
@@ -774,7 +774,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
|
||||
|
||||
func TestInternalCacheWrites(t *testing.T) {
|
||||
id := "ticw"
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"cache-writes": "true"})
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
cfs, err := runInstance.getCacheFs(rootFs)
|
||||
@@ -793,7 +793,7 @@ func TestInternalCacheWrites(t *testing.T) {
|
||||
|
||||
func TestInternalMaxChunkSizeRespected(t *testing.T) {
|
||||
id := fmt.Sprintf("timcsr%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"cache-workers": "1"})
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"workers": "1"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
cfs, err := runInstance.getCacheFs(rootFs)
|
||||
@@ -868,7 +868,7 @@ func TestInternalBug2117(t *testing.T) {
|
||||
|
||||
id := fmt.Sprintf("tib2117%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil,
|
||||
map[string]string{"cache-info-age": "72h", "cache-chunk-clean-interval": "15m"})
|
||||
map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
if runInstance.rootIsCrypt {
|
||||
@@ -918,10 +918,7 @@ func TestInternalBug2117(t *testing.T) {
|
||||
// run holds the remotes for a test run
|
||||
type run struct {
|
||||
okDiff time.Duration
|
||||
allCfgMap map[string]string
|
||||
allFlagMap map[string]string
|
||||
runDefaultCfgMap map[string]string
|
||||
runDefaultFlagMap map[string]string
|
||||
runDefaultCfgMap configmap.Simple
|
||||
mntDir string
|
||||
tmpUploadDir string
|
||||
useMount bool
|
||||
@@ -945,38 +942,16 @@ func newRun() *run {
|
||||
isMounted: false,
|
||||
}
|
||||
|
||||
r.allCfgMap = map[string]string{
|
||||
"plex_url": "",
|
||||
"plex_username": "",
|
||||
"plex_password": "",
|
||||
"chunk_size": cache.DefCacheChunkSize,
|
||||
"info_age": cache.DefCacheInfoAge,
|
||||
"chunk_total_size": cache.DefCacheTotalChunkSize,
|
||||
// Read in all the defaults for all the options
|
||||
fsInfo, err := fs.Find("cache")
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Couldn't find cache remote: %v", err))
|
||||
}
|
||||
r.allFlagMap = map[string]string{
|
||||
"cache-db-path": filepath.Join(config.CacheDir, "cache-backend"),
|
||||
"cache-chunk-path": filepath.Join(config.CacheDir, "cache-backend"),
|
||||
"cache-db-purge": "true",
|
||||
"cache-chunk-size": cache.DefCacheChunkSize,
|
||||
"cache-total-chunk-size": cache.DefCacheTotalChunkSize,
|
||||
"cache-chunk-clean-interval": cache.DefCacheChunkCleanInterval,
|
||||
"cache-info-age": cache.DefCacheInfoAge,
|
||||
"cache-read-retries": strconv.Itoa(cache.DefCacheReadRetries),
|
||||
"cache-workers": strconv.Itoa(cache.DefCacheTotalWorkers),
|
||||
"cache-chunk-no-memory": "false",
|
||||
"cache-rps": strconv.Itoa(cache.DefCacheRps),
|
||||
"cache-writes": "false",
|
||||
"cache-tmp-upload-path": "",
|
||||
"cache-tmp-wait-time": cache.DefCacheTmpWaitTime,
|
||||
}
|
||||
r.runDefaultCfgMap = make(map[string]string)
|
||||
for key, value := range r.allCfgMap {
|
||||
r.runDefaultCfgMap[key] = value
|
||||
}
|
||||
r.runDefaultFlagMap = make(map[string]string)
|
||||
for key, value := range r.allFlagMap {
|
||||
r.runDefaultFlagMap[key] = value
|
||||
r.runDefaultCfgMap = configmap.Simple{}
|
||||
for _, option := range fsInfo.Options {
|
||||
r.runDefaultCfgMap.Set(option.Name, fmt.Sprint(option.Default))
|
||||
}
|
||||
|
||||
if mountDir == "" {
|
||||
if runtime.GOOS != "windows" {
|
||||
r.mntDir, err = ioutil.TempDir("", "rclonecache-mount")
|
||||
@@ -1086,28 +1061,22 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
|
||||
boltDb, err := cache.GetPersistent(runInstance.dbPath, runInstance.chunkPath, &cache.Features{PurgeDb: true})
|
||||
require.NoError(t, err)
|
||||
|
||||
for k, v := range r.runDefaultCfgMap {
|
||||
if c, ok := cfg[k]; ok {
|
||||
config.FileSet(cacheRemote, k, c)
|
||||
} else {
|
||||
config.FileSet(cacheRemote, k, v)
|
||||
}
|
||||
}
|
||||
for k, v := range r.runDefaultFlagMap {
|
||||
if c, ok := flags[k]; ok {
|
||||
_ = flag.Set(k, c)
|
||||
} else {
|
||||
_ = flag.Set(k, v)
|
||||
}
|
||||
}
|
||||
fs.Config.LowLevelRetries = 1
|
||||
|
||||
m := configmap.Simple{}
|
||||
for k, v := range r.runDefaultCfgMap {
|
||||
m.Set(k, v)
|
||||
}
|
||||
for k, v := range flags {
|
||||
m.Set(k, v)
|
||||
}
|
||||
|
||||
// Instantiate root
|
||||
if purge {
|
||||
boltDb.PurgeTempUploads()
|
||||
_ = os.RemoveAll(path.Join(runInstance.tmpUploadDir, id))
|
||||
}
|
||||
f, err := fs.NewFs(remote + ":" + id)
|
||||
f, err := cache.NewFs(remote, id, m)
|
||||
require.NoError(t, err)
|
||||
cfs, err := r.getCacheFs(f)
|
||||
require.NoError(t, err)
|
||||
@@ -1157,9 +1126,6 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
|
||||
}
|
||||
r.tempFiles = nil
|
||||
debug.FreeOSMemory()
|
||||
for k, v := range r.runDefaultFlagMap {
|
||||
_ = flag.Set(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
|
||||
|
||||
Vendored
+7
-7
@@ -22,7 +22,7 @@ func TestInternalUploadTempDirCreated(t *testing.T) {
|
||||
id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
|
||||
map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
|
||||
@@ -63,7 +63,7 @@ func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
|
||||
id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
|
||||
map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
|
||||
@@ -73,7 +73,7 @@ func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
|
||||
id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
|
||||
map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
|
||||
@@ -83,7 +83,7 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
|
||||
id := fmt.Sprintf("tiumef%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
|
||||
map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
err := rootFs.Mkdir("one")
|
||||
@@ -163,7 +163,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
|
||||
id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
|
||||
map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
err := rootFs.Mkdir("test")
|
||||
@@ -213,7 +213,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
|
||||
id := "tiutfo"
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
|
||||
map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
boltDb.PurgeTempUploads()
|
||||
@@ -343,7 +343,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
|
||||
id := "tiuufo"
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
|
||||
map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
boltDb.PurgeTempUploads()
|
||||
|
||||
+455
@@ -0,0 +1,455 @@
|
||||
// +build !plan9
|
||||
|
||||
package cache_test
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"fmt"
|
||||
|
||||
"github.com/ncw/rclone/backend/cache"
|
||||
_ "github.com/ncw/rclone/backend/drive"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestInternalUploadTempDirCreated(t *testing.T) {
|
||||
id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) {
|
||||
// create some rand test data
|
||||
testSize := int64(524288000)
|
||||
testReader := runInstance.randomReader(t, testSize)
|
||||
bu := runInstance.listenForBackgroundUpload(t, rootFs, "one")
|
||||
runInstance.writeRemoteReader(t, rootFs, "one", testReader)
|
||||
// validate that it exists in temp fs
|
||||
ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
|
||||
require.NoError(t, err)
|
||||
|
||||
if runInstance.rootIsCrypt {
|
||||
require.Equal(t, int64(524416032), ti.Size())
|
||||
} else {
|
||||
require.Equal(t, testSize, ti.Size())
|
||||
}
|
||||
de1, err := runInstance.list(t, rootFs, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, de1, 1)
|
||||
|
||||
runInstance.completeBackgroundUpload(t, "one", bu)
|
||||
// check if it was removed from temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
|
||||
require.True(t, os.IsNotExist(err))
|
||||
|
||||
// check if it can be read
|
||||
data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, data2, 1024)
|
||||
}
|
||||
|
||||
func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
|
||||
id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
|
||||
}
|
||||
|
||||
func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
|
||||
id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
|
||||
}
|
||||
|
||||
func TestInternalUploadMoveExistingFile(t *testing.T) {
|
||||
id := fmt.Sprintf("tiumef%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
err := rootFs.Mkdir("one")
|
||||
require.NoError(t, err)
|
||||
err = rootFs.Mkdir("one/test")
|
||||
require.NoError(t, err)
|
||||
err = rootFs.Mkdir("second")
|
||||
require.NoError(t, err)
|
||||
|
||||
// create some rand test data
|
||||
testSize := int64(10485760)
|
||||
testReader := runInstance.randomReader(t, testSize)
|
||||
runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
|
||||
runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
|
||||
|
||||
de1, err := runInstance.list(t, rootFs, "one/test")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, de1, 1)
|
||||
|
||||
time.Sleep(time.Second * 5)
|
||||
//_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
|
||||
//require.NoError(t, err)
|
||||
|
||||
err = runInstance.dirMove(t, rootFs, "one/test", "second/test")
|
||||
require.NoError(t, err)
|
||||
|
||||
// check if it can be read
|
||||
de1, err = runInstance.list(t, rootFs, "second/test")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, de1, 1)
|
||||
}
|
||||
|
||||
func TestInternalUploadTempPathCleaned(t *testing.T) {
|
||||
id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
err := rootFs.Mkdir("one")
|
||||
require.NoError(t, err)
|
||||
err = rootFs.Mkdir("one/test")
|
||||
require.NoError(t, err)
|
||||
err = rootFs.Mkdir("second")
|
||||
require.NoError(t, err)
|
||||
|
||||
// create some rand test data
|
||||
testSize := int64(1048576)
|
||||
testReader := runInstance.randomReader(t, testSize)
|
||||
testReader2 := runInstance.randomReader(t, testSize)
|
||||
runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
|
||||
runInstance.writeObjectReader(t, rootFs, "second/data.bin", testReader2)
|
||||
|
||||
runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
|
||||
require.True(t, os.IsNotExist(err))
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
|
||||
require.True(t, os.IsNotExist(err))
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second")))
|
||||
require.False(t, os.IsNotExist(err))
|
||||
|
||||
runInstance.completeAllBackgroundUploads(t, rootFs, "second/data.bin")
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/data.bin")))
|
||||
require.True(t, os.IsNotExist(err))
|
||||
|
||||
de1, err := runInstance.list(t, rootFs, "one/test")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, de1, 1)
|
||||
|
||||
// check if it can be read
|
||||
de1, err = runInstance.list(t, rootFs, "second")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, de1, 1)
|
||||
}
|
||||
|
||||
func TestInternalUploadQueueMoreFiles(t *testing.T) {
|
||||
id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
err := rootFs.Mkdir("test")
|
||||
require.NoError(t, err)
|
||||
minSize := 5242880
|
||||
maxSize := 10485760
|
||||
totalFiles := 10
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
lastFile := ""
|
||||
for i := 0; i < totalFiles; i++ {
|
||||
size := int64(rand.Intn(maxSize-minSize) + minSize)
|
||||
testReader := runInstance.randomReader(t, size)
|
||||
remote := "test/" + strconv.Itoa(i) + ".bin"
|
||||
runInstance.writeRemoteReader(t, rootFs, remote, testReader)
|
||||
|
||||
// validate that it exists in temp fs
|
||||
ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote)))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, size, runInstance.cleanSize(t, ti.Size()))
|
||||
|
||||
if runInstance.wrappedIsExternal && i < totalFiles-1 {
|
||||
time.Sleep(time.Second * 3)
|
||||
}
|
||||
lastFile = remote
|
||||
}
|
||||
|
||||
// check if cache lists all files, likely temp upload didn't finish yet
|
||||
de1, err := runInstance.list(t, rootFs, "test")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, de1, totalFiles)
|
||||
|
||||
// wait for background uploader to do its thing
|
||||
runInstance.completeAllBackgroundUploads(t, rootFs, lastFile)
|
||||
|
||||
// retry until we have no more temp files and fail if they don't go down to 0
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test")))
|
||||
require.True(t, os.IsNotExist(err))
|
||||
|
||||
// check if cache lists all files
|
||||
de1, err = runInstance.list(t, rootFs, "test")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, de1, totalFiles)
|
||||
}
|
||||
|
||||
func TestInternalUploadTempFileOperations(t *testing.T) {
|
||||
id := "tiutfo"
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
boltDb.PurgeTempUploads()
|
||||
|
||||
// create some rand test data
|
||||
runInstance.mkdir(t, rootFs, "test")
|
||||
runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
|
||||
|
||||
// check if it can be read
|
||||
data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("one content"), data1)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
|
||||
// test DirMove - allowed
|
||||
err = runInstance.dirMove(t, rootFs, "test", "second")
|
||||
if err != errNotSupported {
|
||||
require.NoError(t, err)
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.Error(t, err)
|
||||
_, err = rootFs.NewObject("second/one")
|
||||
require.NoError(t, err)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.Error(t, err)
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
|
||||
require.NoError(t, err)
|
||||
_, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
|
||||
require.Error(t, err)
|
||||
var started bool
|
||||
started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one")))
|
||||
require.NoError(t, err)
|
||||
require.False(t, started)
|
||||
runInstance.mkdir(t, rootFs, "test")
|
||||
runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
|
||||
}
|
||||
|
||||
// test Rmdir - allowed
|
||||
err = runInstance.rm(t, rootFs, "test")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "directory not empty")
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
|
||||
require.False(t, started)
|
||||
require.NoError(t, err)
|
||||
|
||||
// test Move/Rename -- allowed
|
||||
err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
|
||||
if err != errNotSupported {
|
||||
require.NoError(t, err)
|
||||
// try to read from it
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.Error(t, err)
|
||||
_, err = rootFs.NewObject("test/second")
|
||||
require.NoError(t, err)
|
||||
data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("one content"), data2)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.Error(t, err)
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
|
||||
require.NoError(t, err)
|
||||
runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
|
||||
}
|
||||
|
||||
// test Copy -- allowed
|
||||
err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
|
||||
if err != errNotSupported {
|
||||
require.NoError(t, err)
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
_, err = rootFs.NewObject("test/third")
|
||||
require.NoError(t, err)
|
||||
data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("one content"), data2)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// test Remove -- allowed
|
||||
err = runInstance.rm(t, rootFs, "test/one")
|
||||
require.NoError(t, err)
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.Error(t, err)
|
||||
// validate that it doesn't exist in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.Error(t, err)
|
||||
runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
|
||||
|
||||
// test Update -- allowed
|
||||
firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
|
||||
require.NoError(t, err)
|
||||
err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
|
||||
require.NoError(t, err)
|
||||
obj2, err := rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
|
||||
require.Equal(t, "one content updated", string(data2))
|
||||
tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
if runInstance.rootIsCrypt {
|
||||
require.Equal(t, int64(67), tmpInfo.Size())
|
||||
} else {
|
||||
require.Equal(t, int64(len(data2)), tmpInfo.Size())
|
||||
}
|
||||
|
||||
// test SetModTime -- allowed
|
||||
secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, secondModTime, firstModTime)
|
||||
require.NotEqual(t, time.Time{}, firstModTime)
|
||||
require.NotEqual(t, time.Time{}, secondModTime)
|
||||
}
|
||||
|
||||
func TestInternalUploadUploadingFileOperations(t *testing.T) {
|
||||
id := "tiuufo"
|
||||
rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
|
||||
nil,
|
||||
map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
|
||||
defer runInstance.cleanupFs(t, rootFs, boltDb)
|
||||
|
||||
boltDb.PurgeTempUploads()
|
||||
|
||||
// create some rand test data
|
||||
runInstance.mkdir(t, rootFs, "test")
|
||||
runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
|
||||
|
||||
// check if it can be read
|
||||
data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("one content"), data1)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
|
||||
err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one")))
|
||||
require.NoError(t, err)
|
||||
|
||||
// test DirMove
|
||||
err = runInstance.dirMove(t, rootFs, "test", "second")
|
||||
if err != errNotSupported {
|
||||
require.Error(t, err)
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// test Rmdir
|
||||
err = runInstance.rm(t, rootFs, "test")
|
||||
require.Error(t, err)
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
// validate that it doesn't exist in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
|
||||
// test Move/Rename
|
||||
err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
|
||||
if err != errNotSupported {
|
||||
require.Error(t, err)
|
||||
// try to read from it
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
_, err = rootFs.NewObject("test/second")
|
||||
require.Error(t, err)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// test Copy -- allowed
|
||||
err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
|
||||
if err != errNotSupported {
|
||||
require.NoError(t, err)
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
_, err = rootFs.NewObject("test/third")
|
||||
require.NoError(t, err)
|
||||
data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("one content"), data2)
|
||||
// validate that it exists in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// test Remove
|
||||
err = runInstance.rm(t, rootFs, "test/one")
|
||||
require.Error(t, err)
|
||||
_, err = rootFs.NewObject("test/one")
|
||||
require.NoError(t, err)
|
||||
// validate that it doesn't exist in temp fs
|
||||
_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
require.NoError(t, err)
|
||||
runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
|
||||
|
||||
// test Update - this seems to work. Why? FIXME
|
||||
//firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
|
||||
//require.NoError(t, err)
|
||||
//err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() {
|
||||
// data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true)
|
||||
// require.Equal(t, "one content", string(data2))
|
||||
//
|
||||
// tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
|
||||
// require.NoError(t, err)
|
||||
// if runInstance.rootIsCrypt {
|
||||
// require.Equal(t, int64(67), tmpInfo.Size())
|
||||
// } else {
|
||||
// require.Equal(t, int64(len(data2)), tmpInfo.Size())
|
||||
// }
|
||||
//})
|
||||
//require.Error(t, err)
|
||||
|
||||
// test SetModTime -- seems to work cause of previous
|
||||
//secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
|
||||
//require.NoError(t, err)
|
||||
//require.Equal(t, secondModTime, firstModTime)
|
||||
//require.NotEqual(t, time.Time{}, firstModTime)
|
||||
//require.NotEqual(t, time.Time{}, secondModTime)
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
--- cache_upload_test.go
|
||||
+++ cache_upload_test.go
|
||||
@@ -1500,9 +1469,6 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
|
||||
}
|
||||
r.tempFiles = nil
|
||||
debug.FreeOSMemory()
|
||||
- for k, v := range r.runDefaultFlagMap {
|
||||
- _ = flag.Set(k, v)
|
||||
- }
|
||||
}
|
||||
|
||||
func (r *run) randomBytes(t *testing.T, size int64) []byte {
|
||||
Vendored
+14
-14
@@ -65,14 +65,14 @@ func NewObjectHandle(o *Object, cfs *Fs) *Handle {
|
||||
offset: 0,
|
||||
preloadOffset: -1, // -1 to trigger the first preload
|
||||
|
||||
UseMemory: cfs.chunkMemory,
|
||||
UseMemory: !cfs.opt.ChunkNoMemory,
|
||||
reading: false,
|
||||
}
|
||||
r.seenOffsets = make(map[int64]bool)
|
||||
r.memory = NewMemory(-1)
|
||||
|
||||
// create a larger buffer to queue up requests
|
||||
r.preloadQueue = make(chan int64, r.cfs.totalWorkers*10)
|
||||
r.preloadQueue = make(chan int64, r.cfs.opt.TotalWorkers*10)
|
||||
r.confirmReading = make(chan bool)
|
||||
r.startReadWorkers()
|
||||
return r
|
||||
@@ -98,7 +98,7 @@ func (r *Handle) startReadWorkers() {
|
||||
if r.hasAtLeastOneWorker() {
|
||||
return
|
||||
}
|
||||
totalWorkers := r.cacheFs().totalWorkers
|
||||
totalWorkers := r.cacheFs().opt.TotalWorkers
|
||||
|
||||
if r.cacheFs().plexConnector.isConfigured() {
|
||||
if !r.cacheFs().plexConnector.isConnected() {
|
||||
@@ -156,7 +156,7 @@ func (r *Handle) confirmExternalReading() {
|
||||
return
|
||||
}
|
||||
fs.Infof(r, "confirmed reading by external reader")
|
||||
r.scaleWorkers(r.cacheFs().totalMaxWorkers)
|
||||
r.scaleWorkers(r.cacheFs().opt.TotalWorkers)
|
||||
}
|
||||
|
||||
// queueOffset will send an offset to the workers if it's different from the last one
|
||||
@@ -179,7 +179,7 @@ func (r *Handle) queueOffset(offset int64) {
|
||||
}
|
||||
|
||||
for i := 0; i < len(r.workers); i++ {
|
||||
o := r.preloadOffset + r.cacheFs().chunkSize*int64(i)
|
||||
o := r.preloadOffset + int64(r.cacheFs().opt.ChunkSize)*int64(i)
|
||||
if o < 0 || o >= r.cachedObject.Size() {
|
||||
continue
|
||||
}
|
||||
@@ -211,7 +211,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
|
||||
var err error
|
||||
|
||||
// we calculate the modulus of the requested offset with the size of a chunk
|
||||
offset := chunkStart % r.cacheFs().chunkSize
|
||||
offset := chunkStart % int64(r.cacheFs().opt.ChunkSize)
|
||||
|
||||
// we align the start offset of the first chunk to a likely chunk in the storage
|
||||
chunkStart = chunkStart - offset
|
||||
@@ -228,7 +228,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
|
||||
if !found {
|
||||
// we're gonna give the workers a chance to pickup the chunk
|
||||
// and retry a couple of times
|
||||
for i := 0; i < r.cacheFs().readRetries*8; i++ {
|
||||
for i := 0; i < r.cacheFs().opt.ReadRetries*8; i++ {
|
||||
data, err = r.storage().GetChunk(r.cachedObject, chunkStart)
|
||||
if err == nil {
|
||||
found = true
|
||||
@@ -255,7 +255,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
|
||||
if offset > 0 {
|
||||
if offset > int64(len(data)) {
|
||||
fs.Errorf(r, "unexpected conditions during reading. current position: %v, current chunk position: %v, current chunk size: %v, offset: %v, chunk size: %v, file size: %v",
|
||||
r.offset, chunkStart, len(data), offset, r.cacheFs().chunkSize, r.cachedObject.Size())
|
||||
r.offset, chunkStart, len(data), offset, r.cacheFs().opt.ChunkSize, r.cachedObject.Size())
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
data = data[int(offset):]
|
||||
@@ -338,9 +338,9 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
|
||||
err = errors.Errorf("cache: unimplemented seek whence %v", whence)
|
||||
}
|
||||
|
||||
chunkStart := r.offset - (r.offset % r.cacheFs().chunkSize)
|
||||
if chunkStart >= r.cacheFs().chunkSize {
|
||||
chunkStart = chunkStart - r.cacheFs().chunkSize
|
||||
chunkStart := r.offset - (r.offset % int64(r.cacheFs().opt.ChunkSize))
|
||||
if chunkStart >= int64(r.cacheFs().opt.ChunkSize) {
|
||||
chunkStart = chunkStart - int64(r.cacheFs().opt.ChunkSize)
|
||||
}
|
||||
r.queueOffset(chunkStart)
|
||||
|
||||
@@ -451,7 +451,7 @@ func (w *worker) run() {
|
||||
}
|
||||
}
|
||||
|
||||
chunkEnd := chunkStart + w.r.cacheFs().chunkSize
|
||||
chunkEnd := chunkStart + int64(w.r.cacheFs().opt.ChunkSize)
|
||||
// TODO: Remove this comment if it proves to be reliable for #1896
|
||||
//if chunkEnd > w.r.cachedObject.Size() {
|
||||
// chunkEnd = w.r.cachedObject.Size()
|
||||
@@ -466,7 +466,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
|
||||
var data []byte
|
||||
|
||||
// stop retries
|
||||
if retry >= w.r.cacheFs().readRetries {
|
||||
if retry >= w.r.cacheFs().opt.ReadRetries {
|
||||
return
|
||||
}
|
||||
// back-off between retries
|
||||
@@ -612,7 +612,7 @@ func (b *backgroundWriter) run() {
|
||||
return
|
||||
}
|
||||
|
||||
absPath, err := b.fs.cache.getPendingUpload(b.fs.Root(), b.fs.tempWriteWait)
|
||||
absPath, err := b.fs.cache.getPendingUpload(b.fs.Root(), time.Duration(b.fs.opt.TempWaitTime))
|
||||
if err != nil || absPath == "" || !b.fs.isRootInPath(absPath) {
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
|
||||
Vendored
+5
-5
@@ -44,7 +44,7 @@ func NewObject(f *Fs, remote string) *Object {
|
||||
|
||||
cacheType := objectInCache
|
||||
parentFs := f.UnWrap()
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
_, err := f.cache.SearchPendingUpload(fullRemote)
|
||||
if err == nil { // queued for upload
|
||||
cacheType = objectPendingUpload
|
||||
@@ -75,7 +75,7 @@ func ObjectFromOriginal(f *Fs, o fs.Object) *Object {
|
||||
|
||||
cacheType := objectInCache
|
||||
parentFs := f.UnWrap()
|
||||
if f.tempWritePath != "" {
|
||||
if f.opt.TempWritePath != "" {
|
||||
_, err := f.cache.SearchPendingUpload(fullRemote)
|
||||
if err == nil { // queued for upload
|
||||
cacheType = objectPendingUpload
|
||||
@@ -153,7 +153,7 @@ func (o *Object) Storable() bool {
|
||||
// 2. is not pending a notification from the wrapped fs
|
||||
func (o *Object) refresh() error {
|
||||
isNotified := o.CacheFs.isNotifiedRemote(o.Remote())
|
||||
isExpired := time.Now().After(o.CacheTs.Add(o.CacheFs.fileAge))
|
||||
isExpired := time.Now().After(o.CacheTs.Add(time.Duration(o.CacheFs.opt.InfoAge)))
|
||||
if !isExpired && !isNotified {
|
||||
return nil
|
||||
}
|
||||
@@ -237,7 +237,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
return err
|
||||
}
|
||||
// pause background uploads if active
|
||||
if o.CacheFs.tempWritePath != "" {
|
||||
if o.CacheFs.opt.TempWritePath != "" {
|
||||
o.CacheFs.backgroundRunner.pause()
|
||||
defer o.CacheFs.backgroundRunner.play()
|
||||
// don't allow started uploads
|
||||
@@ -274,7 +274,7 @@ func (o *Object) Remove() error {
|
||||
return err
|
||||
}
|
||||
// pause background uploads if active
|
||||
if o.CacheFs.tempWritePath != "" {
|
||||
if o.CacheFs.opt.TempWritePath != "" {
|
||||
o.CacheFs.backgroundRunner.pause()
|
||||
defer o.CacheFs.backgroundRunner.play()
|
||||
// don't allow started uploads
|
||||
|
||||
Vendored
+4
-4
@@ -16,7 +16,6 @@ import (
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/patrickmn/go-cache"
|
||||
"golang.org/x/net/websocket"
|
||||
)
|
||||
@@ -60,10 +59,11 @@ type plexConnector struct {
|
||||
running bool
|
||||
runningMu sync.Mutex
|
||||
stateCache *cache.Cache
|
||||
saveToken func(string)
|
||||
}
|
||||
|
||||
// newPlexConnector connects to a Plex server and generates a token
|
||||
func newPlexConnector(f *Fs, plexURL, username, password string) (*plexConnector, error) {
|
||||
func newPlexConnector(f *Fs, plexURL, username, password string, saveToken func(string)) (*plexConnector, error) {
|
||||
u, err := url.ParseRequestURI(strings.TrimRight(plexURL, "/"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -76,6 +76,7 @@ func newPlexConnector(f *Fs, plexURL, username, password string) (*plexConnector
|
||||
password: password,
|
||||
token: "",
|
||||
stateCache: cache.New(time.Hour, time.Minute),
|
||||
saveToken: saveToken,
|
||||
}
|
||||
|
||||
return pc, nil
|
||||
@@ -209,8 +210,7 @@ func (p *plexConnector) authenticate() error {
|
||||
}
|
||||
p.token = token
|
||||
if p.token != "" {
|
||||
config.FileSet(p.f.Name(), "plex_token", p.token)
|
||||
config.SaveConfig()
|
||||
p.saveToken(p.token)
|
||||
fs.Infof(p.f.Name(), "Connected to Plex server: %v", p.url.String())
|
||||
}
|
||||
p.listenWebsocket()
|
||||
|
||||
Vendored
+5
-4
@@ -34,7 +34,8 @@ const (
|
||||
|
||||
// Features flags for this storage type
|
||||
type Features struct {
|
||||
PurgeDb bool // purge the db before starting
|
||||
PurgeDb bool // purge the db before starting
|
||||
DbWaitTime time.Duration // time to wait for DB to be available
|
||||
}
|
||||
|
||||
var boltMap = make(map[string]*Persistent)
|
||||
@@ -122,7 +123,7 @@ func (b *Persistent) connect() error {
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create a data directory %q", b.dataPath)
|
||||
}
|
||||
b.db, err = bolt.Open(b.dbPath, 0644, &bolt.Options{Timeout: *cacheDbWaitTime})
|
||||
b.db, err = bolt.Open(b.dbPath, 0644, &bolt.Options{Timeout: b.features.DbWaitTime})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to open a cache connection to %q", b.dbPath)
|
||||
}
|
||||
@@ -342,7 +343,7 @@ func (b *Persistent) RemoveDir(fp string) error {
|
||||
// ExpireDir will flush a CachedDirectory and all its objects from the objects
|
||||
// chunks will remain as they are
|
||||
func (b *Persistent) ExpireDir(cd *Directory) error {
|
||||
t := time.Now().Add(cd.CacheFs.fileAge * -1)
|
||||
t := time.Now().Add(time.Duration(-cd.CacheFs.opt.InfoAge))
|
||||
cd.CacheTs = &t
|
||||
|
||||
// expire all parents
|
||||
@@ -429,7 +430,7 @@ func (b *Persistent) RemoveObject(fp string) error {
|
||||
|
||||
// ExpireObject will flush an Object and all its data if desired
|
||||
func (b *Persistent) ExpireObject(co *Object, withData bool) error {
|
||||
co.CacheTs = time.Now().Add(co.CacheFs.fileAge * -1)
|
||||
co.CacheTs = time.Now().Add(time.Duration(-co.CacheFs.opt.InfoAge))
|
||||
err := b.AddObject(co)
|
||||
if withData {
|
||||
_ = os.RemoveAll(path.Join(b.dataPath, co.abs()))
|
||||
|
||||
+61
-35
@@ -5,24 +5,18 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
cryptShowMapping = flags.BoolP("crypt-show-mapping", "", false, "For all files listed show how the names encrypt.")
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
@@ -30,11 +24,13 @@ func init() {
|
||||
Description: "Encrypt/Decrypt a remote",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "remote",
|
||||
Help: "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
|
||||
Name: "remote",
|
||||
Help: "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "filename_encryption",
|
||||
Help: "How to encrypt the filenames.",
|
||||
Name: "filename_encryption",
|
||||
Help: "How to encrypt the filenames.",
|
||||
Default: "standard",
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "off",
|
||||
@@ -48,8 +44,9 @@ func init() {
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "directory_name_encryption",
|
||||
Help: "Option to either encrypt directory names or leave them intact.",
|
||||
Name: "directory_name_encryption",
|
||||
Help: "Option to either encrypt directory names or leave them intact.",
|
||||
Default: true,
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "true",
|
||||
@@ -68,50 +65,67 @@ func init() {
|
||||
Name: "password2",
|
||||
Help: "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
|
||||
IsPassword: true,
|
||||
Optional: true,
|
||||
}, {
|
||||
Name: "show_mapping",
|
||||
Help: "For all files listed show how the names encrypt.",
|
||||
Default: false,
|
||||
Hide: fs.OptionHideConfigurator,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// NewCipher constructs a Cipher for the given config name
|
||||
func NewCipher(name string) (Cipher, error) {
|
||||
mode, err := NewNameEncryptionMode(config.FileGet(name, "filename_encryption", "standard"))
|
||||
// newCipherForConfig constructs a Cipher for the given config name
|
||||
func newCipherForConfig(opt *Options) (Cipher, error) {
|
||||
mode, err := NewNameEncryptionMode(opt.FilenameEncryption)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirNameEncrypt, err := strconv.ParseBool(config.FileGet(name, "directory_name_encryption", "true"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
password := config.FileGet(name, "password", "")
|
||||
if password == "" {
|
||||
if opt.Password == "" {
|
||||
return nil, errors.New("password not set in config file")
|
||||
}
|
||||
password, err = obscure.Reveal(password)
|
||||
password, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to decrypt password")
|
||||
}
|
||||
salt := config.FileGet(name, "password2", "")
|
||||
if salt != "" {
|
||||
salt, err = obscure.Reveal(salt)
|
||||
var salt string
|
||||
if opt.Password2 != "" {
|
||||
salt, err = obscure.Reveal(opt.Password2)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to decrypt password2")
|
||||
}
|
||||
}
|
||||
cipher, err := newCipher(mode, password, salt, dirNameEncrypt)
|
||||
cipher, err := newCipher(mode, password, salt, opt.DirectoryNameEncryption)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to make cipher")
|
||||
}
|
||||
return cipher, nil
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, rpath string) (fs.Fs, error) {
|
||||
cipher, err := NewCipher(name)
|
||||
// NewCipher constructs a Cipher for the given config
|
||||
func NewCipher(m configmap.Mapper) (Cipher, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := config.FileGet(name, "remote")
|
||||
return newCipherForConfig(opt)
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cipher, err := newCipherForConfig(opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := opt.Remote
|
||||
if strings.HasPrefix(remote, name+":") {
|
||||
return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
|
||||
}
|
||||
@@ -130,6 +144,7 @@ func NewFs(name, rpath string) (fs.Fs, error) {
|
||||
Fs: wrappedFs,
|
||||
name: name,
|
||||
root: rpath,
|
||||
opt: *opt,
|
||||
cipher: cipher,
|
||||
}
|
||||
// the features here are ones we could support, and they are
|
||||
@@ -161,11 +176,22 @@ func NewFs(name, rpath string) (fs.Fs, error) {
|
||||
return f, err
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Remote string `config:"remote"`
|
||||
FilenameEncryption string `config:"filename_encryption"`
|
||||
DirectoryNameEncryption bool `config:"directory_name_encryption"`
|
||||
Password string `config:"password"`
|
||||
Password2 string `config:"password2"`
|
||||
ShowMapping bool `config:"show_mapping"`
|
||||
}
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
fs.Fs
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
features *fs.Features // optional features
|
||||
cipher Cipher
|
||||
}
|
||||
@@ -198,7 +224,7 @@ func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
|
||||
fs.Debugf(remote, "Skipping undecryptable file name: %v", err)
|
||||
return
|
||||
}
|
||||
if *cryptShowMapping {
|
||||
if f.opt.ShowMapping {
|
||||
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
|
||||
}
|
||||
*entries = append(*entries, f.newObject(obj))
|
||||
@@ -212,7 +238,7 @@ func (f *Fs) addDir(entries *fs.DirEntries, dir fs.Directory) {
|
||||
fs.Debugf(remote, "Skipping undecryptable dir name: %v", err)
|
||||
return
|
||||
}
|
||||
if *cryptShowMapping {
|
||||
if f.opt.ShowMapping {
|
||||
fs.Logf(decryptedRemote, "Encrypts to %q", remote)
|
||||
}
|
||||
*entries = append(*entries, f.newDir(dir))
|
||||
|
||||
+209
-89
@@ -23,7 +23,8 @@ import (
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
@@ -49,27 +50,13 @@ const (
|
||||
defaultExtensions = "docx,xlsx,pptx,svg"
|
||||
scopePrefix = "https://www.googleapis.com/auth/"
|
||||
defaultScope = "drive"
|
||||
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
|
||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||
defaultChunkSize = fs.SizeSuffix(8 * 1024 * 1024)
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
driveAuthOwnerOnly = flags.BoolP("drive-auth-owner-only", "", false, "Only consider files owned by the authenticated user.")
|
||||
driveUseTrash = flags.BoolP("drive-use-trash", "", true, "Send files to the trash instead of deleting permanently.")
|
||||
driveSkipGdocs = flags.BoolP("drive-skip-gdocs", "", false, "Skip google documents in all listings.")
|
||||
driveSharedWithMe = flags.BoolP("drive-shared-with-me", "", false, "Only show files that are shared with me")
|
||||
driveTrashedOnly = flags.BoolP("drive-trashed-only", "", false, "Only show files that are in the trash")
|
||||
driveExtensions = flags.StringP("drive-formats", "", defaultExtensions, "Comma separated list of preferred formats for downloading Google docs.")
|
||||
driveUseCreatedDate = flags.BoolP("drive-use-created-date", "", false, "Use created date instead of modified date.")
|
||||
driveListChunk = flags.Int64P("drive-list-chunk", "", 1000, "Size of listing chunk 100-1000. 0 to disable.")
|
||||
driveImpersonate = flags.StringP("drive-impersonate", "", "", "Impersonate this user when using a service account.")
|
||||
driveAlternateExport = flags.BoolP("drive-alternate-export", "", false, "Use alternate export URLs for google documents export.")
|
||||
driveAcknowledgeAbuse = flags.BoolP("drive-acknowledge-abuse", "", false, "Set to allow files which return cannotDownloadAbusiveFile to be downloaded.")
|
||||
driveKeepRevisionForever = flags.BoolP("drive-keep-revision-forever", "", false, "Keep new head revision forever.")
|
||||
// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
|
||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||
chunkSize = fs.SizeSuffix(8 * 1024 * 1024)
|
||||
driveUploadCutoff = chunkSize
|
||||
// Description of how to auth for this app
|
||||
driveConfig = &oauth2.Config{
|
||||
Scopes: []string{scopePrefix + "drive"},
|
||||
@@ -112,38 +99,43 @@ func init() {
|
||||
Name: "drive",
|
||||
Description: "Google Drive",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
var err error
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
|
||||
return
|
||||
}
|
||||
// Fill in the scopes
|
||||
scope := config.FileGet(name, "scope")
|
||||
if scope == "" {
|
||||
scope = defaultScope
|
||||
if opt.Scope == "" {
|
||||
opt.Scope = defaultScope
|
||||
}
|
||||
driveConfig.Scopes = nil
|
||||
for _, scope := range strings.Split(scope, ",") {
|
||||
for _, scope := range strings.Split(opt.Scope, ",") {
|
||||
driveConfig.Scopes = append(driveConfig.Scopes, scopePrefix+strings.TrimSpace(scope))
|
||||
// Set the root_folder_id if using drive.appfolder
|
||||
if scope == "drive.appfolder" {
|
||||
config.FileSet(name, "root_folder_id", "appDataFolder")
|
||||
m.Set("root_folder_id", "appDataFolder")
|
||||
}
|
||||
}
|
||||
if config.FileGet(name, "service_account_file") == "" {
|
||||
err = oauthutil.Config("drive", name, driveConfig)
|
||||
if opt.ServiceAccountFile == "" {
|
||||
err = oauthutil.Config("drive", name, m, driveConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
}
|
||||
err = configTeamDrive(name)
|
||||
err = configTeamDrive(opt, m, name)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure team drive: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Google Application Client Id - leave blank normally.",
|
||||
Help: "Google Application Client Id\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Google Application Client Secret - leave blank normally.",
|
||||
Help: "Google Application Client Secret\nLeave blank normally.",
|
||||
}, {
|
||||
Name: "scope",
|
||||
Help: "Scope that rclone should use when requesting access from drive.",
|
||||
@@ -165,14 +157,97 @@ func init() {
|
||||
}},
|
||||
}, {
|
||||
Name: "root_folder_id",
|
||||
Help: "ID of the root folder - leave blank normally. Fill in to access \"Computers\" folders. (see docs).",
|
||||
Help: "ID of the root folder\nLeave blank normally.\nFill in to access \"Computers\" folders. (see docs).",
|
||||
}, {
|
||||
Name: "service_account_file",
|
||||
Help: "Service Account Credentials JSON file path - leave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
}, {
|
||||
Name: "service_account_credentials",
|
||||
Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
Hide: fs.OptionHideBoth,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "team_drive",
|
||||
Help: "ID of the Team Drive",
|
||||
Hide: fs.OptionHideBoth,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "auth_owner_only",
|
||||
Default: false,
|
||||
Help: "Only consider files owned by the authenticated user.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_trash",
|
||||
Default: true,
|
||||
Help: "Send files to the trash instead of deleting permanently.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_gdocs",
|
||||
Default: false,
|
||||
Help: "Skip google documents in all listings.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "shared_with_me",
|
||||
Default: false,
|
||||
Help: "Only show files that are shared with me",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "trashed_only",
|
||||
Default: false,
|
||||
Help: "Only show files that are in the trash",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "formats",
|
||||
Default: defaultExtensions,
|
||||
Help: "Comma separated list of preferred formats for downloading Google docs.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_created_date",
|
||||
Default: false,
|
||||
Help: "Use created date instead of modified date.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "list_chunk",
|
||||
Default: 1000,
|
||||
Help: "Size of listing chunk 100-1000. 0 to disable.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "impersonate",
|
||||
Default: "",
|
||||
Help: "Impersonate this user when using a service account.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "alternate_export",
|
||||
Default: false,
|
||||
Help: "Use alternate export URLs for google documents export.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Default: defaultChunkSize,
|
||||
Help: "Cutoff for switching to chunked upload",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Default: defaultChunkSize,
|
||||
Help: "Upload chunk size. Must a power of 2 >= 256k.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "acknowledge_abuse",
|
||||
Default: false,
|
||||
Help: "Set to allow files which return cannotDownloadAbusiveFile to be downloaded.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "keep_revision_forever",
|
||||
Default: false,
|
||||
Help: "Keep new head revision forever.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "untrash",
|
||||
Default: false,
|
||||
Help: "Untrash any trashed files - use with --drive-trashed-only.",
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&driveUploadCutoff, "drive-upload-cutoff", "", "Cutoff for switching to chunked upload")
|
||||
flags.VarP(&chunkSize, "drive-chunk-size", "", "Upload chunk size. Must a power of 2 >= 256k.")
|
||||
|
||||
// Invert mimeTypeToExtension
|
||||
extensionToMimeType = make(map[string]string, len(mimeTypeToExtension))
|
||||
@@ -181,10 +256,35 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Scope string `config:"scope"`
|
||||
RootFolderID string `config:"root_folder_id"`
|
||||
ServiceAccountFile string `config:"service_account_file"`
|
||||
ServiceAccountCredentials string `config:"service_account_credentials"`
|
||||
TeamDriveID string `config:"team_drive"`
|
||||
AuthOwnerOnly bool `config:"auth_owner_only"`
|
||||
UseTrash bool `config:"use_trash"`
|
||||
SkipGdocs bool `config:"skip_gdocs"`
|
||||
SharedWithMe bool `config:"shared_with_me"`
|
||||
TrashedOnly bool `config:"trashed_only"`
|
||||
Extensions string `config:"formats"`
|
||||
UseCreatedDate bool `config:"use_created_date"`
|
||||
ListChunk int64 `config:"list_chunk"`
|
||||
Impersonate string `config:"impersonate"`
|
||||
AlternateExport bool `config:"alternate_export"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
AcknowledgeAbuse bool `config:"acknowledge_abuse"`
|
||||
KeepRevisionForever bool `config:"keep_revision_forever"`
|
||||
Untrash bool `config:"untrash"`
|
||||
}
|
||||
|
||||
// Fs represents a remote drive server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
svc *drive.Service // the connection to the drive server
|
||||
client *http.Client // authorized client
|
||||
@@ -192,7 +292,6 @@ type Fs struct {
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
pacer *pacer.Pacer // To pace the API calls
|
||||
extensions []string // preferred extensions to download docs
|
||||
teamDriveID string // team drive ID, may be ""
|
||||
isTeamDrive bool // true if this is a team drive
|
||||
}
|
||||
|
||||
@@ -274,8 +373,8 @@ type listFn func(*drive.File) bool
|
||||
func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bool, includeAll bool, fn listFn) (found bool, err error) {
|
||||
var query []string
|
||||
if !includeAll {
|
||||
q := "trashed=" + strconv.FormatBool(*driveTrashedOnly)
|
||||
if *driveTrashedOnly {
|
||||
q := "trashed=" + strconv.FormatBool(f.opt.TrashedOnly)
|
||||
if f.opt.TrashedOnly {
|
||||
q = fmt.Sprintf("(mimeType='%s' or %s)", driveFolderType, q)
|
||||
}
|
||||
query = append(query, q)
|
||||
@@ -283,10 +382,10 @@ func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bo
|
||||
// Search with sharedWithMe will always return things listed in "Shared With Me" (without any parents)
|
||||
// We must not filter with parent when we try list "ROOT" with drive-shared-with-me
|
||||
// If we need to list file inside those shared folders, we must search it without sharedWithMe
|
||||
if *driveSharedWithMe && dirID == f.rootFolderID {
|
||||
if f.opt.SharedWithMe && dirID == f.rootFolderID {
|
||||
query = append(query, "sharedWithMe=true")
|
||||
}
|
||||
if dirID != "" && !(*driveSharedWithMe && dirID == f.rootFolderID) {
|
||||
if dirID != "" && !(f.opt.SharedWithMe && dirID == f.rootFolderID) {
|
||||
query = append(query, fmt.Sprintf("'%s' in parents", dirID))
|
||||
}
|
||||
if title != "" {
|
||||
@@ -308,11 +407,11 @@ func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bo
|
||||
list.Q(strings.Join(query, " and "))
|
||||
// fmt.Printf("list Query = %q\n", query)
|
||||
}
|
||||
if *driveListChunk > 0 {
|
||||
list.PageSize(*driveListChunk)
|
||||
if f.opt.ListChunk > 0 {
|
||||
list.PageSize(f.opt.ListChunk)
|
||||
}
|
||||
if f.isTeamDrive {
|
||||
list.TeamDriveId(f.teamDriveID)
|
||||
list.TeamDriveId(f.opt.TeamDriveID)
|
||||
list.SupportsTeamDrives(true)
|
||||
list.IncludeTeamDriveItems(true)
|
||||
list.Corpora("teamDrive")
|
||||
@@ -324,7 +423,7 @@ func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bo
|
||||
|
||||
var fields = partialFields
|
||||
|
||||
if *driveAuthOwnerOnly {
|
||||
if f.opt.AuthOwnerOnly {
|
||||
fields += ",owners"
|
||||
}
|
||||
|
||||
@@ -395,17 +494,16 @@ func (f *Fs) parseExtensions(extensions string) error {
|
||||
}
|
||||
|
||||
// Figure out if the user wants to use a team drive
|
||||
func configTeamDrive(name string) error {
|
||||
teamDrive := config.FileGet(name, "team_drive")
|
||||
if teamDrive == "" {
|
||||
func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
|
||||
if opt.TeamDriveID == "" {
|
||||
fmt.Printf("Configure this as a team drive?\n")
|
||||
} else {
|
||||
fmt.Printf("Change current team drive ID %q?\n", teamDrive)
|
||||
fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID)
|
||||
}
|
||||
if !config.ConfirmWithDefault(false) {
|
||||
return nil
|
||||
}
|
||||
client, err := createOAuthClient(name)
|
||||
client, err := createOAuthClient(opt, name, m)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "config team drive failed to create oauth client")
|
||||
}
|
||||
@@ -440,7 +538,8 @@ func configTeamDrive(name string) error {
|
||||
} else {
|
||||
driveID = config.Choose("Enter a Team Drive ID", driveIDs, driveNames, true)
|
||||
}
|
||||
config.FileSet(name, "team_drive", driveID)
|
||||
m.Set("team_drive", driveID)
|
||||
opt.TeamDriveID = driveID
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -449,39 +548,37 @@ func newPacer() *pacer.Pacer {
|
||||
return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
|
||||
}
|
||||
|
||||
func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
|
||||
func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
|
||||
conf, err := google.JWTConfigFromJSON(credentialsData, driveConfig.Scopes...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error processing credentials")
|
||||
}
|
||||
if *driveImpersonate != "" {
|
||||
conf.Subject = *driveImpersonate
|
||||
if opt.Impersonate != "" {
|
||||
conf.Subject = opt.Impersonate
|
||||
}
|
||||
ctxWithSpecialClient := oauthutil.Context(fshttp.NewClient(fs.Config))
|
||||
return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
|
||||
}
|
||||
|
||||
func createOAuthClient(name string) (*http.Client, error) {
|
||||
func createOAuthClient(opt *Options, name string, m configmap.Mapper) (*http.Client, error) {
|
||||
var oAuthClient *http.Client
|
||||
var err error
|
||||
|
||||
// try loading service account credentials from env variable, then from a file
|
||||
serviceAccountCreds := []byte(config.FileGet(name, "service_account_credentials"))
|
||||
serviceAccountPath := config.FileGet(name, "service_account_file")
|
||||
if len(serviceAccountCreds) == 0 && serviceAccountPath != "" {
|
||||
loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(serviceAccountPath))
|
||||
if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
|
||||
loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error opening service account credentials file")
|
||||
}
|
||||
serviceAccountCreds = loadedCreds
|
||||
opt.ServiceAccountCredentials = string(loadedCreds)
|
||||
}
|
||||
if len(serviceAccountCreds) > 0 {
|
||||
oAuthClient, err = getServiceAccountClient(serviceAccountCreds)
|
||||
if opt.ServiceAccountCredentials != "" {
|
||||
oAuthClient, err = getServiceAccountClient(opt, []byte(opt.ServiceAccountCredentials))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create oauth client from service account")
|
||||
}
|
||||
} else {
|
||||
oAuthClient, _, err = oauthutil.NewClient(name, driveConfig)
|
||||
oAuthClient, _, err = oauthutil.NewClient(name, m, driveConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create oauth client")
|
||||
}
|
||||
@@ -491,15 +588,21 @@ func createOAuthClient(name string) (*http.Client, error) {
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, path string) (fs.Fs, error) {
|
||||
if !isPowerOfTwo(int64(chunkSize)) {
|
||||
return nil, errors.Errorf("drive: chunk size %v isn't a power of two", chunkSize)
|
||||
func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if chunkSize < 256*1024 {
|
||||
return nil, errors.Errorf("drive: chunk size can't be less than 256k - was %v", chunkSize)
|
||||
if !isPowerOfTwo(int64(opt.ChunkSize)) {
|
||||
return nil, errors.Errorf("drive: chunk size %v isn't a power of two", opt.ChunkSize)
|
||||
}
|
||||
if opt.ChunkSize < 256*1024 {
|
||||
return nil, errors.Errorf("drive: chunk size can't be less than 256k - was %v", opt.ChunkSize)
|
||||
}
|
||||
|
||||
oAuthClient, err := createOAuthClient(name)
|
||||
oAuthClient, err := createOAuthClient(opt, name, m)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "drive: failed when making oauth client")
|
||||
}
|
||||
@@ -512,10 +615,10 @@ func NewFs(name, path string) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: newPacer(),
|
||||
}
|
||||
f.teamDriveID = config.FileGet(name, "team_drive")
|
||||
f.isTeamDrive = f.teamDriveID != ""
|
||||
f.isTeamDrive = opt.TeamDriveID != ""
|
||||
f.features = (&fs.Features{
|
||||
DuplicateFiles: true,
|
||||
ReadMimeType: true,
|
||||
@@ -532,20 +635,20 @@ func NewFs(name, path string) (fs.Fs, error) {
|
||||
|
||||
// set root folder for a team drive or query the user root folder
|
||||
if f.isTeamDrive {
|
||||
f.rootFolderID = f.teamDriveID
|
||||
f.rootFolderID = f.opt.TeamDriveID
|
||||
} else {
|
||||
f.rootFolderID = "root"
|
||||
}
|
||||
|
||||
// override root folder if set in the config
|
||||
if rootID := config.FileGet(name, "root_folder_id"); rootID != "" {
|
||||
f.rootFolderID = rootID
|
||||
if opt.RootFolderID != "" {
|
||||
f.rootFolderID = opt.RootFolderID
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, f.rootFolderID, f)
|
||||
|
||||
// Parse extensions
|
||||
err = f.parseExtensions(*driveExtensions)
|
||||
err = f.parseExtensions(opt.Extensions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -722,6 +825,23 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
var iErr error
|
||||
_, err = f.list(directoryID, "", false, false, false, func(item *drive.File) bool {
|
||||
remote := path.Join(dir, item.Name)
|
||||
|
||||
// Untrash all trashed files if required
|
||||
if f.opt.Untrash && item.Trashed {
|
||||
fs.Infof(remote, "Untrashing")
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
info := drive.File{
|
||||
Trashed: false,
|
||||
ForceSendFields: []string{"Trashed"},
|
||||
}
|
||||
_, err = f.svc.Files.Update(item.Id, &info).Fields("").SupportsTeamDrives(f.isTeamDrive).Do()
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Errorf(remote, "Untrashing failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case item.MimeType == driveFolderType:
|
||||
// cache the directory ID for later lookups
|
||||
@@ -729,7 +849,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
|
||||
d := fs.NewDir(remote, when).SetID(item.Id)
|
||||
entries = append(entries, d)
|
||||
case *driveAuthOwnerOnly && !isAuthOwned(item):
|
||||
case f.opt.AuthOwnerOnly && !isAuthOwned(item):
|
||||
// ignore object
|
||||
case item.Md5Checksum != "" || item.Size > 0:
|
||||
// If item has MD5 sum or a length it is a file stored on drive
|
||||
@@ -739,7 +859,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
return true
|
||||
}
|
||||
entries = append(entries, o)
|
||||
case *driveSkipGdocs:
|
||||
case f.opt.SkipGdocs:
|
||||
fs.Debugf(remote, "Skipping google document type %q", item.MimeType)
|
||||
default:
|
||||
exportMimeTypes, isDocument := f.exportFormats()[item.MimeType]
|
||||
@@ -760,7 +880,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
}
|
||||
obj := o.(*Object)
|
||||
obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
|
||||
if *driveAlternateExport {
|
||||
if f.opt.AlternateExport {
|
||||
switch item.MimeType {
|
||||
case "application/vnd.google-apps.drawing":
|
||||
obj.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", item.Id, extension)
|
||||
@@ -854,11 +974,11 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOpt
|
||||
}
|
||||
|
||||
var info *drive.File
|
||||
if size == 0 || size < int64(driveUploadCutoff) {
|
||||
if size == 0 || size < int64(f.opt.UploadCutoff) {
|
||||
// Make the API request to upload metadata and file data.
|
||||
// Don't retry, return a retry error instead
|
||||
err = f.pacer.CallNoRetry(func() (bool, error) {
|
||||
info, err = f.svc.Files.Create(createInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
|
||||
info, err = f.svc.Files.Create(createInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(f.opt.KeepRevisionForever).Do()
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -972,7 +1092,7 @@ func (f *Fs) Rmdir(dir string) error {
|
||||
// trash the directory if it had trashed files
|
||||
// in or the user wants to trash, otherwise
|
||||
// delete it.
|
||||
err = f.rmdir(directoryID, trashedFiles || *driveUseTrash)
|
||||
err = f.rmdir(directoryID, trashedFiles || f.opt.UseTrash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1015,7 +1135,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||
|
||||
var info *drive.File
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
|
||||
info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(f.opt.KeepRevisionForever).Do()
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1040,7 +1160,7 @@ func (f *Fs) Purge() error {
|
||||
return err
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
if *driveUseTrash {
|
||||
if f.opt.UseTrash {
|
||||
info := drive.File{
|
||||
Trashed: true,
|
||||
}
|
||||
@@ -1316,11 +1436,11 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInter
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
changesCall := f.svc.Changes.List(pageToken).Fields("nextPageToken,newStartPageToken,changes(fileId,file(name,parents,mimeType))")
|
||||
if *driveListChunk > 0 {
|
||||
changesCall.PageSize(*driveListChunk)
|
||||
if f.opt.ListChunk > 0 {
|
||||
changesCall.PageSize(f.opt.ListChunk)
|
||||
}
|
||||
if f.isTeamDrive {
|
||||
changesCall.TeamDriveId(f.teamDriveID)
|
||||
changesCall.TeamDriveId(f.opt.TeamDriveID)
|
||||
changesCall.SupportsTeamDrives(true)
|
||||
changesCall.IncludeTeamDriveItems(true)
|
||||
}
|
||||
@@ -1444,7 +1564,7 @@ func (o *Object) setMetaData(info *drive.File) {
|
||||
o.url = fmt.Sprintf("%sfiles/%s?alt=media", o.fs.svc.BasePath, info.Id)
|
||||
o.md5sum = strings.ToLower(info.Md5Checksum)
|
||||
o.bytes = info.Size
|
||||
if *driveUseCreatedDate {
|
||||
if o.fs.opt.UseCreatedDate {
|
||||
o.modifiedDate = info.CreatedTime
|
||||
} else {
|
||||
o.modifiedDate = info.ModifiedTime
|
||||
@@ -1617,7 +1737,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
_, res, err := o.httpResponse("GET", options)
|
||||
if err != nil {
|
||||
if isGoogleError(err, "cannotDownloadAbusiveFile") {
|
||||
if *driveAcknowledgeAbuse {
|
||||
if o.fs.opt.AcknowledgeAbuse {
|
||||
// Retry acknowledging abuse
|
||||
if strings.ContainsRune(o.url, '?') {
|
||||
o.url += "&"
|
||||
@@ -1663,10 +1783,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
// Make the API request to upload metadata and file data.
|
||||
var err error
|
||||
var info *drive.File
|
||||
if size == 0 || size < int64(driveUploadCutoff) {
|
||||
if size == 0 || size < int64(o.fs.opt.UploadCutoff) {
|
||||
// Don't retry, return a retry error instead
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
info, err = o.fs.svc.Files.Update(o.id, updateInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
|
||||
info, err = o.fs.svc.Files.Update(o.id, updateInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).KeepRevisionForever(o.fs.opt.KeepRevisionForever).Do()
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1690,7 +1810,7 @@ func (o *Object) Remove() error {
|
||||
}
|
||||
var err error
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
if *driveUseTrash {
|
||||
if o.fs.opt.UseTrash {
|
||||
info := drive.File{
|
||||
Trashed: true,
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string,
|
||||
if f.isTeamDrive {
|
||||
params.Set("supportsTeamDrives", "true")
|
||||
}
|
||||
if *driveKeepRevisionForever {
|
||||
if f.opt.KeepRevisionForever {
|
||||
params.Set("keepRevisionForever", "true")
|
||||
}
|
||||
urls := "https://www.googleapis.com/upload/drive/v3/files"
|
||||
@@ -197,11 +197,11 @@ func (rx *resumableUpload) Upload() (*drive.File, error) {
|
||||
start := int64(0)
|
||||
var StatusCode int
|
||||
var err error
|
||||
buf := make([]byte, int(chunkSize))
|
||||
buf := make([]byte, int(rx.f.opt.ChunkSize))
|
||||
for start < rx.ContentLength {
|
||||
reqSize := rx.ContentLength - start
|
||||
if reqSize >= int64(chunkSize) {
|
||||
reqSize = int64(chunkSize)
|
||||
if reqSize >= int64(rx.f.opt.ChunkSize) {
|
||||
reqSize = int64(rx.f.opt.ChunkSize)
|
||||
}
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
|
||||
|
||||
|
||||
+53
-34
@@ -37,7 +37,8 @@ import (
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -55,24 +56,6 @@ const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
var (
|
||||
// Description of how to auth for this app
|
||||
dropboxConfig = &oauth2.Config{
|
||||
Scopes: []string{},
|
||||
// Endpoint: oauth2.Endpoint{
|
||||
// AuthURL: "https://www.dropbox.com/1/oauth2/authorize",
|
||||
// TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
|
||||
// },
|
||||
Endpoint: dropbox.OAuthEndpoint(""),
|
||||
ClientID: rcloneClientID,
|
||||
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
}
|
||||
// A regexp matching path names for files Dropbox ignores
|
||||
// See https://www.dropbox.com/en/help/145 - Ignored files
|
||||
ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
|
||||
// Upload chunk size - setting too small makes uploads slow.
|
||||
// Chunks are buffered into memory for retries.
|
||||
//
|
||||
@@ -96,8 +79,26 @@ var (
|
||||
// Choose 48MB which is 91% of Maximum speed. rclone by
|
||||
// default does 4 transfers so this should use 4*48MB = 192MB
|
||||
// by default.
|
||||
uploadChunkSize = fs.SizeSuffix(48 * 1024 * 1024)
|
||||
maxUploadChunkSize = fs.SizeSuffix(150 * 1024 * 1024)
|
||||
defaultChunkSize = 48 * 1024 * 1024
|
||||
maxChunkSize = 150 * 1024 * 1024
|
||||
)
|
||||
|
||||
var (
|
||||
// Description of how to auth for this app
|
||||
dropboxConfig = &oauth2.Config{
|
||||
Scopes: []string{},
|
||||
// Endpoint: oauth2.Endpoint{
|
||||
// AuthURL: "https://www.dropbox.com/1/oauth2/authorize",
|
||||
// TokenURL: "https://api.dropboxapi.com/1/oauth2/token",
|
||||
// },
|
||||
Endpoint: dropbox.OAuthEndpoint(""),
|
||||
ClientID: rcloneClientID,
|
||||
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
}
|
||||
// A regexp matching path names for files Dropbox ignores
|
||||
// See https://www.dropbox.com/en/help/145 - Ignored files
|
||||
ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -106,27 +107,37 @@ func init() {
|
||||
Name: "dropbox",
|
||||
Description: "Dropbox",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
err := oauthutil.ConfigNoOffline("dropbox", name, dropboxConfig)
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.ConfigNoOffline("dropbox", name, m, dropboxConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Dropbox App Client Id - leave blank normally.",
|
||||
Help: "Dropbox App Client Id\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Dropbox App Client Secret - leave blank normally.",
|
||||
Help: "Dropbox App Client Secret\nLeave blank normally.",
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: fmt.Sprintf("Upload chunk size. Max %v.", fs.SizeSuffix(maxChunkSize)),
|
||||
Default: fs.SizeSuffix(defaultChunkSize),
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&uploadChunkSize, "dropbox-chunk-size", "", fmt.Sprintf("Upload chunk size. Max %v.", maxUploadChunkSize))
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dropbox server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
srv files.Client // the connection to the dropbox server
|
||||
sharing sharing.Client // as above, but for generating sharing links
|
||||
@@ -185,15 +196,22 @@ func shouldRetry(err error) (bool, error) {
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
if uploadChunkSize > maxUploadChunkSize {
|
||||
return nil, errors.Errorf("chunk size too big, must be < %v", maxUploadChunkSize)
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.ChunkSize > maxChunkSize {
|
||||
return nil, errors.Errorf("chunk size too big, must be < %v", maxChunkSize)
|
||||
}
|
||||
|
||||
// Convert the old token if it exists. The old token was just
|
||||
// just a string, the new one is a JSON blob
|
||||
oldToken := strings.TrimSpace(config.FileGet(name, config.ConfigToken))
|
||||
if oldToken != "" && oldToken[0] != '{' {
|
||||
oldToken, ok := m.Get(config.ConfigToken)
|
||||
oldToken = strings.TrimSpace(oldToken)
|
||||
if ok && oldToken != "" && oldToken[0] != '{' {
|
||||
fs.Infof(name, "Converting token to new format")
|
||||
newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
|
||||
err := config.SetValueAndSave(name, config.ConfigToken, newToken)
|
||||
@@ -202,13 +220,14 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
}
|
||||
}
|
||||
|
||||
oAuthClient, _, err := oauthutil.NewClient(name, dropboxConfig)
|
||||
oAuthClient, _, err := oauthutil.NewClient(name, m, dropboxConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to configure dropbox")
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
}
|
||||
config := dropbox.Config{
|
||||
@@ -911,7 +930,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
// unknown (i.e. -1) or smaller than uploadChunkSize, the method incurs an
|
||||
// avoidable request to the Dropbox API that does not carry payload.
|
||||
func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size int64) (entry *files.FileMetadata, err error) {
|
||||
chunkSize := int64(uploadChunkSize)
|
||||
chunkSize := int64(o.fs.opt.ChunkSize)
|
||||
chunks := 0
|
||||
if size != -1 {
|
||||
chunks = int(size/chunkSize) + 1
|
||||
@@ -1026,7 +1045,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
size := src.Size()
|
||||
var err error
|
||||
var entry *files.FileMetadata
|
||||
if size > int64(uploadChunkSize) || size == -1 {
|
||||
if size > int64(o.fs.opt.ChunkSize) || size == -1 {
|
||||
entry, err = o.uploadChunked(in, commitInfo, size)
|
||||
} else {
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
|
||||
+28
-40
@@ -4,16 +4,15 @@ package ftp
|
||||
import (
|
||||
"io"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/ncw/rclone/lib/readers"
|
||||
@@ -30,33 +29,40 @@ func init() {
|
||||
{
|
||||
Name: "host",
|
||||
Help: "FTP host to connect to",
|
||||
Optional: false,
|
||||
Required: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "ftp.example.com",
|
||||
Help: "Connect to ftp.example.com",
|
||||
}},
|
||||
}, {
|
||||
Name: "user",
|
||||
Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
|
||||
Optional: true,
|
||||
Name: "user",
|
||||
Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
|
||||
}, {
|
||||
Name: "port",
|
||||
Help: "FTP port, leave blank to use default (21) ",
|
||||
Optional: true,
|
||||
Name: "port",
|
||||
Help: "FTP port, leave blank to use default (21)",
|
||||
}, {
|
||||
Name: "pass",
|
||||
Help: "FTP password",
|
||||
IsPassword: true,
|
||||
Optional: false,
|
||||
Required: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Host string `config:"host"`
|
||||
User string `config:"user"`
|
||||
Pass string `config:"pass"`
|
||||
Port string `config:"port"`
|
||||
}
|
||||
|
||||
// Fs represents a remote FTP server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
url string
|
||||
user string
|
||||
@@ -161,51 +167,33 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (ff fs.Fs, err error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
|
||||
// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
|
||||
// FIXME Convert the old scheme used for the first beta - remove after release
|
||||
if ftpURL := config.FileGet(name, "url"); ftpURL != "" {
|
||||
fs.Infof(name, "Converting old configuration")
|
||||
u, err := url.Parse(ftpURL)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "Failed to parse old url %q", ftpURL)
|
||||
}
|
||||
parts := strings.Split(u.Host, ":")
|
||||
config.FileSet(name, "host", parts[0])
|
||||
if len(parts) > 1 {
|
||||
config.FileSet(name, "port", parts[1])
|
||||
}
|
||||
config.FileSet(name, "host", u.Host)
|
||||
config.FileSet(name, "user", config.FileGet(name, "username"))
|
||||
config.FileSet(name, "pass", config.FileGet(name, "password"))
|
||||
config.FileDeleteKey(name, "username")
|
||||
config.FileDeleteKey(name, "password")
|
||||
config.FileDeleteKey(name, "url")
|
||||
config.SaveConfig()
|
||||
if u.Path != "" && u.Path != "/" {
|
||||
fs.Errorf(name, "Path %q in FTP URL no longer supported - put it on the end of the remote %s:%s", u.Path, name, u.Path)
|
||||
}
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err = configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
host := config.FileGet(name, "host")
|
||||
user := config.FileGet(name, "user")
|
||||
pass := config.FileGet(name, "pass")
|
||||
port := config.FileGet(name, "port")
|
||||
pass, err = obscure.Reveal(pass)
|
||||
pass, err := obscure.Reveal(opt.Pass)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "NewFS decrypt password")
|
||||
}
|
||||
user := opt.User
|
||||
if user == "" {
|
||||
user = os.Getenv("USER")
|
||||
}
|
||||
port := opt.Port
|
||||
if port == "" {
|
||||
port = "21"
|
||||
}
|
||||
|
||||
dialAddr := host + ":" + port
|
||||
dialAddr := opt.Host + ":" + port
|
||||
u := "ftp://" + path.Join(dialAddr+"/", root)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
url: u,
|
||||
user: user,
|
||||
pass: pass,
|
||||
|
||||
@@ -29,7 +29,8 @@ import (
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
@@ -55,8 +56,6 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
gcsLocation = flags.StringP("gcs-location", "", "", "Default location for buckets (us|eu|asia|us-central1|us-east1|us-east4|us-west1|asia-east1|asia-noetheast1|asia-southeast1|australia-southeast1|europe-west1|europe-west2).")
|
||||
gcsStorageClass = flags.StringP("gcs-storage-class", "", "", "Default storage class for buckets (MULTI_REGIONAL|REGIONAL|STANDARD|NEARLINE|COLDLINE|DURABLE_REDUCED_AVAILABILITY).")
|
||||
// Description of how to auth for this app
|
||||
storageConfig = &oauth2.Config{
|
||||
Scopes: []string{storage.DevstorageFullControlScope},
|
||||
@@ -71,29 +70,36 @@ var (
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "google cloud storage",
|
||||
Prefix: "gcs",
|
||||
Description: "Google Cloud Storage (this is not Google Drive)",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
if config.FileGet(name, "service_account_file") != "" {
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
saFile, _ := m.Get("service_account_file")
|
||||
saCreds, _ := m.Get("service_account_credentials")
|
||||
if saFile != "" || saCreds != "" {
|
||||
return
|
||||
}
|
||||
err := oauthutil.Config("google cloud storage", name, storageConfig)
|
||||
err := oauthutil.Config("google cloud storage", name, m, storageConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Google Application Client Id - leave blank normally.",
|
||||
Help: "Google Application Client Id\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Google Application Client Secret - leave blank normally.",
|
||||
Help: "Google Application Client Secret\nLeave blank normally.",
|
||||
}, {
|
||||
Name: "project_number",
|
||||
Help: "Project number optional - needed only for list/create/delete buckets - see your developer console.",
|
||||
Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.",
|
||||
}, {
|
||||
Name: "service_account_file",
|
||||
Help: "Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login.",
|
||||
Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
}, {
|
||||
Name: "service_account_credentials",
|
||||
Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: "object_acl",
|
||||
Help: "Access Control List for new objects.",
|
||||
@@ -207,22 +213,29 @@ func init() {
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
ProjectNumber string `config:"project_number"`
|
||||
ServiceAccountFile string `config:"service_account_file"`
|
||||
ServiceAccountCredentials string `config:"service_account_credentials"`
|
||||
ObjectACL string `config:"object_acl"`
|
||||
BucketACL string `config:"bucket_acl"`
|
||||
Location string `config:"location"`
|
||||
StorageClass string `config:"storage_class"`
|
||||
}
|
||||
|
||||
// Fs represents a remote storage server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
features *fs.Features // optional features
|
||||
svc *storage.Service // the connection to the storage server
|
||||
client *http.Client // authorized client
|
||||
bucket string // the bucket we are working on
|
||||
bucketOKMu sync.Mutex // mutex to protect bucket OK
|
||||
bucketOK bool // true if we have created the bucket
|
||||
projectNumber string // used for finding buckets
|
||||
objectACL string // used when creating new objects
|
||||
bucketACL string // used when creating new buckets
|
||||
location string // location of new buckets
|
||||
storageClass string // storage class of new buckets
|
||||
pacer *pacer.Pacer // To pace the API calls
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
svc *storage.Service // the connection to the storage server
|
||||
client *http.Client // authorized client
|
||||
bucket string // the bucket we are working on
|
||||
bucketOKMu sync.Mutex // mutex to protect bucket OK
|
||||
bucketOK bool // true if we have created the bucket
|
||||
pacer *pacer.Pacer // To pace the API calls
|
||||
}
|
||||
|
||||
// Object describes a storage object
|
||||
@@ -315,27 +328,37 @@ func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, bucket:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
var oAuthClient *http.Client
|
||||
var err error
|
||||
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.ObjectACL == "" {
|
||||
opt.ObjectACL = "private"
|
||||
}
|
||||
if opt.BucketACL == "" {
|
||||
opt.BucketACL = "private"
|
||||
}
|
||||
|
||||
// try loading service account credentials from env variable, then from a file
|
||||
serviceAccountCreds := []byte(config.FileGet(name, "service_account_credentials"))
|
||||
serviceAccountPath := config.FileGet(name, "service_account_file")
|
||||
if len(serviceAccountCreds) == 0 && serviceAccountPath != "" {
|
||||
loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(serviceAccountPath))
|
||||
if opt.ServiceAccountCredentials != "" && opt.ServiceAccountFile != "" {
|
||||
loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error opening service account credentials file")
|
||||
}
|
||||
serviceAccountCreds = loadedCreds
|
||||
opt.ServiceAccountCredentials = string(loadedCreds)
|
||||
}
|
||||
if len(serviceAccountCreds) > 0 {
|
||||
oAuthClient, err = getServiceAccountClient(serviceAccountCreds)
|
||||
if opt.ServiceAccountCredentials != "" {
|
||||
oAuthClient, err = getServiceAccountClient([]byte(opt.ServiceAccountCredentials))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account")
|
||||
}
|
||||
} else {
|
||||
oAuthClient, _, err = oauthutil.NewClient(name, storageConfig)
|
||||
oAuthClient, _, err = oauthutil.NewClient(name, m, storageConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
|
||||
}
|
||||
@@ -347,33 +370,17 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
bucket: bucket,
|
||||
root: directory,
|
||||
projectNumber: config.FileGet(name, "project_number"),
|
||||
objectACL: config.FileGet(name, "object_acl"),
|
||||
bucketACL: config.FileGet(name, "bucket_acl"),
|
||||
location: config.FileGet(name, "location"),
|
||||
storageClass: config.FileGet(name, "storage_class"),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
|
||||
name: name,
|
||||
bucket: bucket,
|
||||
root: directory,
|
||||
opt: *opt,
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
ReadMimeType: true,
|
||||
WriteMimeType: true,
|
||||
BucketBased: true,
|
||||
}).Fill(f)
|
||||
if f.objectACL == "" {
|
||||
f.objectACL = "private"
|
||||
}
|
||||
if f.bucketACL == "" {
|
||||
f.bucketACL = "private"
|
||||
}
|
||||
if *gcsLocation != "" {
|
||||
f.location = *gcsLocation
|
||||
}
|
||||
if *gcsStorageClass != "" {
|
||||
f.storageClass = *gcsStorageClass
|
||||
}
|
||||
|
||||
// Create a new authorized Drive client.
|
||||
f.client = oAuthClient
|
||||
@@ -550,10 +557,10 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
|
||||
if dir != "" {
|
||||
return nil, fs.ErrorListBucketRequired
|
||||
}
|
||||
if f.projectNumber == "" {
|
||||
if f.opt.ProjectNumber == "" {
|
||||
return nil, errors.New("can't list buckets without project number")
|
||||
}
|
||||
listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks)
|
||||
listBuckets := f.svc.Buckets.List(f.opt.ProjectNumber).MaxResults(listChunks)
|
||||
for {
|
||||
var buckets *storage.Buckets
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -672,17 +679,17 @@ func (f *Fs) Mkdir(dir string) (err error) {
|
||||
return errors.Wrap(err, "failed to get bucket")
|
||||
}
|
||||
|
||||
if f.projectNumber == "" {
|
||||
if f.opt.ProjectNumber == "" {
|
||||
return errors.New("can't make bucket without project number")
|
||||
}
|
||||
|
||||
bucket := storage.Bucket{
|
||||
Name: f.bucket,
|
||||
Location: f.location,
|
||||
StorageClass: f.storageClass,
|
||||
Location: f.opt.Location,
|
||||
StorageClass: f.opt.StorageClass,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.svc.Buckets.Insert(f.projectNumber, &bucket).PredefinedAcl(f.bucketACL).Do()
|
||||
_, err = f.svc.Buckets.Insert(f.opt.ProjectNumber, &bucket).PredefinedAcl(f.opt.BucketACL).Do()
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err == nil {
|
||||
@@ -948,7 +955,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
}
|
||||
var newObject *storage.Object
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.objectACL).Do()
|
||||
newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.opt.ObjectACL).Do()
|
||||
return shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
+21
-7
@@ -14,7 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/ncw/rclone/lib/rest"
|
||||
@@ -35,7 +36,7 @@ func init() {
|
||||
Options: []fs.Option{{
|
||||
Name: "url",
|
||||
Help: "URL of http host to connect to",
|
||||
Optional: false,
|
||||
Required: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "https://example.com",
|
||||
Help: "Connect to example.com",
|
||||
@@ -45,11 +46,17 @@ func init() {
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Endpoint string `config:"url"`
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote HTTP files
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
features *fs.Features // optional features
|
||||
opt Options // options for this backend
|
||||
endpoint *url.URL
|
||||
endpointURL string // endpoint as a string
|
||||
httpClient *http.Client
|
||||
@@ -78,14 +85,20 @@ func statusError(res *http.Response, err error) error {
|
||||
|
||||
// NewFs creates a new Fs object from the name and root. It connects to
|
||||
// the host specified in the config file.
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
endpoint := config.FileGet(name, "url")
|
||||
if !strings.HasSuffix(endpoint, "/") {
|
||||
endpoint += "/"
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(opt.Endpoint, "/") {
|
||||
opt.Endpoint += "/"
|
||||
}
|
||||
|
||||
// Parse the endpoint and stick the root onto it
|
||||
base, err := url.Parse(endpoint)
|
||||
base, err := url.Parse(opt.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -130,6 +143,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
httpClient: client,
|
||||
endpoint: u,
|
||||
endpointURL: u.String(),
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fstest"
|
||||
"github.com/ncw/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -29,7 +30,7 @@ var (
|
||||
)
|
||||
|
||||
// prepareServer the test server and return a function to tidy it up afterwards
|
||||
func prepareServer(t *testing.T) func() {
|
||||
func prepareServer(t *testing.T) (configmap.Simple, func()) {
|
||||
// file server for test/files
|
||||
fileServer := http.FileServer(http.Dir(filesPath))
|
||||
|
||||
@@ -41,19 +42,24 @@ func prepareServer(t *testing.T) func() {
|
||||
// fs.Config.LogLevel = fs.LogLevelDebug
|
||||
// fs.Config.DumpHeaders = true
|
||||
// fs.Config.DumpBodies = true
|
||||
config.FileSet(remoteName, "type", "http")
|
||||
config.FileSet(remoteName, "url", ts.URL)
|
||||
// config.FileSet(remoteName, "type", "http")
|
||||
// config.FileSet(remoteName, "url", ts.URL)
|
||||
|
||||
m := configmap.Simple{
|
||||
"type": "http",
|
||||
"url": ts.URL,
|
||||
}
|
||||
|
||||
// return a function to tidy up
|
||||
return ts.Close
|
||||
return m, ts.Close
|
||||
}
|
||||
|
||||
// prepare the test server and return a function to tidy it up afterwards
|
||||
func prepare(t *testing.T) (fs.Fs, func()) {
|
||||
tidy := prepareServer(t)
|
||||
m, tidy := prepareServer(t)
|
||||
|
||||
// Instantiate it
|
||||
f, err := NewFs(remoteName, "")
|
||||
f, err := NewFs(remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
return f, tidy
|
||||
@@ -177,20 +183,20 @@ func TestMimeType(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIsAFileRoot(t *testing.T) {
|
||||
tidy := prepareServer(t)
|
||||
m, tidy := prepareServer(t)
|
||||
defer tidy()
|
||||
|
||||
f, err := NewFs(remoteName, "one%.txt")
|
||||
f, err := NewFs(remoteName, "one%.txt", m)
|
||||
assert.Equal(t, err, fs.ErrorIsFile)
|
||||
|
||||
testListRoot(t, f)
|
||||
}
|
||||
|
||||
func TestIsAFileSubDir(t *testing.T) {
|
||||
tidy := prepareServer(t)
|
||||
m, tidy := prepareServer(t)
|
||||
defer tidy()
|
||||
|
||||
f, err := NewFs(remoteName, "three/underthree.txt")
|
||||
f, err := NewFs(remoteName, "three/underthree.txt", m)
|
||||
assert.Equal(t, err, fs.ErrorIsFile)
|
||||
|
||||
entries, err := f.List("")
|
||||
|
||||
+16
-7
@@ -16,6 +16,8 @@ import (
|
||||
"github.com/ncw/rclone/backend/swift"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/lib/oauthutil"
|
||||
@@ -52,18 +54,18 @@ func init() {
|
||||
Name: "hubic",
|
||||
Description: "Hubic",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
err := oauthutil.Config("hubic", name, oauthConfig)
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("hubic", name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Hubic Client Id - leave blank normally.",
|
||||
Help: "Hubic Client Id\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Hubic Client Secret - leave blank normally.",
|
||||
Help: "Hubic Client Secret\nLeave blank normally.",
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -145,8 +147,8 @@ func (f *Fs) getCredentials() (err error) {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
client, _, err := oauthutil.NewClient(name, oauthConfig)
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
client, _, err := oauthutil.NewClient(name, m, oauthConfig)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to configure Hubic")
|
||||
}
|
||||
@@ -167,8 +169,15 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
return nil, errors.Wrap(err, "error authenticating swift connection")
|
||||
}
|
||||
|
||||
// Parse config into swift.Options struct
|
||||
opt := new(swift.Options)
|
||||
err = configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Make inner swift Fs from the connection
|
||||
swiftFs, err := swift.NewFsWithConnection(name, root, c, true)
|
||||
swiftFs, err := swift.NewFsWithConnection(opt, name, root, c, true)
|
||||
if err != nil && err != fs.ErrorIsFile {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+63
-28
@@ -16,19 +16,11 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/ncw/rclone/lib/readers"
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/appengine/log"
|
||||
)
|
||||
|
||||
var (
|
||||
followSymlinks = flags.BoolP("copy-links", "L", false, "Follow symlinks and copy the pointed to item.")
|
||||
skipSymlinks = flags.BoolP("skip-links", "", false, "Don't warn about skipped symlinks.")
|
||||
noUTFNorm = flags.BoolP("local-no-unicode-normalization", "", false, "Don't apply unicode normalization to paths and filenames")
|
||||
noCheckUpdated = flags.BoolP("local-no-check-updated", "", false, "Don't check to see if the files change during upload")
|
||||
)
|
||||
|
||||
// Constants
|
||||
@@ -41,29 +33,68 @@ func init() {
|
||||
Description: "Local Disk",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "nounc",
|
||||
Help: "Disable UNC (long path names) conversion on Windows",
|
||||
Optional: true,
|
||||
Name: "nounc",
|
||||
Help: "Disable UNC (long path names) conversion on Windows",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "true",
|
||||
Help: "Disables long file names",
|
||||
}},
|
||||
}, {
|
||||
Name: "copy_links",
|
||||
Help: "Follow symlinks and copy the pointed to item.",
|
||||
Default: false,
|
||||
NoPrefix: true,
|
||||
ShortOpt: "L",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_links",
|
||||
Help: "Don't warn about skipped symlinks.",
|
||||
Default: false,
|
||||
NoPrefix: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "no_unicode_normalization",
|
||||
Help: "Don't apply unicode normalization to paths and filenames",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "no_check_updated",
|
||||
Help: "Don't check to see if the files change during upload",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "one_file_system",
|
||||
Help: "Don't cross filesystem boundaries (unix/macOS only).",
|
||||
Default: false,
|
||||
NoPrefix: true,
|
||||
ShortOpt: "x",
|
||||
Advanced: true,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
FollowSymlinks bool `config:"copy_links"`
|
||||
SkipSymlinks bool `config:"skip_links"`
|
||||
NoUTFNorm bool `config:"no_unicode_normalization"`
|
||||
NoCheckUpdated bool `config:"no_check_updated"`
|
||||
NoUNC bool `config:"nounc"`
|
||||
OneFileSystem bool `config:"one_file_system"`
|
||||
}
|
||||
|
||||
// Fs represents a local filesystem rooted at root
|
||||
type Fs struct {
|
||||
name string // the name of the remote
|
||||
root string // The root directory (OS path)
|
||||
opt Options // parsed config options
|
||||
features *fs.Features // optional features
|
||||
dev uint64 // device number of root node
|
||||
precisionOk sync.Once // Whether we need to read the precision
|
||||
precision time.Duration // precision of local filesystem
|
||||
wmu sync.Mutex // used for locking access to 'warned'.
|
||||
warned map[string]struct{} // whether we have warned about this string
|
||||
nounc bool // Skip UNC conversion on Windows
|
||||
// do os.Lstat or os.Stat
|
||||
lstat func(name string) (os.FileInfo, error)
|
||||
dirNames *mapper // directory name mapping
|
||||
@@ -84,18 +115,22 @@ type Object struct {
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// NewFs constructs an Fs from the path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
var err error
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if *noUTFNorm {
|
||||
log.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
|
||||
if opt.NoUTFNorm {
|
||||
fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
|
||||
}
|
||||
|
||||
nounc := config.FileGet(name, "nounc")
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
warned: make(map[string]struct{}),
|
||||
nounc: nounc == "true",
|
||||
dev: devUnset,
|
||||
lstat: os.Lstat,
|
||||
dirNames: newMapper(),
|
||||
@@ -105,14 +140,14 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
CaseInsensitive: f.caseInsensitive(),
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(f)
|
||||
if *followSymlinks {
|
||||
if opt.FollowSymlinks {
|
||||
f.lstat = os.Stat
|
||||
}
|
||||
|
||||
// Check to see if this points to a file
|
||||
fi, err := f.lstat(f.root)
|
||||
if err == nil {
|
||||
f.dev = readDevice(fi)
|
||||
f.dev = readDevice(fi, f.opt.OneFileSystem)
|
||||
}
|
||||
if err == nil && fi.Mode().IsRegular() {
|
||||
// It is a file, so use the parent as the root
|
||||
@@ -243,7 +278,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
newRemote := path.Join(remote, name)
|
||||
newPath := filepath.Join(fsDirPath, name)
|
||||
// Follow symlinks if required
|
||||
if *followSymlinks && (mode&os.ModeSymlink) != 0 {
|
||||
if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
|
||||
fi, err = os.Stat(newPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -253,7 +288,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
if fi.IsDir() {
|
||||
// Ignore directories which are symlinks. These are junction points under windows which
|
||||
// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
|
||||
if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi) {
|
||||
if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) {
|
||||
d := fs.NewDir(f.dirNames.Save(newRemote, f.cleanRemote(newRemote)), fi.ModTime())
|
||||
entries = append(entries, d)
|
||||
}
|
||||
@@ -357,7 +392,7 @@ func (f *Fs) Mkdir(dir string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.dev = readDevice(fi)
|
||||
f.dev = readDevice(fi, f.opt.OneFileSystem)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -643,7 +678,7 @@ func (o *Object) Storable() bool {
|
||||
}
|
||||
mode := o.mode
|
||||
if mode&os.ModeSymlink != 0 {
|
||||
if !*skipSymlinks {
|
||||
if !o.fs.opt.SkipSymlinks {
|
||||
fs.Logf(o, "Can't follow symlink without -L/--copy-links")
|
||||
}
|
||||
return false
|
||||
@@ -668,7 +703,7 @@ type localOpenFile struct {
|
||||
|
||||
// Read bytes from the object - see io.Reader
|
||||
func (file *localOpenFile) Read(p []byte) (n int, err error) {
|
||||
if !*noCheckUpdated {
|
||||
if !file.o.fs.opt.NoCheckUpdated {
|
||||
// Check if file has the same size and modTime
|
||||
fi, err := file.fd.Stat()
|
||||
if err != nil {
|
||||
@@ -878,7 +913,7 @@ func (f *Fs) cleanPath(s string) string {
|
||||
s = s2
|
||||
}
|
||||
}
|
||||
if !f.nounc {
|
||||
if !f.opt.NoUNC {
|
||||
// Convert to UNC
|
||||
s = uncPath(s)
|
||||
}
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestUpdatingCheck(t *testing.T) {
|
||||
|
||||
fi, err := fd.Stat()
|
||||
require.NoError(t, err)
|
||||
o := &Object{size: fi.Size(), modTime: fi.ModTime()}
|
||||
o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
|
||||
wrappedFd := readers.NewLimitedReadCloser(fd, -1)
|
||||
hash, err := hash.NewMultiHasherTypes(hash.Supported)
|
||||
require.NoError(t, err)
|
||||
@@ -65,11 +65,7 @@ func TestUpdatingCheck(t *testing.T) {
|
||||
require.Errorf(t, err, "can't copy - source file is being updated")
|
||||
|
||||
// turn the checking off and try again
|
||||
|
||||
*noCheckUpdated = true
|
||||
defer func() {
|
||||
*noCheckUpdated = false
|
||||
}()
|
||||
in.o.fs.opt.NoCheckUpdated = true
|
||||
|
||||
r.WriteFile(filePath, "content updated", time.Now())
|
||||
_, err = in.Read(buf)
|
||||
|
||||
@@ -8,6 +8,6 @@ import "os"
|
||||
|
||||
// readDevice turns a valid os.FileInfo into a device number,
|
||||
// returning devUnset if it fails.
|
||||
func readDevice(fi os.FileInfo) uint64 {
|
||||
func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
|
||||
return devUnset
|
||||
}
|
||||
@@ -9,17 +9,12 @@ import (
|
||||
"syscall"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
)
|
||||
|
||||
var (
|
||||
oneFileSystem = flags.BoolP("one-file-system", "x", false, "Don't cross filesystem boundaries.")
|
||||
)
|
||||
|
||||
// readDevice turns a valid os.FileInfo into a device number,
|
||||
// returning devUnset if it fails.
|
||||
func readDevice(fi os.FileInfo) uint64 {
|
||||
if !*oneFileSystem {
|
||||
func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
|
||||
if !oneFileSystem {
|
||||
return devUnset
|
||||
}
|
||||
statT, ok := fi.Sys().(*syscall.Stat_t)
|
||||
|
||||
+32
-15
@@ -24,8 +24,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -44,7 +44,6 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
megaDebug = flags.BoolP("mega-debug", "", false, "If set then output more debug from mega.")
|
||||
megaCacheMu sync.Mutex // mutex for the below
|
||||
megaCache = map[string]*mega.Mega{} // cache logged in Mega's by user
|
||||
)
|
||||
@@ -58,20 +57,33 @@ func init() {
|
||||
Options: []fs.Option{{
|
||||
Name: "user",
|
||||
Help: "User name",
|
||||
Optional: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "pass",
|
||||
Help: "Password.",
|
||||
Optional: true,
|
||||
Required: true,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "debug",
|
||||
Help: "If set then output more debug from mega.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
User string `config:"user"`
|
||||
Pass string `config:"pass"`
|
||||
Debug bool `config:"debug"`
|
||||
}
|
||||
|
||||
// Fs represents a remote mega
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed config options
|
||||
features *fs.Features // optional features
|
||||
srv *mega.Mega // the connection to the server
|
||||
pacer *pacer.Pacer // pacer for API calls
|
||||
@@ -145,12 +157,16 @@ func (f *Fs) readMetaDataForPath(remote string) (info *mega.Node, err error) {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
user := config.FileGet(name, "user")
|
||||
pass := config.FileGet(name, "pass")
|
||||
if pass != "" {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.Pass != "" {
|
||||
var err error
|
||||
pass, err = obscure.Reveal(pass)
|
||||
opt.Pass, err = obscure.Reveal(opt.Pass)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't decrypt password")
|
||||
}
|
||||
@@ -163,30 +179,31 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
// them up between different remotes.
|
||||
megaCacheMu.Lock()
|
||||
defer megaCacheMu.Unlock()
|
||||
srv := megaCache[user]
|
||||
srv := megaCache[opt.User]
|
||||
if srv == nil {
|
||||
srv = mega.New().SetClient(fshttp.NewClient(fs.Config))
|
||||
srv.SetRetries(fs.Config.LowLevelRetries) // let mega do the low level retries
|
||||
srv.SetLogger(func(format string, v ...interface{}) {
|
||||
fs.Infof("*go-mega*", format, v...)
|
||||
})
|
||||
if *megaDebug {
|
||||
if opt.Debug {
|
||||
srv.SetDebugger(func(format string, v ...interface{}) {
|
||||
fs.Debugf("*go-mega*", format, v...)
|
||||
})
|
||||
}
|
||||
|
||||
err := srv.Login(user, pass)
|
||||
err := srv.Login(opt.User, opt.Pass)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't login")
|
||||
}
|
||||
megaCache[user] = srv
|
||||
megaCache[opt.User] = srv
|
||||
}
|
||||
|
||||
root = parsePath(root)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: srv,
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
}
|
||||
@@ -196,7 +213,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
}).Fill(f)
|
||||
|
||||
// Find the root node and check if it is a file or not
|
||||
_, err := f.findRoot(false)
|
||||
_, err = f.findRoot(false)
|
||||
switch err {
|
||||
case nil:
|
||||
// root node found and is a directory
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
|
||||
package api
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
timeFormat = `"` + time.RFC3339 + `"`
|
||||
@@ -93,6 +96,22 @@ type ItemReference struct {
|
||||
Path string `json:"path"` // Path that used to navigate to the item. Read/Write.
|
||||
}
|
||||
|
||||
// RemoteItemFacet groups data needed to reference a OneDrive remote item
|
||||
type RemoteItemFacet struct {
|
||||
ID string `json:"id"` // The unique identifier of the item within the remote Drive. Read-only.
|
||||
Name string `json:"name"` // The name of the item (filename and extension). Read-write.
|
||||
CreatedBy IdentitySet `json:"createdBy"` // Identity of the user, device, and application which created the item. Read-only.
|
||||
LastModifiedBy IdentitySet `json:"lastModifiedBy"` // Identity of the user, device, and application which last modified the item. Read-only.
|
||||
CreatedDateTime Timestamp `json:"createdDateTime"` // Date and time of item creation. Read-only.
|
||||
LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // Date and time the item was last modified. Read-only.
|
||||
Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only.
|
||||
File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only.
|
||||
FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write.
|
||||
ParentReference *ItemReference `json:"parentReference"` // Parent information, if the item has a parent. Read-write.
|
||||
Size int64 `json:"size"` // Size of the item in bytes. Read-only.
|
||||
WebURL string `json:"webUrl"` // URL that displays the resource in the browser. Read-only.
|
||||
}
|
||||
|
||||
// FolderFacet groups folder-related data on OneDrive into a single structure
|
||||
type FolderFacet struct {
|
||||
ChildCount int64 `json:"childCount"` // Number of children contained immediately within this container.
|
||||
@@ -143,6 +162,7 @@ type Item struct {
|
||||
Description string `json:"description"` // Provide a user-visible description of the item. Read-write.
|
||||
Folder *FolderFacet `json:"folder"` // Folder metadata, if the item is a folder. Read-only.
|
||||
File *FileFacet `json:"file"` // File metadata, if the item is a file. Read-only.
|
||||
RemoteItem *RemoteItemFacet `json:"remoteItem"` // Remote Item metadata, if the item is a remote shared item. Read-only.
|
||||
FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` // File system information on client. Read-write.
|
||||
// Image *ImageFacet `json:"image"` // Image metadata, if the item is an image. Read-only.
|
||||
// Photo *PhotoFacet `json:"photo"` // Photo metadata, if the item is a photo. Read-only.
|
||||
@@ -228,3 +248,112 @@ type AsyncOperationStatus struct {
|
||||
PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete.
|
||||
Status string `json:"status"` // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting"
|
||||
}
|
||||
|
||||
// GetID returns a normalized ID of the item
|
||||
// If DriveID is known it will be prefixed to the ID with # seperator
|
||||
func (i *Item) GetID() string {
|
||||
if i.IsRemote() && i.RemoteItem.ID != "" {
|
||||
return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID
|
||||
} else if i.ParentReference != nil && strings.Index(i.ID, "#") == -1 {
|
||||
return i.ParentReference.DriveID + "#" + i.ID
|
||||
}
|
||||
return i.ID
|
||||
}
|
||||
|
||||
// GetDriveID returns a normalized ParentReferance of the item
|
||||
func (i *Item) GetDriveID() string {
|
||||
return i.GetParentReferance().DriveID
|
||||
}
|
||||
|
||||
// GetName returns a normalized Name of the item
|
||||
func (i *Item) GetName() string {
|
||||
if i.IsRemote() && i.RemoteItem.Name != "" {
|
||||
return i.RemoteItem.Name
|
||||
}
|
||||
return i.Name
|
||||
}
|
||||
|
||||
// GetFolder returns a normalized Folder of the item
|
||||
func (i *Item) GetFolder() *FolderFacet {
|
||||
if i.IsRemote() && i.RemoteItem.Folder != nil {
|
||||
return i.RemoteItem.Folder
|
||||
}
|
||||
return i.Folder
|
||||
}
|
||||
|
||||
// GetFile returns a normalized File of the item
|
||||
func (i *Item) GetFile() *FileFacet {
|
||||
if i.IsRemote() && i.RemoteItem.File != nil {
|
||||
return i.RemoteItem.File
|
||||
}
|
||||
return i.File
|
||||
}
|
||||
|
||||
// GetFileSystemInfo returns a normalized FileSystemInfo of the item
|
||||
func (i *Item) GetFileSystemInfo() *FileSystemInfoFacet {
|
||||
if i.IsRemote() && i.RemoteItem.FileSystemInfo != nil {
|
||||
return i.RemoteItem.FileSystemInfo
|
||||
}
|
||||
return i.FileSystemInfo
|
||||
}
|
||||
|
||||
// GetSize returns a normalized Size of the item
|
||||
func (i *Item) GetSize() int64 {
|
||||
if i.IsRemote() && i.RemoteItem.Size != 0 {
|
||||
return i.RemoteItem.Size
|
||||
}
|
||||
return i.Size
|
||||
}
|
||||
|
||||
// GetWebURL returns a normalized WebURL of the item
|
||||
func (i *Item) GetWebURL() string {
|
||||
if i.IsRemote() && i.RemoteItem.WebURL != "" {
|
||||
return i.RemoteItem.WebURL
|
||||
}
|
||||
return i.WebURL
|
||||
}
|
||||
|
||||
// GetCreatedBy returns a normalized CreatedBy of the item
|
||||
func (i *Item) GetCreatedBy() IdentitySet {
|
||||
if i.IsRemote() && i.RemoteItem.CreatedBy != (IdentitySet{}) {
|
||||
return i.RemoteItem.CreatedBy
|
||||
}
|
||||
return i.CreatedBy
|
||||
}
|
||||
|
||||
// GetLastModifiedBy returns a normalized LastModifiedBy of the item
|
||||
func (i *Item) GetLastModifiedBy() IdentitySet {
|
||||
if i.IsRemote() && i.RemoteItem.LastModifiedBy != (IdentitySet{}) {
|
||||
return i.RemoteItem.LastModifiedBy
|
||||
}
|
||||
return i.LastModifiedBy
|
||||
}
|
||||
|
||||
// GetCreatedDateTime returns a normalized CreatedDateTime of the item
|
||||
func (i *Item) GetCreatedDateTime() Timestamp {
|
||||
if i.IsRemote() && i.RemoteItem.CreatedDateTime != (Timestamp{}) {
|
||||
return i.RemoteItem.CreatedDateTime
|
||||
}
|
||||
return i.CreatedDateTime
|
||||
}
|
||||
|
||||
// GetLastModifiedDateTime returns a normalized LastModifiedDateTime of the item
|
||||
func (i *Item) GetLastModifiedDateTime() Timestamp {
|
||||
if i.IsRemote() && i.RemoteItem.LastModifiedDateTime != (Timestamp{}) {
|
||||
return i.RemoteItem.LastModifiedDateTime
|
||||
}
|
||||
return i.LastModifiedDateTime
|
||||
}
|
||||
|
||||
// GetParentReferance returns a normalized ParentReferance of the item
|
||||
func (i *Item) GetParentReferance() *ItemReference {
|
||||
if i.IsRemote() && i.ParentReference == nil {
|
||||
return i.RemoteItem.ParentReference
|
||||
}
|
||||
return i.ParentReference
|
||||
}
|
||||
|
||||
// IsRemote checks if item is a remote item
|
||||
func (i *Item) IsRemote() bool {
|
||||
return i.RemoteItem != nil
|
||||
}
|
||||
+167
-94
@@ -18,7 +18,8 @@ import (
|
||||
"github.com/ncw/rclone/backend/onedrive/api"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -73,8 +74,7 @@ var (
|
||||
RedirectURL: oauthutil.RedirectLocalhostURL,
|
||||
}
|
||||
oauthBusinessResource = oauth2.SetAuthURLParam("resource", discoveryServiceURL)
|
||||
|
||||
chunkSize = fs.SizeSuffix(10 * 1024 * 1024)
|
||||
sharedURL = "https://api.onedrive.com/v1.0/drives" // root URL for remote shared resources
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -83,7 +83,7 @@ func init() {
|
||||
Name: "onedrive",
|
||||
Description: "Microsoft OneDrive",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
// choose account type
|
||||
fmt.Printf("Choose OneDrive account type?\n")
|
||||
fmt.Printf(" * Say b for a OneDrive business account\n")
|
||||
@@ -92,12 +92,12 @@ func init() {
|
||||
|
||||
if isPersonal {
|
||||
// for personal accounts we don't safe a field about the account
|
||||
err := oauthutil.Config("onedrive", name, oauthPersonalConfig)
|
||||
err := oauthutil.Config("onedrive", name, m, oauthPersonalConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
} else {
|
||||
err := oauthutil.ConfigErrorCheck("onedrive", name, func(req *http.Request) oauthutil.AuthError {
|
||||
err := oauthutil.ConfigErrorCheck("onedrive", name, m, func(req *http.Request) oauthutil.AuthError {
|
||||
var resp oauthutil.AuthError
|
||||
|
||||
resp.Name = req.URL.Query().Get("error")
|
||||
@@ -112,7 +112,7 @@ func init() {
|
||||
}
|
||||
|
||||
// Are we running headless?
|
||||
if config.FileGet(name, config.ConfigAutomatic) != "" {
|
||||
if automatic, _ := m.Get(config.ConfigAutomatic); automatic != "" {
|
||||
// Yes, okay we are done
|
||||
return
|
||||
}
|
||||
@@ -126,7 +126,7 @@ func init() {
|
||||
Services []serviceResource `json:"value"`
|
||||
}
|
||||
|
||||
oAuthClient, _, err := oauthutil.NewClient(name, oauthBusinessConfig)
|
||||
oAuthClient, _, err := oauthutil.NewClient(name, m, oauthBusinessConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure OneDrive: %v", err)
|
||||
return
|
||||
@@ -171,13 +171,13 @@ func init() {
|
||||
foundService = config.Choose("Choose resource URL", resourcesID, resourcesURL, false)
|
||||
}
|
||||
|
||||
config.FileSet(name, configResourceURL, foundService)
|
||||
m.Set(configResourceURL, foundService)
|
||||
oauthBusinessResource = oauth2.SetAuthURLParam("resource", foundService)
|
||||
|
||||
// get the token from the inital config
|
||||
// we need to update the token with a resource
|
||||
// specific token we will query now
|
||||
token, err := oauthutil.GetToken(name)
|
||||
token, err := oauthutil.GetToken(name, m)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Error while getting token: %s", err)
|
||||
return
|
||||
@@ -220,7 +220,7 @@ func init() {
|
||||
token.RefreshToken = jsonToken.RefreshToken
|
||||
|
||||
// finally save them in the config
|
||||
err = oauthutil.PutToken(name, token, true)
|
||||
err = oauthutil.PutToken(name, m, token, true)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Error while setting token: %s", err)
|
||||
}
|
||||
@@ -228,20 +228,30 @@ func init() {
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Microsoft App Client Id - leave blank normally.",
|
||||
Help: "Microsoft App Client Id\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Microsoft App Client Secret - leave blank normally.",
|
||||
Help: "Microsoft App Client Secret\nLeave blank normally.",
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "Chunk size to upload files with - must be multiple of 320k.",
|
||||
Default: fs.SizeSuffix(10 * 1024 * 1024),
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
flags.VarP(&chunkSize, "onedrive-chunk-size", "", "Above this size files will be chunked - must be multiple of 320k.")
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
ResourceURL string `config:"resource_url"`
|
||||
}
|
||||
|
||||
// Fs represents a remote one drive
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
srv *rest.Client // the connection to the one drive server
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
@@ -325,6 +335,7 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon
|
||||
resp, err = f.srv.CallJSON(&opts, nil, &info)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
return info, resp, err
|
||||
}
|
||||
|
||||
@@ -343,26 +354,35 @@ func errorHandler(resp *http.Response) error {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
// get the resource URL from the config file0
|
||||
resourceURL := config.FileGet(name, configResourceURL, "")
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.ChunkSize%(320*1024) != 0 {
|
||||
return nil, errors.Errorf("chunk size %d is not a multiple of 320k", opt.ChunkSize)
|
||||
}
|
||||
// if we have a resource URL it's a business account otherwise a personal one
|
||||
isBusiness := opt.ResourceURL != ""
|
||||
var rootURL string
|
||||
var oauthConfig *oauth2.Config
|
||||
if resourceURL == "" {
|
||||
if !isBusiness {
|
||||
// personal account setup
|
||||
oauthConfig = oauthPersonalConfig
|
||||
rootURL = rootURLPersonal
|
||||
} else {
|
||||
// business account setup
|
||||
oauthConfig = oauthBusinessConfig
|
||||
rootURL = resourceURL + "_api/v2.0/drives/me"
|
||||
rootURL = opt.ResourceURL + "_api/v2.0/drives/me"
|
||||
sharedURL = opt.ResourceURL + "_api/v2.0/drives"
|
||||
|
||||
// update the URL in the AuthOptions
|
||||
oauthBusinessResource = oauth2.SetAuthURLParam("resource", resourceURL)
|
||||
oauthBusinessResource = oauth2.SetAuthURLParam("resource", opt.ResourceURL)
|
||||
}
|
||||
root = parsePath(root)
|
||||
oAuthClient, ts, err := oauthutil.NewClient(name, oauthConfig)
|
||||
oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure OneDrive: %v", err)
|
||||
}
|
||||
@@ -370,9 +390,10 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
isBusiness: resourceURL != "",
|
||||
isBusiness: isBusiness,
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: true,
|
||||
@@ -482,21 +503,18 @@ func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err er
|
||||
}
|
||||
return "", false, err
|
||||
}
|
||||
if info.Folder == nil {
|
||||
if info.GetFolder() == nil {
|
||||
return "", false, errors.New("found file when looking for folder")
|
||||
}
|
||||
return info.ID, true, nil
|
||||
return info.GetID(), true, nil
|
||||
}
|
||||
|
||||
// CreateDir makes a directory with pathID as parent and name leaf
|
||||
func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
|
||||
// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
|
||||
func (f *Fs) CreateDir(dirID, leaf string) (newID string, err error) {
|
||||
// fs.Debugf(f, "CreateDir(%q, %q)\n", dirID, leaf)
|
||||
var resp *http.Response
|
||||
var info *api.Item
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/items/" + pathID + "/children",
|
||||
}
|
||||
opts := newOptsCall(dirID, "POST", "/children")
|
||||
mkdir := api.CreateItemRequest{
|
||||
Name: replaceReservedChars(leaf),
|
||||
ConflictBehavior: "fail",
|
||||
@@ -509,8 +527,9 @@ func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
|
||||
//fmt.Printf("...Error %v\n", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
//fmt.Printf("...Id %q\n", *info.Id)
|
||||
return info.ID, nil
|
||||
return info.GetID(), nil
|
||||
}
|
||||
|
||||
// list the objects into the function supplied
|
||||
@@ -527,10 +546,8 @@ type listAllFn func(*api.Item) bool
|
||||
func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
|
||||
// Top parameter asks for bigger pages of data
|
||||
// https://dev.onedrive.com/odata/optional-query-parameters.htm
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/items/" + dirID + "/children?top=1000",
|
||||
}
|
||||
opts := newOptsCall(dirID, "GET", "/children?top=1000")
|
||||
|
||||
OUTER:
|
||||
for {
|
||||
var result api.ListChildrenResponse
|
||||
@@ -547,7 +564,7 @@ OUTER:
|
||||
}
|
||||
for i := range result.Value {
|
||||
item := &result.Value[i]
|
||||
isFolder := item.Folder != nil
|
||||
isFolder := item.GetFolder() != nil
|
||||
if isFolder {
|
||||
if filesOnly {
|
||||
continue
|
||||
@@ -560,7 +577,7 @@ OUTER:
|
||||
if item.Deleted != nil {
|
||||
continue
|
||||
}
|
||||
item.Name = restoreReservedChars(item.Name)
|
||||
item.Name = restoreReservedChars(item.GetName())
|
||||
if fn(item) {
|
||||
found = true
|
||||
break OUTER
|
||||
@@ -595,13 +612,15 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
|
||||
}
|
||||
var iErr error
|
||||
_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool {
|
||||
remote := path.Join(dir, info.Name)
|
||||
if info.Folder != nil {
|
||||
remote := path.Join(dir, info.GetName())
|
||||
folder := info.GetFolder()
|
||||
if folder != nil {
|
||||
// cache the directory ID for later lookups
|
||||
f.dirCache.Put(remote, info.ID)
|
||||
d := fs.NewDir(remote, time.Time(info.LastModifiedDateTime)).SetID(info.ID)
|
||||
if info.Folder != nil {
|
||||
d.SetItems(info.Folder.ChildCount)
|
||||
id := info.GetID()
|
||||
f.dirCache.Put(remote, id)
|
||||
d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
|
||||
if folder != nil {
|
||||
d.SetItems(folder.ChildCount)
|
||||
}
|
||||
entries = append(entries, d)
|
||||
} else {
|
||||
@@ -674,11 +693,9 @@ func (f *Fs) Mkdir(dir string) error {
|
||||
|
||||
// deleteObject removes an object by ID
|
||||
func (f *Fs) deleteObject(id string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/items/" + id,
|
||||
NoResponse: true,
|
||||
}
|
||||
opts := newOptsCall(id, "DELETE", "")
|
||||
opts.NoResponse = true
|
||||
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.Call(&opts)
|
||||
return shouldRetry(resp, err)
|
||||
@@ -814,17 +831,17 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
|
||||
}
|
||||
|
||||
// Copy the object
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/items/" + srcObj.id + "/action.copy",
|
||||
ExtraHeaders: map[string]string{"Prefer": "respond-async"},
|
||||
NoResponse: true,
|
||||
}
|
||||
opts := newOptsCall(srcObj.id, "POST", "/action.copy")
|
||||
opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"}
|
||||
opts.NoResponse = true
|
||||
|
||||
id, _, _ := parseDirID(directoryID)
|
||||
|
||||
replacedLeaf := replaceReservedChars(leaf)
|
||||
copy := api.CopyItemRequest{
|
||||
Name: &replacedLeaf,
|
||||
ParentReference: api.ItemReference{
|
||||
ID: directoryID,
|
||||
ID: id,
|
||||
},
|
||||
}
|
||||
var resp *http.Response
|
||||
@@ -891,14 +908,14 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
|
||||
}
|
||||
|
||||
// Move the object
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/items/" + srcObj.id,
|
||||
}
|
||||
opts := newOptsCall(srcObj.id, "PATCH", "")
|
||||
|
||||
id, _, _ := parseDirID(directoryID)
|
||||
|
||||
move := api.MoveItemRequest{
|
||||
Name: replaceReservedChars(leaf),
|
||||
ParentReference: &api.ItemReference{
|
||||
ID: directoryID,
|
||||
ID: id,
|
||||
},
|
||||
// We set the mod time too as it gets reset otherwise
|
||||
FileSystemInfo: &api.FileSystemInfoFacet{
|
||||
@@ -1013,35 +1030,37 @@ func (o *Object) Size() int64 {
|
||||
|
||||
// setMetaData sets the metadata from info
|
||||
func (o *Object) setMetaData(info *api.Item) (err error) {
|
||||
if info.Folder != nil {
|
||||
if info.GetFolder() != nil {
|
||||
return errors.Wrapf(fs.ErrorNotAFile, "%q", o.remote)
|
||||
}
|
||||
o.hasMetaData = true
|
||||
o.size = info.Size
|
||||
o.size = info.GetSize()
|
||||
|
||||
// Docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes
|
||||
//
|
||||
// We use SHA1 for onedrive personal and QuickXorHash for onedrive for business
|
||||
if info.File != nil {
|
||||
o.mimeType = info.File.MimeType
|
||||
if info.File.Hashes.Sha1Hash != "" {
|
||||
o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash)
|
||||
file := info.GetFile()
|
||||
if file != nil {
|
||||
o.mimeType = file.MimeType
|
||||
if file.Hashes.Sha1Hash != "" {
|
||||
o.sha1 = strings.ToLower(file.Hashes.Sha1Hash)
|
||||
}
|
||||
if info.File.Hashes.QuickXorHash != "" {
|
||||
h, err := base64.StdEncoding.DecodeString(info.File.Hashes.QuickXorHash)
|
||||
if file.Hashes.QuickXorHash != "" {
|
||||
h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash)
|
||||
if err != nil {
|
||||
fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", info.File.Hashes.QuickXorHash, err)
|
||||
fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err)
|
||||
} else {
|
||||
o.quickxorhash = hex.EncodeToString(h)
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.FileSystemInfo != nil {
|
||||
o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime)
|
||||
fileSystemInfo := info.GetFileSystemInfo()
|
||||
if fileSystemInfo != nil {
|
||||
o.modTime = time.Time(fileSystemInfo.LastModifiedDateTime)
|
||||
} else {
|
||||
o.modTime = time.Time(info.LastModifiedDateTime)
|
||||
o.modTime = time.Time(info.GetLastModifiedDateTime())
|
||||
}
|
||||
o.id = info.ID
|
||||
o.id = info.GetID()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1080,9 +1099,20 @@ func (o *Object) ModTime() time.Time {
|
||||
|
||||
// setModTime sets the modification time of the local fs object
|
||||
func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/root:/" + rest.URLPathEscape(o.srvPath()),
|
||||
var opts rest.Opts
|
||||
_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
|
||||
_, drive, rootURL := parseDirID(directoryID)
|
||||
if drive != "" {
|
||||
opts = rest.Opts{
|
||||
Method: "PATCH",
|
||||
RootURL: rootURL,
|
||||
Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()),
|
||||
}
|
||||
} else {
|
||||
opts = rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/root:/" + rest.URLPathEscape(o.srvPath()),
|
||||
}
|
||||
}
|
||||
update := api.SetFileSystemInfo{
|
||||
FileSystemInfo: api.FileSystemInfoFacet{
|
||||
@@ -1119,11 +1149,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
}
|
||||
fs.FixRangeOption(options, o.size)
|
||||
var resp *http.Response
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/items/" + o.id + "/content",
|
||||
Options: options,
|
||||
}
|
||||
opts := newOptsCall(o.id, "GET", "/content")
|
||||
opts.Options = options
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(&opts)
|
||||
return shouldRetry(resp, err)
|
||||
@@ -1141,9 +1169,20 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
|
||||
// createUploadSession creates an upload session for the object
|
||||
func (o *Object) createUploadSession(modTime time.Time) (response *api.CreateUploadResponse, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/upload.createSession",
|
||||
leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
|
||||
id, drive, rootURL := parseDirID(directoryID)
|
||||
var opts rest.Opts
|
||||
if drive != "" {
|
||||
opts = rest.Opts{
|
||||
Method: "POST",
|
||||
RootURL: rootURL,
|
||||
Path: "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(leaf) + ":/upload.createSession",
|
||||
}
|
||||
} else {
|
||||
opts = rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/upload.createSession",
|
||||
}
|
||||
}
|
||||
createRequest := api.CreateUploadRequest{}
|
||||
createRequest.Item.FileSystemInfo.CreatedDateTime = api.Timestamp(modTime)
|
||||
@@ -1204,10 +1243,6 @@ func (o *Object) cancelUploadSession(url string) (err error) {
|
||||
|
||||
// uploadMultipart uploads a file using multipart upload
|
||||
func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
|
||||
if chunkSize%(320*1024) != 0 {
|
||||
return nil, errors.Errorf("chunk size %d is not a multiple of 320k", chunkSize)
|
||||
}
|
||||
|
||||
// Create upload session
|
||||
fs.Debugf(o, "Starting multipart upload")
|
||||
session, err := o.createUploadSession(modTime)
|
||||
@@ -1231,7 +1266,7 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
|
||||
remaining := size
|
||||
position := int64(0)
|
||||
for remaining > 0 {
|
||||
n := int64(chunkSize)
|
||||
n := int64(o.fs.opt.ChunkSize)
|
||||
if remaining < n {
|
||||
n = remaining
|
||||
}
|
||||
@@ -1251,11 +1286,24 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
|
||||
// uploadSinglepart uploads a file as a single part
|
||||
func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
|
||||
var resp *http.Response
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
|
||||
ContentLength: &size,
|
||||
Body: in,
|
||||
var opts rest.Opts
|
||||
_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
|
||||
_, drive, rootURL := parseDirID(directoryID)
|
||||
if drive != "" {
|
||||
opts = rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: rootURL,
|
||||
Path: "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
|
||||
ContentLength: &size,
|
||||
Body: in,
|
||||
}
|
||||
} else {
|
||||
opts = rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
|
||||
ContentLength: &size,
|
||||
Body: in,
|
||||
}
|
||||
}
|
||||
// for go1.8 (see release notes) we must nil the Body if we want a
|
||||
// "Content-Length: 0" header which onedrive requires for all files.
|
||||
@@ -1269,6 +1317,7 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = o.setMetaData(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1315,6 +1364,30 @@ func (o *Object) ID() string {
|
||||
return o.id
|
||||
}
|
||||
|
||||
func newOptsCall(id string, method string, route string) (opts rest.Opts) {
|
||||
id, drive, rootURL := parseDirID(id)
|
||||
|
||||
if drive != "" {
|
||||
return rest.Opts{
|
||||
Method: method,
|
||||
RootURL: rootURL,
|
||||
Path: "/" + drive + "/items/" + id + route,
|
||||
}
|
||||
}
|
||||
return rest.Opts{
|
||||
Method: method,
|
||||
Path: "/items/" + id + route,
|
||||
}
|
||||
}
|
||||
|
||||
func parseDirID(ID string) (string, string, string) {
|
||||
if strings.Index(ID, "#") >= 0 {
|
||||
s := strings.Split(ID, "#")
|
||||
return s[1], s[0], sharedURL
|
||||
}
|
||||
return ID, "", ""
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
|
||||
@@ -12,7 +12,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
@@ -37,23 +38,30 @@ func init() {
|
||||
Description: "OpenDrive",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "username",
|
||||
Help: "Username",
|
||||
Name: "username",
|
||||
Help: "Username",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "password",
|
||||
Help: "Password.",
|
||||
IsPassword: true,
|
||||
Required: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
UserName string `config:"username"`
|
||||
Password string `config:"password"`
|
||||
}
|
||||
|
||||
// Fs represents a remote server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
username string // account name
|
||||
password string // auth key0
|
||||
srv *rest.Client // the connection to the server
|
||||
pacer *pacer.Pacer // To pace and retry the API calls
|
||||
session UserSessionInfo // contains the session data
|
||||
@@ -110,27 +118,31 @@ func (f *Fs) DirCacheFlush() {
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, bucket:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root = parsePath(root)
|
||||
username := config.FileGet(name, "username")
|
||||
if username == "" {
|
||||
if opt.UserName == "" {
|
||||
return nil, errors.New("username not found")
|
||||
}
|
||||
password, err := obscure.Reveal(config.FileGet(name, "password"))
|
||||
opt.Password, err = obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, errors.New("password coudl not revealed")
|
||||
return nil, errors.New("password could not revealed")
|
||||
}
|
||||
if password == "" {
|
||||
if opt.Password == "" {
|
||||
return nil, errors.New("password not found")
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
username: username,
|
||||
password: password,
|
||||
root: root,
|
||||
srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, "0", f)
|
||||
@@ -141,7 +153,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
// get sessionID
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
account := Account{Username: username, Password: password}
|
||||
account := Account{Username: opt.UserName, Password: opt.Password}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
|
||||
@@ -23,6 +23,8 @@ import (
|
||||
"github.com/ncw/rclone/backend/pcloud/api"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -65,26 +67,31 @@ func init() {
|
||||
Name: "pcloud",
|
||||
Description: "Pcloud",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
err := oauthutil.Config("pcloud", name, oauthConfig)
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("pcloud", name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Pcloud App Client Id - leave blank normally.",
|
||||
Help: "Pcloud App Client Id\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Pcloud App Client Secret - leave blank normally.",
|
||||
Help: "Pcloud App Client Secret\nLeave blank normally.",
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
}
|
||||
|
||||
// Fs represents a remote pcloud
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
srv *rest.Client // the connection to the server
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
@@ -229,9 +236,15 @@ func errorHandler(resp *http.Response) error {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root = parsePath(root)
|
||||
oAuthClient, ts, err := oauthutil.NewClient(name, oauthConfig)
|
||||
oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure Pcloud: %v", err)
|
||||
}
|
||||
@@ -239,6 +252,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
}
|
||||
|
||||
@@ -17,7 +17,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/ncw/rclone/fs/walk"
|
||||
@@ -34,49 +35,43 @@ func init() {
|
||||
Description: "QingCloud Object Storage",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "env_auth",
|
||||
Help: "Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank.",
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "false",
|
||||
Help: "Enter QingStor credentials in the next step",
|
||||
}, {
|
||||
Value: "true",
|
||||
Help: "Get QingStor credentials from the environment (env vars or IAM)",
|
||||
},
|
||||
},
|
||||
Name: "env_auth",
|
||||
Help: "Get QingStor credentials from runtime. Only applies if access_key_id and secret_access_key is blank.",
|
||||
Default: false,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "false",
|
||||
Help: "Enter QingStor credentials in the next step",
|
||||
}, {
|
||||
Value: "true",
|
||||
Help: "Get QingStor credentials from the environment (env vars or IAM)",
|
||||
}},
|
||||
}, {
|
||||
Name: "access_key_id",
|
||||
Help: "QingStor Access Key ID - leave blank for anonymous access or runtime credentials.",
|
||||
Help: "QingStor Access Key ID\nLeave blank for anonymous access or runtime credentials.",
|
||||
}, {
|
||||
Name: "secret_access_key",
|
||||
Help: "QingStor Secret Access Key (password) - leave blank for anonymous access or runtime credentials.",
|
||||
Help: "QingStor Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Enter a endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
|
||||
}, {
|
||||
Name: "zone",
|
||||
Help: "Choose or Enter a zone to connect. Default is \"pek3a\".",
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "pek3a",
|
||||
|
||||
Help: "The Beijing (China) Three Zone\nNeeds location constraint pek3a.",
|
||||
},
|
||||
{
|
||||
Value: "sh1a",
|
||||
|
||||
Help: "The Shanghai (China) First Zone\nNeeds location constraint sh1a.",
|
||||
},
|
||||
{
|
||||
Value: "gd2a",
|
||||
|
||||
Help: "The Guangdong (China) Second Zone\nNeeds location constraint gd2a.",
|
||||
},
|
||||
},
|
||||
Help: "Zone to connect to.\nDefault is \"pek3a\".",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "pek3a",
|
||||
Help: "The Beijing (China) Three Zone\nNeeds location constraint pek3a.",
|
||||
}, {
|
||||
Value: "sh1a",
|
||||
Help: "The Shanghai (China) First Zone\nNeeds location constraint sh1a.",
|
||||
}, {
|
||||
Value: "gd2a",
|
||||
Help: "The Guangdong (China) Second Zone\nNeeds location constraint gd2a.",
|
||||
}},
|
||||
}, {
|
||||
Name: "connection_retries",
|
||||
Help: "Number of connnection retry.\nLeave blank will use the default value \"3\".",
|
||||
Name: "connection_retries",
|
||||
Help: "Number of connnection retries.",
|
||||
Default: 3,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -95,17 +90,28 @@ func timestampToTime(tp int64) time.Time {
|
||||
return tm.UTC()
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
AccessKeyID string `config:"access_key_id"`
|
||||
SecretAccessKey string `config:"secret_access_key"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
Zone string `config:"zone"`
|
||||
ConnectionRetries int `config:"connection_retries"`
|
||||
}
|
||||
|
||||
// Fs represents a remote qingstor server
|
||||
type Fs struct {
|
||||
name string // The name of the remote
|
||||
root string // The root is a subdir, is a special object
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
svc *qs.Service // The connection to the qingstor server
|
||||
zone string // The zone we are working on
|
||||
bucket string // The bucket we are working on
|
||||
bucketOKMu sync.Mutex // mutex to protect bucketOK and bucketDeleted
|
||||
bucketOK bool // true if we have created the bucket
|
||||
bucketDeleted bool // true if we have deleted the bucket
|
||||
root string // The root is a subdir, is a special object
|
||||
features *fs.Features // optional features
|
||||
svc *qs.Service // The connection to the qingstor server
|
||||
}
|
||||
|
||||
// Object describes a qingstor object
|
||||
@@ -165,12 +171,12 @@ func qsParseEndpoint(endpoint string) (protocol, host, port string, err error) {
|
||||
}
|
||||
|
||||
// qsConnection makes a connection to qingstor
|
||||
func qsServiceConnection(name string) (*qs.Service, error) {
|
||||
accessKeyID := config.FileGet(name, "access_key_id")
|
||||
secretAccessKey := config.FileGet(name, "secret_access_key")
|
||||
func qsServiceConnection(opt *Options) (*qs.Service, error) {
|
||||
accessKeyID := opt.AccessKeyID
|
||||
secretAccessKey := opt.SecretAccessKey
|
||||
|
||||
switch {
|
||||
case config.FileGetBool(name, "env_auth", false):
|
||||
case opt.EnvAuth:
|
||||
// No need for empty checks if "env_auth" is true
|
||||
case accessKeyID == "" && secretAccessKey == "":
|
||||
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
|
||||
@@ -184,7 +190,7 @@ func qsServiceConnection(name string) (*qs.Service, error) {
|
||||
host := "qingstor.com"
|
||||
port := 443
|
||||
|
||||
endpoint := config.FileGet(name, "endpoint", "")
|
||||
endpoint := opt.Endpoint
|
||||
if endpoint != "" {
|
||||
_protocol, _host, _port, err := qsParseEndpoint(endpoint)
|
||||
|
||||
@@ -204,48 +210,49 @@ func qsServiceConnection(name string) (*qs.Service, error) {
|
||||
|
||||
}
|
||||
|
||||
connectionRetries := 3
|
||||
retries := config.FileGet(name, "connection_retries", "")
|
||||
if retries != "" {
|
||||
connectionRetries, _ = strconv.Atoi(retries)
|
||||
}
|
||||
|
||||
cf, err := qsConfig.NewDefault()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cf.AccessKeyID = accessKeyID
|
||||
cf.SecretAccessKey = secretAccessKey
|
||||
cf.Protocol = protocol
|
||||
cf.Host = host
|
||||
cf.Port = port
|
||||
cf.ConnectionRetries = connectionRetries
|
||||
cf.ConnectionRetries = opt.ConnectionRetries
|
||||
cf.Connection = fshttp.NewClient(fs.Config)
|
||||
|
||||
svc, _ := qs.Init(cf)
|
||||
|
||||
return svc, err
|
||||
return qs.Init(cf)
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, bucket:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket, key, err := qsParsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
svc, err := qsServiceConnection(name)
|
||||
svc, err := qsServiceConnection(opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
zone := config.FileGet(name, "zone")
|
||||
if zone == "" {
|
||||
zone = "pek3a"
|
||||
if opt.Zone == "" {
|
||||
opt.Zone = "pek3a"
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
zone: zone,
|
||||
root: key,
|
||||
bucket: bucket,
|
||||
opt: *opt,
|
||||
svc: svc,
|
||||
zone: opt.Zone,
|
||||
bucket: bucket,
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
ReadMimeType: true,
|
||||
@@ -258,7 +265,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
f.root += "/"
|
||||
}
|
||||
//Check to see if the object exists
|
||||
bucketInit, err := svc.Bucket(bucket, zone)
|
||||
bucketInit, err := svc.Bucket(bucket, opt.Zone)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+101
-82
@@ -37,8 +37,8 @@ import (
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/ncw/rclone/fs/walk"
|
||||
@@ -82,8 +82,9 @@ func init() {
|
||||
Help: "Any other S3 compatible provider",
|
||||
}},
|
||||
}, {
|
||||
Name: "env_auth",
|
||||
Help: "Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.",
|
||||
Name: "env_auth",
|
||||
Help: "Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).\nOnly applies if access_key_id and secret_access_key is blank.",
|
||||
Default: false,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "false",
|
||||
Help: "Enter AWS credentials in the next step",
|
||||
@@ -93,10 +94,10 @@ func init() {
|
||||
}},
|
||||
}, {
|
||||
Name: "access_key_id",
|
||||
Help: "AWS Access Key ID - leave blank for anonymous access or runtime credentials.",
|
||||
Help: "AWS Access Key ID.\nLeave blank for anonymous access or runtime credentials.",
|
||||
}, {
|
||||
Name: "secret_access_key",
|
||||
Help: "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.",
|
||||
Help: "AWS Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.",
|
||||
@@ -146,7 +147,7 @@ func init() {
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to. Leave blank if you are using an S3 clone and you don't have a region.",
|
||||
Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
|
||||
Provider: "!AWS",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
@@ -293,7 +294,7 @@ func init() {
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must be set to match the Region. Used when creating buckets only.",
|
||||
Help: "Location constraint - must be set to match the Region.\nUsed when creating buckets only.",
|
||||
Provider: "AWS",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
@@ -340,7 +341,7 @@ func init() {
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter",
|
||||
Help: "Location constraint - must match endpoint when using IBM Cloud Public.\nFor on-prem COS, do not make a selection from this list, hit enter",
|
||||
Provider: "IBMCOS",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "us-standard",
|
||||
@@ -441,7 +442,7 @@ func init() {
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must be set to match the Region. Leave blank if not sure. Used when creating buckets only.",
|
||||
Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
|
||||
Provider: "!AWS,IBMCOS",
|
||||
}, {
|
||||
Name: "acl",
|
||||
@@ -518,10 +519,28 @@ func init() {
|
||||
Value: "ONEZONE_IA",
|
||||
Help: "One Zone Infrequent Access storage class",
|
||||
}},
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "Chunk size to use for uploading",
|
||||
Default: fs.SizeSuffix(s3manager.MinUploadPartSize),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "disable_checksum",
|
||||
Help: "Don't store MD5 checksum with object metadata",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "session_token",
|
||||
Help: "An AWS session token",
|
||||
Hide: fs.OptionHideBoth,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
Help: "Concurrency for multipart uploads.",
|
||||
Default: 2,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&s3ChunkSize, "s3-chunk-size", "", "Chunk size to use for uploading")
|
||||
}
|
||||
|
||||
// Constants
|
||||
@@ -534,31 +553,36 @@ const (
|
||||
maxFileSize = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
// Flags
|
||||
s3ACL = flags.StringP("s3-acl", "", "", "Canned ACL used when creating buckets and/or storing objects in S3")
|
||||
s3StorageClass = flags.StringP("s3-storage-class", "", "", "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA|ONEZONE_IA)")
|
||||
s3ChunkSize = fs.SizeSuffix(s3manager.MinUploadPartSize)
|
||||
s3DisableChecksum = flags.BoolP("s3-disable-checksum", "", false, "Don't store MD5 checksum with object metadata")
|
||||
s3UploadConcurrency = flags.IntP("s3-upload-concurrency", "", 2, "Concurrency for multipart uploads")
|
||||
)
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Provider string `config:"provider"`
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
AccessKeyID string `config:"access_key_id"`
|
||||
SecretAccessKey string `config:"secret_access_key"`
|
||||
Region string `config:"region"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
LocationConstraint string `config:"location_constraint"`
|
||||
ACL string `config:"acl"`
|
||||
ServerSideEncryption string `config:"server_side_encryption"`
|
||||
StorageClass string `config:"storage_class"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
DisableChecksum bool `config:"disable_checksum"`
|
||||
SessionToken string `config:"session_token"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
}
|
||||
|
||||
// Fs represents a remote s3 server
|
||||
type Fs struct {
|
||||
name string // the name of the remote
|
||||
root string // root of the bucket - ignore all objects above this
|
||||
features *fs.Features // optional features
|
||||
c *s3.S3 // the connection to the s3 server
|
||||
ses *session.Session // the s3 session
|
||||
bucket string // the bucket we are working on
|
||||
bucketOKMu sync.Mutex // mutex to protect bucket OK
|
||||
bucketOK bool // true if we have created the bucket
|
||||
bucketDeleted bool // true if we have deleted the bucket
|
||||
acl string // ACL for new buckets / objects
|
||||
locationConstraint string // location constraint of new buckets
|
||||
sse string // the type of server-side encryption
|
||||
storageClass string // storage class
|
||||
name string // the name of the remote
|
||||
root string // root of the bucket - ignore all objects above this
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
c *s3.S3 // the connection to the s3 server
|
||||
ses *session.Session // the s3 session
|
||||
bucket string // the bucket we are working on
|
||||
bucketOKMu sync.Mutex // mutex to protect bucket OK
|
||||
bucketOK bool // true if we have created the bucket
|
||||
bucketDeleted bool // true if we have deleted the bucket
|
||||
}
|
||||
|
||||
// Object describes a s3 object
|
||||
@@ -620,12 +644,12 @@ func s3ParsePath(path string) (bucket, directory string, err error) {
|
||||
}
|
||||
|
||||
// s3Connection makes a connection to s3
|
||||
func s3Connection(name string) (*s3.S3, *session.Session, error) {
|
||||
func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
|
||||
// Make the auth
|
||||
v := credentials.Value{
|
||||
AccessKeyID: config.FileGet(name, "access_key_id"),
|
||||
SecretAccessKey: config.FileGet(name, "secret_access_key"),
|
||||
SessionToken: config.FileGet(name, "session_token"),
|
||||
AccessKeyID: opt.AccessKeyID,
|
||||
SecretAccessKey: opt.SecretAccessKey,
|
||||
SessionToken: opt.SessionToken,
|
||||
}
|
||||
|
||||
lowTimeoutClient := &http.Client{Timeout: 1 * time.Second} // low timeout to ec2 metadata service
|
||||
@@ -660,7 +684,7 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) {
|
||||
cred := credentials.NewChainCredentials(providers)
|
||||
|
||||
switch {
|
||||
case config.FileGetBool(name, "env_auth", false):
|
||||
case opt.EnvAuth:
|
||||
// No need for empty checks if "env_auth" is true
|
||||
case v.AccessKeyID == "" && v.SecretAccessKey == "":
|
||||
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
|
||||
@@ -671,26 +695,24 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) {
|
||||
return nil, nil, errors.New("secret_access_key not found")
|
||||
}
|
||||
|
||||
endpoint := config.FileGet(name, "endpoint")
|
||||
region := config.FileGet(name, "region")
|
||||
if region == "" && endpoint == "" {
|
||||
endpoint = "https://s3.amazonaws.com/"
|
||||
if opt.Region == "" && opt.Endpoint == "" {
|
||||
opt.Endpoint = "https://s3.amazonaws.com/"
|
||||
}
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
if opt.Region == "" {
|
||||
opt.Region = "us-east-1"
|
||||
}
|
||||
awsConfig := aws.NewConfig().
|
||||
WithRegion(region).
|
||||
WithRegion(opt.Region).
|
||||
WithMaxRetries(maxRetries).
|
||||
WithCredentials(cred).
|
||||
WithEndpoint(endpoint).
|
||||
WithEndpoint(opt.Endpoint).
|
||||
WithHTTPClient(fshttp.NewClient(fs.Config)).
|
||||
WithS3ForcePathStyle(true)
|
||||
// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
|
||||
ses := session.New()
|
||||
c := s3.New(ses, awsConfig)
|
||||
if region == "other-v2-signature" {
|
||||
fs.Debugf(name, "Using v2 auth")
|
||||
if opt.Region == "other-v2-signature" {
|
||||
fs.Debugf(nil, "Using v2 auth")
|
||||
signer := func(req *request.Request) {
|
||||
// Ignore AnonymousCredentials object
|
||||
if req.Config.Credentials == credentials.AnonymousCredentials {
|
||||
@@ -706,40 +728,37 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, bucket:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.ChunkSize < fs.SizeSuffix(s3manager.MinUploadPartSize) {
|
||||
return nil, errors.Errorf("s3 chunk size (%v) must be >= %v", opt.ChunkSize, fs.SizeSuffix(s3manager.MinUploadPartSize))
|
||||
}
|
||||
bucket, directory, err := s3ParsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, ses, err := s3Connection(name)
|
||||
c, ses, err := s3Connection(opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &Fs{
|
||||
name: name,
|
||||
c: c,
|
||||
bucket: bucket,
|
||||
ses: ses,
|
||||
acl: config.FileGet(name, "acl"),
|
||||
root: directory,
|
||||
locationConstraint: config.FileGet(name, "location_constraint"),
|
||||
sse: config.FileGet(name, "server_side_encryption"),
|
||||
storageClass: config.FileGet(name, "storage_class"),
|
||||
name: name,
|
||||
root: directory,
|
||||
opt: *opt,
|
||||
c: c,
|
||||
bucket: bucket,
|
||||
ses: ses,
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
ReadMimeType: true,
|
||||
WriteMimeType: true,
|
||||
BucketBased: true,
|
||||
}).Fill(f)
|
||||
if *s3ACL != "" {
|
||||
f.acl = *s3ACL
|
||||
}
|
||||
if *s3StorageClass != "" {
|
||||
f.storageClass = *s3StorageClass
|
||||
}
|
||||
if s3ChunkSize < fs.SizeSuffix(s3manager.MinUploadPartSize) {
|
||||
return nil, errors.Errorf("s3 chunk size must be >= %v", fs.SizeSuffix(s3manager.MinUploadPartSize))
|
||||
}
|
||||
if f.root != "" {
|
||||
f.root += "/"
|
||||
// Check to see if the object exists
|
||||
@@ -1064,11 +1083,11 @@ func (f *Fs) Mkdir(dir string) error {
|
||||
}
|
||||
req := s3.CreateBucketInput{
|
||||
Bucket: &f.bucket,
|
||||
ACL: &f.acl,
|
||||
ACL: &f.opt.ACL,
|
||||
}
|
||||
if f.locationConstraint != "" {
|
||||
if f.opt.LocationConstraint != "" {
|
||||
req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
|
||||
LocationConstraint: &f.locationConstraint,
|
||||
LocationConstraint: &f.opt.LocationConstraint,
|
||||
}
|
||||
}
|
||||
_, err := f.c.CreateBucket(&req)
|
||||
@@ -1297,7 +1316,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
|
||||
directive := s3.MetadataDirectiveReplace // replace metadata with that passed in
|
||||
req := s3.CopyObjectInput{
|
||||
Bucket: &o.fs.bucket,
|
||||
ACL: &o.fs.acl,
|
||||
ACL: &o.fs.opt.ACL,
|
||||
Key: &key,
|
||||
ContentType: &mimeType,
|
||||
CopySource: aws.String(pathEscape(sourceKey)),
|
||||
@@ -1353,10 +1372,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
size := src.Size()
|
||||
|
||||
uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
|
||||
u.Concurrency = *s3UploadConcurrency
|
||||
u.Concurrency = o.fs.opt.UploadConcurrency
|
||||
u.LeavePartsOnError = false
|
||||
u.S3 = o.fs.c
|
||||
u.PartSize = int64(s3ChunkSize)
|
||||
u.PartSize = int64(o.fs.opt.ChunkSize)
|
||||
|
||||
if size == -1 {
|
||||
// Make parts as small as possible while still being able to upload to the
|
||||
@@ -1376,7 +1395,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
metaMtime: aws.String(swift.TimeToFloatString(modTime)),
|
||||
}
|
||||
|
||||
if !*s3DisableChecksum && size > uploader.PartSize {
|
||||
if !o.fs.opt.DisableChecksum && size > uploader.PartSize {
|
||||
hash, err := src.Hash(hash.MD5)
|
||||
|
||||
if err == nil && matchMd5.MatchString(hash) {
|
||||
@@ -1394,18 +1413,18 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
key := o.fs.root + o.remote
|
||||
req := s3manager.UploadInput{
|
||||
Bucket: &o.fs.bucket,
|
||||
ACL: &o.fs.acl,
|
||||
ACL: &o.fs.opt.ACL,
|
||||
Key: &key,
|
||||
Body: in,
|
||||
ContentType: &mimeType,
|
||||
Metadata: metadata,
|
||||
//ContentLength: &size,
|
||||
}
|
||||
if o.fs.sse != "" {
|
||||
req.ServerSideEncryption = &o.fs.sse
|
||||
if o.fs.opt.ServerSideEncryption != "" {
|
||||
req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
|
||||
}
|
||||
if o.fs.storageClass != "" {
|
||||
req.StorageClass = &o.fs.storageClass
|
||||
if o.fs.opt.StorageClass != "" {
|
||||
req.StorageClass = &o.fs.opt.StorageClass
|
||||
}
|
||||
_, err = uploader.Upload(&req)
|
||||
if err != nil {
|
||||
|
||||
+86
-72
@@ -20,7 +20,8 @@ import (
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -38,10 +39,6 @@ const (
|
||||
|
||||
var (
|
||||
currentUser = readCurrentUser()
|
||||
|
||||
// Flags
|
||||
sftpAskPassword = flags.BoolP("sftp-ask-password", "", false, "Allow asking for SFTP password when needed.")
|
||||
sshPathOverride = flags.StringP("ssh-path-override", "", "", "Override path used by SSH connection.")
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -52,32 +49,28 @@ func init() {
|
||||
Options: []fs.Option{{
|
||||
Name: "host",
|
||||
Help: "SSH host to connect to",
|
||||
Optional: false,
|
||||
Required: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "example.com",
|
||||
Help: "Connect to example.com",
|
||||
}},
|
||||
}, {
|
||||
Name: "user",
|
||||
Help: "SSH username, leave blank for current username, " + currentUser,
|
||||
Optional: true,
|
||||
Name: "user",
|
||||
Help: "SSH username, leave blank for current username, " + currentUser,
|
||||
}, {
|
||||
Name: "port",
|
||||
Help: "SSH port, leave blank to use default (22)",
|
||||
Optional: true,
|
||||
Name: "port",
|
||||
Help: "SSH port, leave blank to use default (22)",
|
||||
}, {
|
||||
Name: "pass",
|
||||
Help: "SSH password, leave blank to use ssh-agent.",
|
||||
Optional: true,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "key_file",
|
||||
Help: "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
|
||||
Optional: true,
|
||||
Name: "key_file",
|
||||
Help: "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
|
||||
}, {
|
||||
Name: "use_insecure_cipher",
|
||||
Help: "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
|
||||
Optional: true,
|
||||
Name: "use_insecure_cipher",
|
||||
Help: "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
|
||||
Default: false,
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "false",
|
||||
@@ -88,30 +81,56 @@ func init() {
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "disable_hashcheck",
|
||||
Help: "Disable the execution of SSH commands to determine if remote file hashing is available. Leave blank or set to false to enable hashing (recommended), set to true to disable hashing.",
|
||||
Optional: true,
|
||||
Name: "disable_hashcheck",
|
||||
Default: false,
|
||||
Help: "Disable the execution of SSH commands to determine if remote file hashing is available.\nLeave blank or set to false to enable hashing (recommended), set to true to disable hashing.",
|
||||
}, {
|
||||
Name: "ask_password",
|
||||
Default: false,
|
||||
Help: "Allow asking for SFTP password when needed.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "path_override",
|
||||
Default: "",
|
||||
Help: "Override path used by SSH connection.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "set_modtime",
|
||||
Default: true,
|
||||
Help: "Set the modified time on the remote if set.",
|
||||
Advanced: true,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Host string `config:"host"`
|
||||
User string `config:"user"`
|
||||
Port string `config:"port"`
|
||||
Pass string `config:"pass"`
|
||||
KeyFile string `config:"key_file"`
|
||||
UseInsecureCipher bool `config:"use_insecure_cipher"`
|
||||
DisableHashCheck bool `config:"disable_hashcheck"`
|
||||
AskPassword bool `config:"ask_password"`
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote SFTP files
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
features *fs.Features // optional features
|
||||
config *ssh.ClientConfig
|
||||
host string
|
||||
port string
|
||||
url string
|
||||
mkdirLock *stringLock
|
||||
cachedHashes *hash.Set
|
||||
hashcheckDisabled bool
|
||||
setModtime bool
|
||||
poolMu sync.Mutex
|
||||
pool []*conn
|
||||
connLimit *rate.Limiter // for limiting number of connections per second
|
||||
name string
|
||||
root string
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
config *ssh.ClientConfig
|
||||
url string
|
||||
mkdirLock *stringLock
|
||||
cachedHashes *hash.Set
|
||||
poolMu sync.Mutex
|
||||
pool []*conn
|
||||
connLimit *rate.Limiter // for limiting number of connections per second
|
||||
}
|
||||
|
||||
// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
|
||||
@@ -197,7 +216,7 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
|
||||
c = &conn{
|
||||
err: make(chan error, 1),
|
||||
}
|
||||
c.sshClient, err = Dial("tcp", f.host+":"+f.port, f.config)
|
||||
c.sshClient, err = Dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't connect SSH")
|
||||
}
|
||||
@@ -270,35 +289,33 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
|
||||
|
||||
// NewFs creates a new Fs object from the name and root. It connects to
|
||||
// the host specified in the config file.
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
user := config.FileGet(name, "user")
|
||||
host := config.FileGet(name, "host")
|
||||
port := config.FileGet(name, "port")
|
||||
pass := config.FileGet(name, "pass")
|
||||
keyFile := config.FileGet(name, "key_file")
|
||||
insecureCipher := config.FileGetBool(name, "use_insecure_cipher")
|
||||
hashcheckDisabled := config.FileGetBool(name, "disable_hashcheck")
|
||||
setModtime := config.FileGetBool(name, "set_modtime", true)
|
||||
if user == "" {
|
||||
user = currentUser
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if port == "" {
|
||||
port = "22"
|
||||
if opt.User == "" {
|
||||
opt.User = currentUser
|
||||
}
|
||||
if opt.Port == "" {
|
||||
opt.Port = "22"
|
||||
}
|
||||
sshConfig := &ssh.ClientConfig{
|
||||
User: user,
|
||||
User: opt.User,
|
||||
Auth: []ssh.AuthMethod{},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: fs.Config.ConnectTimeout,
|
||||
}
|
||||
|
||||
if insecureCipher {
|
||||
if opt.UseInsecureCipher {
|
||||
sshConfig.Config.SetDefaults()
|
||||
sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
|
||||
}
|
||||
|
||||
// Add ssh agent-auth if no password or file specified
|
||||
if pass == "" && keyFile == "" {
|
||||
if opt.Pass == "" && opt.KeyFile == "" {
|
||||
sshAgentClient, _, err := sshagent.New()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
|
||||
@@ -311,8 +328,8 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
// Load key file if specified
|
||||
if keyFile != "" {
|
||||
key, err := ioutil.ReadFile(keyFile)
|
||||
if opt.KeyFile != "" {
|
||||
key, err := ioutil.ReadFile(opt.KeyFile)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read private key file")
|
||||
}
|
||||
@@ -324,8 +341,8 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
// Auth from password if specified
|
||||
if pass != "" {
|
||||
clearpass, err := obscure.Reveal(pass)
|
||||
if opt.Pass != "" {
|
||||
clearpass, err := obscure.Reveal(opt.Pass)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -333,23 +350,20 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
// Ask for password if none was defined and we're allowed to
|
||||
if pass == "" && *sftpAskPassword {
|
||||
if opt.Pass == "" && opt.AskPassword {
|
||||
_, _ = fmt.Fprint(os.Stderr, "Enter SFTP password: ")
|
||||
clearpass := config.ReadPassword()
|
||||
sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
config: sshConfig,
|
||||
host: host,
|
||||
port: port,
|
||||
url: "sftp://" + user + "@" + host + ":" + port + "/" + root,
|
||||
hashcheckDisabled: hashcheckDisabled,
|
||||
setModtime: setModtime,
|
||||
mkdirLock: newStringLock(),
|
||||
connLimit: rate.NewLimiter(rate.Limit(connectionsPerSecond), 1),
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
config: sshConfig,
|
||||
url: "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root,
|
||||
mkdirLock: newStringLock(),
|
||||
connLimit: rate.NewLimiter(rate.Limit(connectionsPerSecond), 1),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
@@ -663,7 +677,7 @@ func (f *Fs) Hashes() hash.Set {
|
||||
return *f.cachedHashes
|
||||
}
|
||||
|
||||
if f.hashcheckDisabled {
|
||||
if f.opt.DisableHashCheck {
|
||||
return hash.Set(hash.None)
|
||||
}
|
||||
|
||||
@@ -758,8 +772,8 @@ func (o *Object) Hash(r hash.Type) (string, error) {
|
||||
session.Stdout = &stdout
|
||||
session.Stderr = &stderr
|
||||
escapedPath := shellEscape(o.path())
|
||||
if *sshPathOverride != "" {
|
||||
escapedPath = shellEscape(path.Join(*sshPathOverride, o.remote))
|
||||
if o.fs.opt.PathOverride != "" {
|
||||
escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote))
|
||||
}
|
||||
err = session.Run(hashCmd + " " + escapedPath)
|
||||
if err != nil {
|
||||
@@ -852,7 +866,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "SetModTime")
|
||||
}
|
||||
if o.fs.setModtime {
|
||||
if o.fs.opt.SetModTime {
|
||||
err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
|
||||
o.fs.putSftpConnection(&c, err)
|
||||
if err != nil {
|
||||
|
||||
+66
-37
@@ -14,8 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/flags"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -31,11 +31,6 @@ const (
|
||||
listChunks = 1000 // chunk size to read directory listings
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
chunkSize = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
@@ -43,8 +38,9 @@ func init() {
|
||||
Description: "Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "env_auth",
|
||||
Help: "Get swift credentials from environment variables in standard OpenStack form.",
|
||||
Name: "env_auth",
|
||||
Help: "Get swift credentials from environment variables in standard OpenStack form.",
|
||||
Default: false,
|
||||
Examples: []fs.OptionExample{
|
||||
{
|
||||
Value: "false",
|
||||
@@ -107,11 +103,13 @@ func init() {
|
||||
Name: "auth_token",
|
||||
Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)",
|
||||
}, {
|
||||
Name: "auth_version",
|
||||
Help: "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)",
|
||||
Name: "auth_version",
|
||||
Help: "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)",
|
||||
Default: 0,
|
||||
}, {
|
||||
Name: "endpoint_type",
|
||||
Help: "Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE)",
|
||||
Name: "endpoint_type",
|
||||
Help: "Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE)",
|
||||
Default: "public",
|
||||
Examples: []fs.OptionExample{{
|
||||
Help: "Public (default, choose this if not sure)",
|
||||
Value: "public",
|
||||
@@ -122,10 +120,32 @@ func init() {
|
||||
Help: "Admin",
|
||||
Value: "admin",
|
||||
}},
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "Above this size files will be chunked into a _segments container.",
|
||||
Default: fs.SizeSuffix(5 * 1024 * 1024 * 1024),
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
flags.VarP(&chunkSize, "swift-chunk-size", "", "Above this size files will be chunked into a _segments container.")
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
User string `config:"user"`
|
||||
Key string `config:"key"`
|
||||
Auth string `config:"auth"`
|
||||
UserID string `config:"user_id"`
|
||||
Domain string `config:"domain"`
|
||||
Tenant string `config:"tenant"`
|
||||
TenantID string `config:"tenant_id"`
|
||||
TenantDomain string `config:"tenant_domain"`
|
||||
Region string `config:"region"`
|
||||
StorageURL string `config:"storage_url"`
|
||||
AuthToken string `config:"auth_token"`
|
||||
AuthVersion int `config:"auth_version"`
|
||||
EndpointType string `config:"endpoint_type"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
}
|
||||
|
||||
// Fs represents a remote swift server
|
||||
@@ -133,6 +153,7 @@ type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
features *fs.Features // optional features
|
||||
opt Options // options for this backend
|
||||
c *swift.Connection // the connection to the swift server
|
||||
container string // the container we are working on
|
||||
containerOKMu sync.Mutex // mutex to protect container OK
|
||||
@@ -195,27 +216,27 @@ func parsePath(path string) (container, directory string, err error) {
|
||||
}
|
||||
|
||||
// swiftConnection makes a connection to swift
|
||||
func swiftConnection(name string) (*swift.Connection, error) {
|
||||
func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
|
||||
c := &swift.Connection{
|
||||
// Keep these in the same order as the Config for ease of checking
|
||||
UserName: config.FileGet(name, "user"),
|
||||
ApiKey: config.FileGet(name, "key"),
|
||||
AuthUrl: config.FileGet(name, "auth"),
|
||||
UserId: config.FileGet(name, "user_id"),
|
||||
Domain: config.FileGet(name, "domain"),
|
||||
Tenant: config.FileGet(name, "tenant"),
|
||||
TenantId: config.FileGet(name, "tenant_id"),
|
||||
TenantDomain: config.FileGet(name, "tenant_domain"),
|
||||
Region: config.FileGet(name, "region"),
|
||||
StorageUrl: config.FileGet(name, "storage_url"),
|
||||
AuthToken: config.FileGet(name, "auth_token"),
|
||||
AuthVersion: config.FileGetInt(name, "auth_version", 0),
|
||||
EndpointType: swift.EndpointType(config.FileGet(name, "endpoint_type", "public")),
|
||||
UserName: opt.User,
|
||||
ApiKey: opt.Key,
|
||||
AuthUrl: opt.Auth,
|
||||
UserId: opt.UserID,
|
||||
Domain: opt.Domain,
|
||||
Tenant: opt.Tenant,
|
||||
TenantId: opt.TenantID,
|
||||
TenantDomain: opt.TenantDomain,
|
||||
Region: opt.Region,
|
||||
StorageUrl: opt.StorageURL,
|
||||
AuthToken: opt.AuthToken,
|
||||
AuthVersion: opt.AuthVersion,
|
||||
EndpointType: swift.EndpointType(opt.EndpointType),
|
||||
ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
|
||||
Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport
|
||||
Transport: fshttp.NewTransport(fs.Config),
|
||||
}
|
||||
if config.FileGetBool(name, "env_auth", false) {
|
||||
if opt.EnvAuth {
|
||||
err := c.ApplyEnvironment()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read environment variables")
|
||||
@@ -251,13 +272,14 @@ func swiftConnection(name string) (*swift.Connection, error) {
|
||||
//
|
||||
// if noCheckContainer is set then the Fs won't check the container
|
||||
// exists before creating it.
|
||||
func NewFsWithConnection(name, root string, c *swift.Connection, noCheckContainer bool) (fs.Fs, error) {
|
||||
func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, noCheckContainer bool) (fs.Fs, error) {
|
||||
container, directory, err := parsePath(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
c: c,
|
||||
container: container,
|
||||
segmentsContainer: container + "_segments",
|
||||
@@ -288,12 +310,19 @@ func NewFsWithConnection(name, root string, c *swift.Connection, noCheckContaine
|
||||
}
|
||||
|
||||
// NewFs contstructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
c, err := swiftConnection(name)
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewFsWithConnection(name, root, c, false)
|
||||
|
||||
c, err := swiftConnection(opt, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewFsWithConnection(opt, name, root, c, false)
|
||||
}
|
||||
|
||||
// Return an Object from a path
|
||||
@@ -871,7 +900,7 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
|
||||
fs.Debugf(o, "Uploading segments into %q seems done (%v)", o.fs.segmentsContainer, err)
|
||||
break
|
||||
}
|
||||
n := int64(chunkSize)
|
||||
n := int64(o.fs.opt.ChunkSize)
|
||||
if size != -1 {
|
||||
n = min(left, n)
|
||||
headers["Content-Length"] = strconv.FormatInt(n, 10) // set Content-Length as we know it
|
||||
@@ -921,7 +950,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
|
||||
contentType := fs.MimeType(src)
|
||||
headers := m.ObjectHeaders()
|
||||
uniquePrefix := ""
|
||||
if size > int64(chunkSize) || size == -1 {
|
||||
if size > int64(o.fs.opt.ChunkSize) || size == -1 {
|
||||
uniquePrefix, err = o.updateChunks(in, headers, size, contentType)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+38
-31
@@ -32,6 +32,8 @@ import (
|
||||
"github.com/ncw/rclone/backend/webdav/odrvcookie"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fserrors"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
@@ -56,15 +58,14 @@ func init() {
|
||||
Options: []fs.Option{{
|
||||
Name: "url",
|
||||
Help: "URL of http host to connect to",
|
||||
Optional: false,
|
||||
Required: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "https://example.com",
|
||||
Help: "Connect to example.com",
|
||||
}},
|
||||
}, {
|
||||
Name: "vendor",
|
||||
Help: "Name of the Webdav site/service/software you are using",
|
||||
Optional: false,
|
||||
Name: "vendor",
|
||||
Help: "Name of the Webdav site/service/software you are using",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "nextcloud",
|
||||
Help: "Nextcloud",
|
||||
@@ -79,34 +80,37 @@ func init() {
|
||||
Help: "Other site/service or software",
|
||||
}},
|
||||
}, {
|
||||
Name: "user",
|
||||
Help: "User name",
|
||||
Optional: true,
|
||||
Name: "user",
|
||||
Help: "User name",
|
||||
}, {
|
||||
Name: "pass",
|
||||
Help: "Password.",
|
||||
Optional: true,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "bearer_token",
|
||||
Help: "Bearer token instead of user/pass (eg a Macaroon)",
|
||||
Optional: true,
|
||||
Name: "bearer_token",
|
||||
Help: "Bearer token instead of user/pass (eg a Macaroon)",
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
URL string `config:"url"`
|
||||
Vendor string `config:"vendor"`
|
||||
User string `config:"user"`
|
||||
Pass string `config:"pass"`
|
||||
}
|
||||
|
||||
// Fs represents a remote webdav
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
endpoint *url.URL // URL of the host
|
||||
endpointURL string // endpoint as a string
|
||||
srv *rest.Client // the connection to the one drive server
|
||||
pacer *pacer.Pacer // pacer for API calls
|
||||
user string // username
|
||||
pass string // password
|
||||
vendor string // name of the vendor
|
||||
precision time.Duration // mod time precision
|
||||
canStream bool // set if can stream
|
||||
useOCMtime bool // set if can use X-OC-Mtime
|
||||
@@ -264,10 +268,12 @@ func (o *Object) filePath() string {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
endpoint := config.FileGet(name, "url")
|
||||
if !strings.HasSuffix(endpoint, "/") {
|
||||
endpoint += "/"
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rootIsDir := strings.HasSuffix(root, "/")
|
||||
root = strings.Trim(root, "/")
|
||||
@@ -275,17 +281,23 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
user := config.FileGet(name, "user")
|
||||
pass := config.FileGet(name, "pass")
|
||||
bearerToken := config.FileGet(name, "bearer_token")
|
||||
if pass != "" {
|
||||
if !strings.HasSuffix(opt.URL, "/") {
|
||||
opt.URL += "/"
|
||||
}
|
||||
if opt.Pass != "" {
|
||||
var err error
|
||||
pass, err = obscure.Reveal(pass)
|
||||
opt.Pass, err = obscure.Reveal(opt.Pass)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "couldn't decrypt password")
|
||||
}
|
||||
}
|
||||
vendor := config.FileGet(name, "vendor")
|
||||
if opt.Vendor == "" {
|
||||
opt.Vendor = "other"
|
||||
}
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
// Parse the endpoint
|
||||
u, err := url.Parse(endpoint)
|
||||
u, err := url.Parse(opt.URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -293,24 +305,23 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
endpoint: u,
|
||||
endpointURL: u.String(),
|
||||
srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
|
||||
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
|
||||
user: user,
|
||||
pass: pass,
|
||||
precision: fs.ModTimeNotSupported,
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(f)
|
||||
if user != "" || pass != "" {
|
||||
f.srv.SetUserPass(user, pass)
|
||||
f.srv.SetUserPass(opt.User, opt.Pass)
|
||||
} else if bearerToken != "" {
|
||||
f.srv.SetHeader("Authorization", "BEARER "+bearerToken)
|
||||
}
|
||||
f.srv.SetErrorHandler(errorHandler)
|
||||
err = f.setQuirks(vendor)
|
||||
err = f.setQuirks(opt.Vendor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -339,10 +350,6 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
|
||||
// setQuirks adjusts the Fs for the vendor passed in
|
||||
func (f *Fs) setQuirks(vendor string) error {
|
||||
if vendor == "" {
|
||||
vendor = "other"
|
||||
}
|
||||
f.vendor = vendor
|
||||
switch vendor {
|
||||
case "owncloud":
|
||||
f.canStream = true
|
||||
@@ -355,7 +362,7 @@ func (f *Fs) setQuirks(vendor string) error {
|
||||
// To mount sharepoint, two Cookies are required
|
||||
// They have to be set instead of BasicAuth
|
||||
f.srv.RemoveHeader("Authorization") // We don't need this Header if using cookies
|
||||
spCk := odrvcookie.New(f.user, f.pass, f.endpointURL)
|
||||
spCk := odrvcookie.New(f.opt.User, f.opt.Pass, f.endpointURL)
|
||||
spCookies, err := spCk.Cookies()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+26
-12
@@ -16,6 +16,8 @@ import (
|
||||
yandex "github.com/ncw/rclone/backend/yandex/api"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
@@ -51,29 +53,35 @@ func init() {
|
||||
Name: "yandex",
|
||||
Description: "Yandex Disk",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string) {
|
||||
err := oauthutil.Config("yandex", name, oauthConfig)
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
err := oauthutil.Config("yandex", name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: config.ConfigClientID,
|
||||
Help: "Yandex Client Id - leave blank normally.",
|
||||
Help: "Yandex Client Id\nLeave blank normally.",
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Yandex Client Secret - leave blank normally.",
|
||||
Help: "Yandex Client Secret\nLeave blank normally.",
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Token string `config:"token"`
|
||||
}
|
||||
|
||||
// Fs represents a remote yandex
|
||||
type Fs struct {
|
||||
name string
|
||||
root string //root path
|
||||
root string // root path
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
yd *yandex.Client // client for rest api
|
||||
diskRoot string //root path with "disk:/" container name
|
||||
diskRoot string // root path with "disk:/" container name
|
||||
}
|
||||
|
||||
// Object describes a swift object
|
||||
@@ -109,11 +117,9 @@ func (f *Fs) Features() *fs.Features {
|
||||
}
|
||||
|
||||
// read access token from ConfigFile string
|
||||
func getAccessToken(name string) (*oauth2.Token, error) {
|
||||
// Read the token from the config file
|
||||
tokenConfig := config.FileGet(name, "token")
|
||||
func getAccessToken(opt *Options) (*oauth2.Token, error) {
|
||||
//Get access token from config string
|
||||
decoder := json.NewDecoder(strings.NewReader(tokenConfig))
|
||||
decoder := json.NewDecoder(strings.NewReader(opt.Token))
|
||||
var result *oauth2.Token
|
||||
err := decoder.Decode(&result)
|
||||
if err != nil {
|
||||
@@ -123,9 +129,16 @@ func getAccessToken(name string) (*oauth2.Token, error) {
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string) (fs.Fs, error) {
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
//read access token from config
|
||||
token, err := getAccessToken(name)
|
||||
token, err := getAccessToken(opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -135,6 +148,7 @@ func NewFs(name, root string) (fs.Fs, error) {
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
yd: yandexDisk,
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
|
||||
"github.com/ncw/rclone/backend/cache"
|
||||
"github.com/ncw/rclone/cmd"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -27,17 +25,6 @@ Print cache stats for a remote in JSON format
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
|
||||
_, configName, _, err := fs.ParseRemote(args[0])
|
||||
if err != nil {
|
||||
fs.Errorf("cachestats", "%s", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if !config.FileGetBool(configName, "read_only", false) {
|
||||
config.FileSet(configName, "read_only", "true")
|
||||
defer config.FileDeleteKey(configName, "read_only")
|
||||
}
|
||||
|
||||
fsrc := cmd.NewFsSrc(args)
|
||||
cmd.Run(false, false, command, func() error {
|
||||
var fsCache *cache.Fs
|
||||
|
||||
+51
-2
@@ -16,6 +16,7 @@ import (
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -151,12 +152,12 @@ func ShowVersion() {
|
||||
// It returns a string with the file name if points to a file
|
||||
// otherwise "".
|
||||
func NewFsFile(remote string) (fs.Fs, string) {
|
||||
fsInfo, configName, fsPath, err := fs.ParseRemote(remote)
|
||||
_, _, fsPath, err := fs.ParseRemote(remote)
|
||||
if err != nil {
|
||||
fs.CountError(err)
|
||||
log.Fatalf("Failed to create file system for %q: %v", remote, err)
|
||||
}
|
||||
f, err := fsInfo.NewFs(configName, fsPath)
|
||||
f, err := fs.NewFs(remote)
|
||||
switch err {
|
||||
case fs.ErrorIsFile:
|
||||
return f, path.Base(fsPath)
|
||||
@@ -496,3 +497,51 @@ func resolveExitCode(err error) {
|
||||
os.Exit(exitCodeUsageError)
|
||||
}
|
||||
}
|
||||
|
||||
// AddBackendFlags creates flags for all the backend options
|
||||
func AddBackendFlags() {
|
||||
for _, fsInfo := range fs.Registry {
|
||||
done := map[string]struct{}{}
|
||||
for i := range fsInfo.Options {
|
||||
opt := &fsInfo.Options[i]
|
||||
// Skip if done already (eg with Provider options)
|
||||
if _, doneAlready := done[opt.Name]; doneAlready {
|
||||
continue
|
||||
}
|
||||
done[opt.Name] = struct{}{}
|
||||
// Make a flag from each option
|
||||
name := strings.Replace(opt.Name, "_", "-", -1) // convert snake_case to kebab-case
|
||||
if !opt.NoPrefix {
|
||||
name = fsInfo.Prefix + "-" + name
|
||||
}
|
||||
found := pflag.CommandLine.Lookup(name) != nil
|
||||
if !found {
|
||||
// Take first line of help only
|
||||
help := strings.TrimSpace(opt.Help)
|
||||
if nl := strings.IndexRune(help, '\n'); nl >= 0 {
|
||||
help = help[:nl]
|
||||
}
|
||||
help = strings.TrimSpace(help)
|
||||
flag := pflag.CommandLine.VarPF(opt, name, string(opt.ShortOpt), help)
|
||||
if _, isBool := opt.Default.(bool); isBool {
|
||||
flag.NoOptDefVal = "true"
|
||||
}
|
||||
// Hide on the command line if requested
|
||||
if opt.Hide&fs.OptionHideCommandLine != 0 {
|
||||
flag.Hidden = true
|
||||
}
|
||||
} else {
|
||||
fs.Errorf(nil, "Not adding duplicate flag --%s", name)
|
||||
}
|
||||
//flag.Hidden = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Main runs rclone interpreting flags and commands out of os.Args
|
||||
func Main() {
|
||||
AddBackendFlags()
|
||||
if err := Root.Execute(); err != nil {
|
||||
log.Fatalf("Fatal error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -40,14 +40,14 @@ use it like this
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(2, 11, command, args)
|
||||
cmd.Run(false, false, command, func() error {
|
||||
fsInfo, configName, _, err := fs.ParseRemote(args[0])
|
||||
fsInfo, _, _, config, err := fs.ConfigFs(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fsInfo.Name != "crypt" {
|
||||
return errors.New("The remote needs to be of type \"crypt\"")
|
||||
}
|
||||
cipher, err := crypt.NewCipher(configName)
|
||||
cipher, err := crypt.NewCipher(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -102,14 +102,14 @@ can be processed line by line as each item is written one to a line.
|
||||
fsrc := cmd.NewFsSrc(args)
|
||||
var cipher crypt.Cipher
|
||||
if showEncrypted {
|
||||
fsInfo, configName, _, err := fs.ParseRemote(args[0])
|
||||
fsInfo, _, _, config, err := fs.ConfigFs(args[0])
|
||||
if err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
}
|
||||
if fsInfo.Name != "crypt" {
|
||||
log.Fatalf("The remote needs to be of type \"crypt\"")
|
||||
}
|
||||
cipher, err = crypt.NewCipher(configName)
|
||||
cipher, err = crypt.NewCipher(config)
|
||||
if err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
}
|
||||
|
||||
@@ -79,9 +79,6 @@ func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
|
||||
resp.Flags |= fuse.OpenNonSeekable
|
||||
}
|
||||
|
||||
// Keep the file in the cache - equivalent of kernel_cache
|
||||
resp.Flags |= fuse.OpenKeepCache
|
||||
|
||||
return &FileHandle{handle}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ Rclone
|
||||
|
||||
Rclone is a command line program to sync files and directories to and from:
|
||||
|
||||
* {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}}
|
||||
* {{< provider name="Amazon Drive" home="https://www.amazon.com/clouddrive" config="/amazonclouddrive/" >}} ([See note](/amazonclouddrive/#status))
|
||||
* {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
|
||||
* {{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}}
|
||||
* {{< provider name="Box" home="https://www.box.com/" config="/box/" >}}
|
||||
|
||||
@@ -7,9 +7,24 @@ date: "2017-06-10"
|
||||
<i class="fa fa-amazon"></i> Amazon Drive
|
||||
-----------------------------------------
|
||||
|
||||
Paths are specified as `remote:path`
|
||||
Amazon Drive, formerly known as Amazon Cloud Drive, is a cloud storage
|
||||
service run by Amazon for consumers.
|
||||
|
||||
Paths may be as deep as required, eg `remote:directory/subdirectory`.
|
||||
## Status
|
||||
|
||||
**Important:** rclone supports Amazon Drive only if you have your own
|
||||
set of API keys. Unfortunately the [Amazon Drive developer
|
||||
program](https://developer.amazon.com/amazon-drive) is now closed to
|
||||
new entries so if you don't already have your own set of keys you will
|
||||
not be able to use rclone with Amazon Drive.
|
||||
|
||||
For the history on why rclone no longer has a set of Amazon Drive API
|
||||
keys see [the forum](https://forum.rclone.org/t/rclone-has-been-banned-from-amazon-drive/2314).
|
||||
|
||||
If you happen to know anyone who works at Amazon then please ask them
|
||||
to re-instate rclone into the Amazon Drive developer program - thanks!
|
||||
|
||||
## Setup
|
||||
|
||||
The initial setup for Amazon Drive involves getting a token from
|
||||
Amazon which you need to do in your browser. `rclone config` walks
|
||||
@@ -21,10 +36,8 @@ Amazon credentials out of the source code. The proxy runs in Google's
|
||||
very secure App Engine environment and doesn't store any credentials
|
||||
which pass through it.
|
||||
|
||||
**NB** rclone doesn't not currently have its own Amazon Drive
|
||||
credentials (see [the
|
||||
forum](https://forum.rclone.org/t/rclone-has-been-banned-from-amazon-drive/)
|
||||
for why) so you will either need to have your own `client_id` and
|
||||
Since rclone doesn't currently have its own Amazon Drive credentials
|
||||
so you will either need to have your own `client_id` and
|
||||
`client_secret` with Amazon Drive, or use a a third party ouath proxy
|
||||
in which case you will need to enter `client_id`, `client_secret`,
|
||||
`auth_url` and `token_url`.
|
||||
|
||||
@@ -171,3 +171,4 @@ Contributors
|
||||
* themylogin <themylogin@gmail.com>
|
||||
* Onno Zweers <onno.zweers@surfsara.nl>
|
||||
* Jasper Lievisse Adriaanse <jasper@humppa.nl>
|
||||
* sandeepkru <sandeep.ummadi@gmail.com>
|
||||
@@ -125,23 +125,20 @@ Rclone has 3 ways of authenticating with Azure Blob Storage:
|
||||
|
||||
This is the most straight forward and least flexible way. Just fill in the `account` and `key` lines and leave the rest blank.
|
||||
|
||||
#### Connection string
|
||||
|
||||
This supports all the possible connection string variants. Leave `account`, `key` and `sas_url` blank and put the connection string into the `connection_string` configuration parameter.
|
||||
|
||||
Use this method if using an account level SAS; the Azure Portal shows connection strings you can cut and paste.
|
||||
|
||||
#### SAS URL
|
||||
|
||||
This only for a container level SAS URL - it does not work with an account level SAS URL. For account level SAS use the connection string method.
|
||||
This can be an account level SAS URL or container level SAS URL
|
||||
|
||||
To use it leave `account`, `key` and `connection_string` blank and fill in `sas_url`.
|
||||
To use it leave `account`, `key` blank and fill in `sas_url`.
|
||||
|
||||
Account level SAS URL or container level SAS URL can be obtained from Azure portal or Azure Storage Explorer.
|
||||
To get a container level SAS URL right click on a container in the Azure Blob explorer in the Azure portal.
|
||||
|
||||
You will only be able to use the container specified in the SAS URL with rclone, eg
|
||||
If You use container level SAS URL, rclone operations are permitted only on particular container, eg
|
||||
|
||||
rclone ls azureblob:container
|
||||
rclone ls azureblob:container or rclone ls azureblob:
|
||||
|
||||
Since container name already exists in SAS URL, you can leave it empty as well.
|
||||
|
||||
However these will not work
|
||||
|
||||
|
||||
+23
-1
@@ -2,6 +2,7 @@ package fs
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -14,7 +15,15 @@ var (
|
||||
//
|
||||
// This is a function pointer to decouple the config
|
||||
// implementation from the fs
|
||||
ConfigFileGet = func(section, key string, defaultVal ...string) string { return "" }
|
||||
ConfigFileGet = func(section, key string) (string, bool) { return "", false }
|
||||
|
||||
// Set a value into the config file
|
||||
//
|
||||
// This is a function pointer to decouple the config
|
||||
// implementation from the fs
|
||||
ConfigFileSet = func(section, key, value string) {
|
||||
Errorf(nil, "No config handler to set %q = %q in section %q of the config file", key, value, section)
|
||||
}
|
||||
|
||||
// CountError counts an error. If any errors have been
|
||||
// counted then it will exit with a non zero error code.
|
||||
@@ -103,3 +112,16 @@ func NewConfig() *ConfigInfo {
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// ConfigToEnv converts an config section and name, eg ("myremote",
|
||||
// "ignore-size") into an environment name
|
||||
// "RCLONE_CONFIG_MYREMOTE_IGNORE_SIZE"
|
||||
func ConfigToEnv(section, name string) string {
|
||||
return "RCLONE_CONFIG_" + strings.ToUpper(strings.Replace(section+"_"+name, "-", "_", -1))
|
||||
}
|
||||
|
||||
// OptionToEnv converts an option name, eg "ignore-size" into an
|
||||
// environment name "RCLONE_IGNORE_SIZE"
|
||||
func OptionToEnv(name string) string {
|
||||
return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
|
||||
}
|
||||
+141
-95
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/Unknwon/goconfig"
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/accounting"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/config/obscure"
|
||||
"github.com/ncw/rclone/fs/driveletter"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
@@ -81,8 +82,9 @@ var (
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Set the function pointer up in fs
|
||||
fs.ConfigFileGet = FileGet
|
||||
// Set the function pointers up in fs
|
||||
fs.ConfigFileGet = FileGetFlag
|
||||
fs.ConfigFileSet = FileSet
|
||||
}
|
||||
|
||||
func getConfigData() *goconfig.ConfigFile {
|
||||
@@ -152,11 +154,24 @@ func makeConfigPath() string {
|
||||
return homeconf
|
||||
}
|
||||
|
||||
// Check to see if user supplied a --config variable or environment
|
||||
// variable. We can't use pflag for this because it isn't initialised
|
||||
// yet so we search the command line manually.
|
||||
_, configSupplied := os.LookupEnv("RCLONE_CONFIG")
|
||||
for _, item := range os.Args {
|
||||
if item == "--config" || strings.HasPrefix(item, "--config=") {
|
||||
configSupplied = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Default to ./.rclone.conf (current working directory)
|
||||
fs.Errorf(nil, "Couldn't find home directory or read HOME or XDG_CONFIG_HOME environment variables.")
|
||||
fs.Errorf(nil, "Defaulting to storing config in current directory.")
|
||||
fs.Errorf(nil, "Use --config flag to workaround.")
|
||||
fs.Errorf(nil, "Error was: %v", err)
|
||||
if !configSupplied {
|
||||
fs.Errorf(nil, "Couldn't find home directory or read HOME or XDG_CONFIG_HOME environment variables.")
|
||||
fs.Errorf(nil, "Defaulting to storing config in current directory.")
|
||||
fs.Errorf(nil, "Use --config flag to workaround.")
|
||||
fs.Errorf(nil, "Error was: %v", err)
|
||||
}
|
||||
return hiddenConfigFileName
|
||||
}
|
||||
|
||||
@@ -692,7 +707,8 @@ func RemoteConfig(name string) {
|
||||
fmt.Printf("Remote config\n")
|
||||
f := MustFindByName(name)
|
||||
if f.Config != nil {
|
||||
f.Config(name)
|
||||
m := fs.ConfigMap(f, name)
|
||||
f.Config(name, m)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -732,7 +748,7 @@ func ChooseOption(o *fs.Option, name string) string {
|
||||
fmt.Println(o.Help)
|
||||
if o.IsPassword {
|
||||
actions := []string{"yYes type in my own password", "gGenerate random password"}
|
||||
if o.Optional {
|
||||
if !o.Required {
|
||||
actions = append(actions, "nNo leave this optional password blank")
|
||||
}
|
||||
var password string
|
||||
@@ -766,19 +782,52 @@ func ChooseOption(o *fs.Option, name string) string {
|
||||
}
|
||||
return obscure.MustObscure(password)
|
||||
}
|
||||
if len(o.Examples) > 0 {
|
||||
var values []string
|
||||
var help []string
|
||||
for _, example := range o.Examples {
|
||||
if matchProvider(example.Provider, subProvider) {
|
||||
values = append(values, example.Value)
|
||||
help = append(help, example.Help)
|
||||
}
|
||||
}
|
||||
return Choose(o.Name, values, help, true)
|
||||
what := fmt.Sprintf("%T value", o.Default)
|
||||
switch o.Default.(type) {
|
||||
case bool:
|
||||
what = "boolean value (true or false)"
|
||||
case fs.SizeSuffix:
|
||||
what = "size with suffix k,M,G,T"
|
||||
case fs.Duration:
|
||||
what = "duration s,m,h,d,w,M,y"
|
||||
case int, int8, int16, int32, int64:
|
||||
what = "signed integer"
|
||||
case uint, byte, uint16, uint32, uint64:
|
||||
what = "unsigned integer"
|
||||
}
|
||||
fmt.Printf("%s> ", o.Name)
|
||||
return ReadLine()
|
||||
var in string
|
||||
for {
|
||||
fmt.Printf("Enter a %s. Press Enter for the default (%q).\n", what, fmt.Sprint(o.Default))
|
||||
if len(o.Examples) > 0 {
|
||||
var values []string
|
||||
var help []string
|
||||
for _, example := range o.Examples {
|
||||
if matchProvider(example.Provider, subProvider) {
|
||||
values = append(values, example.Value)
|
||||
help = append(help, example.Help)
|
||||
}
|
||||
}
|
||||
in = Choose(o.Name, values, help, true)
|
||||
} else {
|
||||
fmt.Printf("%s> ", o.Name)
|
||||
in = ReadLine()
|
||||
}
|
||||
if in == "" {
|
||||
if o.Required && fmt.Sprint(o.Default) == "" {
|
||||
fmt.Printf("This value is required and it has no default.\n")
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
newIn, err := configstruct.StringToInterface(o.Default, in)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to parse %q: %v\n", in, err)
|
||||
continue
|
||||
}
|
||||
in = fmt.Sprint(newIn) // canonicalise
|
||||
break
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// UpdateRemote adds the keyValues passed in to the remote of name.
|
||||
@@ -847,8 +896,9 @@ func JSONListProviders() error {
|
||||
// fsOption returns an Option describing the possible remotes
|
||||
func fsOption() *fs.Option {
|
||||
o := &fs.Option{
|
||||
Name: "Storage",
|
||||
Help: "Type of storage to configure.",
|
||||
Name: "Storage",
|
||||
Help: "Type of storage to configure.",
|
||||
Default: "",
|
||||
}
|
||||
for _, item := range fs.Registry {
|
||||
example := fs.OptionExample{
|
||||
@@ -880,17 +930,61 @@ func NewRemoteName() (name string) {
|
||||
}
|
||||
}
|
||||
|
||||
// NewRemote make a new remote from its name
|
||||
func NewRemote(name string) {
|
||||
newType := ChooseOption(fsOption(), name)
|
||||
getConfigData().SetValue(name, "type", newType)
|
||||
ri := fs.MustFind(newType)
|
||||
for _, option := range ri.Options {
|
||||
subProvider := getConfigData().MustValue(name, fs.ConfigProvider, "")
|
||||
if matchProvider(option.Provider, subProvider) {
|
||||
getConfigData().SetValue(name, option.Name, ChooseOption(&option, name))
|
||||
// editOptions edits the options. If new is true then it just allows
|
||||
// entry and doesn't show any old values.
|
||||
func editOptions(ri *fs.RegInfo, name string, new bool) {
|
||||
hasAdvanced := false
|
||||
for _, advanced := range []bool{false, true} {
|
||||
if advanced {
|
||||
if !hasAdvanced {
|
||||
break
|
||||
}
|
||||
fmt.Printf("Edit advanced config? (y/n)\n")
|
||||
if !Confirm() {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, option := range ri.Options {
|
||||
hasAdvanced = hasAdvanced || option.Advanced
|
||||
if option.Advanced != advanced {
|
||||
continue
|
||||
}
|
||||
subProvider := getConfigData().MustValue(name, fs.ConfigProvider, "")
|
||||
if matchProvider(option.Provider, subProvider) {
|
||||
if !new {
|
||||
fmt.Printf("Value %q = %q\n", option.Name, FileGet(name, option.Name))
|
||||
fmt.Printf("Edit? (y/n)>\n")
|
||||
if !Confirm() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
FileSet(name, option.Name, ChooseOption(&option, name))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewRemote make a new remote from its name
|
||||
func NewRemote(name string) {
|
||||
var (
|
||||
newType string
|
||||
ri *fs.RegInfo
|
||||
err error
|
||||
)
|
||||
|
||||
// Set the type first
|
||||
for {
|
||||
newType = ChooseOption(fsOption(), name)
|
||||
ri, err = fs.Find(newType)
|
||||
if err != nil {
|
||||
fmt.Printf("Bad remote %q: %v\n", newType, err)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
getConfigData().SetValue(name, "type", newType)
|
||||
|
||||
editOptions(ri, name, true)
|
||||
RemoteConfig(name)
|
||||
if OkRemote(name) {
|
||||
SaveConfig()
|
||||
@@ -903,25 +997,8 @@ func NewRemote(name string) {
|
||||
func EditRemote(ri *fs.RegInfo, name string) {
|
||||
ShowRemote(name)
|
||||
fmt.Printf("Edit remote\n")
|
||||
subProvider := getConfigData().MustValue(name, fs.ConfigProvider, "")
|
||||
for {
|
||||
for _, option := range ri.Options {
|
||||
key := option.Name
|
||||
value := FileGet(name, key)
|
||||
if !matchProvider(option.Provider, subProvider) {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("Value %q = %q\n", key, value)
|
||||
fmt.Printf("Edit? (y/n)>\n")
|
||||
if Confirm() {
|
||||
newValue := ChooseOption(&option, name)
|
||||
getConfigData().SetValue(name, key, newValue)
|
||||
// Update subProvider if it changed
|
||||
if key == fs.ConfigProvider {
|
||||
subProvider = newValue
|
||||
}
|
||||
}
|
||||
}
|
||||
editOptions(ri, name, false)
|
||||
if OkRemote(name) {
|
||||
break
|
||||
}
|
||||
@@ -1076,8 +1153,8 @@ func Authorize(args []string) {
|
||||
log.Fatalf("Invalid number of arguments: %d", len(args))
|
||||
}
|
||||
newType := args[0]
|
||||
fs := fs.MustFind(newType)
|
||||
if fs.Config == nil {
|
||||
f := fs.MustFind(newType)
|
||||
if f.Config == nil {
|
||||
log.Fatalf("Can't authorize fs %q", newType)
|
||||
}
|
||||
// Name used for temporary fs
|
||||
@@ -1092,14 +1169,15 @@ func Authorize(args []string) {
|
||||
getConfigData().SetValue(name, ConfigClientID, args[1])
|
||||
getConfigData().SetValue(name, ConfigClientSecret, args[2])
|
||||
}
|
||||
fs.Config(name)
|
||||
m := fs.ConfigMap(f, name)
|
||||
f.Config(name, m)
|
||||
}
|
||||
|
||||
// configToEnv converts an config section and name, eg ("myremote",
|
||||
// "ignore-size") into an environment name
|
||||
// "RCLONE_CONFIG_MYREMOTE_IGNORE_SIZE"
|
||||
func configToEnv(section, name string) string {
|
||||
return "RCLONE_CONFIG_" + strings.ToUpper(strings.Replace(section+"_"+name, "-", "_", -1))
|
||||
// FileGetFlag gets the config key under section returning the
|
||||
// the value and true if found and or ("", false) otherwise
|
||||
func FileGetFlag(section, key string) (string, bool) {
|
||||
newValue, err := getConfigData().GetValue(section, key)
|
||||
return newValue, err == nil
|
||||
}
|
||||
|
||||
// FileGet gets the config key under section returning the
|
||||
@@ -1107,7 +1185,7 @@ func configToEnv(section, name string) string {
|
||||
//
|
||||
// It looks up defaults in the environment if they are present
|
||||
func FileGet(section, key string, defaultVal ...string) string {
|
||||
envKey := configToEnv(section, key)
|
||||
envKey := fs.ConfigToEnv(section, key)
|
||||
newValue, found := os.LookupEnv(envKey)
|
||||
if found {
|
||||
defaultVal = []string{newValue}
|
||||
@@ -1115,46 +1193,14 @@ func FileGet(section, key string, defaultVal ...string) string {
|
||||
return getConfigData().MustValue(section, key, defaultVal...)
|
||||
}
|
||||
|
||||
// FileGetBool gets the config key under section returning the
|
||||
// default or false if not set.
|
||||
//
|
||||
// It looks up defaults in the environment if they are present
|
||||
func FileGetBool(section, key string, defaultVal ...bool) bool {
|
||||
envKey := configToEnv(section, key)
|
||||
newValue, found := os.LookupEnv(envKey)
|
||||
if found {
|
||||
newBool, err := strconv.ParseBool(newValue)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Couldn't parse %q into bool - ignoring: %v", envKey, err)
|
||||
} else {
|
||||
defaultVal = []bool{newBool}
|
||||
}
|
||||
}
|
||||
return getConfigData().MustBool(section, key, defaultVal...)
|
||||
}
|
||||
|
||||
// FileGetInt gets the config key under section returning the
|
||||
// default or 0 if not set.
|
||||
//
|
||||
// It looks up defaults in the environment if they are present
|
||||
func FileGetInt(section, key string, defaultVal ...int) int {
|
||||
envKey := configToEnv(section, key)
|
||||
newValue, found := os.LookupEnv(envKey)
|
||||
if found {
|
||||
newInt, err := strconv.Atoi(newValue)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Couldn't parse %q into int - ignoring: %v", envKey, err)
|
||||
} else {
|
||||
defaultVal = []int{newInt}
|
||||
}
|
||||
}
|
||||
return getConfigData().MustInt(section, key, defaultVal...)
|
||||
}
|
||||
|
||||
// FileSet sets the key in section to value. It doesn't save
|
||||
// the config file.
|
||||
func FileSet(section, key, value string) {
|
||||
getConfigData().SetValue(section, key, value)
|
||||
if value != "" {
|
||||
getConfigData().SetValue(section, key, value)
|
||||
} else {
|
||||
FileDeleteKey(section, key)
|
||||
}
|
||||
}
|
||||
|
||||
// FileDeleteKey deletes the config key in the config file.
|
||||
|
||||
@@ -52,8 +52,8 @@ func AddFlags(flagSet *pflag.FlagSet) {
|
||||
flags.BoolVarP(flagSet, &fs.Config.InsecureSkipVerify, "no-check-certificate", "", fs.Config.InsecureSkipVerify, "Do not verify the server SSL certificate. Insecure.")
|
||||
flags.BoolVarP(flagSet, &fs.Config.AskPassword, "ask-password", "", fs.Config.AskPassword, "Allow prompt for password for encrypted configuration.")
|
||||
flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transfering")
|
||||
flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer (default)")
|
||||
flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transfering")
|
||||
flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer")
|
||||
flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transfering (default)")
|
||||
flags.IntVar64P(flagSet, &fs.Config.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes")
|
||||
flags.BoolVarP(flagSet, &fs.Config.TrackRenames, "track-renames", "", fs.Config.TrackRenames, "When synchronizing, track file renames and do a server side move if possible")
|
||||
flags.IntVarP(flagSet, &fs.Config.LowLevelRetries, "low-level-retries", "", fs.Config.LowLevelRetries, "Number of low level retries to do.")
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
// Package configmap provides an abstraction for reading and writing config
|
||||
package configmap
|
||||
|
||||
// Getter provides an interface to get config items
|
||||
type Getter interface {
|
||||
// Get should get an item with the key passed in and return
|
||||
// the value. If the item is found then it should return true,
|
||||
// otherwise false.
|
||||
Get(key string) (value string, ok bool)
|
||||
}
|
||||
|
||||
// Setter provides an interface to set config items
|
||||
type Setter interface {
|
||||
// Set should set an item into persistent config store.
|
||||
Set(key, value string)
|
||||
}
|
||||
|
||||
// Mapper provides an interface to read and write config
|
||||
type Mapper interface {
|
||||
Getter
|
||||
Setter
|
||||
}
|
||||
|
||||
// Map provides a wrapper around multiple Setter and
|
||||
// Getter interfaces.
|
||||
type Map struct {
|
||||
setters []Setter
|
||||
getters []Getter
|
||||
}
|
||||
|
||||
// New returns an empty Map
|
||||
func New() *Map {
|
||||
return &Map{}
|
||||
}
|
||||
|
||||
// AddGetter appends a getter onto the end of the getters
|
||||
func (c *Map) AddGetter(getter Getter) *Map {
|
||||
c.getters = append(c.getters, getter)
|
||||
return c
|
||||
}
|
||||
|
||||
// AddGetters appends multiple getters onto the end of the getters
|
||||
func (c *Map) AddGetters(getters ...Getter) *Map {
|
||||
c.getters = append(c.getters, getters...)
|
||||
return c
|
||||
}
|
||||
|
||||
// AddSetter appends a setter onto the end of the setters
|
||||
func (c *Map) AddSetter(setter Setter) *Map {
|
||||
c.setters = append(c.setters, setter)
|
||||
return c
|
||||
}
|
||||
|
||||
// Get gets an item with the key passed in and return the value from
|
||||
// the first getter. If the item is found then it returns true,
|
||||
// otherwise false.
|
||||
func (c *Map) Get(key string) (value string, ok bool) {
|
||||
for _, do := range c.getters {
|
||||
value, ok = do.Get(key)
|
||||
if ok {
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Set sets an item into all the stored setters.
|
||||
func (c *Map) Set(key, value string) {
|
||||
for _, do := range c.setters {
|
||||
do.Set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
// Simple is a simple Mapper for testing
|
||||
type Simple map[string]string
|
||||
|
||||
// Get the value
|
||||
func (c Simple) Get(key string) (value string, ok bool) {
|
||||
value, ok = c[key]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
// Set the value
|
||||
func (c Simple) Set(key, value string) {
|
||||
c[key] = value
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package configmap
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var (
|
||||
_ Mapper = (Simple)(nil)
|
||||
_ Getter = (Simple)(nil)
|
||||
_ Setter = (Simple)(nil)
|
||||
)
|
||||
|
||||
func TestConfigMapGet(t *testing.T) {
|
||||
m := New()
|
||||
|
||||
value, found := m.Get("config1")
|
||||
assert.Equal(t, "", value)
|
||||
assert.Equal(t, false, found)
|
||||
|
||||
value, found = m.Get("config2")
|
||||
assert.Equal(t, "", value)
|
||||
assert.Equal(t, false, found)
|
||||
|
||||
m1 := Simple{
|
||||
"config1": "one",
|
||||
}
|
||||
|
||||
m.AddGetter(m1)
|
||||
|
||||
value, found = m.Get("config1")
|
||||
assert.Equal(t, "one", value)
|
||||
assert.Equal(t, true, found)
|
||||
|
||||
value, found = m.Get("config2")
|
||||
assert.Equal(t, "", value)
|
||||
assert.Equal(t, false, found)
|
||||
|
||||
m2 := Simple{
|
||||
"config1": "one2",
|
||||
"config2": "two2",
|
||||
}
|
||||
|
||||
m.AddGetter(m2)
|
||||
|
||||
value, found = m.Get("config1")
|
||||
assert.Equal(t, "one", value)
|
||||
assert.Equal(t, true, found)
|
||||
|
||||
value, found = m.Get("config2")
|
||||
assert.Equal(t, "two2", value)
|
||||
assert.Equal(t, true, found)
|
||||
|
||||
}
|
||||
|
||||
func TestConfigMapSet(t *testing.T) {
|
||||
m := New()
|
||||
|
||||
m1 := Simple{
|
||||
"config1": "one",
|
||||
}
|
||||
m2 := Simple{
|
||||
"config1": "one2",
|
||||
"config2": "two2",
|
||||
}
|
||||
|
||||
m.AddSetter(m1).AddSetter(m2)
|
||||
|
||||
m.Set("config2", "potato")
|
||||
|
||||
assert.Equal(t, Simple{
|
||||
"config1": "one",
|
||||
"config2": "potato",
|
||||
}, m1)
|
||||
assert.Equal(t, Simple{
|
||||
"config1": "one2",
|
||||
"config2": "potato",
|
||||
}, m2)
|
||||
|
||||
m.Set("config1", "beetroot")
|
||||
|
||||
assert.Equal(t, Simple{
|
||||
"config1": "beetroot",
|
||||
"config2": "potato",
|
||||
}, m1)
|
||||
assert.Equal(t, Simple{
|
||||
"config1": "beetroot",
|
||||
"config2": "potato",
|
||||
}, m2)
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
// Package configstruct parses unstructured maps into structures
|
||||
package configstruct
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var matchUpper = regexp.MustCompile("([A-Z]+)")
|
||||
|
||||
// camelToSnake converts CamelCase to snake_case
|
||||
func camelToSnake(in string) string {
|
||||
out := matchUpper.ReplaceAllString(in, "_$1")
|
||||
out = strings.ToLower(out)
|
||||
out = strings.Trim(out, "_")
|
||||
return out
|
||||
}
|
||||
|
||||
// StringToInterface turns in into an interface{} the same type as def
|
||||
func StringToInterface(def interface{}, in string) (newValue interface{}, err error) {
|
||||
typ := reflect.TypeOf(def)
|
||||
switch typ.Kind() {
|
||||
case reflect.String:
|
||||
// Pass strings unmodified
|
||||
return in, nil
|
||||
}
|
||||
// Otherwise parse with Sscanln
|
||||
//
|
||||
// This means any types we use here must implement fmt.Scanner
|
||||
o := reflect.New(typ)
|
||||
n, err := fmt.Sscanln(in, o.Interface())
|
||||
if err != nil {
|
||||
return newValue, errors.Wrapf(err, "parsing %q as %T failed", in, def)
|
||||
}
|
||||
if n != 1 {
|
||||
return newValue, errors.New("no items parsed")
|
||||
}
|
||||
return o.Elem().Interface(), nil
|
||||
}
|
||||
|
||||
// Item descripts a single entry in the options structure
|
||||
type Item struct {
|
||||
Name string // snake_case
|
||||
Field string // CamelCase
|
||||
Num int // number of the field in the struct
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
// Items parses the opt struct and returns a slice of Item objects.
|
||||
//
|
||||
// opt must be a pointer to a struct. The struct should have entirely
|
||||
// public fields.
|
||||
//
|
||||
// The config_name is looked up in a struct tag called "config" or if
|
||||
// not found is the field name converted from CamelCase to snake_case.
|
||||
func Items(opt interface{}) (items []Item, err error) {
|
||||
def := reflect.ValueOf(opt)
|
||||
if def.Kind() != reflect.Ptr {
|
||||
return nil, errors.New("argument must be a pointer")
|
||||
}
|
||||
def = def.Elem() // indirect the pointer
|
||||
if def.Kind() != reflect.Struct {
|
||||
return nil, errors.New("argument must be a pointer to a struct")
|
||||
}
|
||||
defType := def.Type()
|
||||
for i := 0; i < def.NumField(); i++ {
|
||||
field := defType.Field(i)
|
||||
fieldName := field.Name
|
||||
configName, ok := field.Tag.Lookup("config")
|
||||
if !ok {
|
||||
configName = camelToSnake(fieldName)
|
||||
}
|
||||
defaultItem := Item{
|
||||
Name: configName,
|
||||
Field: fieldName,
|
||||
Num: i,
|
||||
Value: def.Field(i).Interface(),
|
||||
}
|
||||
items = append(items, defaultItem)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// Set interprets the field names in defaults and looks up config
|
||||
// values in the config passed in. Any values found in config will be
|
||||
// set in the opt structure.
|
||||
//
|
||||
// opt must be a pointer to a struct. The struct should have entirely
|
||||
// public fields. The field names are converted from CamelCase to
|
||||
// snake_case and looked up in the config supplied or a
|
||||
// `config:"field_name"` is looked up.
|
||||
//
|
||||
// If items are found then they are converted from string to native
|
||||
// types and set in opt.
|
||||
//
|
||||
// All the field types in the struct must implement fmt.Scanner.
|
||||
func Set(config configmap.Getter, opt interface{}) (err error) {
|
||||
defaultItems, err := Items(opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defStruct := reflect.ValueOf(opt).Elem()
|
||||
for _, defaultItem := range defaultItems {
|
||||
newValue := defaultItem.Value
|
||||
if configValue, ok := config.Get(defaultItem.Name); ok {
|
||||
var newNewValue interface{}
|
||||
newNewValue, err = StringToInterface(newValue, configValue)
|
||||
if err != nil {
|
||||
// Mask errors if setting an empty string as
|
||||
// it isn't valid for all types. This makes
|
||||
// empty string be the equivalent of unset.
|
||||
if configValue != "" {
|
||||
return errors.Wrapf(err, "couldn't parse config item %q = %q as %T", defaultItem.Name, configValue, defaultItem.Value)
|
||||
}
|
||||
} else {
|
||||
newValue = newNewValue
|
||||
}
|
||||
}
|
||||
defStruct.Field(defaultItem.Num).Set(reflect.ValueOf(newValue))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package configstruct_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type conf struct {
|
||||
A string
|
||||
B string
|
||||
}
|
||||
|
||||
type conf2 struct {
|
||||
PotatoPie string `config:"spud_pie"`
|
||||
BeanStew bool
|
||||
RaisinRoll int
|
||||
SausageOnStick int64
|
||||
ForbiddenFruit uint
|
||||
CookingTime fs.Duration
|
||||
TotalWeight fs.SizeSuffix
|
||||
}
|
||||
|
||||
func TestItemsError(t *testing.T) {
|
||||
_, err := configstruct.Items(nil)
|
||||
assert.EqualError(t, err, "argument must be a pointer")
|
||||
_, err = configstruct.Items(new(int))
|
||||
assert.EqualError(t, err, "argument must be a pointer to a struct")
|
||||
}
|
||||
|
||||
func TestItems(t *testing.T) {
|
||||
in := &conf2{
|
||||
PotatoPie: "yum",
|
||||
BeanStew: true,
|
||||
RaisinRoll: 42,
|
||||
SausageOnStick: 101,
|
||||
ForbiddenFruit: 6,
|
||||
CookingTime: fs.Duration(42 * time.Second),
|
||||
TotalWeight: fs.SizeSuffix(17 << 20),
|
||||
}
|
||||
got, err := configstruct.Items(in)
|
||||
require.NoError(t, err)
|
||||
want := []configstruct.Item{
|
||||
{Name: "spud_pie", Field: "PotatoPie", Num: 0, Value: string("yum")},
|
||||
{Name: "bean_stew", Field: "BeanStew", Num: 1, Value: true},
|
||||
{Name: "raisin_roll", Field: "RaisinRoll", Num: 2, Value: int(42)},
|
||||
{Name: "sausage_on_stick", Field: "SausageOnStick", Num: 3, Value: int64(101)},
|
||||
{Name: "forbidden_fruit", Field: "ForbiddenFruit", Num: 4, Value: uint(6)},
|
||||
{Name: "cooking_time", Field: "CookingTime", Num: 5, Value: fs.Duration(42 * time.Second)},
|
||||
{Name: "total_weight", Field: "TotalWeight", Num: 6, Value: fs.SizeSuffix(17 << 20)},
|
||||
}
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
func TestSetBasics(t *testing.T) {
|
||||
c := &conf{A: "one", B: "two"}
|
||||
err := configstruct.Set(configMap{}, c)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, &conf{A: "one", B: "two"}, c)
|
||||
}
|
||||
|
||||
// a simple configmap.Getter for testing
|
||||
type configMap map[string]string
|
||||
|
||||
// Get the value
|
||||
func (c configMap) Get(key string) (value string, ok bool) {
|
||||
value, ok = c[key]
|
||||
return value, ok
|
||||
}
|
||||
|
||||
func TestSetMore(t *testing.T) {
|
||||
c := &conf{A: "one", B: "two"}
|
||||
m := configMap{
|
||||
"a": "ONE",
|
||||
}
|
||||
err := configstruct.Set(m, c)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, &conf{A: "ONE", B: "two"}, c)
|
||||
}
|
||||
|
||||
func TestSetFull(t *testing.T) {
|
||||
in := &conf2{
|
||||
PotatoPie: "yum",
|
||||
BeanStew: true,
|
||||
RaisinRoll: 42,
|
||||
SausageOnStick: 101,
|
||||
ForbiddenFruit: 6,
|
||||
CookingTime: fs.Duration(42 * time.Second),
|
||||
TotalWeight: fs.SizeSuffix(17 << 20),
|
||||
}
|
||||
m := configMap{
|
||||
"spud_pie": "YUM",
|
||||
"bean_stew": "FALSE",
|
||||
"raisin_roll": "43 ",
|
||||
"sausage_on_stick": " 102 ",
|
||||
"forbidden_fruit": "0x7",
|
||||
"cooking_time": "43s",
|
||||
"total_weight": "18M",
|
||||
}
|
||||
want := &conf2{
|
||||
PotatoPie: "YUM",
|
||||
BeanStew: false,
|
||||
RaisinRoll: 43,
|
||||
SausageOnStick: 102,
|
||||
ForbiddenFruit: 7,
|
||||
CookingTime: fs.Duration(43 * time.Second),
|
||||
TotalWeight: fs.SizeSuffix(18 << 20),
|
||||
}
|
||||
err := configstruct.Set(m, in)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, in)
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package configstruct
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCamelToSnake(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"", ""},
|
||||
{"Type", "type"},
|
||||
{"AuthVersion", "auth_version"},
|
||||
{"AccessKeyID", "access_key_id"},
|
||||
} {
|
||||
got := camelToSnake(test.in)
|
||||
assert.Equal(t, test.want, got, test.in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStringToInterface(t *testing.T) {
|
||||
item := struct{ A int }{2}
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
def interface{}
|
||||
want interface{}
|
||||
err string
|
||||
}{
|
||||
{"", string(""), "", ""},
|
||||
{" string ", string(""), " string ", ""},
|
||||
{"123", int(0), int(123), ""},
|
||||
{"0x123", int(0), int(0x123), ""},
|
||||
{" 0x123 ", int(0), int(0x123), ""},
|
||||
{"-123", int(0), int(-123), ""},
|
||||
{"0", false, false, ""},
|
||||
{"1", false, true, ""},
|
||||
{"FALSE", false, false, ""},
|
||||
{"true", false, true, ""},
|
||||
{"123", uint(0), uint(123), ""},
|
||||
{"123", int64(0), int64(123), ""},
|
||||
{"123x", int64(0), nil, "parsing \"123x\" as int64 failed: expected newline"},
|
||||
{"truth", false, nil, "parsing \"truth\" as bool failed: syntax error scanning boolean"},
|
||||
{"struct", item, nil, "parsing \"struct\" as struct { A int } failed: can't scan type: *struct { A int }"},
|
||||
} {
|
||||
what := fmt.Sprintf("parse %q as %T", test.in, test.def)
|
||||
got, err := StringToInterface(test.def, test.in)
|
||||
if test.err == "" {
|
||||
require.NoError(t, err, what)
|
||||
assert.Equal(t, test.want, got, what)
|
||||
} else {
|
||||
assert.Nil(t, got)
|
||||
assert.EqualError(t, err, test.err, what)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,23 +5,16 @@ package flags
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// optionToEnv converts an option name, eg "ignore-size" into an
|
||||
// environment name "RCLONE_IGNORE_SIZE"
|
||||
func optionToEnv(name string) string {
|
||||
return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
|
||||
}
|
||||
|
||||
// setDefaultFromEnv constructs a name from the flag passed in and
|
||||
// sets the default from the environment if possible.
|
||||
func setDefaultFromEnv(name string) {
|
||||
key := optionToEnv(name)
|
||||
key := fs.OptionToEnv(name)
|
||||
newValue, found := os.LookupEnv(key)
|
||||
if found {
|
||||
flag := pflag.Lookup(name)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
@@ -13,6 +14,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/config/configstruct"
|
||||
"github.com/ncw/rclone/fs/fspath"
|
||||
"github.com/ncw/rclone/fs/hash"
|
||||
"github.com/pkg/errors"
|
||||
@@ -68,24 +71,87 @@ type RegInfo struct {
|
||||
Name string
|
||||
// Description of this fs - defaults to Name
|
||||
Description string
|
||||
// Prefix for command line flags for this fs - defaults to Name if not set
|
||||
Prefix string
|
||||
// Create a new file system. If root refers to an existing
|
||||
// object, then it should return a Fs which which points to
|
||||
// the parent of that object and ErrorIsFile.
|
||||
NewFs func(name string, root string) (Fs, error) `json:"-"`
|
||||
NewFs func(name string, root string, config configmap.Mapper) (Fs, error) `json:"-"`
|
||||
// Function to call to help with config
|
||||
Config func(string) `json:"-"`
|
||||
Config func(name string, config configmap.Mapper) `json:"-"`
|
||||
// Options for the Fs configuration
|
||||
Options []Option
|
||||
Options Options
|
||||
}
|
||||
|
||||
// Options is a slice of configuration Option for a backend
|
||||
type Options []Option
|
||||
|
||||
// Set the default values for the options
|
||||
func (os Options) setValues() {
|
||||
for i := range os {
|
||||
o := &os[i]
|
||||
if o.Default == nil {
|
||||
o.Default = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OptionVisibility controls whether the options are visible in the
|
||||
// configurator or the command line.
|
||||
type OptionVisibility byte
|
||||
|
||||
// Constants Option.Hide
|
||||
const (
|
||||
OptionHideCommandLine OptionVisibility = 1 << iota
|
||||
OptionHideConfigurator
|
||||
OptionHideBoth = OptionHideCommandLine | OptionHideConfigurator
|
||||
)
|
||||
|
||||
// Option is describes an option for the config wizard
|
||||
//
|
||||
// This also describes command line options and environment variables
|
||||
type Option struct {
|
||||
Name string
|
||||
Help string
|
||||
Provider string
|
||||
Optional bool
|
||||
IsPassword bool
|
||||
Examples OptionExamples `json:",omitempty"`
|
||||
Name string // name of the option in snake_case
|
||||
Help string // Help, the first line only is used for the command line help
|
||||
Provider string // Set to filter on provider
|
||||
Default interface{} // default value, nil => ""
|
||||
Value interface{} // value to be set by flags
|
||||
Examples OptionExamples `json:",omitempty"` // config examples
|
||||
ShortOpt string // the short option for this if required
|
||||
Hide OptionVisibility // set this to hide the config from the configurator or the command line
|
||||
Required bool // this option is required
|
||||
IsPassword bool // set if the option is a password
|
||||
NoPrefix bool // set if the option for this should not use the backend prefix
|
||||
Advanced bool // set if this is an advanced config option
|
||||
}
|
||||
|
||||
// Gets the current current value which is the default if not set
|
||||
func (o *Option) value() interface{} {
|
||||
val := o.Value
|
||||
if val == nil {
|
||||
val = o.Default
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// String turns Option into a string
|
||||
func (o *Option) String() string {
|
||||
return fmt.Sprint(o.value())
|
||||
}
|
||||
|
||||
// Set a Option from a string
|
||||
func (o *Option) Set(s string) (err error) {
|
||||
newValue, err := configstruct.StringToInterface(o.value(), s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.Value = newValue
|
||||
return nil
|
||||
}
|
||||
|
||||
// Type of the value
|
||||
func (o *Option) Type() string {
|
||||
return reflect.TypeOf(o.value()).Name()
|
||||
}
|
||||
|
||||
// OptionExamples is a slice of examples
|
||||
@@ -114,6 +180,10 @@ type OptionExample struct {
|
||||
//
|
||||
// Fs modules should use this in an init() function
|
||||
func Register(info *RegInfo) {
|
||||
info.Options.setValues()
|
||||
if info.Prefix == "" {
|
||||
info.Prefix = info.Name
|
||||
}
|
||||
Registry = append(Registry, info)
|
||||
}
|
||||
|
||||
@@ -790,9 +860,11 @@ func MustFind(name string) *RegInfo {
|
||||
func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err error) {
|
||||
configName, fsPath = fspath.Parse(path)
|
||||
var fsName string
|
||||
var ok bool
|
||||
if configName != "" {
|
||||
fsName = ConfigFileGet(configName, "type")
|
||||
if fsName == "" {
|
||||
m := ConfigMap(nil, configName)
|
||||
fsName, ok = m.Get("type")
|
||||
if !ok {
|
||||
return nil, "", "", ErrorNotFoundInConfigFile
|
||||
}
|
||||
} else {
|
||||
@@ -803,6 +875,119 @@ func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err e
|
||||
return fsInfo, configName, fsPath, err
|
||||
}
|
||||
|
||||
// A configmap.Getter to read from the environment RCLONE_CONFIG_backend_option_name
|
||||
type configEnvVars string
|
||||
|
||||
// Get a config item from the environment variables if possible
|
||||
func (configName configEnvVars) Get(key string) (value string, ok bool) {
|
||||
return os.LookupEnv(ConfigToEnv(string(configName), key))
|
||||
}
|
||||
|
||||
// A configmap.Getter to read from the environment RCLONE_option_name
|
||||
type optionEnvVars string
|
||||
|
||||
// Get a config item from the option environment variables if possible
|
||||
func (prefix optionEnvVars) Get(key string) (value string, ok bool) {
|
||||
return os.LookupEnv(OptionToEnv(string(prefix) + "-" + key))
|
||||
}
|
||||
|
||||
// A configmap.Getter to read either the default value or the set
|
||||
// value from the RegInfo.Options
|
||||
type regInfoValues struct {
|
||||
fsInfo *RegInfo
|
||||
useDefault bool
|
||||
}
|
||||
|
||||
// override the values in configMap with the either the flag values or
|
||||
// the default values
|
||||
func (r *regInfoValues) Get(key string) (value string, ok bool) {
|
||||
for i := range r.fsInfo.Options {
|
||||
o := &r.fsInfo.Options[i]
|
||||
if o.Name == key {
|
||||
if r.useDefault || o.Value != nil {
|
||||
return o.String(), true
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// A configmap.Setter to read from the config file
|
||||
type setConfigFile string
|
||||
|
||||
// Set a config item into the config file
|
||||
func (section setConfigFile) Set(key, value string) {
|
||||
Debugf(nil, "Saving config %q = %q in section %q of the config file", key, value, section)
|
||||
ConfigFileSet(string(section), key, value)
|
||||
}
|
||||
|
||||
// A configmap.Getter to read from the config file
|
||||
type getConfigFile string
|
||||
|
||||
// Get a config item from the config file
|
||||
func (section getConfigFile) Get(key string) (value string, ok bool) {
|
||||
value, ok = ConfigFileGet(string(section), key)
|
||||
// Ignore empty lines in the config file
|
||||
if value == "" {
|
||||
ok = false
|
||||
}
|
||||
return value, ok
|
||||
}
|
||||
|
||||
// ConfigMap creates a configmap.Map from the *RegInfo and the
|
||||
// configName passed in.
|
||||
//
|
||||
// If fsInfo is nil then the returned configmap.Map should only be
|
||||
// used for reading non backend specific parameters, such as "type".
|
||||
func ConfigMap(fsInfo *RegInfo, configName string) (config *configmap.Map) {
|
||||
// Create the config
|
||||
config = configmap.New()
|
||||
|
||||
// Read the config, more specific to least specific
|
||||
|
||||
// flag values
|
||||
if fsInfo != nil {
|
||||
config.AddGetter(®InfoValues{fsInfo, false})
|
||||
}
|
||||
|
||||
// remote specific environment vars
|
||||
config.AddGetter(configEnvVars(configName))
|
||||
|
||||
// backend specific environment vars
|
||||
if fsInfo != nil {
|
||||
config.AddGetter(optionEnvVars(fsInfo.Prefix))
|
||||
}
|
||||
|
||||
// config file
|
||||
config.AddGetter(getConfigFile(configName))
|
||||
|
||||
// default values
|
||||
if fsInfo != nil {
|
||||
config.AddGetter(®InfoValues{fsInfo, true})
|
||||
}
|
||||
|
||||
// Set Config
|
||||
config.AddSetter(setConfigFile(configName))
|
||||
return config
|
||||
}
|
||||
|
||||
// ConfigFs makes the config for calling NewFs with.
|
||||
//
|
||||
// It parses the path which is of the form remote:path
|
||||
//
|
||||
// Remotes are looked up in the config file. If the remote isn't
|
||||
// found then NotFoundInConfigFile will be returned.
|
||||
func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config *configmap.Map, err error) {
|
||||
// Parse the remote path
|
||||
fsInfo, configName, fsPath, err = ParseRemote(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
config = ConfigMap(fsInfo, configName)
|
||||
return
|
||||
}
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
//
|
||||
// The path is of the form remote:path
|
||||
@@ -813,11 +998,11 @@ func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err e
|
||||
// On Windows avoid single character remote names as they can be mixed
|
||||
// up with drive letters.
|
||||
func NewFs(path string) (Fs, error) {
|
||||
fsInfo, configName, fsPath, err := ParseRemote(path)
|
||||
fsInfo, configName, fsPath, config, err := ConfigFs(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fsInfo.NewFs(configName, fsPath)
|
||||
return fsInfo.NewFs(configName, fsPath, config)
|
||||
}
|
||||
|
||||
// TemporaryLocalFs creates a local FS in the OS's temporary directory.
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
@@ -52,3 +53,20 @@ func TestFeaturesDisableList(t *testing.T) {
|
||||
assert.False(t, ft.CaseInsensitive)
|
||||
assert.False(t, ft.DuplicateFiles)
|
||||
}
|
||||
|
||||
// Check it satisfies the interface
|
||||
var _ pflag.Value = (*Option)(nil)
|
||||
|
||||
func TestOption(t *testing.T) {
|
||||
d := &Option{
|
||||
Name: "potato",
|
||||
Value: SizeSuffix(17 << 20),
|
||||
}
|
||||
assert.Equal(t, "17M", d.String())
|
||||
assert.Equal(t, "SizeSuffix", d.Type())
|
||||
err := d.Set("18M")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, SizeSuffix(18<<20), d.Value)
|
||||
err = d.Set("sdfsdf")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -188,6 +188,12 @@ func Cause(cause error) (retriable bool, err error) {
|
||||
|
||||
// Unwrap 1 level if possible
|
||||
err = errors.Cause(err)
|
||||
if err == nil {
|
||||
// errors.Cause can return nil which isn't
|
||||
// desirable so pick the previous error in
|
||||
// this case.
|
||||
err = prev
|
||||
}
|
||||
if err == prev {
|
||||
// Unpack any struct or *struct with a field
|
||||
// of name Err which satisfies the error
|
||||
@@ -196,11 +202,11 @@ func Cause(cause error) (retriable bool, err error) {
|
||||
// others in the stdlib
|
||||
errType := reflect.TypeOf(err)
|
||||
errValue := reflect.ValueOf(err)
|
||||
if errType.Kind() == reflect.Ptr {
|
||||
if errValue.IsValid() && errType.Kind() == reflect.Ptr {
|
||||
errType = errType.Elem()
|
||||
errValue = errValue.Elem()
|
||||
}
|
||||
if errType.Kind() == reflect.Struct {
|
||||
if errValue.IsValid() && errType.Kind() == reflect.Struct {
|
||||
if errField := errValue.FieldByName("Err"); errField.IsValid() {
|
||||
errFieldValue := errField.Interface()
|
||||
if newErr, ok := errFieldValue.(error); ok {
|
||||
|
||||
@@ -39,7 +39,15 @@ type myError2 struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *myError2) Error() string { return e.Err.Error() }
|
||||
func (e *myError2) Error() string {
|
||||
if e == nil {
|
||||
return "myError2(nil)"
|
||||
}
|
||||
if e.Err == nil {
|
||||
return "myError2{Err: nil}"
|
||||
}
|
||||
return e.Err.Error()
|
||||
}
|
||||
|
||||
type myError3 struct {
|
||||
Err int
|
||||
@@ -53,11 +61,23 @@ type myError4 struct {
|
||||
|
||||
func (e *myError4) Error() string { return e.e.Error() }
|
||||
|
||||
type errorCause struct {
|
||||
e error
|
||||
}
|
||||
|
||||
func (e *errorCause) Error() string { return fmt.Sprintf("%#v", e) }
|
||||
|
||||
func (e *errorCause) Cause() error { return e.e }
|
||||
|
||||
func TestCause(t *testing.T) {
|
||||
e3 := &myError3{3}
|
||||
e4 := &myError4{io.EOF}
|
||||
|
||||
eNil1 := &myError2{nil}
|
||||
eNil2 := &myError2{Err: (*myError2)(nil)}
|
||||
errPotato := errors.New("potato")
|
||||
nilCause1 := &errorCause{nil}
|
||||
nilCause2 := &errorCause{(*myError2)(nil)}
|
||||
|
||||
for i, test := range []struct {
|
||||
err error
|
||||
wantRetriable bool
|
||||
@@ -70,10 +90,15 @@ func TestCause(t *testing.T) {
|
||||
{errUseOfClosedNetworkConnection, false, errUseOfClosedNetworkConnection},
|
||||
{makeNetErr(syscall.EAGAIN), true, syscall.EAGAIN},
|
||||
{makeNetErr(syscall.Errno(123123123)), false, syscall.Errno(123123123)},
|
||||
{eNil1, false, eNil1},
|
||||
{eNil2, false, eNil2.Err},
|
||||
{myError1{io.EOF}, false, io.EOF},
|
||||
{&myError2{io.EOF}, false, io.EOF},
|
||||
{e3, false, e3},
|
||||
{e4, false, e4},
|
||||
{&errorCause{errPotato}, false, errPotato},
|
||||
{nilCause1, false, nilCause1},
|
||||
{nilCause2, false, nilCause2.e},
|
||||
} {
|
||||
gotRetriable, gotErr := Cause(test.err)
|
||||
what := fmt.Sprintf("test #%d: %v", i, test.err)
|
||||
|
||||
+24
-4
@@ -1,6 +1,8 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -17,6 +19,13 @@ func (d Duration) String() string {
|
||||
if d == DurationOff {
|
||||
return "off"
|
||||
}
|
||||
for i := len(ageSuffixes) - 2; i >= 0; i-- {
|
||||
ageSuffix := &ageSuffixes[i]
|
||||
if math.Abs(float64(d)) >= float64(ageSuffix.Multiplier) {
|
||||
timeUnits := float64(d) / float64(ageSuffix.Multiplier)
|
||||
return strconv.FormatFloat(timeUnits, 'f', -1, 64) + ageSuffix.Suffix
|
||||
}
|
||||
}
|
||||
return time.Duration(d).String()
|
||||
}
|
||||
|
||||
@@ -30,10 +39,6 @@ var ageSuffixes = []struct {
|
||||
Suffix string
|
||||
Multiplier time.Duration
|
||||
}{
|
||||
{Suffix: "ms", Multiplier: time.Millisecond},
|
||||
{Suffix: "s", Multiplier: time.Second},
|
||||
{Suffix: "m", Multiplier: time.Minute},
|
||||
{Suffix: "h", Multiplier: time.Hour},
|
||||
{Suffix: "d", Multiplier: time.Hour * 24},
|
||||
{Suffix: "w", Multiplier: time.Hour * 24 * 7},
|
||||
{Suffix: "M", Multiplier: time.Hour * 24 * 30},
|
||||
@@ -51,6 +56,12 @@ func ParseDuration(age string) (time.Duration, error) {
|
||||
return time.Duration(DurationOff), nil
|
||||
}
|
||||
|
||||
// Attempt to parse as a time.Duration first
|
||||
d, err := time.ParseDuration(age)
|
||||
if err == nil {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
for _, ageSuffix := range ageSuffixes {
|
||||
if strings.HasSuffix(age, ageSuffix.Suffix) {
|
||||
numberString := age[:len(age)-len(ageSuffix.Suffix)]
|
||||
@@ -81,3 +92,12 @@ func (d *Duration) Set(s string) error {
|
||||
func (d Duration) Type() string {
|
||||
return "duration"
|
||||
}
|
||||
|
||||
// Scan implements the fmt.Scanner interface
|
||||
func (d *Duration) Scan(s fmt.ScanState, ch rune) error {
|
||||
token, err := s.Token(true, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return d.Set(string(token))
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -23,6 +24,7 @@ func TestParseDuration(t *testing.T) {
|
||||
{"1ms", time.Millisecond, false},
|
||||
{"1s", time.Second, false},
|
||||
{"1m", time.Minute, false},
|
||||
{"1.5m", (3 * time.Minute) / 2, false},
|
||||
{"1h", time.Hour, false},
|
||||
{"1d", time.Hour * 24, false},
|
||||
{"1w", time.Hour * 24 * 7, false},
|
||||
@@ -33,6 +35,7 @@ func TestParseDuration(t *testing.T) {
|
||||
{"1.s", time.Second, false},
|
||||
{"1x", 0, true},
|
||||
{"off", time.Duration(DurationOff), false},
|
||||
{"1h2m3s", time.Hour + 2*time.Minute + 3*time.Second, false},
|
||||
} {
|
||||
duration, err := ParseDuration(test.in)
|
||||
if test.err {
|
||||
@@ -52,9 +55,39 @@ func TestDurationString(t *testing.T) {
|
||||
{time.Duration(0), "0s"},
|
||||
{time.Second, "1s"},
|
||||
{time.Minute, "1m0s"},
|
||||
{time.Millisecond, "1ms"},
|
||||
{time.Second, "1s"},
|
||||
{(3 * time.Minute) / 2, "1m30s"},
|
||||
{time.Hour, "1h0m0s"},
|
||||
{time.Hour * 24, "1d"},
|
||||
{time.Hour * 24 * 7, "1w"},
|
||||
{time.Hour * 24 * 30, "1M"},
|
||||
{time.Hour * 24 * 365, "1y"},
|
||||
{time.Hour * 24 * 365 * 3 / 2, "1.5y"},
|
||||
{-time.Second, "-1s"},
|
||||
{time.Second, "1s"},
|
||||
{time.Duration(DurationOff), "off"},
|
||||
{time.Hour + 2*time.Minute + 3*time.Second, "1h2m3s"},
|
||||
{time.Hour * 24, "1d"},
|
||||
{time.Hour * 24 * 7, "1w"},
|
||||
{time.Hour * 24 * 30, "1M"},
|
||||
{time.Hour * 24 * 365, "1y"},
|
||||
{time.Hour * 24 * 365 * 3 / 2, "1.5y"},
|
||||
{-time.Hour * 24 * 365 * 3 / 2, "-1.5y"},
|
||||
} {
|
||||
got := Duration(test.in).String()
|
||||
assert.Equal(t, test.want, got)
|
||||
// Test the reverse
|
||||
reverse, err := ParseDuration(test.want)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.in, reverse)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurationScan(t *testing.T) {
|
||||
var v Duration
|
||||
n, err := fmt.Sscan(" 17m ", &v)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
assert.Equal(t, Duration(17*60*time.Second), v)
|
||||
}
|
||||
@@ -110,3 +110,12 @@ func (x *SizeSuffix) Set(s string) error {
|
||||
func (x *SizeSuffix) Type() string {
|
||||
return "int64"
|
||||
}
|
||||
|
||||
// Scan implements the fmt.Scanner interface
|
||||
func (x *SizeSuffix) Scan(s fmt.ScanState, ch rune) error {
|
||||
token, err := s.Token(true, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return x.Set(string(token))
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
@@ -93,3 +94,11 @@ func TestSizeSuffixSet(t *testing.T) {
|
||||
assert.Equal(t, test.want, int64(ss))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeSuffixScan(t *testing.T) {
|
||||
var v SizeSuffix
|
||||
n, err := fmt.Sscan(" 17M ", &v)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, n)
|
||||
assert.Equal(t, SizeSuffix(17<<20), v)
|
||||
}
|
||||
+14
-8
@@ -522,13 +522,18 @@ func copyEmptyDirectories(f fs.Fs, entries map[string]fs.DirEntry) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func parentDirCheck(entries map[string]fs.DirEntry, entry fs.DirEntry) {
|
||||
func (s *syncCopyMove) srcParentDirCheck(entry fs.DirEntry) {
|
||||
// If we are moving files then we don't want to remove directories with files in them
|
||||
// from the srcEmptyDirs as we are about to move them making the directory empty.
|
||||
if s.DoMove {
|
||||
return
|
||||
}
|
||||
parentDir := path.Dir(entry.Remote())
|
||||
if parentDir == "." {
|
||||
parentDir = ""
|
||||
}
|
||||
if _, ok := entries[parentDir]; ok {
|
||||
delete(entries, parentDir)
|
||||
if _, ok := s.srcEmptyDirs[parentDir]; ok {
|
||||
delete(s.srcEmptyDirs, parentDir)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -771,10 +776,11 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
|
||||
}
|
||||
switch x := src.(type) {
|
||||
case fs.Object:
|
||||
// Remove parent directory from srcEmptyDirs
|
||||
// If it's a copy operation,
|
||||
// remove parent directory from srcEmptyDirs
|
||||
// since it's not really empty
|
||||
s.srcEmptyDirsMu.Lock()
|
||||
parentDirCheck(s.srcEmptyDirs, src)
|
||||
s.srcParentDirCheck(src)
|
||||
s.srcEmptyDirsMu.Unlock()
|
||||
|
||||
if s.trackRenames {
|
||||
@@ -796,7 +802,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
|
||||
// Do the same thing to the entire contents of the directory
|
||||
// Record the directory for deletion
|
||||
s.srcEmptyDirsMu.Lock()
|
||||
parentDirCheck(s.srcEmptyDirs, src)
|
||||
s.srcParentDirCheck(src)
|
||||
s.srcEmptyDirs[src.Remote()] = src
|
||||
s.srcEmptyDirsMu.Unlock()
|
||||
return true
|
||||
@@ -811,7 +817,7 @@ func (s *syncCopyMove) Match(dst, src fs.DirEntry) (recurse bool) {
|
||||
switch srcX := src.(type) {
|
||||
case fs.Object:
|
||||
s.srcEmptyDirsMu.Lock()
|
||||
parentDirCheck(s.srcEmptyDirs, src)
|
||||
s.srcParentDirCheck(src)
|
||||
s.srcEmptyDirsMu.Unlock()
|
||||
|
||||
if s.deleteMode == fs.DeleteModeOnly {
|
||||
@@ -836,7 +842,7 @@ func (s *syncCopyMove) Match(dst, src fs.DirEntry) (recurse bool) {
|
||||
if ok {
|
||||
// Record the src directory for deletion
|
||||
s.srcEmptyDirsMu.Lock()
|
||||
parentDirCheck(s.srcEmptyDirs, src)
|
||||
s.srcParentDirCheck(src)
|
||||
s.srcEmptyDirs[src.Remote()] = src
|
||||
s.srcEmptyDirsMu.Unlock()
|
||||
return true
|
||||
|
||||
@@ -947,6 +947,52 @@ func testServerSideMove(t *testing.T, r *fstest.Run, withFilter, testDeleteEmpty
|
||||
}
|
||||
}
|
||||
|
||||
// Test move
|
||||
func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||
file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
|
||||
r.Mkdir(r.Fremote)
|
||||
|
||||
// run move with --delete-empty-src-dirs
|
||||
err := MoveDir(r.Fremote, r.Flocal, true)
|
||||
require.NoError(t, err)
|
||||
|
||||
fstest.CheckListingWithPrecision(
|
||||
t,
|
||||
r.Flocal,
|
||||
nil,
|
||||
[]string{},
|
||||
fs.GetModifyWindow(r.Flocal),
|
||||
)
|
||||
fstest.CheckItems(t, r.Fremote, file1, file2)
|
||||
}
|
||||
|
||||
func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
|
||||
file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
|
||||
r.Mkdir(r.Fremote)
|
||||
|
||||
err := MoveDir(r.Fremote, r.Flocal, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
fstest.CheckListingWithPrecision(
|
||||
t,
|
||||
r.Flocal,
|
||||
nil,
|
||||
[]string{
|
||||
"sub dir",
|
||||
"nested",
|
||||
"nested/sub dir",
|
||||
},
|
||||
fs.GetModifyWindow(r.Flocal),
|
||||
)
|
||||
fstest.CheckItems(t, r.Fremote, file1, file2)
|
||||
}
|
||||
|
||||
// Test a server side move if possible, or the backup path if not
|
||||
func TestServerSideMove(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
|
||||
+39
-35
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"github.com/ncw/rclone/fs"
|
||||
"github.com/ncw/rclone/fs/config"
|
||||
"github.com/ncw/rclone/fs/config/configmap"
|
||||
"github.com/ncw/rclone/fs/fshttp"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/skratchdot/open-golang/open"
|
||||
@@ -85,9 +86,9 @@ type oldToken struct {
|
||||
|
||||
// GetToken returns the token saved in the config file under
|
||||
// section name.
|
||||
func GetToken(name string) (*oauth2.Token, error) {
|
||||
tokenString := config.FileGet(name, config.ConfigToken)
|
||||
if tokenString == "" {
|
||||
func GetToken(name string, m configmap.Mapper) (*oauth2.Token, error) {
|
||||
tokenString, ok := m.Get(config.ConfigToken)
|
||||
if !ok || tokenString == "" {
|
||||
return nil, errors.New("empty token found - please run rclone config again")
|
||||
}
|
||||
token := new(oauth2.Token)
|
||||
@@ -110,7 +111,7 @@ func GetToken(name string) (*oauth2.Token, error) {
|
||||
token.RefreshToken = oldtoken.RefreshToken
|
||||
token.Expiry = oldtoken.Expiry
|
||||
// Save new format in config file
|
||||
err = PutToken(name, token, false)
|
||||
err = PutToken(name, m, token, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -120,14 +121,14 @@ func GetToken(name string) (*oauth2.Token, error) {
|
||||
// PutToken stores the token in the config file
|
||||
//
|
||||
// This saves the config file if it changes
|
||||
func PutToken(name string, token *oauth2.Token, newSection bool) error {
|
||||
func PutToken(name string, m configmap.Mapper, token *oauth2.Token, newSection bool) error {
|
||||
tokenBytes, err := json.Marshal(token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tokenString := string(tokenBytes)
|
||||
old := config.FileGet(name, config.ConfigToken)
|
||||
if tokenString != old {
|
||||
old, ok := m.Get(config.ConfigToken)
|
||||
if !ok || tokenString != old {
|
||||
err = config.SetValueAndSave(name, config.ConfigToken, tokenString)
|
||||
if newSection && err != nil {
|
||||
fs.Debugf(name, "Added new token to config, still needs to be saved")
|
||||
@@ -144,6 +145,7 @@ func PutToken(name string, token *oauth2.Token, newSection bool) error {
|
||||
type TokenSource struct {
|
||||
mu sync.Mutex
|
||||
name string
|
||||
m configmap.Mapper
|
||||
tokenSource oauth2.TokenSource
|
||||
token *oauth2.Token
|
||||
config *oauth2.Config
|
||||
@@ -176,7 +178,7 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) {
|
||||
if ts.expiryTimer != nil {
|
||||
ts.expiryTimer.Reset(ts.timeToExpiry())
|
||||
}
|
||||
err = PutToken(ts.name, token, false)
|
||||
err = PutToken(ts.name, ts.m, token, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -229,27 +231,27 @@ func Context(client *http.Client) context.Context {
|
||||
// config file if they are not blank.
|
||||
// If any value is overridden, true is returned.
|
||||
// the origConfig is copied
|
||||
func overrideCredentials(name string, origConfig *oauth2.Config) (newConfig *oauth2.Config, changed bool) {
|
||||
func overrideCredentials(name string, m configmap.Mapper, origConfig *oauth2.Config) (newConfig *oauth2.Config, changed bool) {
|
||||
newConfig = new(oauth2.Config)
|
||||
*newConfig = *origConfig
|
||||
changed = false
|
||||
ClientID := config.FileGet(name, config.ConfigClientID)
|
||||
if ClientID != "" {
|
||||
ClientID, ok := m.Get(config.ConfigClientID)
|
||||
if ok && ClientID != "" {
|
||||
newConfig.ClientID = ClientID
|
||||
changed = true
|
||||
}
|
||||
ClientSecret := config.FileGet(name, config.ConfigClientSecret)
|
||||
if ClientSecret != "" {
|
||||
ClientSecret, ok := m.Get(config.ConfigClientSecret)
|
||||
if ok && ClientSecret != "" {
|
||||
newConfig.ClientSecret = ClientSecret
|
||||
changed = true
|
||||
}
|
||||
AuthURL := config.FileGet(name, config.ConfigAuthURL)
|
||||
if AuthURL != "" {
|
||||
AuthURL, ok := m.Get(config.ConfigAuthURL)
|
||||
if ok && AuthURL != "" {
|
||||
newConfig.Endpoint.AuthURL = AuthURL
|
||||
changed = true
|
||||
}
|
||||
TokenURL := config.FileGet(name, config.ConfigTokenURL)
|
||||
if TokenURL != "" {
|
||||
TokenURL, ok := m.Get(config.ConfigTokenURL)
|
||||
if ok && TokenURL != "" {
|
||||
newConfig.Endpoint.TokenURL = TokenURL
|
||||
changed = true
|
||||
}
|
||||
@@ -260,9 +262,9 @@ func overrideCredentials(name string, origConfig *oauth2.Config) (newConfig *oau
|
||||
// configures a Client with it. It returns the client and a
|
||||
// TokenSource which Invalidate may need to be called on. It uses the
|
||||
// httpClient passed in as the base client.
|
||||
func NewClientWithBaseClient(name string, config *oauth2.Config, baseClient *http.Client) (*http.Client, *TokenSource, error) {
|
||||
config, _ = overrideCredentials(name, config)
|
||||
token, err := GetToken(name)
|
||||
func NewClientWithBaseClient(name string, m configmap.Mapper, config *oauth2.Config, baseClient *http.Client) (*http.Client, *TokenSource, error) {
|
||||
config, _ = overrideCredentials(name, m, config)
|
||||
token, err := GetToken(name, m)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -274,6 +276,7 @@ func NewClientWithBaseClient(name string, config *oauth2.Config, baseClient *htt
|
||||
// tokens in the config file
|
||||
ts := &TokenSource{
|
||||
name: name,
|
||||
m: m,
|
||||
token: token,
|
||||
config: config,
|
||||
ctx: ctx,
|
||||
@@ -284,36 +287,37 @@ func NewClientWithBaseClient(name string, config *oauth2.Config, baseClient *htt
|
||||
|
||||
// NewClient gets a token from the config file and configures a Client
|
||||
// with it. It returns the client and a TokenSource which Invalidate may need to be called on
|
||||
func NewClient(name string, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) {
|
||||
return NewClientWithBaseClient(name, oauthConfig, fshttp.NewClient(fs.Config))
|
||||
func NewClient(name string, m configmap.Mapper, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) {
|
||||
return NewClientWithBaseClient(name, m, oauthConfig, fshttp.NewClient(fs.Config))
|
||||
}
|
||||
|
||||
// Config does the initial creation of the token
|
||||
//
|
||||
// It may run an internal webserver to receive the results
|
||||
func Config(id, name string, config *oauth2.Config, opts ...oauth2.AuthCodeOption) error {
|
||||
return doConfig(id, name, nil, config, true, opts)
|
||||
func Config(id, name string, m configmap.Mapper, config *oauth2.Config, opts ...oauth2.AuthCodeOption) error {
|
||||
return doConfig(id, name, m, nil, config, true, opts)
|
||||
}
|
||||
|
||||
// ConfigNoOffline does the same as Config but does not pass the
|
||||
// "access_type=offline" parameter.
|
||||
func ConfigNoOffline(id, name string, config *oauth2.Config, opts ...oauth2.AuthCodeOption) error {
|
||||
return doConfig(id, name, nil, config, false, opts)
|
||||
func ConfigNoOffline(id, name string, m configmap.Mapper, config *oauth2.Config, opts ...oauth2.AuthCodeOption) error {
|
||||
return doConfig(id, name, m, nil, config, false, opts)
|
||||
}
|
||||
|
||||
// ConfigErrorCheck does the same as Config, but allows the backend to pass a error handling function
|
||||
// This function gets called with the request made to rclone as a parameter if no code was found
|
||||
func ConfigErrorCheck(id, name string, errorHandler func(*http.Request) AuthError, config *oauth2.Config, opts ...oauth2.AuthCodeOption) error {
|
||||
return doConfig(id, name, errorHandler, config, true, opts)
|
||||
func ConfigErrorCheck(id, name string, m configmap.Mapper, errorHandler func(*http.Request) AuthError, config *oauth2.Config, opts ...oauth2.AuthCodeOption) error {
|
||||
return doConfig(id, name, m, errorHandler, config, true, opts)
|
||||
}
|
||||
|
||||
func doConfig(id, name string, errorHandler func(*http.Request) AuthError, oauthConfig *oauth2.Config, offline bool, opts []oauth2.AuthCodeOption) error {
|
||||
oauthConfig, changed := overrideCredentials(name, oauthConfig)
|
||||
automatic := config.FileGet(name, config.ConfigAutomatic) != ""
|
||||
func doConfig(id, name string, m configmap.Mapper, errorHandler func(*http.Request) AuthError, oauthConfig *oauth2.Config, offline bool, opts []oauth2.AuthCodeOption) error {
|
||||
oauthConfig, changed := overrideCredentials(name, m, oauthConfig)
|
||||
auto, ok := m.Get(config.ConfigAutomatic)
|
||||
automatic := ok && auto != ""
|
||||
|
||||
// See if already have a token
|
||||
tokenString := config.FileGet(name, "token")
|
||||
if tokenString != "" {
|
||||
tokenString, ok := m.Get("token")
|
||||
if ok && tokenString != "" {
|
||||
fmt.Printf("Already have a token - refresh?\n")
|
||||
if !config.Confirm() {
|
||||
return nil
|
||||
@@ -354,7 +358,7 @@ func doConfig(id, name string, errorHandler func(*http.Request) AuthError, oauth
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return PutToken(name, token, false)
|
||||
return PutToken(name, m, token, false)
|
||||
}
|
||||
case TitleBarRedirectURL:
|
||||
useWebServer = automatic
|
||||
@@ -436,7 +440,7 @@ func doConfig(id, name string, errorHandler func(*http.Request) AuthError, oauth
|
||||
}
|
||||
fmt.Printf("Paste the following into your remote machine --->\n%s\n<---End paste", result)
|
||||
}
|
||||
return PutToken(name, token, true)
|
||||
return PutToken(name, m, token, true)
|
||||
}
|
||||
|
||||
// Local web server for collecting auth
|
||||
|
||||
@@ -4,8 +4,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/ncw/rclone/cmd"
|
||||
|
||||
_ "github.com/ncw/rclone/backend/all" // import all backends
|
||||
@@ -13,7 +11,5 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := cmd.Root.Execute(); err != nil {
|
||||
log.Fatalf("Fatal error: %v", err)
|
||||
}
|
||||
cmd.Main()
|
||||
}
|
||||
+288
@@ -0,0 +1,288 @@
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
|
||||
# Visual Studio 2015 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUNIT
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
**/Properties/launchSettings.json
|
||||
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_i.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# JustCode is a .NET coding add-in
|
||||
.JustCode
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# TODO: Comment the next line if you want to checkin your web deploy settings
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/packages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/packages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/packages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Typescript v1 declaration files
|
||||
typings/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# JetBrains Rider
|
||||
.idea/
|
||||
*.sln.iml
|
||||
|
||||
# CodeRush
|
||||
.cr/
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
|
||||
# Contributing
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
the rights to use your contribution. For details, visit https://cla.microsoft.com.
|
||||
|
||||
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
|
||||
a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
|
||||
provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
|
||||
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
+255
@@ -0,0 +1,255 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The Factory interface represents an object that can create its Policy object. Each HTTP request sent
|
||||
// requires that this Factory create a new instance of its Policy object.
|
||||
type Factory interface {
|
||||
New(next Policy, po *PolicyOptions) Policy
|
||||
}
|
||||
|
||||
// FactoryFunc is an adapter that allows the use of an ordinary function as a Factory interface.
|
||||
type FactoryFunc func(next Policy, po *PolicyOptions) PolicyFunc
|
||||
|
||||
// New calls f(next,po).
|
||||
func (f FactoryFunc) New(next Policy, po *PolicyOptions) Policy {
|
||||
return f(next, po)
|
||||
}
|
||||
|
||||
// The Policy interface represents a mutable Policy object created by a Factory. The object can mutate/process
|
||||
// the HTTP request and then forward it on to the next Policy object in the linked-list. The returned
|
||||
// Response goes backward through the linked-list for additional processing.
|
||||
// NOTE: Request is passed by value so changes do not change the caller's version of
|
||||
// the request. However, Request has some fields that reference mutable objects (not strings).
|
||||
// These references are copied; a deep copy is not performed. Specifically, this means that
|
||||
// you should avoid modifying the objects referred to by these fields: URL, Header, Body,
|
||||
// GetBody, TransferEncoding, Form, MultipartForm, Trailer, TLS, Cancel, and Response.
|
||||
type Policy interface {
|
||||
Do(ctx context.Context, request Request) (Response, error)
|
||||
}
|
||||
|
||||
// PolicyFunc is an adapter that allows the use of an ordinary function as a Policy interface.
|
||||
type PolicyFunc func(ctx context.Context, request Request) (Response, error)
|
||||
|
||||
// Do calls f(ctx, request).
|
||||
func (f PolicyFunc) Do(ctx context.Context, request Request) (Response, error) {
|
||||
return f(ctx, request)
|
||||
}
|
||||
|
||||
// Options configures a Pipeline's behavior.
|
||||
type Options struct {
|
||||
HTTPSender Factory // If sender is nil, then the pipeline's default client is used to send the HTTP requests.
|
||||
Log LogOptions
|
||||
}
|
||||
|
||||
// LogLevel tells a logger the minimum level to log. When code reports a log entry,
|
||||
// the LogLevel indicates the level of the log entry. The logger only records entries
|
||||
// whose level is at least the level it was told to log. See the Log* constants.
|
||||
// For example, if a logger is configured with LogError, then LogError, LogPanic,
|
||||
// and LogFatal entries will be logged; lower level entries are ignored.
|
||||
type LogLevel uint32
|
||||
|
||||
const (
|
||||
// LogNone tells a logger not to log any entries passed to it.
|
||||
LogNone LogLevel = iota
|
||||
|
||||
// LogFatal tells a logger to log all LogFatal entries passed to it.
|
||||
LogFatal
|
||||
|
||||
// LogPanic tells a logger to log all LogPanic and LogFatal entries passed to it.
|
||||
LogPanic
|
||||
|
||||
// LogError tells a logger to log all LogError, LogPanic and LogFatal entries passed to it.
|
||||
LogError
|
||||
|
||||
// LogWarning tells a logger to log all LogWarning, LogError, LogPanic and LogFatal entries passed to it.
|
||||
LogWarning
|
||||
|
||||
// LogInfo tells a logger to log all LogInfo, LogWarning, LogError, LogPanic and LogFatal entries passed to it.
|
||||
LogInfo
|
||||
|
||||
// LogDebug tells a logger to log all LogDebug, LogInfo, LogWarning, LogError, LogPanic and LogFatal entries passed to it.
|
||||
LogDebug
|
||||
)
|
||||
|
||||
// LogOptions configures the pipeline's logging mechanism & level filtering.
|
||||
type LogOptions struct {
|
||||
Log func(level LogLevel, message string)
|
||||
|
||||
// ShouldLog is called periodically allowing you to return whether the specified LogLevel should be logged or not.
|
||||
// An application can return different values over the its lifetime; this allows the application to dynamically
|
||||
// alter what is logged. NOTE: This method can be called by multiple goroutines simultaneously so make sure
|
||||
// you implement it in a goroutine-safe way. If nil, nothing is logged (the equivalent of returning LogNone).
|
||||
// Usually, the function will be implemented simply like this: return level <= LogWarning
|
||||
ShouldLog func(level LogLevel) bool
|
||||
}
|
||||
|
||||
type pipeline struct {
|
||||
factories []Factory
|
||||
options Options
|
||||
}
|
||||
|
||||
// The Pipeline interface represents an ordered list of Factory objects and an object implementing the HTTPSender interface.
|
||||
// You construct a Pipeline by calling the pipeline.NewPipeline function. To send an HTTP request, call pipeline.NewRequest
|
||||
// and then call Pipeline's Do method passing a context, the request, and a method-specific Factory (or nil). Passing a
|
||||
// method-specific Factory allows this one call to Do to inject a Policy into the linked-list. The policy is injected where
|
||||
// the MethodFactoryMarker (see the pipeline.MethodFactoryMarker function) is in the slice of Factory objects.
|
||||
//
|
||||
// When Do is called, the Pipeline object asks each Factory object to construct its Policy object and adds each Policy to a linked-list.
|
||||
// THen, Do sends the Context and Request through all the Policy objects. The final Policy object sends the request over the network
|
||||
// (via the HTTPSender object passed to NewPipeline) and the response is returned backwards through all the Policy objects.
|
||||
// Since Pipeline and Factory objects are goroutine-safe, you typically create 1 Pipeline object and reuse it to make many HTTP requests.
|
||||
type Pipeline interface {
|
||||
Do(ctx context.Context, methodFactory Factory, request Request) (Response, error)
|
||||
}
|
||||
|
||||
// NewPipeline creates a new goroutine-safe Pipeline object from the slice of Factory objects and the specified options.
|
||||
func NewPipeline(factories []Factory, o Options) Pipeline {
|
||||
if o.HTTPSender == nil {
|
||||
o.HTTPSender = newDefaultHTTPClientFactory()
|
||||
}
|
||||
if o.Log.Log == nil {
|
||||
o.Log.Log = func(LogLevel, string) {} // No-op logger
|
||||
}
|
||||
return &pipeline{factories: factories, options: o}
|
||||
}
|
||||
|
||||
// Do is called for each and every HTTP request. It tells each Factory to create its own (mutable) Policy object
|
||||
// replacing a MethodFactoryMarker factory (if it exists) with the methodFactory passed in. Then, the Context and Request
|
||||
// are sent through the pipeline of Policy objects (which can transform the Request's URL/query parameters/headers) and
|
||||
// ultimately sends the transformed HTTP request over the network.
|
||||
func (p *pipeline) Do(ctx context.Context, methodFactory Factory, request Request) (Response, error) {
|
||||
response, err := p.newPolicies(methodFactory).Do(ctx, request)
|
||||
request.close()
|
||||
return response, err
|
||||
}
|
||||
|
||||
func (p *pipeline) newPolicies(methodFactory Factory) Policy {
|
||||
// The last Policy is the one that actually sends the request over the wire and gets the response.
|
||||
// It is overridable via the Options' HTTPSender field.
|
||||
po := &PolicyOptions{pipeline: p} // One object shared by all policy objects
|
||||
next := p.options.HTTPSender.New(nil, po)
|
||||
|
||||
// Walk over the slice of Factory objects in reverse (from wire to API)
|
||||
markers := 0
|
||||
for i := len(p.factories) - 1; i >= 0; i-- {
|
||||
factory := p.factories[i]
|
||||
if _, ok := factory.(methodFactoryMarker); ok {
|
||||
markers++
|
||||
if markers > 1 {
|
||||
panic("MethodFactoryMarker can only appear once in the pipeline")
|
||||
}
|
||||
if methodFactory != nil {
|
||||
// Replace MethodFactoryMarker with passed-in methodFactory
|
||||
next = methodFactory.New(next, po)
|
||||
}
|
||||
} else {
|
||||
// Use the slice's Factory to construct its Policy
|
||||
next = factory.New(next, po)
|
||||
}
|
||||
}
|
||||
|
||||
// Each Factory has created its Policy
|
||||
if markers == 0 && methodFactory != nil {
|
||||
panic("Non-nil methodFactory requires MethodFactoryMarker in the pipeline")
|
||||
}
|
||||
return next // Return head of the Policy object linked-list
|
||||
}
|
||||
|
||||
// A PolicyOptions represents optional information that can be used by a node in the
|
||||
// linked-list of Policy objects. A PolicyOptions is passed to the Factory's New method
|
||||
// which passes it (if desired) to the Policy object it creates. Today, the Policy object
|
||||
// uses the options to perform logging. But, in the future, this could be used for more.
|
||||
type PolicyOptions struct {
|
||||
pipeline *pipeline
|
||||
}
|
||||
|
||||
// ShouldLog returns true if the specified log level should be logged.
|
||||
func (po *PolicyOptions) ShouldLog(level LogLevel) bool {
|
||||
if po.pipeline.options.Log.ShouldLog != nil {
|
||||
return po.pipeline.options.Log.ShouldLog(level)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Log logs a string to the Pipeline's Logger.
|
||||
func (po *PolicyOptions) Log(level LogLevel, msg string) {
|
||||
if !po.ShouldLog(level) {
|
||||
return // Short circuit message formatting if we're not logging it
|
||||
}
|
||||
|
||||
// We are logging it, ensure trailing newline
|
||||
if len(msg) == 0 || msg[len(msg)-1] != '\n' {
|
||||
msg += "\n" // Ensure trailing newline
|
||||
}
|
||||
po.pipeline.options.Log.Log(level, msg)
|
||||
|
||||
// If logger doesn't handle fatal/panic, we'll do it here.
|
||||
if level == LogFatal {
|
||||
os.Exit(1)
|
||||
} else if level == LogPanic {
|
||||
panic(msg)
|
||||
}
|
||||
}
|
||||
|
||||
var pipelineHTTPClient = newDefaultHTTPClient()
|
||||
|
||||
func newDefaultHTTPClient() *http.Client {
|
||||
// We want the Transport to have a large connection pool
|
||||
return &http.Client{
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
// We use Dial instead of DialContext as DialContext has been reported to cause slower performance.
|
||||
Dial /*Context*/ : (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
DualStack: true,
|
||||
}).Dial, /*Context*/
|
||||
MaxIdleConns: 0, // No limit
|
||||
MaxIdleConnsPerHost: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
DisableKeepAlives: false,
|
||||
DisableCompression: false,
|
||||
MaxResponseHeaderBytes: 0,
|
||||
//ResponseHeaderTimeout: time.Duration{},
|
||||
//ExpectContinueTimeout: time.Duration{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newDefaultHTTPClientFactory creates a DefaultHTTPClientPolicyFactory object that sends HTTP requests to a Go's default http.Client.
|
||||
func newDefaultHTTPClientFactory() Factory {
|
||||
return FactoryFunc(func(next Policy, po *PolicyOptions) PolicyFunc {
|
||||
return func(ctx context.Context, request Request) (Response, error) {
|
||||
r, err := pipelineHTTPClient.Do(request.WithContext(ctx))
|
||||
if err != nil {
|
||||
err = NewError(err, "HTTP request failed")
|
||||
}
|
||||
return NewHTTPResponse(r), err
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var mfm = methodFactoryMarker{} // Singleton
|
||||
|
||||
// MethodFactoryMarker returns a special marker Factory object. When Pipeline's Do method is called, any
|
||||
// MethodMarkerFactory object is replaced with the specified methodFactory object. If nil is passed fro Do's
|
||||
// methodFactory parameter, then the MethodFactoryMarker is ignored as the linked-list of Policy objects is created.
|
||||
func MethodFactoryMarker() Factory {
|
||||
return mfm
|
||||
}
|
||||
|
||||
type methodFactoryMarker struct {
|
||||
}
|
||||
|
||||
func (methodFactoryMarker) New(next Policy, po *PolicyOptions) Policy {
|
||||
panic("methodFactoryMarker policy should have been replaced with a method policy")
|
||||
}
|
||||
Generated
Vendored
Executable
+33
@@ -0,0 +1,33 @@
|
||||
// +build !windows,!nacl,!plan9
|
||||
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"log"
|
||||
"log/syslog"
|
||||
)
|
||||
|
||||
// ForceLog should rarely be used. It forceable logs an entry to the
|
||||
// Windows Event Log (on Windows) or to the SysLog (on Linux)
|
||||
func ForceLog(level LogLevel, msg string) {
|
||||
if defaultLogger == nil {
|
||||
return // Return fast if we failed to create the logger.
|
||||
}
|
||||
// We are logging it, ensure trailing newline
|
||||
if len(msg) == 0 || msg[len(msg)-1] != '\n' {
|
||||
msg += "\n" // Ensure trailing newline
|
||||
}
|
||||
switch level {
|
||||
case LogFatal:
|
||||
defaultLogger.Fatal(msg)
|
||||
case LogPanic:
|
||||
defaultLogger.Panic(msg)
|
||||
case LogError, LogWarning, LogInfo:
|
||||
defaultLogger.Print(msg)
|
||||
}
|
||||
}
|
||||
|
||||
var defaultLogger = func() *log.Logger {
|
||||
l, _ := syslog.NewLogger(syslog.LOG_USER|syslog.LOG_WARNING, log.LstdFlags)
|
||||
return l
|
||||
}()
|
||||
Generated
Vendored
Executable
+61
@@ -0,0 +1,61 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// ForceLog should rarely be used. It forceable logs an entry to the
|
||||
// Windows Event Log (on Windows) or to the SysLog (on Linux)
|
||||
func ForceLog(level LogLevel, msg string) {
|
||||
var el eventType
|
||||
switch level {
|
||||
case LogError, LogFatal, LogPanic:
|
||||
el = elError
|
||||
case LogWarning:
|
||||
el = elWarning
|
||||
case LogInfo:
|
||||
el = elInfo
|
||||
}
|
||||
// We are logging it, ensure trailing newline
|
||||
if len(msg) == 0 || msg[len(msg)-1] != '\n' {
|
||||
msg += "\n" // Ensure trailing newline
|
||||
}
|
||||
reportEvent(el, 0, msg)
|
||||
}
|
||||
|
||||
type eventType int16
|
||||
|
||||
const (
|
||||
elSuccess eventType = 0
|
||||
elError eventType = 1
|
||||
elWarning eventType = 2
|
||||
elInfo eventType = 4
|
||||
)
|
||||
|
||||
var reportEvent = func() func(eventType eventType, eventID int32, msg string) {
|
||||
advAPI32 := syscall.MustLoadDLL("AdvAPI32.dll")
|
||||
registerEventSource := advAPI32.MustFindProc("RegisterEventSourceW")
|
||||
|
||||
sourceName, _ := os.Executable()
|
||||
sourceNameUTF16, _ := syscall.UTF16PtrFromString(sourceName)
|
||||
handle, _, lastErr := registerEventSource.Call(uintptr(0), uintptr(unsafe.Pointer(sourceNameUTF16)))
|
||||
if lastErr == nil { // On error, logging is a no-op
|
||||
return func(eventType eventType, eventID int32, msg string) {}
|
||||
}
|
||||
reportEvent := advAPI32.MustFindProc("ReportEventW")
|
||||
return func(eventType eventType, eventID int32, msg string) {
|
||||
s, _ := syscall.UTF16PtrFromString(msg)
|
||||
_, _, _ = reportEvent.Call(
|
||||
uintptr(handle), // HANDLE hEventLog
|
||||
uintptr(eventType), // WORD wType
|
||||
uintptr(0), // WORD wCategory
|
||||
uintptr(eventID), // DWORD dwEventID
|
||||
uintptr(0), // PSID lpUserSid
|
||||
uintptr(1), // WORD wNumStrings
|
||||
uintptr(0), // DWORD dwDataSize
|
||||
uintptr(unsafe.Pointer(&s)), // LPCTSTR *lpStrings
|
||||
uintptr(0)) // LPVOID lpRawData
|
||||
}
|
||||
}()
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
// Copyright 2017 Microsoft Corporation. All rights reserved.
|
||||
// Use of this source code is governed by an MIT
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
/*
|
||||
Package pipeline implements an HTTP request/response middleware pipeline whose
|
||||
policy objects mutate an HTTP request's URL, query parameters, and/or headers before
|
||||
the request is sent over the wire.
|
||||
|
||||
Not all policy objects mutate an HTTP request; some policy objects simply impact the
|
||||
flow of requests/responses by performing operations such as logging, retry policies,
|
||||
timeouts, failure injection, and deserialization of response payloads.
|
||||
|
||||
Implementing the Policy Interface
|
||||
|
||||
To implement a policy, define a struct that implements the pipeline.Policy interface's Do method. Your Do
|
||||
method is called when an HTTP request wants to be sent over the network. Your Do method can perform any
|
||||
operation(s) it desires. For example, it can log the outgoing request, mutate the URL, headers, and/or query
|
||||
parameters, inject a failure, etc. Your Do method must then forward the HTTP request to next Policy object
|
||||
in a linked-list ensuring that the remaining Policy objects perform their work. Ultimately, the last Policy
|
||||
object sends the HTTP request over the network (by calling the HTTPSender's Do method).
|
||||
|
||||
When an HTTP response comes back, each Policy object in the linked-list gets a chance to process the response
|
||||
(in reverse order). The Policy object can log the response, retry the operation if due to a transient failure
|
||||
or timeout, deserialize the response body, etc. Ultimately, the last Policy object returns the HTTP response
|
||||
to the code that initiated the original HTTP request.
|
||||
|
||||
Here is a template for how to define a pipeline.Policy object:
|
||||
|
||||
type myPolicy struct {
|
||||
node PolicyNode
|
||||
// TODO: Add configuration/setting fields here (if desired)...
|
||||
}
|
||||
|
||||
func (p *myPolicy) Do(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
|
||||
// TODO: Mutate/process the HTTP request here...
|
||||
response, err := p.node.Do(ctx, request) // Forward HTTP request to next Policy & get HTTP response
|
||||
// TODO: Mutate/process the HTTP response here...
|
||||
return response, err // Return response/error to previous Policy
|
||||
}
|
||||
|
||||
Implementing the Factory Interface
|
||||
|
||||
Each Policy struct definition requires a factory struct definition that implements the pipeline.Factory interface's New
|
||||
method. The New method is called when application code wants to initiate a new HTTP request. Factory's New method is
|
||||
passed a pipeline.PolicyNode object which contains a reference to the owning pipeline.Pipeline object (discussed later) and
|
||||
a reference to the next Policy object in the linked list. The New method should create its corresponding Policy object
|
||||
passing it the PolicyNode and any other configuration/settings fields appropriate for the specific Policy object.
|
||||
|
||||
Here is a template for how to define a pipeline.Policy object:
|
||||
|
||||
// NOTE: Once created & initialized, Factory objects should be goroutine-safe (ex: immutable);
|
||||
// this allows reuse (efficient use of memory) and makes these objects usable by multiple goroutines concurrently.
|
||||
type myPolicyFactory struct {
|
||||
// TODO: Add any configuration/setting fields if desired...
|
||||
}
|
||||
|
||||
func (f *myPolicyFactory) New(node pipeline.PolicyNode) Policy {
|
||||
return &myPolicy{node: node} // TODO: Also initialize any configuration/setting fields here (if desired)...
|
||||
}
|
||||
|
||||
Using your Factory and Policy objects via a Pipeline
|
||||
|
||||
To use the Factory and Policy objects, an application constructs a slice of Factory objects and passes
|
||||
this slice to the pipeline.NewPipeline function.
|
||||
|
||||
func NewPipeline(factories []pipeline.Factory, sender pipeline.HTTPSender) Pipeline
|
||||
|
||||
This function also requires an object implementing the HTTPSender interface. For simple scenarios,
|
||||
passing nil for HTTPSender causes a standard Go http.Client object to be created and used to actually
|
||||
send the HTTP response over the network. For more advanced scenarios, you can pass your own HTTPSender
|
||||
object in. This allows sharing of http.Client objects or the use of custom-configured http.Client objects
|
||||
or other objects that can simulate the network requests for testing purposes.
|
||||
|
||||
Now that you have a pipeline.Pipeline object, you can create a pipeline.Request object (which is a simple
|
||||
wrapper around Go's standard http.Request object) and pass it to Pipeline's Do method along with passing a
|
||||
context.Context for cancelling the HTTP request (if desired).
|
||||
|
||||
type Pipeline interface {
|
||||
Do(ctx context.Context, methodFactory pipeline.Factory, request pipeline.Request) (pipeline.Response, error)
|
||||
}
|
||||
|
||||
Do iterates over the slice of Factory objects and tells each one to create its corresponding
|
||||
Policy object. After the linked-list of Policy objects have been created, Do calls the first
|
||||
Policy object passing it the Context & HTTP request parameters. These parameters now flow through
|
||||
all the Policy objects giving each object a chance to look at and/or mutate the HTTP request.
|
||||
The last Policy object sends the message over the network.
|
||||
|
||||
When the network operation completes, the HTTP response and error return values pass
|
||||
back through the same Policy objects in reverse order. Most Policy objects ignore the
|
||||
response/error but some log the result, retry the operation (depending on the exact
|
||||
reason the operation failed), or deserialize the response's body. Your own Policy
|
||||
objects can do whatever they like when processing outgoing requests or incoming responses.
|
||||
|
||||
Note that after an I/O request runs to completion, the Policy objects for that request
|
||||
are garbage collected. However, Pipeline object (like Factory objects) are goroutine-safe allowing
|
||||
them to be created once and reused over many I/O operations. This allows for efficient use of
|
||||
memory and also makes them safely usable by multiple goroutines concurrently.
|
||||
|
||||
Inserting a Method-Specific Factory into the Linked-List of Policy Objects
|
||||
|
||||
While Pipeline and Factory objects can be reused over many different operations, it is
|
||||
common to have special behavior for a specific operation/method. For example, a method
|
||||
may need to deserialize the response's body to an instance of a specific data type.
|
||||
To accommodate this, the Pipeline's Do method takes an additional method-specific
|
||||
Factory object. The Do method tells this Factory to create a Policy object and
|
||||
injects this method-specific Policy object into the linked-list of Policy objects.
|
||||
|
||||
When creating a Pipeline object, the slice of Factory objects passed must have 1
|
||||
(and only 1) entry marking where the method-specific Factory should be injected.
|
||||
The Factory marker is obtained by calling the pipeline.MethodFactoryMarker() function:
|
||||
|
||||
func MethodFactoryMarker() pipeline.Factory
|
||||
|
||||
Creating an HTTP Request Object
|
||||
|
||||
The HTTP request object passed to Pipeline's Do method is not Go's http.Request struct.
|
||||
Instead, it is a pipeline.Request struct which is a simple wrapper around Go's standard
|
||||
http.Request. You create a pipeline.Request object by calling the pipeline.NewRequest function:
|
||||
|
||||
func NewRequest(method string, url url.URL, options pipeline.RequestOptions) (request pipeline.Request, err error)
|
||||
|
||||
To this function, you must pass a pipeline.RequestOptions that looks like this:
|
||||
|
||||
type RequestOptions struct {
|
||||
// The readable and seekable stream to be sent to the server as the request's body.
|
||||
Body io.ReadSeeker
|
||||
|
||||
// The callback method (if not nil) to be invoked to report progress as the stream is uploaded in the HTTP request.
|
||||
Progress ProgressReceiver
|
||||
}
|
||||
|
||||
The method and struct ensure that the request's body stream is a read/seekable stream.
|
||||
A seekable stream is required so that upon retry, the final Policy object can seek
|
||||
the stream back to the beginning before retrying the network request and re-uploading the
|
||||
body. In addition, you can associate a ProgressReceiver callback function which will be
|
||||
invoked periodically to report progress while bytes are being read from the body stream
|
||||
and sent over the network.
|
||||
|
||||
Processing the HTTP Response
|
||||
|
||||
When an HTTP response comes in from the network, a reference to Go's http.Response struct is
|
||||
embedded in a struct that implements the pipeline.Response interface:
|
||||
|
||||
type Response interface {
|
||||
Response() *http.Response
|
||||
}
|
||||
|
||||
This interface is returned through all the Policy objects. Each Policy object can call the Response
|
||||
interface's Response method to examine (or mutate) the embedded http.Response object.
|
||||
|
||||
A Policy object can internally define another struct (implementing the pipeline.Response interface)
|
||||
that embeds an http.Response and adds additional fields and return this structure to other Policy
|
||||
objects. This allows a Policy object to deserialize the body to some other struct and return the
|
||||
original http.Response and the additional struct back through the Policy chain. Other Policy objects
|
||||
can see the Response but cannot see the additional struct with the deserialized body. After all the
|
||||
Policy objects have returned, the pipeline.Response interface is returned by Pipeline's Do method.
|
||||
The caller of this method can perform a type assertion attempting to get back to the struct type
|
||||
really returned by the Policy object. If the type assertion is successful, the caller now has
|
||||
access to both the http.Response and the deserialized struct object.*/
|
||||
package pipeline
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
type causer interface {
|
||||
Cause() error
|
||||
}
|
||||
|
||||
// ErrorNode can be an embedded field in a private error object. This field
|
||||
// adds Program Counter support and a 'cause' (reference to a preceding error).
|
||||
// When initializing a error type with this embedded field, initialize the
|
||||
// ErrorNode field by calling ErrorNode{}.Initialize(cause).
|
||||
type ErrorNode struct {
|
||||
pc uintptr // Represents a Program Counter that you can get symbols for.
|
||||
cause error // Refers to the preceding error (or nil)
|
||||
}
|
||||
|
||||
// Error returns a string with the PC's symbols or "" if the PC is invalid.
|
||||
// When defining a new error type, have its Error method call this one passing
|
||||
// it the string representation of the error.
|
||||
func (e *ErrorNode) Error(msg string) string {
|
||||
s := ""
|
||||
if fn := runtime.FuncForPC(e.pc); fn != nil {
|
||||
file, line := fn.FileLine(e.pc)
|
||||
s = fmt.Sprintf("-> %v, %v:%v\n", fn.Name(), file, line)
|
||||
}
|
||||
s += msg + "\n\n"
|
||||
if e.cause != nil {
|
||||
s += e.cause.Error() + "\n"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Cause returns the error that preceded this error.
|
||||
func (e *ErrorNode) Cause() error { return e.cause }
|
||||
|
||||
// Temporary returns true if the error occurred due to a temporary condition.
|
||||
func (e ErrorNode) Temporary() bool {
|
||||
type temporary interface {
|
||||
Temporary() bool
|
||||
}
|
||||
|
||||
for err := e.cause; err != nil; {
|
||||
if t, ok := err.(temporary); ok {
|
||||
return t.Temporary()
|
||||
}
|
||||
|
||||
if cause, ok := err.(causer); ok {
|
||||
err = cause.Cause()
|
||||
} else {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Timeout returns true if the error occurred due to time expiring.
|
||||
func (e ErrorNode) Timeout() bool {
|
||||
type timeout interface {
|
||||
Timeout() bool
|
||||
}
|
||||
|
||||
for err := e.cause; err != nil; {
|
||||
if t, ok := err.(timeout); ok {
|
||||
return t.Timeout()
|
||||
}
|
||||
|
||||
if cause, ok := err.(causer); ok {
|
||||
err = cause.Cause()
|
||||
} else {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Initialize is used to initialize an embedded ErrorNode field.
|
||||
// It captures the caller's program counter and saves the cause (preceding error).
|
||||
// To initialize the field, use "ErrorNode{}.Initialize(cause, 3)". A callersToSkip
|
||||
// value of 3 is very common; but, depending on your code nesting, you may need
|
||||
// a different value.
|
||||
func (ErrorNode) Initialize(cause error, callersToSkip int) ErrorNode {
|
||||
// Get the PC of Initialize method's caller.
|
||||
pc := [1]uintptr{}
|
||||
_ = runtime.Callers(callersToSkip, pc[:])
|
||||
return ErrorNode{pc: pc[0], cause: cause}
|
||||
}
|
||||
|
||||
// Cause walks all the preceding errors and return the originating error.
|
||||
func Cause(err error) error {
|
||||
for err != nil {
|
||||
cause, ok := err.(causer)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
err = cause.Cause()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// NewError creates a simple string error (like Error.New). But, this
|
||||
// error also captures the caller's Program Counter and the preceding error.
|
||||
func NewError(cause error, msg string) error {
|
||||
return &pcError{
|
||||
ErrorNode: ErrorNode{}.Initialize(cause, 3),
|
||||
msg: msg,
|
||||
}
|
||||
}
|
||||
|
||||
// pcError is a simple string error (like error.New) with an ErrorNode (PC & cause).
|
||||
type pcError struct {
|
||||
ErrorNode
|
||||
msg string
|
||||
}
|
||||
|
||||
// Error satisfies the error interface. It shows the error with Program Counter
|
||||
// symbols and calls Error on the preceding error so you can see the full error chain.
|
||||
func (e *pcError) Error() string { return e.ErrorNode.Error(e.msg) }
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package pipeline_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Azure/azure-pipeline-go/pipeline"
|
||||
)
|
||||
|
||||
// Here is the template for defining your own Factory & Policy:
|
||||
|
||||
// newMyPolicyFactory creates a 'My' policy factory. Make this function
|
||||
// public if this should be callable from another package; everything
|
||||
// else about the factory/policy should remain private to the package.
|
||||
func newMyPolicyFactory( /* Desired parameters */ ) pipeline.Factory {
|
||||
return &myPolicyFactory{ /* Set desired fields */ }
|
||||
}
|
||||
|
||||
type myPolicyFactory struct {
|
||||
// Desired fields (goroutine-safe because the factory is shared by many Policy objects)
|
||||
}
|
||||
|
||||
// New initializes a Xxx policy object.
|
||||
func (f *myPolicyFactory) New(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.Policy {
|
||||
return &myPolicy{next: next, po: po /* Set desired fields */}
|
||||
}
|
||||
|
||||
type myPolicy struct {
|
||||
next pipeline.Policy
|
||||
po *pipeline.PolicyOptions // Optional private field
|
||||
// Additional desired fields (mutable for use by this specific Policy object)
|
||||
}
|
||||
|
||||
func (p *myPolicy) Do(ctx context.Context, request pipeline.Request) (response pipeline.Response, err error) {
|
||||
// TODO: Put your policy behavior code here
|
||||
// Your code should NOT mutate the ctx or request parameters
|
||||
// However, you can make a copy of the request and mutate the copy
|
||||
// You can also pass a different Context on.
|
||||
// You can optionally use po (PolicyOptions) in this func.
|
||||
|
||||
// Forward the request to the next node in the pipeline:
|
||||
response, err = p.next.Do(ctx, request)
|
||||
|
||||
// Process the response here. You can deserialize the body into an object.
|
||||
// If you do this, also define a struct that wraps an http.Response & your
|
||||
// deserialized struct. Have your wrapper struct implement the
|
||||
// pipeline.Response interface and then return your struct (via the interface)
|
||||
// After the pipeline completes, take response and perform a type assertion
|
||||
// to get back to the wrapper struct so you can access the deserialized object.
|
||||
|
||||
return // Return the response & err
|
||||
}
|
||||
|
||||
func newMyPolicyFactory2( /* Desired parameters */ ) pipeline.Factory {
|
||||
return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
|
||||
return func(ctx context.Context, request pipeline.Request) (response pipeline.Response, err error) {
|
||||
// TODO: Put your policy behavior code here
|
||||
// Your code should NOT mutate the ctx or request parameters
|
||||
// However, you can make a copy of the request and mutate the copy
|
||||
// You can also pass a different Context on.
|
||||
// You can optionally use po (PolicyOptions) in this func.
|
||||
|
||||
// Forward the request to the next node in the pipeline:
|
||||
response, err = next.Do(ctx, request)
|
||||
|
||||
// Process the response here. You can deserialize the body into an object.
|
||||
// If you do this, also define a struct that wraps an http.Response & your
|
||||
// deserialized struct. Have your wrapper struct implement the
|
||||
// pipeline.Response interface and then return your struct (via the interface)
|
||||
// After the pipeline completes, take response and perform a type assertion
|
||||
// to get back to the wrapper struct so you can access the deserialized object.
|
||||
|
||||
return // Return the response & err
|
||||
}
|
||||
})
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package pipeline
|
||||
|
||||
import "io"
|
||||
|
||||
// ********** The following is common between the request body AND the response body.
|
||||
|
||||
// ProgressReceiver defines the signature of a callback function invoked as progress is reported.
|
||||
type ProgressReceiver func(bytesTransferred int64)
|
||||
|
||||
// ********** The following are specific to the request body (a ReadSeekCloser)
|
||||
|
||||
// This struct is used when sending a body to the network
|
||||
type requestBodyProgress struct {
|
||||
requestBody io.ReadSeeker // Seeking is required to support retries
|
||||
pr ProgressReceiver
|
||||
}
|
||||
|
||||
// NewRequestBodyProgress adds progress reporting to an HTTP request's body stream.
|
||||
func NewRequestBodyProgress(requestBody io.ReadSeeker, pr ProgressReceiver) io.ReadSeeker {
|
||||
if pr == nil {
|
||||
panic("pr must not be nil")
|
||||
}
|
||||
return &requestBodyProgress{requestBody: requestBody, pr: pr}
|
||||
}
|
||||
|
||||
// Read reads a block of data from an inner stream and reports progress
|
||||
func (rbp *requestBodyProgress) Read(p []byte) (n int, err error) {
|
||||
n, err = rbp.requestBody.Read(p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Invokes the user's callback method to report progress
|
||||
position, err := rbp.requestBody.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
rbp.pr(position)
|
||||
return
|
||||
}
|
||||
|
||||
func (rbp *requestBodyProgress) Seek(offset int64, whence int) (offsetFromStart int64, err error) {
|
||||
return rbp.requestBody.Seek(offset, whence)
|
||||
}
|
||||
|
||||
// requestBodyProgress supports Close but the underlying stream may not; if it does, Close will close it.
|
||||
func (rbp *requestBodyProgress) Close() error {
|
||||
if c, ok := rbp.requestBody.(io.Closer); ok {
|
||||
return c.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ********** The following are specific to the response body (a ReadCloser)
|
||||
|
||||
// This struct is used when sending a body to the network
|
||||
type responseBodyProgress struct {
|
||||
responseBody io.ReadCloser
|
||||
pr ProgressReceiver
|
||||
offset int64
|
||||
}
|
||||
|
||||
// NewResponseBodyProgress adds progress reporting to an HTTP response's body stream.
|
||||
func NewResponseBodyProgress(responseBody io.ReadCloser, pr ProgressReceiver) io.ReadCloser {
|
||||
if pr == nil {
|
||||
panic("pr must not be nil")
|
||||
}
|
||||
return &responseBodyProgress{responseBody: responseBody, pr: pr, offset: 0}
|
||||
}
|
||||
|
||||
// Read reads a block of data from an inner stream and reports progress
|
||||
func (rbp *responseBodyProgress) Read(p []byte) (n int, err error) {
|
||||
n, err = rbp.responseBody.Read(p)
|
||||
rbp.offset += int64(n)
|
||||
|
||||
// Invokes the user's callback method to report progress
|
||||
rbp.pr(rbp.offset)
|
||||
return
|
||||
}
|
||||
|
||||
func (rbp *responseBodyProgress) Close() error {
|
||||
return rbp.responseBody.Close()
|
||||
}
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Request is a thin wrapper over an http.Request. The wrapper provides several helper methods.
|
||||
type Request struct {
|
||||
*http.Request
|
||||
}
|
||||
|
||||
// NewRequest initializes a new HTTP request object with any desired options.
|
||||
func NewRequest(method string, url url.URL, body io.ReadSeeker) (request Request, err error) {
|
||||
// Note: the url is passed by value so that any pipeline operations that modify it do so on a copy.
|
||||
|
||||
// This code to construct an http.Request is copied from http.NewRequest(); we intentionally omitted removeEmptyPort for now.
|
||||
request.Request = &http.Request{
|
||||
Method: method,
|
||||
URL: &url,
|
||||
Proto: "HTTP/1.1",
|
||||
ProtoMajor: 1,
|
||||
ProtoMinor: 1,
|
||||
Header: make(http.Header),
|
||||
Host: url.Host,
|
||||
}
|
||||
|
||||
if body != nil {
|
||||
err = request.SetBody(body)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// SetBody sets the body and content length, assumes body is not nil.
|
||||
func (r Request) SetBody(body io.ReadSeeker) error {
|
||||
size, err := body.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
body.Seek(0, io.SeekStart)
|
||||
r.ContentLength = size
|
||||
r.Header["Content-Length"] = []string{strconv.FormatInt(size, 10)}
|
||||
|
||||
if size != 0 {
|
||||
r.Body = &retryableRequestBody{body: body}
|
||||
r.GetBody = func() (io.ReadCloser, error) {
|
||||
_, err := body.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.Body, nil
|
||||
}
|
||||
} else {
|
||||
// in case the body is an empty stream, we need to use http.NoBody to explicitly provide no content
|
||||
r.Body = http.NoBody
|
||||
r.GetBody = func() (io.ReadCloser, error) {
|
||||
return http.NoBody, nil
|
||||
}
|
||||
|
||||
// close the user-provided empty body
|
||||
if c, ok := body.(io.Closer); ok {
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy makes a copy of an http.Request. Specifically, it makes a deep copy
|
||||
// of its Method, URL, Host, Proto(Major/Minor), Header. ContentLength, Close,
|
||||
// RemoteAddr, RequestURI. Copy makes a shallow copy of the Body, GetBody, TLS,
|
||||
// Cancel, Response, and ctx fields. Copy panics if any of these fields are
|
||||
// not nil: TransferEncoding, Form, PostForm, MultipartForm, or Trailer.
|
||||
func (r Request) Copy() Request {
|
||||
if r.TransferEncoding != nil || r.Form != nil || r.PostForm != nil || r.MultipartForm != nil || r.Trailer != nil {
|
||||
panic("Can't make a deep copy of the http.Request because at least one of the following is not nil:" +
|
||||
"TransferEncoding, Form, PostForm, MultipartForm, or Trailer.")
|
||||
}
|
||||
copy := *r.Request // Copy the request
|
||||
urlCopy := *(r.Request.URL) // Copy the URL
|
||||
copy.URL = &urlCopy
|
||||
copy.Header = http.Header{} // Copy the header
|
||||
for k, vs := range r.Header {
|
||||
for _, value := range vs {
|
||||
copy.Header.Add(k, value)
|
||||
}
|
||||
}
|
||||
return Request{Request: ©} // Return the copy
|
||||
}
|
||||
|
||||
func (r Request) close() error {
|
||||
if r.Body != nil && r.Body != http.NoBody {
|
||||
c, ok := r.Body.(*retryableRequestBody)
|
||||
if !ok {
|
||||
panic("unexpected request body type (should be *retryableReadSeekerCloser)")
|
||||
}
|
||||
return c.realClose()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RewindBody seeks the request's Body stream back to the beginning so it can be resent when retrying an operation.
|
||||
func (r Request) RewindBody() error {
|
||||
if r.Body != nil && r.Body != http.NoBody {
|
||||
s, ok := r.Body.(io.Seeker)
|
||||
if !ok {
|
||||
panic("unexpected request body type (should be io.Seeker)")
|
||||
}
|
||||
|
||||
// Reset the stream back to the beginning
|
||||
_, err := s.Seek(0, io.SeekStart)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ********** The following type/methods implement the retryableRequestBody (a ReadSeekCloser)
|
||||
|
||||
// This struct is used when sending a body to the network
|
||||
type retryableRequestBody struct {
|
||||
body io.ReadSeeker // Seeking is required to support retries
|
||||
}
|
||||
|
||||
// Read reads a block of data from an inner stream and reports progress
|
||||
func (b *retryableRequestBody) Read(p []byte) (n int, err error) {
|
||||
return b.body.Read(p)
|
||||
}
|
||||
|
||||
func (b *retryableRequestBody) Seek(offset int64, whence int) (offsetFromStart int64, err error) {
|
||||
return b.body.Seek(offset, whence)
|
||||
}
|
||||
|
||||
func (b *retryableRequestBody) Close() error {
|
||||
// We don't want the underlying transport to close the request body on transient failures so this is a nop.
|
||||
// The pipeline closes the request body upon success.
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *retryableRequestBody) realClose() error {
|
||||
if c, ok := b.body.(io.Closer); ok {
|
||||
return c.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The Response interface exposes an http.Response object as it returns through the pipeline of Policy objects.
|
||||
// This ensures that Policy objects have access to the HTTP response. However, the object this interface encapsulates
|
||||
// might be a struct with additional fields that is created by a Policy object (typically a method-specific Factory).
|
||||
// The method that injected the method-specific Factory gets this returned Response and performs a type assertion
|
||||
// to the expected struct and returns the struct to its caller.
|
||||
type Response interface {
|
||||
Response() *http.Response
|
||||
}
|
||||
|
||||
// This is the default struct that has the http.Response.
|
||||
// A method can replace this struct with its own struct containing an http.Response
|
||||
// field and any other additional fields.
|
||||
type httpResponse struct {
|
||||
response *http.Response
|
||||
}
|
||||
|
||||
// NewHTTPResponse is typically called by a Policy object to return a Response object.
|
||||
func NewHTTPResponse(response *http.Response) Response {
|
||||
return &httpResponse{response: response}
|
||||
}
|
||||
|
||||
// This method satisfies the public Response interface's Response method
|
||||
func (r httpResponse) Response() *http.Response {
|
||||
return r.response
|
||||
}
|
||||
|
||||
// WriteRequestWithResponse appends a formatted HTTP request into a Buffer. If request and/or err are
|
||||
// not nil, then these are also written into the Buffer.
|
||||
func WriteRequestWithResponse(b *bytes.Buffer, request *http.Request, response *http.Response, err error) {
|
||||
// Write the request into the buffer.
|
||||
fmt.Fprint(b, " "+request.Method+" "+request.URL.String()+"\n")
|
||||
writeHeader(b, request.Header)
|
||||
if response != nil {
|
||||
fmt.Fprintln(b, " --------------------------------------------------------------------------------")
|
||||
fmt.Fprint(b, " RESPONSE Status: "+response.Status+"\n")
|
||||
writeHeader(b, response.Header)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(b, " --------------------------------------------------------------------------------")
|
||||
fmt.Fprint(b, " ERROR:\n"+err.Error()+"\n")
|
||||
}
|
||||
}
|
||||
|
||||
// formatHeaders appends an HTTP request's or response's header into a Buffer.
|
||||
func writeHeader(b *bytes.Buffer, header map[string][]string) {
|
||||
if len(header) == 0 {
|
||||
b.WriteString(" (no headers)\n")
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(header))
|
||||
// Alphabetize the headers
|
||||
for k := range header {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
// Redact the value of any Authorization header to prevent security information from persisting in logs
|
||||
value := interface{}("REDACTED")
|
||||
if !strings.EqualFold(k, "Authorization") {
|
||||
value = header[k]
|
||||
}
|
||||
fmt.Fprintf(b, " %s: %+v\n", k, value)
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package pipeline
|
||||
|
||||
const (
|
||||
// UserAgent is the string to be used in the user agent string when making requests.
|
||||
UserAgent = "azure-pipeline-go/" + Version
|
||||
|
||||
// Version is the semantic version (see http://semver.org) of the pipeline package.
|
||||
Version = "0.1.0"
|
||||
)
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
Thanks you for your contribution to the Azure-SDK-for-Go! We will triage and review it as quickly as we can.
|
||||
|
||||
As part of your submission, please make sure that you can make the following assertions:
|
||||
|
||||
- [ ] I'm not making changes to Auto-Generated files which will just get erased next time there's a release.
|
||||
- If that's what you want to do, consider making a contribution here: https://github.com/Azure/autorest.go
|
||||
- [ ] I've tested my changes, adding unit tests where applicable.
|
||||
- [ ] I've added Apache 2.0 Headers to the top of any new source files.
|
||||
- [ ] I'm submitting this PR to the `dev` branch, or I'm fixing a bug that warrants its own release and I'm targeting the `master` branch.
|
||||
- [ ] If I'm targeting the `master` branch, I've also added a note to [CHANGELOG.md](https://github.com/Azure/azure-sdk-for-go/blob/master/README.md).
|
||||
- [ ] I've mentioned any relevant open issues in this PR, making clear the context for the contribution.
|
||||
|
||||
-32
@@ -1,32 +0,0 @@
|
||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
# *.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
*.test
|
||||
*.prof
|
||||
|
||||
# Editor swap files
|
||||
*.swp
|
||||
*~
|
||||
.DS_Store
|
||||
|
||||
# ignore vendor/
|
||||
vendor/
|
||||
-37
@@ -1,37 +0,0 @@
|
||||
sudo: false
|
||||
|
||||
language: go
|
||||
go:
|
||||
- 1.8.x
|
||||
- 1.9.x
|
||||
- 1.10.x
|
||||
- master
|
||||
|
||||
env:
|
||||
global:
|
||||
- DEP_VERSION="0.4.1"
|
||||
- DEP_SHA=31144e465e52ffbc0035248a10ddea61a09bf28b00784fd3fdd9882c8cbb2315
|
||||
- IGNORE_BREAKING_CHANGES=true
|
||||
matrix:
|
||||
- GOARCH="amd64"
|
||||
- GOARCH="386"
|
||||
|
||||
matrix:
|
||||
allow_failures:
|
||||
- go: master
|
||||
|
||||
before_install:
|
||||
- curl -fL -o $GOPATH/bin/dep https://github.com/golang/dep/releases/download/v$DEP_VERSION/dep-linux-amd64 && echo "$DEP_SHA $GOPATH/bin/dep" | sha256sum -c - && chmod +x $GOPATH/bin/dep
|
||||
|
||||
install:
|
||||
- go get -u github.com/golang/lint/golint
|
||||
- dep ensure
|
||||
|
||||
script:
|
||||
- bash rungas.sh
|
||||
- grep -L -r --include *.go --exclude-dir vendor -P "Copyright (\d{4}|\(c\)) Microsoft" ./ | tee /dev/stderr | test -z "$(< /dev/stdin)"
|
||||
- go build $(go list ./... | grep -v vendor)
|
||||
- test -z "$(go fmt $(go list ./... | grep -v vendor) | tee /dev/stderr)"
|
||||
- go vet $(go list ./... | grep -v vendor)
|
||||
- go test $(sh ./findTestedPackages.sh)
|
||||
- go run ./tools/apidiff/main.go packages ./services FETCH_HEAD~1 FETCH_HEAD --copyrepo --breakingchanges || $IGNORE_BREAKING_CHANGES
|
||||
-597
@@ -1,597 +0,0 @@
|
||||
# CHANGELOG
|
||||
|
||||
## `v18.0.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ----------------------: | :-------------------: |
|
||||
| apimanagement | 2018-06-01-preview |
|
||||
| hdinsight | 2018-06-01-preview |
|
||||
| computervision | v2.0 |
|
||||
| recoveryservices/backup | 2016-12-01 |
|
||||
| storagedatalake | 2018-06-17 |
|
||||
| devspaces | 2018-06-01-preview |
|
||||
| automation | 2017-05-15-preview |
|
||||
| keyvault | v7.0 <br/> 2018-02-14 |
|
||||
| automation | 2018-01-preview |
|
||||
| media | 2018-06-01-preview |
|
||||
| datafactory | 2018-06-01 |
|
||||
|
||||
### Updates Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -------------: | :--------------------------------: |
|
||||
| face | v1.0 |
|
||||
| luis/authoring | v2.0 |
|
||||
| luis/runtime | v2.0 |
|
||||
| textanalytics | v2.0 |
|
||||
| eventhub | 2017-04-01 <br/>2018-01-01-preview |
|
||||
| monitor | 2017-05-01-preview |
|
||||
| signalr | 2018-03-01-preview |
|
||||
| sql | 2017-10-01-preview |
|
||||
| storage | 2018-03-01-preview <br/>2018-02-01 |
|
||||
| servicebus | 2017-04-01 |
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
| Package Name | API Version |
|
||||
| ----------------------: | :-------------------: |
|
||||
| adhybridhealthservice | 2014-01-01 |
|
||||
| advisor | 2017-04-19 |
|
||||
| appinsights | 2015-05-01 <br/> v1 |
|
||||
| automation | 2015-10-31/automation |
|
||||
| batchai | 2018-05-01/batchai |
|
||||
| computervision | v1.0 |
|
||||
| consumption | 2018-05-31 |
|
||||
| network | 2018-04-01 |
|
||||
| operationalinsights | 2015-03-20<br/>v1 |
|
||||
| recoveryservices/backup | 2017-07-01 |
|
||||
| reservations | 2018-06-01 |
|
||||
|
||||
## `v17.4.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ---------------: | :---------: |
|
||||
| autosuggest | v1.0 |
|
||||
| containerservice | 2018-03-31 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ---------------: | :---------------------------------------: |
|
||||
| consumption | 2018-05-31 |
|
||||
| documentdb | 2015-04-08 |
|
||||
| network | 2018-02-01<br/>2018-04-01<br/>2018-05-01 |
|
||||
| notificationhubs | 2017-04-01 |
|
||||
| sql | 2017-03-01-preview<br/>2017-10-01-preview |
|
||||
|
||||
## `v17.3.1`
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------: | :-----------------------------------------------------------------------------------: |
|
||||
| network | 2017-10-01<br/>2017-11-01<br/>2018-01-01<br/>2018-02-01<br/>2018-04-01<br/>2018-05-01 |
|
||||
| servicefabric | 5.6<br/>6.0<br/>6.1<br/>6.2 |
|
||||
|
||||
## `v17.3.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ----------------: | :---------: |
|
||||
| containerinstance | 2018-06-01 |
|
||||
| reservations | 2018-06-01 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -----------: | :---------: |
|
||||
| datafactory | 2017-09-01 |
|
||||
| network | 2015-06-15 |
|
||||
|
||||
## `v17.2.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------------: | :---------: |
|
||||
| managedapplications | 2018-06-01 |
|
||||
| resources | 2018-05-01 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -------------------: | :---------: |
|
||||
| networksecuritygroup | classic |
|
||||
|
||||
## `v17.1.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -----------: | :-----------------------: |
|
||||
| iotcentral | 2017-07-01-privatepreview |
|
||||
|
||||
## `v17.0.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| --------------------: | :-----------------: |
|
||||
| batchai | 2018-05-01 |
|
||||
| adhybridhealthservice | 2014-01-01 |
|
||||
| consumption | 2018-05-31 |
|
||||
| customimagesearch | v1.0 |
|
||||
| luis/authoring | v2.0 |
|
||||
| management | 2018-03-01-preview |
|
||||
| maps | 2017-01-01-preview |
|
||||
| maps | 2018-05-01 |
|
||||
| network | 2018-05-01 |
|
||||
| policy | "2018-03-01 |
|
||||
| servicefabric | "2018-02-01 |
|
||||
| services | "2018-03-01-preview |
|
||||
| storage | 2018-03-01-preview |
|
||||
| trafficmanager | 2018-02-01 |
|
||||
| workspaces | 2016-04-01 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------------: | :----------------: |
|
||||
| aad | 2017-01-01 |
|
||||
| aad | 2017-06-01 |
|
||||
| account | 2016-11-01 |
|
||||
| advisor | 2017-04-19 |
|
||||
| analysisservices | 2016-05-16 |
|
||||
| analysisservices | 2017-07-14 |
|
||||
| analysisservices | 2017-08-01 |
|
||||
| apimanagement | 2016-07-07 |
|
||||
| apimanagement | 2016-10-10 |
|
||||
| apimanagement | 2017-03-01 |
|
||||
| apimanagement | 2018-01-01 |
|
||||
| batch | 2015-12-01 |
|
||||
| batch | 2017-01-01 |
|
||||
| batch | 2017-05-01 |
|
||||
| batch | 2017-09-01 |
|
||||
| batchai | 2018-03-01 |
|
||||
| batchai | 2018-05-01 |
|
||||
| botservices | 2017-12-01 |
|
||||
| catalog | 2016-11-01-preview |
|
||||
| cdn | 2015-06-01 |
|
||||
| cdn | 2016-04-02 |
|
||||
| cdn | 2016-10-02 |
|
||||
| cdn | 2017-04-02 |
|
||||
| cdn | 2017-10-12 |
|
||||
| compute | 2015-06-15 |
|
||||
| compute | 2016-03-30 |
|
||||
| compute | 2017-03-30 |
|
||||
| compute | 2018-04-01 |
|
||||
| computervision | v1.0 |
|
||||
| consumption | 2018-03-31 |
|
||||
| containerinstance | 2018-04-01 |
|
||||
| containerregistry | 2017-03-01 |
|
||||
| containerregistry | 2017-10-01 |
|
||||
| containerregistry | 2018-02-01 |
|
||||
| containerservice | 2016-03-30 |
|
||||
| containerservice | 2016-09-30 |
|
||||
| containerservice | 2017-01-31 |
|
||||
| containerservice | 2017-07-01 |
|
||||
| containerservice | 2017-08-31 |
|
||||
| containerservice | 2017-09-30 |
|
||||
| customerinsights | 2017-01-01 |
|
||||
| customerinsights | 2017-04-26 |
|
||||
| databox | 2018-01-01 |
|
||||
| databricks | 2018-04-01 |
|
||||
| datacatalog | 2016-03-30 |
|
||||
| datafactory | 2017-09-01-preview |
|
||||
| datamigration | 2018-03-31-preview |
|
||||
| devices | 2016-02-03 |
|
||||
| devices | 2017-01-19 |
|
||||
| devices | 2017-07-01 |
|
||||
| devices | 2018-01-22 |
|
||||
| devices | 2018-04-01 |
|
||||
| dns | 2016-04-01 |
|
||||
| dns | 2017-09-01 |
|
||||
| dns | 2017-10-01 |
|
||||
| documentdb | 2015-04-08 |
|
||||
| dtl | 2016-05-15 |
|
||||
| eventgrid | 2018-01-01 |
|
||||
| eventgrid | 2018-05-01-preview |
|
||||
| eventhub | 2015-08-01 |
|
||||
| eventhub | 2017-04-01 |
|
||||
| eventhub | 2018-01-01-preview |
|
||||
| insights | 2015-05-01 |
|
||||
| iothub | 2017-08-21-preview |
|
||||
| iothub | 2017-11-15 |
|
||||
| iothub | 2018-01-22 |
|
||||
| iotspaces | 2017-10-01-preview |
|
||||
| logic | 2016-06-01 |
|
||||
| managedapplications | 2016-09-01-preview |
|
||||
| managedapplications | 2017-09-01 |
|
||||
| management | 2018-01-01-preview |
|
||||
| media | 2018-03-30-preview |
|
||||
| mysql | 2017-12-01 |
|
||||
| network | 2015-06-15 |
|
||||
| network | 2016-03-30 |
|
||||
| network | 2016-06-01 |
|
||||
| network | 2016-09-01 |
|
||||
| network | 2016-12-01 |
|
||||
| network | 2017-03-01 |
|
||||
| network | 2017-06-01 |
|
||||
| network | 2017-08-01 |
|
||||
| network | 2017-09-01 |
|
||||
| network | 2017-10-01 |
|
||||
| network | 2017-11-01 |
|
||||
| network | 2018-01-01 |
|
||||
| network | 2018-02-01 |
|
||||
| network | 2018-04-01 |
|
||||
| notificationhubs | 2014-09-01 |
|
||||
| notificationhubs | 2016-03-01 |
|
||||
| notificationhubs | 2017-04-01 |
|
||||
| operationalinsights | 2015-03-20 |
|
||||
| operationalinsights | v1 |
|
||||
| postgresql | 2017-12-01 |
|
||||
| powerbidedicated | 2017-10-01 |
|
||||
| powerbiembedded | 2016-01-29 |
|
||||
| prediction | customvision |
|
||||
| luis/programmatic | v2.0 |
|
||||
| redis | 2016-04-01 |
|
||||
| redis | 2017-02-01 |
|
||||
| redis | 2017-10-01 |
|
||||
| redis | 2018-03-01 |
|
||||
| relay | 2016-07-01 |
|
||||
| relay | 2017-04-01 |
|
||||
| reservations | 2017-11-01 |
|
||||
| resources | 2015-11-01 |
|
||||
| resources | 2016-02-01 |
|
||||
| resources | 2016-07-01 |
|
||||
| resources | 2016-09-01 |
|
||||
| resources | 2017-05-10 |
|
||||
| resources | 2018-02-01 |
|
||||
| luis/runtime | v2.0 |
|
||||
| scheduler | 2016-03-01 |
|
||||
| search | 2015-08-19 |
|
||||
| servermanagement | 2015-07-01-preview |
|
||||
| servermanagement | 2016-07-01-preview |
|
||||
| servicebus | 2015-08-01 |
|
||||
| servicebus | 2017-04-01 |
|
||||
| servicefabric | 2016-09-01 |
|
||||
| servicefabric | 2017-07-01-preview |
|
||||
| signalr | 2018-03-01-preview |
|
||||
| siterecovery | 2016-08-10 |
|
||||
| siterecovery | 2018-01-10 |
|
||||
| sql | 2014-04-01 |
|
||||
| sql | 2015-05-01-preview |
|
||||
| sql | 2017-03-01-preview |
|
||||
| sql | 2017-10-01-preview |
|
||||
| storage | 2015-05-01-preview |
|
||||
| storage | 2015-06-15 |
|
||||
| storage | 2016-01-01 |
|
||||
| storage | 2016-05-01 |
|
||||
| storage | 2016-12-01 |
|
||||
| storage | 2017-06-01 |
|
||||
| storage | 2017-10-01 |
|
||||
| storage | 2018-02-01 |
|
||||
| storsimple | 2017-06-01 |
|
||||
| streamanalytics | 2016-03-01 |
|
||||
| subscription | 2017-11-01-preview |
|
||||
| subscription | 2018-03-01-preview |
|
||||
| timeseriesinsights | 2017-02-28-preview |
|
||||
| timeseriesinsights | 2017-11-15 |
|
||||
| trafficmanager | 2018-03-01 |
|
||||
| training | customvision |
|
||||
| visualstudio | 2014-04-01-preview |
|
||||
| web | 2015-08-preview |
|
||||
| web | 2016-09-01 |
|
||||
| web | 2018-02-01 |
|
||||
| webservices | 2017-01-01 |
|
||||
|
||||
## `v16.2.2`
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -----------: | :----------------: |
|
||||
| signalr | 2018-03-01-preview |
|
||||
|
||||
## `v16.2.1`
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -----------: | :-----------------------: |
|
||||
| web | 2016-09-01<br/>2018-02-01 |
|
||||
|
||||
## `v16.2.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -------------: | :----------------: |
|
||||
| eventgrid | 2018-05-01-preview |
|
||||
| trafficmanager | 2018-03-01 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------: | :-----------------------: |
|
||||
| redis | 2017-10-01<br/>2018-03-01 |
|
||||
| textanalytics | v2.0 |
|
||||
| web | 2016-09-01<br/>2018-02-01 |
|
||||
|
||||
## `v16.1.0`
|
||||
|
||||
- Added a `NewAuthorizerFromEnvironment()` function for the keyvault service.
|
||||
|
||||
## `v16.0.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------------: | :----------------: |
|
||||
| batchai | 2018-05-01 |
|
||||
| botservices | 2017-12-01 |
|
||||
| containerinstance | 2018-04-01 |
|
||||
| containerregistry | 2018-02-01 |
|
||||
| keyvault | v7.0 |
|
||||
| managedapplications | 2017-09-01 |
|
||||
| network | 2018-04-01 |
|
||||
| policyinsights | 2018-04-04 |
|
||||
| signalr | 2018-03-01-preview |
|
||||
| storage | 2018-02-0 |
|
||||
| visualsearch | v1.0 |
|
||||
| web | 2018-02-01 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------------: | :--------------------------------------------------------------: |
|
||||
| apimanagement | 2018-01-01 |
|
||||
| automation | 2015-10-31<br/>2017-05-15-preview |
|
||||
| billing | 2018-03-01-preview |
|
||||
| botservices | 2017-12-01 |
|
||||
| catalog | 2016-11-01-preview |
|
||||
| cognitiveservices | 2017-04-18 |
|
||||
| commerce | 2015-06-01-preview |
|
||||
| compute | 2018-04-01 |
|
||||
| consumption | 2018-03-31 |
|
||||
| contentmoderator | v1.0 |
|
||||
| datafactory | 2017-09-01-preview |
|
||||
| datamigration | 2017-11-15-preview |
|
||||
| eventhub | 2017-04-01 |
|
||||
| insights | 2015-05-0 |
|
||||
| iothub | 2017-11-15 |
|
||||
| network | 2018-02-01 |
|
||||
| operationalinsights | 2015-03-20<br/>2015-11-01-preview |
|
||||
| servicebus | 2017-04-01 |
|
||||
| siterecovery | 2018-01-10 |
|
||||
| sql | 2017-03-01-preview<br/>2017-10-01-preview<br/>2015-05-01-preview |
|
||||
| timeseriesinsights | 2017-11-15 |
|
||||
| luis/runtime | v2.0 |
|
||||
| web | 2016-09-01 |
|
||||
|
||||
## `v15.3.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------: | :----------------: |
|
||||
| databox | 2018-01-01 |
|
||||
| devices | 2018-04-01 |
|
||||
| media | 2018-03-30-preview |
|
||||
| servicefabric | 6.2 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ----------------: | :------------: |
|
||||
| apimanagement | 2018-01-01 |
|
||||
| batch | 2018-03-01.6.1 |
|
||||
| containerregistry | 2017-10-01 |
|
||||
| documentdb | 2015-04-08 |
|
||||
| servicebus | 2017-04-01 |
|
||||
|
||||
## `v15.2.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -----------: | :---------: |
|
||||
| addons | 2017-05-15 |
|
||||
| addons | 2018-03-01 |
|
||||
|
||||
### Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------: | :---------: |
|
||||
| apimanagement | 2017-03-01 |
|
||||
| apimanagement | 2018-01-01 |
|
||||
| insights | 2015-05-01 |
|
||||
|
||||
## `v15.1.1`
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Drain the response body when retrying a request.
|
||||
- Avoid allocating when draining a response body.
|
||||
|
||||
## `v15.1.0`
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| ----------------: | :----------------: |
|
||||
| datamigration | 2018-03-31-preview |
|
||||
| devices | 2018-01-22 |
|
||||
| network | 2018-02-01 |
|
||||
| cognitiveservices | customvision |
|
||||
|
||||
## Updated Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -----------: | :----------------------------------------------------------------------------: |
|
||||
| compute | 2015-06-15<br/>2016-03-30<br/>2016-04-30-preview<br/>2017-03-30<br/>2018-04-01 |
|
||||
| datafactory | 2017-09-01-preview |
|
||||
| insights | 2018-03-01 |
|
||||
| mysql | 2017-12-01 |
|
||||
| postgresql | 2017-12-01 |
|
||||
| web | 2016-09-01 |
|
||||
|
||||
## `v15.0.1`
|
||||
|
||||
Fixing some build issues, and temporarily reverting CI.
|
||||
|
||||
## `v15.0.0`
|
||||
|
||||
NOTE: There is a new subdirectory, ./services/preview, which going forward will be used for segregating pre-GA packages.
|
||||
|
||||
### New Features
|
||||
|
||||
- Added helper func per enum type that returns a slice of all possible values for that type.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Removed the "arm-" prefix from user-agent strings as not all services are for ARM.
|
||||
- Fixed missing marshaller for structs with flattened fields.
|
||||
- Fixed an issue where the incorrect content MIME type was being specified.
|
||||
- Marshalling of empty values for enum types now works as expected.
|
||||
|
||||
### New Services
|
||||
|
||||
| Package Name | API Version |
|
||||
| -------------: | :--------------------------------------------------------------: |
|
||||
| apimanagement | 2017-03-01 |
|
||||
| azurestack | 2017-06-01 |
|
||||
| billing | 2018-03-01-preview |
|
||||
| compute | 2018-04-01 |
|
||||
| consumption | 2018-03-31 |
|
||||
| databricks | 2018-04-01 |
|
||||
| dns | 2017-10-01 |
|
||||
| insights | 2018-03-01 |
|
||||
| iothub | 2018-01-22 |
|
||||
| iotspaces | 2017-10-01-preview |
|
||||
| management | 2018-01-01-preview |
|
||||
| migrate | 2018-02-02 |
|
||||
| policyinsights | 2017-08-09-preview<br/>2017-10-17-preview<br/>2017-12-12-preview |
|
||||
| resources | 2018-02-01 |
|
||||
| siterecovery | 2018-01-10 |
|
||||
| sql | 2017-10-01-preview |
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
| Package Name | API Version |
|
||||
| ------------------: | :----------------------------------------: |
|
||||
| automation | 2017-05-15-preview |
|
||||
| advisor | 2017-04-19 |
|
||||
| cognitiveservices | 2017-04-18 |
|
||||
| compute | 2017-03-30<br/>2017-12-01 |
|
||||
| consumption | 2018-01-31 |
|
||||
| containerinstance | 2018-02-01-preview |
|
||||
| containerservice | 2017-08-31<br/>2017-09-30 |
|
||||
| customsearch | v1.0 |
|
||||
| datafactory | 2017-09-01-preview |
|
||||
| datamigration | 2017-11-15-preview |
|
||||
| dns | 2018-03-01-preview |
|
||||
| entitysearch | v1.0 |
|
||||
| imagesearch | v1.0 |
|
||||
| insights | 2017-05-01-preview |
|
||||
| iothub | 2017-11-15 |
|
||||
| management | 2017-08-31-preview<br/>2017-11-01-preview |
|
||||
| mysql | 2017-12-01-preview |
|
||||
| newssearch | v1.0 |
|
||||
| operationalinsights | 2015-03-20 |
|
||||
| postgresql | 2017-12-01-preview |
|
||||
| servicebus | 2015-08-01 |
|
||||
| servicefabric | 2017-07-01-preview<br/>5.6<br/>6.0<br/>6.1 |
|
||||
| servicemap | 2015-11-01-preview |
|
||||
| spellcheck | v1.0 |
|
||||
| timeseriesinsights | 2017-02-28-preview<br/>2017-11-15 |
|
||||
| videosearch | v1.0 |
|
||||
| web | 2016-09-01 |
|
||||
| websearch | v1.0 |
|
||||
|
||||
## `v14.6.0`
|
||||
|
||||
### New Services
|
||||
|
||||
- Batch 2018-03-01.6.1
|
||||
- BatchAI 2018-03-01
|
||||
- Cognitive services custom vision prediction v1.1
|
||||
- Eventhub 2018-01-01-preview
|
||||
- MySQL 2017-12-01
|
||||
- PostgreSQL 2017-12-01
|
||||
- Redis 2018-03-01
|
||||
- Subscription 2018-03-01-preview
|
||||
|
||||
## `v14.5.0`
|
||||
|
||||
### Changes
|
||||
|
||||
- Added new preview packages for apimanagement and dns.
|
||||
|
||||
## `v14.4.0`
|
||||
|
||||
### Changes
|
||||
|
||||
- Added new preview API versions for mysql and postgresql.
|
||||
|
||||
## `v14.3.0`
|
||||
|
||||
### Changes
|
||||
|
||||
- Add exports for max file range and sizes for files in storage.
|
||||
- Updated README regarding blob storage support.
|
||||
- Add godoc indexer tool.
|
||||
- Add apidiff tool.
|
||||
|
||||
## `v14.2.0`
|
||||
|
||||
### Changes
|
||||
|
||||
- For blob storage, added GetProperties() method to Container.
|
||||
- Added PublicAccess field to ContainerProperties struct.
|
||||
|
||||
## `v14.1.1`
|
||||
|
||||
- Fixing timestamp marshalling bug in the `storage` package.
|
||||
- Updating `profileBuilder` to clear-output folders when it is run by `go generate`.
|
||||
- Tweaking Swagger -> SDK config to use "latest" instead of "nightly" and be tied to a particular version of autorest.go.
|
||||
|
||||
## `v14.1.0`
|
||||
|
||||
### Changes
|
||||
|
||||
- Update README with details on new authentication helpers.
|
||||
- Update `latest` profile to point to latest stable API versions.
|
||||
- Add new API version for Azure Monitoring service and for Batch Data plane service.
|
||||
|
||||
## `v14.0.2`
|
||||
|
||||
### Changes
|
||||
|
||||
- Updating `profileBuilder list` to accept an `input` flag instead of reading from `stdin`.
|
||||
- Simplifying CI to have less chatter, be faster, and have no special cases.
|
||||
|
||||
## `v14.0.1`
|
||||
|
||||
### Changes
|
||||
|
||||
- Removed the ./services/search/2016-09-01/search package, it was never intended to be included and doesn't work.
|
||||
|
||||
## `v14.0.0`
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
- Removed the ./arm, ./datalake-store and ./dataplane directories. You can find the same packages under ./services
|
||||
- The management package was moved to ./services/classic/management
|
||||
- Renamed package ./services/redis/mgmt/2017-10-01/cache to ./services/redis/mgmt/2017-10-01/redis
|
||||
|
||||
### Changes
|
||||
|
||||
- Removed the "-beta" suffix.
|
||||
- Added various new services.
|
||||
- Added ./version package for centralized SDK versioning.
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
/documentation/ @mcardosos @marstr @joshgav
|
||||
/profiles/ @marstr @jhendrixMSFT
|
||||
/services/ @jhendrixMSFT @marstr @mcardosos @vladbarosan
|
||||
/storage/ @mcardosos @jhendrixMSFT
|
||||
/tools/apidiff/ @jhendrixMSFT
|
||||
/tools/generator/ @marstr
|
||||
/tools/indexer/ @jhendrixMSFT
|
||||
/tools/profileBuilder/ @marstr @jhendrixMSFT
|
||||
/version/ @jhendrixMSFT @marstr @mcardosos @vladbarosan
|
||||
.travis.yml @marstr @jhendrixMSFT @mcardosos @vladbarosan
|
||||
doc.go @joshgav
|
||||
findTestedPackages.sh @marstr
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
# Contributing
|
||||
|
||||
While in preview we maintain only a `master` branch for releases and `dev` branch for development.
|
||||
|
||||
After release we will maintain a branch for each major version to backport important fixes.
|
||||
|
||||
Please submit pull requests to `dev` to update the latest version or to a version branch for backports.
|
||||
|
||||
Also see these [guidelines][] about contributing to Azure projects.
|
||||
|
||||
This project follows the [Microsoft Open Source Code of Conduct][CoC]. For more information see the [Code of Conduct FAQ][CoCfaq]. Contact [opencode@microsoft.com][CoCmail] with questions and comments.
|
||||
|
||||
[guidelines]: http://azure.github.io/guidelines/
|
||||
[CoC]: https://opensource.microsoft.com/codeofconduct/
|
||||
[CoCfaq]: https://opensource.microsoft.com/codeofconduct/faq/
|
||||
[CoCmail]: mailto:opencode@microsoft.com
|
||||
-246
@@ -1,246 +0,0 @@
|
||||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/Azure/go-autorest"
|
||||
packages = [
|
||||
"autorest",
|
||||
"autorest/adal",
|
||||
"autorest/azure",
|
||||
"autorest/azure/auth",
|
||||
"autorest/date",
|
||||
"autorest/to",
|
||||
"autorest/validation"
|
||||
]
|
||||
revision = "1f7cd6cfe0adea687ad44a512dfe76140f804318"
|
||||
version = "v10.12.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/dgrijalva/jwt-go"
|
||||
packages = ["."]
|
||||
revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
|
||||
version = "v3.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/dimchansky/utfbom"
|
||||
packages = ["."]
|
||||
revision = "6c6132ff69f0f6c088739067407b5d32c52e1d0f"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/dnaeon/go-vcr"
|
||||
packages = [
|
||||
"cassette",
|
||||
"recorder"
|
||||
]
|
||||
revision = "8b144be0744f013a1b44386058f1fcb3ba98177d"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/fsnotify/fsnotify"
|
||||
packages = ["."]
|
||||
revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9"
|
||||
version = "v1.4.7"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/globalsign/mgo"
|
||||
packages = [
|
||||
".",
|
||||
"bson",
|
||||
"internal/json",
|
||||
"internal/sasl",
|
||||
"internal/scram"
|
||||
]
|
||||
revision = "113d3961e7311526535a1ef7042196563d442761"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/hashicorp/hcl"
|
||||
packages = [
|
||||
".",
|
||||
"hcl/ast",
|
||||
"hcl/parser",
|
||||
"hcl/printer",
|
||||
"hcl/scanner",
|
||||
"hcl/strconv",
|
||||
"hcl/token",
|
||||
"json/parser",
|
||||
"json/scanner",
|
||||
"json/token"
|
||||
]
|
||||
revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/inconshreveable/mousetrap"
|
||||
packages = ["."]
|
||||
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
|
||||
version = "v1.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/kr/pretty"
|
||||
packages = ["."]
|
||||
revision = "73f6ac0b30a98e433b289500d779f50c1a6f0712"
|
||||
version = "v0.1.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/kr/text"
|
||||
packages = ["."]
|
||||
revision = "e2ffdb16a802fe2bb95e2e35ff34f0e53aeef34f"
|
||||
version = "v0.1.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/magiconair/properties"
|
||||
packages = ["."]
|
||||
revision = "c2353362d570a7bfa228149c62842019201cfb71"
|
||||
version = "v1.8.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/marstr/collection"
|
||||
packages = ["."]
|
||||
revision = "871b1cfa2ab97d3d8f54a034280907896190c346"
|
||||
version = "v0.3.3"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/marstr/goalias"
|
||||
packages = ["model"]
|
||||
revision = "8dff9a14db648bfdd58d45515d3eaaee23aad078"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/marstr/guid"
|
||||
packages = ["."]
|
||||
revision = "8bd9a64bf37eb297b492a4101fb28e80ac0b290f"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/marstr/randname"
|
||||
packages = ["."]
|
||||
revision = "48a63b6052f1f9373db9388a658da30c6ab53db1"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/mitchellh/go-homedir"
|
||||
packages = ["."]
|
||||
revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/mitchellh/mapstructure"
|
||||
packages = ["."]
|
||||
revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/pelletier/go-toml"
|
||||
packages = ["."]
|
||||
revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/satori/go.uuid"
|
||||
packages = ["."]
|
||||
revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/shopspring/decimal"
|
||||
packages = ["."]
|
||||
revision = "69b3a8ad1f5f2c8bd855cb6506d18593064a346b"
|
||||
version = "1.0.1"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/spf13/afero"
|
||||
packages = [
|
||||
".",
|
||||
"mem"
|
||||
]
|
||||
revision = "787d034dfe70e44075ccc060d346146ef53270ad"
|
||||
version = "v1.1.1"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/spf13/cast"
|
||||
packages = ["."]
|
||||
revision = "8965335b8c7107321228e3e3702cab9832751bac"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/spf13/cobra"
|
||||
packages = ["."]
|
||||
revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
|
||||
version = "v0.0.3"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/spf13/jwalterweatherman"
|
||||
packages = ["."]
|
||||
revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/spf13/pflag"
|
||||
packages = ["."]
|
||||
revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
|
||||
version = "v1.0.1"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/spf13/viper"
|
||||
packages = ["."]
|
||||
revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736"
|
||||
version = "v1.0.2"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "golang.org/x/crypto"
|
||||
packages = [
|
||||
"pkcs12",
|
||||
"pkcs12/internal/rc2"
|
||||
]
|
||||
revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "golang.org/x/sys"
|
||||
packages = ["unix"]
|
||||
revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56"
|
||||
|
||||
[[projects]]
|
||||
name = "golang.org/x/text"
|
||||
packages = [
|
||||
"internal/gen",
|
||||
"internal/triegen",
|
||||
"internal/ucd",
|
||||
"transform",
|
||||
"unicode/cldr",
|
||||
"unicode/norm"
|
||||
]
|
||||
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "golang.org/x/tools"
|
||||
packages = [
|
||||
"go/ast/astutil",
|
||||
"imports",
|
||||
"internal/fastwalk"
|
||||
]
|
||||
revision = "ffe88906718454e356e61776b54256e873fc133b"
|
||||
|
||||
[[projects]]
|
||||
branch = "v1"
|
||||
name = "gopkg.in/check.v1"
|
||||
packages = ["."]
|
||||
revision = "788fd78401277ebd861206a03c884797c6ec5541"
|
||||
|
||||
[[projects]]
|
||||
name = "gopkg.in/yaml.v2"
|
||||
packages = ["."]
|
||||
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
|
||||
version = "v2.2.1"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "f4bfed9d1dcc4af2e0ff5811bb14dfebe84f87e7aed10199b8ff8d867d947612"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
-69
@@ -1,69 +0,0 @@
|
||||
# Gopkg.toml example
|
||||
#
|
||||
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
|
||||
# for detailed Gopkg.toml documentation.
|
||||
#
|
||||
# required = ["github.com/user/thing/cmd/thing"]
|
||||
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project"
|
||||
# version = "1.0.0"
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project2"
|
||||
# branch = "dev"
|
||||
# source = "github.com/myfork/project2"
|
||||
#
|
||||
# [[override]]
|
||||
# name = "github.com/x/y"
|
||||
# version = "2.4.0"
|
||||
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/Azure/go-autorest"
|
||||
version = "10.12.0"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/dnaeon/go-vcr"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/globalsign/mgo"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/marstr/collection"
|
||||
version = "0.3.3"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/marstr/goalias"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/marstr/guid"
|
||||
version = "1.1.0"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/marstr/randname"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/satori/go.uuid"
|
||||
version = "1.2.0"
|
||||
|
||||
[[constraint]]
|
||||
name = "github.com/shopspring/decimal"
|
||||
version = "1.0.0"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "golang.org/x/crypto"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "golang.org/x/tools"
|
||||
|
||||
[[constraint]]
|
||||
branch = "v1"
|
||||
name = "gopkg.in/check.v1"
|
||||
-202
@@ -1,202 +0,0 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2016 Microsoft Corporation
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-5
@@ -1,5 +0,0 @@
|
||||
Microsoft Azure-SDK-for-Go
|
||||
Copyright 2014-2017 Microsoft
|
||||
|
||||
This product includes software developed at
|
||||
the Microsoft Corporation (https://www.microsoft.com).
|
||||
-374
@@ -1,374 +0,0 @@
|
||||
# Azure SDK for Go
|
||||
|
||||
[](https://godoc.org/github.com/Azure/azure-sdk-for-go)
|
||||
[](https://travis-ci.org/Azure/azure-sdk-for-go)
|
||||
[](https://goreportcard.com/report/github.com/Azure/azure-sdk-for-go)
|
||||
|
||||
azure-sdk-for-go provides Go packages for managing and using Azure services.
|
||||
It is continuously tested with Go 1.8, 1.9, 1.10 and master.
|
||||
|
||||
To be notified about updates and changes, subscribe to the [Azure update
|
||||
feed](https://azure.microsoft.com/updates/).
|
||||
|
||||
Users may prefer to jump right in to our samples repo at
|
||||
[github.com/Azure-Samples/azure-sdk-for-go-samples][samples_repo].
|
||||
|
||||
## Package Updates
|
||||
|
||||
Most packages in the SDK are generated from [Azure API specs][azure_rest_specs]
|
||||
using [Azure/autorest.go][] and [Azure/autorest][]. These generated packages
|
||||
depend on the HTTP client implemented at [Azure/go-autorest][].
|
||||
|
||||
[azure_rest_specs]: https://github.com/Azure/azure-rest-api-specs
|
||||
[azure/autorest]: https://github.com/Azure/autorest
|
||||
[azure/autorest.go]: https://github.com/Azure/autorest.go
|
||||
[azure/go-autorest]: https://github.com/Azure/go-autorest
|
||||
|
||||
The SDK codebase adheres to [semantic versioning](https://semver.org) and thus
|
||||
avoids breaking changes other than at major (x.0.0) releases. Because Azure's
|
||||
APIs are updated frequently, we release a **new major version at the end of
|
||||
each month** with a full changelog. For more details and background see [SDK Update
|
||||
Practices](https://github.com/Azure/azure-sdk-for-go/wiki/SDK-Update-Practices).
|
||||
|
||||
To more reliably manage dependencies like the Azure SDK in your applications we
|
||||
recommend [golang/dep](https://github.com/golang/dep).
|
||||
|
||||
## Other Azure Go Packages
|
||||
|
||||
Azure provides several other packages for using services from Go, listed below.
|
||||
If a package you need isn't available please open an issue and let us know.
|
||||
|
||||
| Service | Import Path/Repo |
|
||||
| -------------------- | -------------------------------------------------------------------------------------------------- |
|
||||
| Storage - Blobs | [github.com/Azure/azure-storage-blob-go](https://github.com/Azure/azure-storage-blob-go) |
|
||||
| Storage - Files | [github.com/Azure/azure-storage-file-go](https://github.com/Azure/azure-storage-file-go) |
|
||||
| Storage - Queues | [github.com/Azure/azure-storage-queue-go](https://github.com/Azure/azure-storage-queue-go) |
|
||||
| Service Bus | [github.com/Azure/azure-service-bus-go](https://github.com/Azure/azure-service-bus-go) |
|
||||
| Event Hubs | [github.com/Azure/azure-event-hubs-go](https://github.com/Azure/azure-event-hubs-go) |
|
||||
| Application Insights | [github.com/Microsoft/ApplicationInsights-go](https://github.com/Microsoft/ApplicationInsights-go) |
|
||||
|
||||
# Install and Use:
|
||||
|
||||
## Install
|
||||
|
||||
```sh
|
||||
$ go get -u github.com/Azure/azure-sdk-for-go/...
|
||||
```
|
||||
|
||||
or if you use dep, within your repo run:
|
||||
|
||||
```sh
|
||||
$ dep ensure -add github.com/Azure/azure-sdk-for-go
|
||||
```
|
||||
|
||||
If you need to install Go, follow [the official instructions](https://golang.org/dl/).
|
||||
|
||||
## Use
|
||||
|
||||
For many more scenarios and examples see
|
||||
[Azure-Samples/azure-sdk-for-go-samples][samples_repo].
|
||||
|
||||
Apply the following general steps to use packages in this repo. For more on
|
||||
authentication and the `Authorizer` interface see [the next
|
||||
section](#authentication).
|
||||
|
||||
1. Import a package from the [services][services_dir] directory.
|
||||
2. Create and authenticate a client with a `New*Client` func, e.g.
|
||||
`c := compute.NewVirtualMachinesClient(...)`.
|
||||
3. Invoke API methods using the client, e.g.
|
||||
`res, err := c.CreateOrUpdate(...)`.
|
||||
4. Handle responses and errors.
|
||||
|
||||
[services_dir]: https://github.com/Azure/azure-sdk-for-go/tree/master/services
|
||||
|
||||
For example, to create a new virtual network (substitute your own values for
|
||||
strings in angle brackets):
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network"
|
||||
|
||||
"github.com/Azure/go-autorest/autorest/azure/auth"
|
||||
"github.com/Azure/go-autorest/autorest/to"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// create a VirtualNetworks client
|
||||
vnetClient := network.NewVirtualNetworksClient("<subscriptionID>")
|
||||
|
||||
// create an authorizer from env vars or Azure Managed Service Idenity
|
||||
authorizer, err := auth.NewAuthorizerFromEnvironment()
|
||||
if err == nil {
|
||||
vnetClient.Authorizer = authorizer
|
||||
}
|
||||
|
||||
// call the VirtualNetworks CreateOrUpdate API
|
||||
vnetClient.CreateOrUpdate(context.Background(),
|
||||
"<resourceGroupName>",
|
||||
"<vnetName>",
|
||||
network.VirtualNetwork{
|
||||
Location: to.StringPtr("<azureRegion>"),
|
||||
VirtualNetworkPropertiesFormat: &network.VirtualNetworkPropertiesFormat{
|
||||
AddressSpace: &network.AddressSpace{
|
||||
AddressPrefixes: &[]string{"10.0.0.0/8"},
|
||||
},
|
||||
Subnets: &[]network.Subnet{
|
||||
{
|
||||
Name: to.StringPtr("<subnet1Name>"),
|
||||
SubnetPropertiesFormat: &network.SubnetPropertiesFormat{
|
||||
AddressPrefix: to.StringPtr("10.0.0.0/16"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: to.StringPtr("<subnet2Name>"),
|
||||
SubnetPropertiesFormat: &network.SubnetPropertiesFormat{
|
||||
AddressPrefix: to.StringPtr("10.1.0.0/16"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
Typical SDK operations must be authenticated and authorized. The _Authorizer_
|
||||
interface allows use of any auth style in requests, such as inserting an OAuth2
|
||||
Authorization header and bearer token received from Azure AD.
|
||||
|
||||
The SDK itself provides a simple way to get an authorizer which first checks
|
||||
for OAuth client credentials in environment variables and then falls back to
|
||||
Azure's [Managed Service Identity]() when available, e.g. when on an Azure
|
||||
VM. The following snippet from [the previous section](#use) demonstrates
|
||||
this helper.
|
||||
|
||||
```go
|
||||
import github.com/Azure/go-autorest/autorest/azure/auth
|
||||
|
||||
// create a VirtualNetworks client
|
||||
vnetClient := network.NewVirtualNetworksClient("<subscriptionID>")
|
||||
|
||||
// create an authorizer from env vars or Azure Managed Service Idenity
|
||||
authorizer, err := auth.NewAuthorizerFromEnvironment()
|
||||
if err == nil {
|
||||
vnetClient.Authorizer = authorizer
|
||||
}
|
||||
|
||||
// call the VirtualNetworks CreateOrUpdate API
|
||||
vnetClient.CreateOrUpdate(context.Background(),
|
||||
// ...
|
||||
```
|
||||
|
||||
The following environment variables help determine authentication configuration:
|
||||
|
||||
- `AZURE_ENVIRONMENT`: Specifies the Azure Environment to use. If not set, it
|
||||
defaults to `AzurePublicCloud`. Not applicable to authentication with Managed
|
||||
Service Identity (MSI).
|
||||
- `AZURE_AD_RESOURCE`: Specifies the AAD resource ID to use. If not set, it
|
||||
defaults to `ResourceManagerEndpoint` for operations with Azure Resource
|
||||
Manager. You can also choose an alternate resource programatically with
|
||||
`auth.NewAuthorizerFromEnvironmentWithResource(resource string)`.
|
||||
|
||||
### More Authentication Details
|
||||
|
||||
The previous is the first and most recommended of several authentication
|
||||
options offered by the SDK because it allows seamless use of both service
|
||||
principals and [Azure Managed Service Identity][]. Other options are listed
|
||||
below.
|
||||
|
||||
> Note: If you need to create a new service principal, run `az ad sp create-for-rbac -n "<app_name>"` in the
|
||||
> [azure-cli](https://github.com/Azure/azure-cli). See [these
|
||||
> docs](https://docs.microsoft.com/cli/azure/create-an-azure-service-principal-azure-cli?view=azure-cli-latest)
|
||||
> for more info. Copy the new principal's ID, secret, and tenant ID for use in
|
||||
> your app, or consider the `--sdk-auth` parameter for serialized output.
|
||||
|
||||
[azure managed service identity]: https://docs.microsoft.com/en-us/azure/active-directory/msi-overview
|
||||
|
||||
- The `auth.NewAuthorizerFromEnvironment()` described above creates an authorizer
|
||||
from the first available of the following configuration:
|
||||
|
||||
1. **Client Credentials**: Azure AD Application ID and Secret.
|
||||
|
||||
- `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate.
|
||||
- `AZURE_CLIENT_ID`: Specifies the app client ID to use.
|
||||
- `AZURE_CLIENT_SECRET`: Specifies the app secret to use.
|
||||
|
||||
2. **Client Certificate**: Azure AD Application ID and X.509 Certificate.
|
||||
|
||||
- `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate.
|
||||
- `AZURE_CLIENT_ID`: Specifies the app client ID to use.
|
||||
- `AZURE_CERTIFICATE_PATH`: Specifies the certificate Path to use.
|
||||
- `AZURE_CERTIFICATE_PASSWORD`: Specifies the certificate password to use.
|
||||
|
||||
3. **Resource Owner Password**: Azure AD User and Password. This grant type is *not
|
||||
recommended*, use device login instead if you need interactive login.
|
||||
|
||||
- `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate.
|
||||
- `AZURE_CLIENT_ID`: Specifies the app client ID to use.
|
||||
- `AZURE_USERNAME`: Specifies the username to use.
|
||||
- `AZURE_PASSWORD`: Specifies the password to use.
|
||||
|
||||
4. **Azure Managed Service Identity**: Delegate credential management to the
|
||||
platform. Requires that code is running in Azure, e.g. on a VM. All
|
||||
configuration is handled by Azure. See [Azure Managed Service
|
||||
Identity](https://docs.microsoft.com/en-us/azure/active-directory/msi-overview)
|
||||
for more details.
|
||||
|
||||
- The `auth.NewAuthorizerFromFile()` method creates an authorizer using
|
||||
credentials from an auth file created by the [Azure CLI][]. Follow these
|
||||
steps to utilize:
|
||||
|
||||
1. Create a service principal and output an auth file using `az ad sp create-for-rbac --sdk-auth > client_credentials.json`.
|
||||
2. Set environment variable `AZURE_AUTH_LOCATION` to the path of the saved
|
||||
output file.
|
||||
3. Use the authorizer returned by `auth.NewAuthorizerFromFile()` in your
|
||||
client as described above.
|
||||
|
||||
[azure cli]: https://github.com/Azure/azure-cli
|
||||
|
||||
- Finally, you can use OAuth's [Device Flow][] by calling
|
||||
`auth.NewDeviceFlowConfig()` and extracting the Authorizer as follows:
|
||||
|
||||
```go
|
||||
config := auth.NewDeviceFlowConfig(clientID, tenantID)
|
||||
a, err = config.Authorizer()
|
||||
```
|
||||
|
||||
[device flow]: https://oauth.net/2/device-flow/
|
||||
|
||||
# Versioning
|
||||
|
||||
azure-sdk-for-go provides at least a basic Go binding for every Azure API. To
|
||||
provide maximum flexibility to users, the SDK even includes previous versions of
|
||||
Azure APIs which are still in use. This enables us to support users of the
|
||||
most updated Azure datacenters, regional datacenters with earlier APIs, and
|
||||
even on-premises installations of Azure Stack.
|
||||
|
||||
**SDK versions** apply globally and are tracked by git
|
||||
[tags](https://github.com/Azure/azure-sdk-for-go/tags). These are in x.y.z form
|
||||
and generally adhere to [semantic versioning](https://semver.org) specifications.
|
||||
|
||||
**Service API versions** are generally represented by a date string and are
|
||||
tracked by offering separate packages for each version. For example, to choose the
|
||||
latest API versions for Compute and Network, use the following imports:
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute"
|
||||
"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network"
|
||||
)
|
||||
```
|
||||
|
||||
Occasionally service-side changes require major changes to existing versions.
|
||||
These cases are noted in the changelog.
|
||||
|
||||
All avilable services and versions are listed under the `services/` path in
|
||||
this repo and in [GoDoc][services_godoc]. Run `find ./services -type d -mindepth 3` to list all available service packages.
|
||||
|
||||
[services_godoc]: https://godoc.org/github.com/Azure/azure-sdk-for-go/services
|
||||
|
||||
### Profiles
|
||||
|
||||
Azure **API profiles** specify subsets of Azure APIs and versions. Profiles can provide:
|
||||
|
||||
- **stability** for your application by locking to specific API versions; and/or
|
||||
- **compatibility** for your application with Azure Stack and regional Azure datacenters.
|
||||
|
||||
In the Go SDK, profiles are available under the `profiles/` path and their
|
||||
component API versions are aliases to the true service package under
|
||||
`services/`. You can use them as follows:
|
||||
|
||||
```go
|
||||
import "github.com/Azure/azure-sdk-for-go/profiles/2017-03-09/compute/mgmt/compute"
|
||||
import "github.com/Azure/azure-sdk-for-go/profiles/2017-03-09/network/mgmt/network"
|
||||
import "github.com/Azure/azure-sdk-for-go/profiles/2017-03-09/storage/mgmt/storage"
|
||||
```
|
||||
|
||||
The 2017-03-09 profile is the only one currently available and is for use in
|
||||
hybrid Azure and Azure Stack environments. More profiles are under development.
|
||||
|
||||
In addition to versioned profiles, we also provide two special profiles
|
||||
`latest` and `preview`. These _always_ include the most recent respective stable or
|
||||
preview API versions for each service, even when updating them to do so causes
|
||||
breaking changes. That is, these do _not_ adhere to semantic versioning rules.
|
||||
|
||||
The `latest` and `preview` profiles can help you stay up to date with API
|
||||
updates as you build applications. Since they are by definition not stable,
|
||||
however, they **should not** be used in production apps. Instead, choose the
|
||||
latest specific API version (or an older one if necessary) from the `services/`
|
||||
path.
|
||||
|
||||
As an example, to automatically use the most recent Compute APIs, use one of
|
||||
the following imports:
|
||||
|
||||
```go
|
||||
import "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute"
|
||||
import "github.com/Azure/azure-sdk-for-go/profiles/preview/compute/mgmt/compute"
|
||||
```
|
||||
|
||||
## Inspecting and Debugging
|
||||
|
||||
All clients implement some handy hooks to help inspect the underlying requests being made to Azure.
|
||||
|
||||
- `RequestInspector`: View and manipulate the go `http.Request` before it's sent
|
||||
- `ResponseInspector`: View the `http.Response` received
|
||||
|
||||
Here is an example of how these can be used with `net/http/httputil` to see requests and responses.
|
||||
|
||||
```go
|
||||
vnetClient := network.NewVirtualNetworksClient("<subscriptionID>")
|
||||
vnetClient.RequestInspector = LogRequest()
|
||||
vnetClient.ResponseInspector = LogResponse()
|
||||
|
||||
...
|
||||
|
||||
func LogRequest() autorest.PrepareDecorator {
|
||||
return func(p autorest.Preparer) autorest.Preparer {
|
||||
return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) {
|
||||
r, err := p.Prepare(r)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
dump, _ := httputil.DumpRequestOut(r, true)
|
||||
log.Println(string(dump))
|
||||
return r, err
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func LogResponse() autorest.RespondDecorator {
|
||||
return func(p autorest.Responder) autorest.Responder {
|
||||
return autorest.ResponderFunc(func(r *http.Response) error {
|
||||
err := p.Respond(r)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
dump, _ := httputil.DumpResponse(r, true)
|
||||
log.Println(string(dump))
|
||||
return err
|
||||
})
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
# Resources
|
||||
|
||||
- SDK docs are at [godoc.org](https://godoc.org/github.com/Azure/azure-sdk-for-go/).
|
||||
- SDK samples are at [Azure-Samples/azure-sdk-for-go-samples](https://github.com/Azure-Samples/azure-sdk-for-go-samples).
|
||||
- SDK notifications are published via the [Azure update feed](https://azure.microsoft.com/updates/).
|
||||
- Azure API docs are at [docs.microsoft.com/rest/api](https://docs.microsoft.com/rest/api/).
|
||||
- General Azure docs are at [docs.microsoft.com/azure](https://docs.microsoft.com/azure).
|
||||
|
||||
## License
|
||||
|
||||
Apache 2.0, see [LICENSE](./LICENSE).
|
||||
|
||||
## Contribute
|
||||
|
||||
See [CONTRIBUTING.md](./CONTRIBUTING.md).
|
||||
|
||||
[samples_repo]: https://github.com/Azure-Samples/azure-sdk-for-go-samples
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
/*
|
||||
Package sdk provides Go packages for managing and using Azure services.
|
||||
|
||||
GitHub repo: https://github.com/Azure/azure-sdk-for-go
|
||||
|
||||
Official documentation: https://docs.microsoft.com/go/azure
|
||||
|
||||
API reference: https://godoc.org/github.com/Azure/azure-sdk-for-go
|
||||
|
||||
Samples: https://github.com/Azure-Samples/azure-sdk-for-go-samples
|
||||
*/
|
||||
package sdk
|
||||
|
||||
// Copyright (c) Microsoft and contributors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
//
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
# Generate code
|
||||
|
||||
## Generate SDK packages
|
||||
|
||||
### Generate an Azure-SDK-for-Go service package
|
||||
|
||||
1. [Install AutoRest](https://github.com/Azure/autorest#installing-autorest).
|
||||
|
||||
1. Call autorest with the following arguments...
|
||||
|
||||
``` cmd
|
||||
autorest path/to/readme/file --go --go-sdk-folder=<your/gopath/src/github.com/Azure/azure-sdk-for-go> --package-version=<version> --user-agent=<Azure-SDK-For-Go/version services> [--tag=choose/a/tag/in/the/readme/file]
|
||||
```
|
||||
|
||||
For example...
|
||||
|
||||
``` cmd
|
||||
autorest C:/azure-rest-api-specs/specification/advisor/resource-manager/readme.md --go --go-sdk-folder=C:/goWorkspace/src/github.com/Azure/azure-sdk-for-go --tag=package-2016-07-preview --package-version=v11.2.0-beta --user-agent='Azure-SDK-For-Go/v11.2.0-beta services'
|
||||
```
|
||||
|
||||
- If you are looking to generate code based on a specific swagger file, you can replace `path/to/readme/file` with `--input-file=path/to/swagger/file`.
|
||||
- If the readme file you want to use as input does not have golang tags yet, you can call autorest like this...
|
||||
|
||||
``` cmd
|
||||
autorest path/to/readme/file --go --license-header=<MICROSOFT_APACHE_NO_VERSION> --namespace=<packageName> --output-folder=<your/gopath/src/github.com/Azure/azure-sdk-for-go/services/serviceName/mgmt/APIversion/packageName> --package-version=<version> --user-agent=<Azure-SDK-For-Go/version services> --clear-output-folder --can-clear-output-folder --tag=<choose/a/tag/in/the/readme/file>
|
||||
```
|
||||
|
||||
For example...
|
||||
|
||||
``` cmd
|
||||
autorest --input-file=https://raw.githubusercontent.com/Azure/azure-rest-api-specs/current/specification/network/resource-manager/Microsoft.Network/2017-10-01/loadBalancer.json --go --license-header=MICROSOFT_APACHE_NO_VERSION --namespace=lb --output-folder=C:/goWorkspace/src/github.com/Azure/azure-sdk-for-go/services/network/mgmt/2017-09-01/network/lb --package-version=v11.2.0-beta --clear-output-folder --can-clear-output-folder
|
||||
```
|
||||
|
||||
1. Run `go fmt` on the generated package folder.
|
||||
|
||||
1. To make sure the SDK has been generated correctly, also run `golint`, `go build` and `go vet`.
|
||||
|
||||
### Generate Azure SDK for Go service packages in bulk
|
||||
|
||||
All services, all API versions.
|
||||
|
||||
1. [Install AutoRest](https://github.com/Azure/autorest#installing-autorest).
|
||||
|
||||
This repo contains a tool to generate the SDK, which depends on the golang tags from the readme files in the Azure REST API specs repo. The tool assumes you have an [Azure REST API specs](https://github.com/Azure/azure-rest-api-specs) clone, and [golint](https://github.com/golang/lint) is installed.
|
||||
|
||||
1. `cd tools/generator`
|
||||
|
||||
1. `go install`
|
||||
|
||||
1. Add `GOPATH/bin` to your `PATH`, in case it was not already there.
|
||||
|
||||
1. Call the generator tool like this...
|
||||
|
||||
``` cmd
|
||||
generator –r [–v] [–l=logs/output/folder] –version=<version> path/to/your/swagger/repo/clone
|
||||
```
|
||||
|
||||
For example...
|
||||
|
||||
``` cmd
|
||||
generator –r –v –l=temp –version=v11.2.0-beta C:/azure-rest-api-specs
|
||||
```
|
||||
|
||||
The generator tool already runs `go fmt`, `golint`, `go build` and `go vet`; so running them is not necessary.
|
||||
|
||||
#### Use the generator tool to generate a single package
|
||||
|
||||
1. Just call the generator tool specifying the service to be generated in the input folder.
|
||||
|
||||
``` cmd
|
||||
generator –r [–v] [–l=logs/output/folder] –version=<version> path/to/your/swagger/repo/clone/specification/service
|
||||
```
|
||||
|
||||
For example...
|
||||
|
||||
``` cmd
|
||||
generator –r –v –l=temp –version=v11.2.0-beta C:/azure-rest-api-specs/specification/network
|
||||
```
|
||||
|
||||
## Include a new package in the SDK
|
||||
|
||||
1. Submit a pull request to the Azure REST API specs repo adding the golang tags for the service and API versions in the service readme file, if the needed tags are not there yet.
|
||||
|
||||
1. Once the tags are available in the Azure REST API specs repo, generate the SDK.
|
||||
|
||||
1. In the changelog file, document the new generated SDK. Include the [autorest.go extension](https://github.com/Azure/autorest.go) version used, and the Azure REST API specs repo commit from where the SDK was generated.
|
||||
|
||||
1. Install [dep](https://github.com/golang/dep).
|
||||
|
||||
1. Run `dep ensure`.
|
||||
|
||||
1. Submit a pull request to this repo, and we will review it.
|
||||
|
||||
## Generate Azure SDK for Go profiles
|
||||
|
||||
Take a look into the [profile generator documentation](tools/profileBuilder)
|
||||
Loaded 100 of 5040 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user