Compare commits

..
Author SHA1 Message Date
Nick Craig-Wood 53bbe891d3 lib/rest: retry errors due to failed JSON decoding - fixes #8383
These errors are almost always caused by an overloaded proxy somewhere
in the chain returning HTML so they are worth retrying.
2025-02-14 14:47:09 +00:00
Nick Craig-Wood 4aaef7472d docs: add FileLu as sponsors and tidy sponsor logos 2025-02-14 12:33:26 +00:00
245 changed files with 1907 additions and 3291 deletions

No files matched your search

+7 -1
View File
@@ -26,7 +26,7 @@ jobs:
strategy:
fail-fast: false
matrix:
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.23']
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.22', 'go1.23']
include:
- job_name: linux
@@ -80,6 +80,12 @@ jobs:
compile_all: true
deploy: true
- job_name: go1.22
os: ubuntu-latest
go: '1.22'
quicktest: true
racequicktest: true
- job_name: go1.23
os: ubuntu-latest
go: '1.23'
@@ -7,43 +7,39 @@ name: Release Build for Docker Plugin
on:
release:
types: [published]
workflow_dispatch:
inputs:
manual:
description: Manual run (bypass default conditions)
type: boolean
default: true
jobs:
build_docker_volume_plugin:
if: inputs.manual || github.repository == 'rclone/rclone'
name: Build docker plugin job
runs-on: ubuntu-latest
steps:
- name: Free some space
shell: bash
run: |
df -h .
# Remove android SDK
sudo rm -rf /usr/local/lib/android || true
# Remove .net runtime
sudo rm -rf /usr/share/dotnet || true
df -h .
- name: Checkout master
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build and publish docker plugin
shell: bash
run: |
VER=${GITHUB_REF#refs/tags/}
PLUGIN_USER=rclone
docker login --username ${{ secrets.DOCKER_HUB_USER }} \
--password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
export PLUGIN_USER PLUGIN_ARCH
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
done
make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}
build_docker_volume_plugin:
if: github.repository == 'rclone/rclone'
needs: build
runs-on: ubuntu-latest
name: Build docker plugin job
steps:
- name: Free some space
shell: bash
run: |
df -h .
# Remove android SDK
sudo rm -rf /usr/local/lib/android || true
# Remove .net runtime
sudo rm -rf /usr/share/dotnet || true
df -h .
- name: Checkout master
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build and publish docker plugin
shell: bash
run: |
VER=${GITHUB_REF#refs/tags/}
PLUGIN_USER=rclone
docker login --username ${{ secrets.DOCKER_HUB_USER }} \
--password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
export PLUGIN_USER PLUGIN_ARCH
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
done
make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest
make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}
+1 -1
View File
@@ -131,8 +131,8 @@ Now
* git co ${BASE_TAG}-stable
* git cherry-pick any fixes
* make startstable
* Do the steps as above
* make startstable
* git co master
* `#` cherry pick the changes to the changelog - check the diff to make sure it is correct
* git checkout ${BASE_TAG}-stable docs/content/changelog.md
+7 -26
View File
@@ -19,7 +19,6 @@ import (
"net/url"
"os"
"path"
"slices"
"sort"
"strconv"
"strings"
@@ -657,33 +656,19 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
var storageErr *azcore.ResponseError
if errors.As(err, &storageErr) {
// General errors from:
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
// Blob specific errors from:
// https://learn.microsoft.com/en-us/rest/api/storageservices/blob-service-error-codes
// FIXME interpret special errors - more to do here
if storageErr, ok := err.(*azcore.ResponseError); ok {
switch storageErr.ErrorCode {
case "InvalidBlobOrBlock":
// These errors happen sometimes in multipart uploads
// because of block concurrency issues
return true, err
case "InternalError":
// The server encountered an internal error. Please retry the request.
return true, err
case "OperationTimedOut":
// The operation could not be completed within the permitted time. The
// operation may or may not have succeeded on the server side. Please query
// the server state before retrying the operation.
return true, err
case "ServerBusy":
// The server is currently unable to receive requests. Please retry your
// request.
return true, err
}
statusCode := storageErr.StatusCode
if slices.Contains(retryErrorCodes, statusCode) {
return true, err
for _, e := range retryErrorCodes {
if statusCode == e {
return true, err
}
}
}
return fserrors.ShouldRetry(err), err
@@ -1887,11 +1872,7 @@ func (f *Fs) copySinglepart(ctx context.Context, remote, dstContainer, dstPath s
pollTime := 100 * time.Millisecond
for copyStatus != nil && string(*copyStatus) == string(container.CopyStatusTypePending) {
time.Sleep(pollTime)
var getMetadata blob.GetPropertiesResponse
err = f.pacer.Call(func() (bool, error) {
getMetadata, err = dstBlobSVC.GetProperties(ctx, &getOptions)
return f.shouldRetry(ctx, err)
})
getMetadata, err := dstBlobSVC.GetProperties(ctx, &getOptions)
if err != nil {
return nil, err
}
@@ -61,7 +61,7 @@ const chars = "abcdefghijklmnopqrstuvwzyxABCDEFGHIJKLMNOPQRSTUVWZYX"
func randomString(charCount int) string {
strBldr := strings.Builder{}
for range charCount {
for i := 0; i < charCount; i++ {
randPos := rand.Int63n(52)
strBldr.WriteByte(chars[randPos])
}
+4 -4
View File
@@ -130,10 +130,10 @@ type AuthorizeAccountResponse struct {
AbsoluteMinimumPartSize int `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file.
AccountID string `json:"accountId"` // The identifier for the account.
Allowed struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
BucketID string `json:"bucketId"` // When present, access is restricted to one bucket.
BucketName string `json:"bucketName"` // When present, name of bucket - may be empty
Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.
NamePrefix any `json:"namePrefix"` // When present, access is restricted to files whose names start with the prefix
BucketID string `json:"bucketId"` // When present, access is restricted to one bucket.
BucketName string `json:"bucketName"` // When present, name of bucket - may be empty
Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.
NamePrefix interface{} `json:"namePrefix"` // When present, access is restricted to files whose names start with the prefix
} `json:"allowed"`
APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files.
AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
+12 -8
View File
@@ -16,7 +16,6 @@ import (
"io"
"net/http"
"path"
"slices"
"strconv"
"strings"
"sync"
@@ -590,7 +589,12 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
// hasPermission returns if the current AuthorizationToken has the selected permission
func (f *Fs) hasPermission(permission string) bool {
return slices.Contains(f.info.Allowed.Capabilities, permission)
for _, capability := range f.info.Allowed.Capabilities {
if capability == permission {
return true
}
}
return false
}
// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
@@ -1271,7 +1275,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
toBeDeleted := make(chan *api.File, f.ci.Transfers)
var wg sync.WaitGroup
wg.Add(f.ci.Transfers)
for range f.ci.Transfers {
for i := 0; i < f.ci.Transfers; i++ {
go func() {
defer wg.Done()
for object := range toBeDeleted {
@@ -1935,7 +1939,7 @@ func init() {
// urlEncode encodes in with % encoding
func urlEncode(in string) string {
var out bytes.Buffer
for i := range len(in) {
for i := 0; i < len(in); i++ {
c := in[i]
if noNeedToEncode[c] {
_ = out.WriteByte(c)
@@ -2256,7 +2260,7 @@ See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
},
}
func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
var newRule api.LifecycleRule
if daysStr := opt["daysFromHidingToDeleting"]; daysStr != "" {
days, err := strconv.Atoi(daysStr)
@@ -2345,7 +2349,7 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
},
}
func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
maxAge := defaultMaxAge
if opt["max-age"] != "" {
maxAge, err = fs.ParseDuration(opt["max-age"])
@@ -2368,7 +2372,7 @@ it would do.
`,
}
func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
return nil, f.cleanUp(ctx, true, false, 0)
}
@@ -2387,7 +2391,7 @@ var commandHelp = []fs.CommandHelp{
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "lifecycle":
return f.lifecycleCommand(ctx, name, arg, opt)
+5 -2
View File
@@ -478,14 +478,17 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
remaining = up.size
)
g.SetLimit(up.f.opt.UploadConcurrency)
for part := range up.parts {
for part := 0; part < up.parts; part++ {
// Fail fast, in case an errgroup managed function returns an error
// gCtx is cancelled. There is no point in copying all the other parts.
if gCtx.Err() != nil {
break
}
reqSize := min(remaining, up.chunkSize)
reqSize := remaining
if reqSize >= up.chunkSize {
reqSize = up.chunkSize
}
part := part // for the closure
g.Go(func() (err error) {
+7 -3
View File
@@ -237,8 +237,8 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
return claims, nil
}
func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]any {
signingHeaders := map[string]any{
func getSigningHeaders(boxConfig *api.ConfigJSON) map[string]interface{} {
signingHeaders := map[string]interface{}{
"kid": boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
}
return signingHeaders
@@ -1343,8 +1343,12 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
nextStreamPosition = streamPosition
for {
limit := f.opt.ListChunk
// box only allows a max of 500 events
limit := min(f.opt.ListChunk, 500)
if limit > 500 {
limit = 500
}
opts := rest.Opts{
Method: "GET",
+6 -3
View File
@@ -105,7 +105,7 @@ func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api
const defaultDelay = 10
var tries int
outer:
for tries = range maxTries {
for tries = 0; tries < maxTries; tries++ {
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
if err != nil {
@@ -203,7 +203,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
errs := make(chan error, 1)
var wg sync.WaitGroup
outer:
for part := range session.TotalParts {
for part := 0; part < session.TotalParts; part++ {
// Check any errors
select {
case err = <-errs:
@@ -211,7 +211,10 @@ outer:
default:
}
reqSize := min(remaining, chunkSize)
reqSize := remaining
if reqSize >= chunkSize {
reqSize = chunkSize
}
// Make a block of memory
buf := make([]byte, reqSize)
+4 -4
View File
@@ -1092,7 +1092,7 @@ func (f *Fs) recurse(ctx context.Context, dir string, list *walk.ListRHelper) er
return err
}
for i := range entries {
for i := 0; i < len(entries); i++ {
innerDir, ok := entries[i].(fs.Directory)
if ok {
err := f.recurse(ctx, innerDir.Remote(), list)
@@ -1428,7 +1428,7 @@ func (f *Fs) cacheReader(u io.Reader, src fs.ObjectInfo, originalRead func(inn i
}()
// wait until both are done
for range 2 {
for c := 0; c < 2; c++ {
<-done
}
}
@@ -1753,7 +1753,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
}
// Stats returns stats about the cache storage
func (f *Fs) Stats() (map[string]map[string]any, error) {
func (f *Fs) Stats() (map[string]map[string]interface{}, error) {
return f.cache.Stats()
}
@@ -1933,7 +1933,7 @@ var commandHelp = []fs.CommandHelp{
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (any, error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
switch name {
case "stats":
return f.Stats()
+8 -8
View File
@@ -360,7 +360,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
require.NoError(t, err)
require.Equal(t, int64(len(checkSample)), o.Size())
for i := range checkSample {
for i := 0; i < len(checkSample); i++ {
require.Equal(t, testData[i], checkSample[i])
}
}
@@ -387,7 +387,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
readData, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
require.NoError(t, err)
for i := range readData {
for i := 0; i < len(readData); i++ {
require.Equalf(t, testData[i], readData[i], "at byte %v", i)
}
}
@@ -688,7 +688,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
co, ok := o.(*cache.Object)
require.True(t, ok)
for i := range 4 { // read first 4
for i := 0; i < 4; i++ { // read first 4
_ = runInstance.readDataFromObj(t, co, chunkSize*int64(i), chunkSize*int64(i+1), false)
}
cfs.CleanUpCache(true)
@@ -971,7 +971,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
f, err := os.CreateTemp("", "rclonecache-tempfile")
require.NoError(t, err)
for range int(cnt) {
for i := 0; i < int(cnt); i++ {
data := randStringBytes(int(chunk))
_, _ = f.Write(data)
}
@@ -1085,9 +1085,9 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
return err
}
func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]any, error) {
func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error) {
var err error
var l []any
var l []interface{}
var list fs.DirEntries
list, err = f.List(context.Background(), remote)
for _, ll := range list {
@@ -1215,7 +1215,7 @@ func (r *run) listenForBackgroundUpload(t *testing.T, f fs.Fs, remote string) ch
var err error
var state cache.BackgroundUploadState
for range 2 {
for i := 0; i < 2; i++ {
select {
case state = <-buCh:
// continue
@@ -1293,7 +1293,7 @@ func (r *run) completeAllBackgroundUploads(t *testing.T, f fs.Fs, lastRemote str
func (r *run) retryBlock(block func() error, maxRetries int, rate time.Duration) error {
var err error
for range maxRetries {
for i := 0; i < maxRetries; i++ {
err = block()
if err == nil {
return nil
+1 -1
View File
@@ -162,7 +162,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
randInstance := rand.New(rand.NewSource(time.Now().Unix()))
lastFile := ""
for i := range totalFiles {
for i := 0; i < totalFiles; i++ {
size := int64(randInstance.Intn(maxSize-minSize) + minSize)
testReader := runInstance.randomReader(t, size)
remote := "test/" + strconv.Itoa(i) + ".bin"
+2 -2
View File
@@ -182,7 +182,7 @@ func (r *Handle) queueOffset(offset int64) {
}
}
for i := range r.workers {
for i := 0; i < r.workers; i++ {
o := r.preloadOffset + int64(r.cacheFs().opt.ChunkSize)*int64(i)
if o < 0 || o >= r.cachedObject.Size() {
continue
@@ -222,7 +222,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
if !found {
// we're gonna give the workers a chance to pickup the chunk
// and retry a couple of times
for i := range r.cacheFs().opt.ReadRetries * 8 {
for i := 0; i < r.cacheFs().opt.ReadRetries*8; i++ {
data, err = r.storage().GetChunk(r.cachedObject, chunkStart)
if err == nil {
found = true
+4 -4
View File
@@ -209,7 +209,7 @@ func (p *plexConnector) authenticate() error {
if err != nil {
return err
}
var data map[string]any
var data map[string]interface{}
err = json.NewDecoder(resp.Body).Decode(&data)
if err != nil {
return fmt.Errorf("failed to obtain token: %w", err)
@@ -273,11 +273,11 @@ func (p *plexConnector) isPlaying(co *Object) bool {
}
// adapted from: https://stackoverflow.com/a/28878037 (credit)
func get(m any, path ...any) (any, bool) {
func get(m interface{}, path ...interface{}) (interface{}, bool) {
for _, p := range path {
switch idx := p.(type) {
case string:
if mm, ok := m.(map[string]any); ok {
if mm, ok := m.(map[string]interface{}); ok {
if val, found := mm[idx]; found {
m = val
continue
@@ -285,7 +285,7 @@ func get(m any, path ...any) (any, bool) {
}
return nil, false
case int:
if mm, ok := m.([]any); ok {
if mm, ok := m.([]interface{}); ok {
if len(mm) > idx {
m = mm[idx]
continue
+5 -6
View File
@@ -18,7 +18,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/walk"
bolt "go.etcd.io/bbolt"
"go.etcd.io/bbolt/errors"
)
// Constants
@@ -598,7 +597,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
})
if err != nil {
if err == errors.ErrDatabaseNotOpen {
if err == bolt.ErrDatabaseNotOpen {
// we're likely a late janitor and we need to end quietly as there's no guarantee of what exists anymore
return
}
@@ -607,16 +606,16 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) {
}
// Stats returns a go map with the stats key values
func (b *Persistent) Stats() (map[string]map[string]any, error) {
r := make(map[string]map[string]any)
r["data"] = make(map[string]any)
func (b *Persistent) Stats() (map[string]map[string]interface{}, error) {
r := make(map[string]map[string]interface{})
r["data"] = make(map[string]interface{})
r["data"]["oldest-ts"] = time.Now()
r["data"]["oldest-file"] = ""
r["data"]["newest-ts"] = time.Now()
r["data"]["newest-file"] = ""
r["data"]["total-chunks"] = 0
r["data"]["total-size"] = int64(0)
r["files"] = make(map[string]any)
r["files"] = make(map[string]interface{})
r["files"]["oldest-ts"] = time.Now()
r["files"]["oldest-name"] = ""
r["files"]["newest-ts"] = time.Now()
+10 -4
View File
@@ -632,7 +632,7 @@ func (f *Fs) parseChunkName(filePath string) (parentPath string, chunkNo int, ct
// forbidChunk prints error message or raises error if file is chunk.
// First argument sets log prefix, use `false` to suppress message.
func (f *Fs) forbidChunk(o any, filePath string) error {
func (f *Fs) forbidChunk(o interface{}, filePath string) error {
if parentPath, _, _, _ := f.parseChunkName(filePath); parentPath != "" {
if f.opt.FailHard {
return fmt.Errorf("chunk overlap with %q", parentPath)
@@ -680,7 +680,7 @@ func (f *Fs) newXactID(ctx context.Context, filePath string) (xactID string, err
circleSec := unixSec % closestPrimeZzzzSeconds
first4chars := strconv.FormatInt(circleSec, 36)
for range maxTransactionProbes {
for tries := 0; tries < maxTransactionProbes; tries++ {
f.xactIDMutex.Lock()
randomness := f.xactIDRand.Int63n(maxTwoBase36Digits + 1)
f.xactIDMutex.Unlock()
@@ -1189,7 +1189,10 @@ func (f *Fs) put(
}
tempRemote := f.makeChunkName(baseRemote, c.chunkNo, "", xactID)
size := min(c.sizeLeft, c.chunkSize)
size := c.sizeLeft
if size > c.chunkSize {
size = c.chunkSize
}
savedReadCount := c.readCount
// If a single chunk is expected, avoid the extra rename operation
@@ -1474,7 +1477,10 @@ func (c *chunkingReader) dummyRead(in io.Reader, size int64) error {
const bufLen = 1048576 // 1 MiB
buf := make([]byte, bufLen)
for size > 0 {
n := min(size, bufLen)
n := size
if n > bufLen {
n = bufLen
}
if _, err := io.ReadFull(in, buf[0:n]); err != nil {
return err
}
+1 -1
View File
@@ -40,7 +40,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
})
}
type settings map[string]any
type settings map[string]interface{}
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
+4 -4
View File
@@ -192,7 +192,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
dirNameEncrypt: dirNameEncrypt,
encryptedSuffix: ".bin",
}
c.buffers.New = func() any {
c.buffers.New = func() interface{} {
return new([blockSize]byte)
}
err := c.Key(password, salt)
@@ -336,7 +336,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
_, _ = result.WriteString(strconv.Itoa(dir) + ".")
// but we'll augment it with the nameKey for real calculation
for i := range len(c.nameKey) {
for i := 0; i < len(c.nameKey); i++ {
dir += int(c.nameKey[i])
}
@@ -418,7 +418,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
}
// add the nameKey to get the real rotate distance
for i := range len(c.nameKey) {
for i := 0; i < len(c.nameKey); i++ {
dir += int(c.nameKey[i])
}
@@ -664,7 +664,7 @@ func (n *nonce) increment() {
// add a uint64 to the nonce
func (n *nonce) add(x uint64) {
carry := uint16(0)
for i := range 8 {
for i := 0; i < 8; i++ {
digit := (*n)[i]
xDigit := byte(x)
x >>= 8
+5 -2
View File
@@ -1307,7 +1307,10 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
open := func(ctx context.Context, underlyingOffset, underlyingLimit int64) (io.ReadCloser, error) {
end := len(ciphertext)
if underlyingLimit >= 0 {
end = min(int(underlyingOffset+underlyingLimit), len(ciphertext))
end = int(underlyingOffset + underlyingLimit)
if end > len(ciphertext) {
end = len(ciphertext)
}
}
reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
return reader, nil
@@ -1487,7 +1490,7 @@ func TestDecrypterRead(t *testing.T) {
assert.NoError(t, err)
// Test truncating the file at each possible point
for i := range len(file16) - 1 {
for i := 0; i < len(file16)-1; i++ {
what := fmt.Sprintf("truncating to %d/%d", i, len(file16))
cd := newCloseDetector(bytes.NewBuffer(file16[:i]))
fh, err := c.newDecrypter(cd)
+1 -1
View File
@@ -924,7 +924,7 @@ Usage Example:
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "decode":
out := make([]string, 0, len(arg))
+2 -2
View File
@@ -25,7 +25,7 @@ func Pad(n int, buf []byte) []byte {
}
length := len(buf)
padding := n - (length % n)
for range padding {
for i := 0; i < padding; i++ {
buf = append(buf, byte(padding))
}
if (len(buf) % n) != 0 {
@@ -54,7 +54,7 @@ func Unpad(n int, buf []byte) ([]byte, error) {
if padding == 0 {
return nil, ErrorPaddingTooShort
}
for i := range padding {
for i := 0; i < padding; i++ {
if buf[length-1-i] != byte(padding) {
return nil, ErrorPaddingNotAllTheSame
}
+33 -12
View File
@@ -18,7 +18,6 @@ import (
"net/http"
"os"
"path"
"slices"
"sort"
"strconv"
"strings"
@@ -200,7 +199,12 @@ func driveScopes(scopesString string) (scopes []string) {
// Returns true if one of the scopes was "drive.appfolder"
func driveScopesContainsAppFolder(scopes []string) bool {
return slices.Contains(scopes, scopePrefix+"drive.appfolder")
for _, scope := range scopes {
if scope == scopePrefix+"drive.appfolder" {
return true
}
}
return false
}
func driveOAuthOptions() []fs.Option {
@@ -954,7 +958,12 @@ func parseDrivePath(path string) (root string, err error) {
type listFn func(*drive.File) bool
func containsString(slice []string, s string) bool {
return slices.Contains(slice, s)
for _, e := range slice {
if e == s {
return true
}
}
return false
}
// getFile returns drive.File for the ID passed and fields passed in
@@ -1143,7 +1152,13 @@ OUTER:
// Check the case of items is correct since
// the `=` operator is case insensitive.
if title != "" && title != item.Name {
found := slices.Contains(stems, item.Name)
found := false
for _, stem := range stems {
if stem == item.Name {
found = true
break
}
}
if !found {
continue
}
@@ -1546,10 +1561,13 @@ func (f *Fs) getFileFields(ctx context.Context) (fields googleapi.Field) {
func (f *Fs) newRegularObject(ctx context.Context, remote string, info *drive.File) (obj fs.Object, err error) {
// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
if f.opt.SkipChecksumGphotos {
if slices.Contains(info.Spaces, "photos") {
info.Md5Checksum = ""
info.Sha1Checksum = ""
info.Sha256Checksum = ""
for _, space := range info.Spaces {
if space == "photos" {
info.Md5Checksum = ""
info.Sha1Checksum = ""
info.Sha256Checksum = ""
break
}
}
}
o := &Object{
@@ -2227,7 +2245,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
wg.Add(1)
in <- listREntry{directoryID, dir}
for range f.ci.Checkers {
for i := 0; i < f.ci.Checkers; i++ {
go f.listRRunner(ctx, &wg, in, out, cb, sendJob)
}
go func() {
@@ -2236,8 +2254,11 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
// if the input channel overflowed add the collected entries to the channel now
for len(overflow) > 0 {
mu.Lock()
l := len(overflow)
// only fill half of the channel to prevent entries being put into overflow again
l := min(len(overflow), listRInputBuffer/2)
if l > listRInputBuffer/2 {
l = listRInputBuffer / 2
}
wg.Add(l)
for _, d := range overflow[:l] {
in <- d
@@ -2257,7 +2278,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
mu.Unlock()
}()
// wait until the all workers to finish
for range f.ci.Checkers {
for i := 0; i < f.ci.Checkers; i++ {
e := <-out
mu.Lock()
// if one worker returns an error early, close the input so all other workers exit
@@ -3893,7 +3914,7 @@ Third delete all orphaned files to the trash
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "get":
out := make(map[string]string)
+3 -2
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"maps"
"strconv"
"strings"
"sync"
@@ -325,7 +324,9 @@ func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err e
metadata := make(fs.Metadata, 16)
// Dump user metadata first as it overrides system metadata
maps.Copy(metadata, info.Properties)
for k, v := range info.Properties {
metadata[k] = v
}
// System metadata
metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission)
+4 -1
View File
@@ -177,7 +177,10 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
if start >= rx.ContentLength {
break
}
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
reqSize = rx.ContentLength - start
if reqSize >= int64(rx.f.opt.ChunkSize) {
reqSize = int64(rx.f.opt.ChunkSize)
}
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
} else {
// If size unknown read into buffer
+4 -1
View File
@@ -55,7 +55,10 @@ func (d *digest) Write(p []byte) (n int, err error) {
n = len(p)
for len(p) > 0 {
d.writtenMore = true
toWrite := min(bytesPerBlock-d.n, len(p))
toWrite := bytesPerBlock - d.n
if toWrite > len(p) {
toWrite = len(p)
}
_, err = d.blockHash.Write(p[:toWrite])
if err != nil {
panic(hashReturnedError)
+1 -1
View File
@@ -11,7 +11,7 @@ import (
func testChunk(t *testing.T, chunk int) {
data := make([]byte, chunk)
for i := range chunk {
for i := 0; i < chunk; i++ {
data[i] = 'A'
}
for _, test := range []struct {
+50 -314
View File
@@ -92,9 +92,6 @@ const (
maxFileNameLength = 255
)
type exportAPIFormat string
type exportExtension string // dotless
var (
// Description of how to auth for this app
dropboxConfig = &oauthutil.Config{
@@ -135,16 +132,6 @@ var (
DefaultTimeoutAsync: 10 * time.Second,
DefaultBatchSizeAsync: 100,
}
exportKnownAPIFormats = map[exportAPIFormat]exportExtension{
"markdown": "md",
"html": "html",
}
// Populated based on exportKnownAPIFormats
exportKnownExtensions = map[exportExtension]exportAPIFormat{}
paperExtension = ".paper"
paperTemplateExtension = ".papert"
)
// Gets an oauth config with the right scopes
@@ -260,61 +247,23 @@ folders.`,
Help: "Specify a different Dropbox namespace ID to use as the root for all paths.",
Default: "",
Advanced: true,
}, {
Name: "export_formats",
Help: `Comma separated list of preferred formats for exporting files
Certain Dropbox files can only be accessed by exporting them to another format.
These include Dropbox Paper documents.
For each such file, rclone will choose the first format on this list that Dropbox
considers valid. If none is valid, it will choose Dropbox's default format.
Known formats include: "html", "md" (markdown)`,
Default: fs.CommaSepList{"html", "md"},
Advanced: true,
}, {
Name: "skip_exports",
Help: "Skip exportable files in all listings.\n\nIf given, exportable files practically become invisible to rclone.",
Default: false,
Advanced: true,
}, {
Name: "show_all_exports",
Default: false,
Help: `Show all exportable files in listings.
Adding this flag will allow all exportable files to be server side copied.
Note that rclone doesn't add extensions to the exportable file names in this mode.
Do **not** use this flag when trying to download exportable files - rclone
will fail to download them.
`,
Advanced: true,
},
}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
}}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
})
for apiFormat, ext := range exportKnownAPIFormats {
exportKnownExtensions[ext] = apiFormat
}
}
// Options defines the configuration for this backend
type Options struct {
ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"`
SharedFiles bool `config:"shared_files"`
SharedFolders bool `config:"shared_folders"`
BatchMode string `config:"batch_mode"`
BatchSize int `config:"batch_size"`
BatchTimeout fs.Duration `config:"batch_timeout"`
AsyncBatch bool `config:"async_batch"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
Enc encoder.MultiEncoder `config:"encoding"`
RootNsid string `config:"root_namespace"`
ExportFormats fs.CommaSepList `config:"export_formats"`
SkipExports bool `config:"skip_exports"`
ShowAllExports bool `config:"show_all_exports"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"`
SharedFiles bool `config:"shared_files"`
SharedFolders bool `config:"shared_folders"`
BatchMode string `config:"batch_mode"`
BatchSize int `config:"batch_size"`
BatchTimeout fs.Duration `config:"batch_timeout"`
AsyncBatch bool `config:"async_batch"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
Enc encoder.MultiEncoder `config:"encoding"`
RootNsid string `config:"root_namespace"`
}
// Fs represents a remote dropbox server
@@ -334,18 +283,8 @@ type Fs struct {
pacer *fs.Pacer // To pace the API calls
ns string // The namespace we are using or "" for none
batcher *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
exportExts []exportExtension
}
type exportType int
const (
notExport exportType = iota // a regular file
exportHide // should be hidden
exportListOnly // listable, but can't export
exportExportable // can export
)
// Object describes a dropbox object
//
// Dropbox Objects always have full metadata
@@ -357,9 +296,6 @@ type Object struct {
bytes int64 // size of the object
modTime time.Time // time it was last modified
hash string // content_hash of the object
exportType exportType
exportAPIFormat exportAPIFormat
}
// Name of the remote (as passed into NewFs)
@@ -500,14 +436,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
HeaderGenerator: f.headerGenerator,
}
for _, e := range opt.ExportFormats {
ext := exportExtension(e)
if exportKnownExtensions[ext] == "" {
return nil, fmt.Errorf("dropbox: unknown export format '%s'", e)
}
f.exportExts = append(f.exportExts, ext)
}
// unauthorized config for endpoints that fail with auth
ucfg := dropbox.Config{
LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
@@ -660,126 +588,38 @@ func (f *Fs) setRoot(root string) {
}
}
type getMetadataResult struct {
entry files.IsMetadata
notFound bool
err error
}
// getMetadata gets the metadata for a file or directory
func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
res.err = f.pacer.Call(func() (bool, error) {
res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
func (f *Fs) getMetadata(ctx context.Context, objPath string) (entry files.IsMetadata, notFound bool, err error) {
err = f.pacer.Call(func() (bool, error) {
entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
Path: f.opt.Enc.FromStandardPath(objPath),
})
return shouldRetry(ctx, res.err)
return shouldRetry(ctx, err)
})
if res.err != nil {
switch e := res.err.(type) {
if err != nil {
switch e := err.(type) {
case files.GetMetadataAPIError:
if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
res.notFound = true
res.err = nil
notFound = true
err = nil
}
}
}
return
}
// Get metadata such that the result would be exported with the given extension
// Return a channel that will eventually receive the metadata
func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportExtension exportExtension) chan getMetadataResult {
ch := make(chan getMetadataResult, 1)
wantDownloadable := (wantExportExtension == "")
go func() {
defer close(ch)
res := f.getMetadata(ctx, filePath)
info, ok := res.entry.(*files.FileMetadata)
if !ok { // Can't check anything about file, just return what we have
ch <- res
return
}
// Return notFound if downloadability or extension doesn't match
if wantDownloadable != info.IsDownloadable {
ch <- getMetadataResult{notFound: true}
return
}
if !info.IsDownloadable {
_, ext := f.chooseExportFormat(info)
if ext != wantExportExtension {
ch <- getMetadataResult{notFound: true}
return
}
}
// Return our real result or error
ch <- res
}()
return ch
}
// For a given rclone-path, figure out what the Dropbox-path may be, in order of preference.
// Multiple paths might be plausible, due to export path munging.
func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-chan getMetadataResult) {
ret = []<-chan getMetadataResult{}
// Prefer an exact match
ret = append(ret, f.getMetadataForExt(ctx, filePath, ""))
// Check if we're plausibly an export path, otherwise we're done
if f.opt.SkipExports || f.opt.ShowAllExports {
return
}
dotted := path.Ext(filePath)
if dotted == "" {
return
}
ext := exportExtension(dotted[1:])
if exportKnownExtensions[ext] == "" {
return
}
// We might be an export path! Try all possibilities
base := strings.TrimSuffix(filePath, dotted)
// `foo.papert.md` will only come from `foo.papert`. Never check something like `foo.papert.paper`
if strings.HasSuffix(base, paperTemplateExtension) {
ret = append(ret, f.getMetadataForExt(ctx, base, ext))
return
}
// Otherwise, try both `foo.md` coming from `foo`, or from `foo.paper`
ret = append(ret, f.getMetadataForExt(ctx, base, ext))
ret = append(ret, f.getMetadataForExt(ctx, base+paperExtension, ext))
return
}
// getFileMetadata gets the metadata for a file
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
var res getMetadataResult
// Try all possible metadatas
possibleMetadatas := f.possibleMetadatas(ctx, filePath)
for _, ch := range possibleMetadatas {
res = <-ch
if res.err != nil {
return nil, res.err
}
if !res.notFound {
break
}
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (fileInfo *files.FileMetadata, err error) {
entry, notFound, err := f.getMetadata(ctx, filePath)
if err != nil {
return nil, err
}
if res.notFound {
if notFound {
return nil, fs.ErrorObjectNotFound
}
fileInfo, ok := res.entry.(*files.FileMetadata)
fileInfo, ok := entry.(*files.FileMetadata)
if !ok {
if _, ok = res.entry.(*files.FolderMetadata); ok {
if _, ok = entry.(*files.FolderMetadata); ok {
return nil, fs.ErrorIsDir
}
return nil, fs.ErrorNotAFile
@@ -788,15 +628,15 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
}
// getDirMetadata gets the metadata for a directory
func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (*files.FolderMetadata, error) {
res := f.getMetadata(ctx, dirPath)
if res.err != nil {
return nil, res.err
func (f *Fs) getDirMetadata(ctx context.Context, dirPath string) (dirInfo *files.FolderMetadata, err error) {
entry, notFound, err := f.getMetadata(ctx, dirPath)
if err != nil {
return nil, err
}
if res.notFound {
if notFound {
return nil, fs.ErrorDirNotFound
}
dirInfo, ok := res.entry.(*files.FolderMetadata)
dirInfo, ok := entry.(*files.FolderMetadata)
if !ok {
return nil, fs.ErrorIsFile
}
@@ -996,15 +836,16 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
var res *files.ListFolderResult
for {
if !started {
arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(root))
arg.Recursive = false
arg.Limit = 1000
arg := files.ListFolderArg{
Path: f.opt.Enc.FromStandardPath(root),
Recursive: false,
Limit: 1000,
}
if root == "/" {
arg.Path = "" // Specify root folder as empty string
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolder(arg)
res, err = f.srv.ListFolder(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1057,9 +898,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if err != nil {
return nil, err
}
if o.(*Object).exportType.listable() {
entries = append(entries, o)
}
entries = append(entries, o)
}
}
if !res.HasMore {
@@ -1145,14 +984,16 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
}
// check directory empty
arg := files.NewListFolderArg(encRoot)
arg.Recursive = false
arg := files.ListFolderArg{
Path: encRoot,
Recursive: false,
}
if root == "/" {
arg.Path = "" // Specify root folder as empty string
}
var res *files.ListFolderResult
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolder(arg)
res, err = f.srv.ListFolder(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1333,16 +1174,6 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
return shouldRetry(ctx, err)
})
if err != nil && createArg.Settings.Expires != nil && strings.Contains(err.Error(), sharing.SharedLinkSettingsErrorNotAuthorized) {
// Some plans can't create links with expiry
fs.Debugf(absPath, "can't create link with expiry, trying without")
createArg.Settings.Expires = nil
err = f.pacer.Call(func() (bool, error) {
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
return shouldRetry(ctx, err)
})
}
if err != nil && strings.Contains(err.Error(),
sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
fs.Debugf(absPath, "has a public link already, attempting to retrieve it")
@@ -1507,14 +1338,16 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
var startCursor *files.ListFolderGetLatestCursorResult
err = f.pacer.Call(func() (bool, error) {
arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(f.slashRoot))
arg.Recursive = true
arg := files.ListFolderArg{
Path: f.opt.Enc.FromStandardPath(f.slashRoot),
Recursive: true,
}
if arg.Path == "/" {
arg.Path = ""
}
startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
startCursor, err = f.srv.ListFolderGetLatestCursor(&arg)
return shouldRetry(ctx, err)
})
@@ -1618,50 +1451,8 @@ func (f *Fs) Shutdown(ctx context.Context) error {
return nil
}
func (f *Fs) chooseExportFormat(info *files.FileMetadata) (exportAPIFormat, exportExtension) {
// Find API export formats Dropbox supports for this file
// Sometimes Dropbox lists a format in ExportAs but not ExportOptions, so check both
ei := info.ExportInfo
dropboxFormatStrings := append([]string{ei.ExportAs}, ei.ExportOptions...)
// Find which extensions these correspond to
exportExtensions := map[exportExtension]exportAPIFormat{}
var dropboxPreferredAPIFormat exportAPIFormat
var dropboxPreferredExtension exportExtension
for _, format := range dropboxFormatStrings {
apiFormat := exportAPIFormat(format)
// Only consider formats we know about
if ext, ok := exportKnownAPIFormats[apiFormat]; ok {
if dropboxPreferredAPIFormat == "" {
dropboxPreferredAPIFormat = apiFormat
dropboxPreferredExtension = ext
}
exportExtensions[ext] = apiFormat
}
}
// See if the user picked a valid extension
for _, ext := range f.exportExts {
if apiFormat, ok := exportExtensions[ext]; ok {
return apiFormat, ext
}
}
// If no matches, prefer the first valid format Dropbox lists
return dropboxPreferredAPIFormat, dropboxPreferredExtension
}
// ------------------------------------------------------------
func (et exportType) listable() bool {
return et != exportHide
}
// something we should _try_ to export
func (et exportType) exportable() bool {
return et == exportExportable || et == exportListOnly
}
// Fs returns the parent Fs
func (o *Object) Fs() fs.Info {
return o.fs
@@ -1705,32 +1496,6 @@ func (o *Object) Size() int64 {
return o.bytes
}
func (o *Object) setMetadataForExport(info *files.FileMetadata) {
o.bytes = -1
o.hash = ""
if o.fs.opt.SkipExports {
o.exportType = exportHide
return
}
if o.fs.opt.ShowAllExports {
o.exportType = exportListOnly
return
}
var exportExt exportExtension
o.exportAPIFormat, exportExt = o.fs.chooseExportFormat(info)
if o.exportAPIFormat == "" {
o.exportType = exportHide
} else {
o.exportType = exportExportable
// get rid of any paper extension, if present
o.remote = strings.TrimSuffix(o.remote, paperExtension)
// add the export extension
o.remote += "." + string(exportExt)
}
}
// setMetadataFromEntry sets the fs data from a files.FileMetadata
//
// This isn't a complete set of metadata and has an inaccurate date
@@ -1739,10 +1504,6 @@ func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
o.bytes = int64(info.Size)
o.modTime = info.ClientModified
o.hash = info.ContentHash
if !info.IsDownloadable {
o.setMetadataForExport(info)
}
return nil
}
@@ -1806,27 +1567,6 @@ func (o *Object) Storable() bool {
return true
}
func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
if o.exportType == exportListOnly || o.exportAPIFormat == "" {
fs.Debugf(o.remote, "No export format found")
return nil, fs.ErrorObjectNotFound
}
arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
var exportResult *files.ExportResult
err = o.fs.pacer.Call(func() (bool, error) {
exportResult, in, err = o.fs.srv.Export(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
o.bytes = int64(exportResult.ExportMetadata.Size)
o.hash = exportResult.ExportMetadata.ExportHash
return
}
// Open an object for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
if o.fs.opt.SharedFiles {
@@ -1846,10 +1586,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return
}
if o.exportType.exportable() {
return o.export(ctx)
}
fs.FixRangeOption(options, o.bytes)
headers := fs.OpenOptionHeaders(options)
arg := files.DownloadArg{
-58
View File
@@ -1,16 +1,9 @@
package dropbox
import (
"context"
"io"
"strings"
"testing"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
"github.com/rclone/rclone/fstest/fstests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestInternalCheckPathLength(t *testing.T) {
@@ -49,54 +42,3 @@ func TestInternalCheckPathLength(t *testing.T) {
assert.Equal(t, test.ok, err == nil, test.in)
}
}
func (f *Fs) importPaperForTest(t *testing.T) {
content := `# test doc
Lorem ipsum __dolor__ sit amet
[link](http://google.com)
`
arg := files.PaperCreateArg{
Path: f.slashRootSlash + "export.paper",
ImportFormat: &files.ImportFormat{Tagged: dropbox.Tagged{Tag: files.ImportFormatMarkdown}},
}
var err error
err = f.pacer.Call(func() (bool, error) {
reader := strings.NewReader(content)
_, err = f.srv.PaperCreate(&arg, reader)
return shouldRetry(context.Background(), err)
})
require.NoError(t, err)
}
func (f *Fs) InternalTestPaperExport(t *testing.T) {
ctx := context.Background()
f.importPaperForTest(t)
f.exportExts = []exportExtension{"html"}
obj, err := f.NewObject(ctx, "export.html")
require.NoError(t, err)
rc, err := obj.Open(ctx)
require.NoError(t, err)
defer func() { require.NoError(t, rc.Close()) }()
buf, err := io.ReadAll(rc)
require.NoError(t, err)
text := string(buf)
for _, excerpt := range []string{
"Lorem ipsum",
"<b>dolor</b>",
`href="http://google.com"`,
} {
require.Contains(t, text, excerpt)
}
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("PaperExport", f.InternalTestPaperExport)
}
var _ fstests.InternalTester = (*Fs)(nil)
+9 -9
View File
@@ -216,11 +216,11 @@ var ItemFields = mustFields(Item{})
// fields returns the JSON fields in use by opt as a | separated
// string.
func fields(opt any) (pipeTags string, err error) {
func fields(opt interface{}) (pipeTags string, err error) {
var tags []string
def := reflect.ValueOf(opt)
defType := def.Type()
for i := range def.NumField() {
for i := 0; i < def.NumField(); i++ {
field := defType.Field(i)
tag, ok := field.Tag.Lookup("json")
if !ok {
@@ -239,7 +239,7 @@ func fields(opt any) (pipeTags string, err error) {
// mustFields returns the JSON fields in use by opt as a | separated
// string. It panics on failure.
func mustFields(opt any) string {
func mustFields(opt interface{}) string {
tags, err := fields(opt)
if err != nil {
panic(err)
@@ -351,12 +351,12 @@ type SpaceInfo struct {
// DeleteResponse is returned from doDeleteFile
type DeleteResponse struct {
Status
Deleted []string `json:"deleted"`
Errors []any `json:"errors"`
ID string `json:"fi_id"`
BackgroundTask int `json:"backgroundtask"`
UsSize string `json:"us_size"`
PaSize string `json:"pa_size"`
Deleted []string `json:"deleted"`
Errors []interface{} `json:"errors"`
ID string `json:"fi_id"`
BackgroundTask int `json:"backgroundtask"`
UsSize string `json:"us_size"`
PaSize string `json:"pa_size"`
//SpaceInfo SpaceInfo `json:"spaceinfo"`
}
+1 -1
View File
@@ -371,7 +371,7 @@ func (f *Fs) getToken(ctx context.Context) (token string, err error) {
}
// params for rpc
type params map[string]any
type params map[string]interface{}
// rpc calls the rpc.php method of the SME file fabric
//
+5 -4
View File
@@ -10,7 +10,6 @@ import (
"net/http"
"net/url"
"path"
"slices"
"strings"
"time"
@@ -170,9 +169,11 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
}
if apiErr, ok := err.(files_sdk.ResponseError); ok {
if slices.Contains(retryErrorCodes, apiErr.HttpCode) {
fs.Debugf(nil, "Retrying API error %v", err)
return true, err
for _, e := range retryErrorCodes {
if apiErr.HttpCode == e {
fs.Debugf(nil, "Retrying API error %v", err)
return true, err
}
}
}
+1 -1
View File
@@ -17,7 +17,7 @@ import (
"github.com/stretchr/testify/require"
)
type settings map[string]any
type settings map[string]interface{}
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
+1 -2
View File
@@ -4,7 +4,6 @@ package googlephotos
import (
"path"
"slices"
"strings"
"sync"
@@ -120,7 +119,7 @@ func (as *albums) _del(album *api.Album) {
dirs := as.path[dir]
for i, dir := range dirs {
if dir == leaf {
dirs = slices.Delete(dirs, i, i+1)
dirs = append(dirs[:i], dirs[i+1:]...)
break
}
}
+2 -2
View File
@@ -388,7 +388,7 @@ func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, e
Method: "GET",
RootURL: "https://accounts.google.com/.well-known/openid-configuration",
}
var openIDconfig map[string]any
var openIDconfig map[string]interface{}
err = f.pacer.Call(func() (bool, error) {
resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
return shouldRetry(ctx, resp, err)
@@ -448,7 +448,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
"token_type_hint": []string{"access_token"},
},
}
var res any
var res interface{}
err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
return shouldRetry(ctx, resp, err)
+1 -1
View File
@@ -24,7 +24,7 @@ import (
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "drop":
return nil, f.db.Stop(true)
+3 -2
View File
@@ -6,7 +6,6 @@ import (
"encoding/gob"
"errors"
"fmt"
"maps"
"strings"
"time"
@@ -196,7 +195,9 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
r.Fp = op.fp
}
maps.Copy(r.Hashes, op.hashes)
for hashType, hashVal := range op.hashes {
r.Hashes[hashType] = hashVal
}
if data, err = r.encode(op.key); err != nil {
return fmt.Errorf("marshal failed: %w", err)
}
+5 -2
View File
@@ -52,7 +52,10 @@ func writeByBlock(p []byte, writer io.Writer, blockSize uint32, bytesInBlock *ui
total := len(p)
nullBytes := make([]byte, blockSize)
for len(p) > 0 {
toWrite := min(int(blockSize-*bytesInBlock), len(p))
toWrite := int(blockSize - *bytesInBlock)
if toWrite > len(p) {
toWrite = len(p)
}
c, err := writer.Write(p[:toWrite])
*bytesInBlock += uint32(c)
*onlyNullBytesInBlock = *onlyNullBytesInBlock && bytes.Equal(nullBytes[:toWrite], p[:toWrite])
@@ -273,7 +276,7 @@ func (h *hidriveHash) Sum(b []byte) []byte {
}
checksum := zeroSum
for i := range h.levels {
for i := 0; i < len(h.levels); i++ {
level := h.levels[i]
if i < len(h.levels)-1 {
// Aggregate non-empty non-final levels.
@@ -216,7 +216,7 @@ func TestLevelWrite(t *testing.T) {
func TestLevelIsFull(t *testing.T) {
content := [hidrivehash.Size]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
l := hidrivehash.NewLevel()
for range 256 {
for i := 0; i < 256; i++ {
assert.False(t, l.(internal.LevelHash).IsFull())
written, err := l.Write(content[:])
assert.Equal(t, len(content), written)
+2 -2
View File
@@ -505,7 +505,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
entries = append(entries, entry)
entriesMu.Unlock()
}
for range checkers {
for i := 0; i < checkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
@@ -740,7 +740,7 @@ It doesn't return anything.
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "set":
newOpt := f.opt
+3 -3
View File
@@ -76,7 +76,7 @@ func (c *Client) DriveService() (*DriveService, error) {
// This function is the main entry point for making requests to the iCloud
// API. If the initial request returns a 401 (Unauthorized), it will try to
// reauthenticate and retry the request.
func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
func (c *Client) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
resp, err = c.Session.Request(ctx, opts, request, response)
if err != nil && resp != nil {
// try to reauth
@@ -100,7 +100,7 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, respo
// This function is useful when you have a session that is already
// authenticated, but you need to make a request without triggering
// a re-authentication.
func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
// Make the request without re-authenticating
resp, err = c.Session.Request(ctx, opts, request, response)
return resp, err
@@ -161,6 +161,6 @@ func newRequestError(Status string, Text string) *RequestError {
}
// newErr orf makes a new error from sprintf parameters.
func newRequestErrorf(Status string, Text string, Parameters ...any) *RequestError {
func newRequestErrorf(Status string, Text string, Parameters ...interface{}) *RequestError {
return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))
}
+6 -6
View File
@@ -476,7 +476,7 @@ func (d *DriveService) MoveItemByDriveID(ctx context.Context, id, etag, dstID st
// CopyDocByItemID copies a document by its item ID.
func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*DriveItemRaw, *http.Response, error) {
// putting name in info doesn't work. extension does work so assume this is a bug in the endpoint
// putting name in info doesnt work. extension does work so assume this is a bug in the endpoint
values := map[string]any{
"info_to_update": map[string]any{},
}
@@ -733,8 +733,8 @@ type DocumentUpdateResponse struct {
StatusCode int `json:"status_code"`
ErrorMessage string `json:"error_message"`
} `json:"status"`
OperationID any `json:"operation_id"`
Document *Document `json:"document"`
OperationID interface{} `json:"operation_id"`
Document *Document `json:"document"`
} `json:"results"`
}
@@ -765,9 +765,9 @@ type Document struct {
IsWritable bool `json:"is_writable"`
IsHidden bool `json:"is_hidden"`
} `json:"file_flags"`
LastOpenedTime int64 `json:"lastOpenedTime"`
RestorePath any `json:"restorePath"`
HasChainedParent bool `json:"hasChainedParent"`
LastOpenedTime int64 `json:"lastOpenedTime"`
RestorePath interface{} `json:"restorePath"`
HasChainedParent bool `json:"hasChainedParent"`
}
// DriveID returns the drive ID of the Document.
+39 -33
View File
@@ -3,13 +3,13 @@ package api
import (
"context"
"fmt"
"maps"
"net/http"
"net/url"
"slices"
"strings"
"github.com/oracle/oci-go-sdk/v65/common"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/lib/rest"
)
@@ -35,7 +35,7 @@ type Session struct {
// }
// Request makes a request
func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) {
func (s *Session) Request(ctx context.Context, opts rest.Opts, request interface{}, response interface{}) (*http.Response, error) {
resp, err := s.srv.CallJSON(ctx, &opts, &request, &response)
if err != nil {
@@ -129,7 +129,7 @@ func (s *Session) AuthWithToken(ctx context.Context) error {
// Validate2FACode validates the 2FA code
func (s *Session) Validate2FACode(ctx context.Context, code string) error {
values := map[string]any{"securityCode": map[string]string{"code": code}}
values := map[string]interface{}{"securityCode": map[string]string{"code": code}}
body, err := IntoReader(values)
if err != nil {
return err
@@ -220,7 +220,9 @@ func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string
"Referer": fmt.Sprintf("%s/", homeEndpoint),
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
}
maps.Copy(headers, overwrite)
for k, v := range overwrite {
headers[k] = v
}
return headers
}
@@ -228,7 +230,9 @@ func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string
func (s *Session) GetHeaders(overwrite map[string]string) map[string]string {
headers := GetCommonHeaders(map[string]string{})
headers["Cookie"] = s.GetCookieString()
maps.Copy(headers, overwrite)
for k, v := range overwrite {
headers[k] = v
}
return headers
}
@@ -250,7 +254,9 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
"Referer": fmt.Sprintf("%s/", baseEndpoint),
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0",
}
maps.Copy(headers, overwrite)
for k, v := range overwrite {
headers[k] = v
}
return headers
}
@@ -332,33 +338,33 @@ type AccountInfo struct {
// ValidateDataDsInfo represents an validation info
type ValidateDataDsInfo struct {
HsaVersion int `json:"hsaVersion"`
LastName string `json:"lastName"`
ICDPEnabled bool `json:"iCDPEnabled"`
TantorMigrated bool `json:"tantorMigrated"`
Dsid string `json:"dsid"`
HsaEnabled bool `json:"hsaEnabled"`
IsHideMyEmailSubscriptionActive bool `json:"isHideMyEmailSubscriptionActive"`
IroncadeMigrated bool `json:"ironcadeMigrated"`
Locale string `json:"locale"`
BrZoneConsolidated bool `json:"brZoneConsolidated"`
ICDRSCapableDeviceList string `json:"ICDRSCapableDeviceList"`
IsManagedAppleID bool `json:"isManagedAppleID"`
IsCustomDomainsFeatureAvailable bool `json:"isCustomDomainsFeatureAvailable"`
IsHideMyEmailFeatureAvailable bool `json:"isHideMyEmailFeatureAvailable"`
ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"`
Gilligvited bool `json:"gilligvited"`
AppleIDAliases []any `json:"appleIdAliases"`
UbiquityEOLEnabled bool `json:"ubiquityEOLEnabled"`
IsPaidDeveloper bool `json:"isPaidDeveloper"`
CountryCode string `json:"countryCode"`
NotificationID string `json:"notificationId"`
PrimaryEmailVerified bool `json:"primaryEmailVerified"`
ADsID string `json:"aDsID"`
Locked bool `json:"locked"`
ICDRSCapableDeviceCount int `json:"ICDRSCapableDeviceCount"`
HasICloudQualifyingDevice bool `json:"hasICloudQualifyingDevice"`
PrimaryEmail string `json:"primaryEmail"`
HsaVersion int `json:"hsaVersion"`
LastName string `json:"lastName"`
ICDPEnabled bool `json:"iCDPEnabled"`
TantorMigrated bool `json:"tantorMigrated"`
Dsid string `json:"dsid"`
HsaEnabled bool `json:"hsaEnabled"`
IsHideMyEmailSubscriptionActive bool `json:"isHideMyEmailSubscriptionActive"`
IroncadeMigrated bool `json:"ironcadeMigrated"`
Locale string `json:"locale"`
BrZoneConsolidated bool `json:"brZoneConsolidated"`
ICDRSCapableDeviceList string `json:"ICDRSCapableDeviceList"`
IsManagedAppleID bool `json:"isManagedAppleID"`
IsCustomDomainsFeatureAvailable bool `json:"isCustomDomainsFeatureAvailable"`
IsHideMyEmailFeatureAvailable bool `json:"isHideMyEmailFeatureAvailable"`
ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"`
Gilligvited bool `json:"gilligvited"`
AppleIDAliases []interface{} `json:"appleIdAliases"`
UbiquityEOLEnabled bool `json:"ubiquityEOLEnabled"`
IsPaidDeveloper bool `json:"isPaidDeveloper"`
CountryCode string `json:"countryCode"`
NotificationID string `json:"notificationId"`
PrimaryEmailVerified bool `json:"primaryEmailVerified"`
ADsID string `json:"aDsID"`
Locked bool `json:"locked"`
ICDRSCapableDeviceCount int `json:"ICDRSCapableDeviceCount"`
HasICloudQualifyingDevice bool `json:"hasICloudQualifyingDevice"`
PrimaryEmail string `json:"primaryEmail"`
AppleIDEntries []struct {
IsPrimary bool `json:"isPrimary"`
Type string `json:"type"`
+6 -2
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"net/http"
"slices"
"strconv"
"time"
@@ -143,7 +142,12 @@ func shouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
if resp == nil {
return false
}
return slices.Contains(retryErrorCodes, resp.StatusCode)
for _, e := range retryErrorCodes {
if resp.StatusCode == e {
return true
}
}
return false
}
func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+12 -5
View File
@@ -13,7 +13,6 @@ import (
"net/url"
"path"
"regexp"
"slices"
"strconv"
"strings"
"time"
@@ -201,7 +200,7 @@ Only enable if you need to be guaranteed to be reflected after write operations.
const iaItemMaxSize int64 = 1099511627776
// metadata keys that are not writeable
var roMetadataKey = map[string]any{
var roMetadataKey = map[string]interface{}{
// do not add mtime here, it's a documented exception
"name": nil, "source": nil, "size": nil, "md5": nil,
"crc32": nil, "sha1": nil, "format": nil, "old_version": nil,
@@ -992,8 +991,10 @@ func (o *Object) Metadata(ctx context.Context) (m fs.Metadata, err error) {
func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
if resp != nil {
if slices.Contains(retryErrorCodes, resp.StatusCode) {
return true, err
for _, e := range retryErrorCodes {
if resp.StatusCode == e {
return true, err
}
}
}
// Ok, not an awserr, check for generic failure conditions
@@ -1146,7 +1147,13 @@ func (f *Fs) waitFileUpload(ctx context.Context, reqPath, tracker string, newSiz
}
fileTrackers, _ := listOrString(iaFile.UpdateTrack)
trackerMatch := slices.Contains(fileTrackers, tracker)
trackerMatch := false
for _, v := range fileTrackers {
if v == tracker {
trackerMatch = true
break
}
}
if !trackerMatch {
continue
}
+20 -20
View File
@@ -70,7 +70,7 @@ func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
// MarshalJSON turns a Rfc3339Time into JSON
func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
return fmt.Appendf(nil, "\"%s\"", t.String()), nil
return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
}
// LoginToken is struct representing the login token generated in the WebUI
@@ -165,25 +165,25 @@ type DeviceRegistrationResponse struct {
// CustomerInfo provides general information about the account. Required for finding the correct internal username.
type CustomerInfo struct {
Username string `json:"username"`
Email string `json:"email"`
Name string `json:"name"`
CountryCode string `json:"country_code"`
LanguageCode string `json:"language_code"`
CustomerGroupCode string `json:"customer_group_code"`
BrandCode string `json:"brand_code"`
AccountType string `json:"account_type"`
SubscriptionType string `json:"subscription_type"`
Usage int64 `json:"usage"`
Quota int64 `json:"quota"`
BusinessUsage int64 `json:"business_usage"`
BusinessQuota int64 `json:"business_quota"`
WriteLocked bool `json:"write_locked"`
ReadLocked bool `json:"read_locked"`
LockedCause any `json:"locked_cause"`
WebHash string `json:"web_hash"`
AndroidHash string `json:"android_hash"`
IOSHash string `json:"ios_hash"`
Username string `json:"username"`
Email string `json:"email"`
Name string `json:"name"`
CountryCode string `json:"country_code"`
LanguageCode string `json:"language_code"`
CustomerGroupCode string `json:"customer_group_code"`
BrandCode string `json:"brand_code"`
AccountType string `json:"account_type"`
SubscriptionType string `json:"subscription_type"`
Usage int64 `json:"usage"`
Quota int64 `json:"quota"`
BusinessUsage int64 `json:"business_usage"`
BusinessQuota int64 `json:"business_quota"`
WriteLocked bool `json:"write_locked"`
ReadLocked bool `json:"read_locked"`
LockedCause interface{} `json:"locked_cause"`
WebHash string `json:"web_hash"`
AndroidHash string `json:"android_hash"`
IOSHash string `json:"ios_hash"`
}
// TrashResponse is returned when emptying the Trash
+1 -1
View File
@@ -193,7 +193,7 @@ func (o *Object) set(e *entity) {
// Call linkbox with the query in opts and return result
//
// This will be checked for error and an error will be returned if Status != 1
func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result any) error {
func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result interface{}) error {
err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, opts, nil, &result)
return f.shouldRetry(ctx, resp, err)
+2 -2
View File
@@ -1046,7 +1046,7 @@ you can try to change the output.`,
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (any, error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
switch name {
case "noop":
if txt, ok := opt["error"]; ok {
@@ -1056,7 +1056,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
return nil, errors.New(txt)
}
if _, ok := opt["echo"]; ok {
out := map[string]any{}
out := map[string]interface{}{}
out["name"] = name
out["arg"] = arg
out["opt"] = opt
+1 -1
View File
@@ -86,7 +86,7 @@ func TestVerifyCopy(t *testing.T) {
require.NoError(t, err)
src.(*Object).fs.opt.NoCheckUpdated = true
for i := range 100 {
for i := 0; i < 100; i++ {
go r.WriteFile(src.Remote(), fmt.Sprintf("some new content %d", i), src.ModTime(context.Background()))
}
_, err = operations.Copy(context.Background(), r.Fremote, nil, filePath+"2", src)
+2 -2
View File
@@ -63,8 +63,8 @@ type UserInfoResponse struct {
Prolong bool `json:"prolong"`
Promocodes struct {
} `json:"promocodes"`
Subscription []any `json:"subscription"`
Version string `json:"version"`
Subscription []interface{} `json:"subscription"`
Version string `json:"version"`
} `json:"billing"`
Bonuses struct {
CameraUpload bool `json:"camera_upload"`
+6 -3
View File
@@ -901,7 +901,7 @@ func (t *treeState) NextRecord() (fs.DirEntry, error) {
return nil, nil
case api.ListParseUnknown15:
skip := int(r.ReadPu32())
for range skip {
for i := 0; i < skip; i++ {
r.ReadPu32()
r.ReadPu32()
}
@@ -1768,7 +1768,7 @@ func (f *Fs) eligibleForSpeedup(remote string, size int64, options ...fs.OpenOpt
func (f *Fs) parseSpeedupPatterns(patternString string) (err error) {
f.speedupGlobs = nil
f.speedupAny = false
uniqueValidPatterns := make(map[string]any)
uniqueValidPatterns := make(map[string]interface{})
for _, pattern := range strings.Split(patternString, ",") {
pattern = strings.ToLower(strings.TrimSpace(pattern))
@@ -2131,7 +2131,10 @@ func getTransferRange(size int64, options ...fs.OpenOption) (start int64, end in
if limit < 0 {
limit = size - offset
}
end = min(offset+limit, size)
end = offset + limit
if end > size {
end = size
}
partial = !(offset == 0 && end == size)
return offset, end, partial
}
+1 -1
View File
@@ -11,7 +11,7 @@ import (
func testChunk(t *testing.T, chunk int) {
data := make([]byte, chunk)
for i := range chunk {
for i := 0; i < chunk; i++ {
data[i] = 'A'
}
for _, test := range []struct {
+8 -6
View File
@@ -21,7 +21,6 @@ import (
"fmt"
"io"
"path"
"slices"
"strings"
"sync"
"time"
@@ -219,11 +218,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
srv = mega.New().SetClient(fshttp.NewClient(ctx))
srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
srv.SetHTTPS(opt.UseHTTPS)
srv.SetLogger(func(format string, v ...any) {
srv.SetLogger(func(format string, v ...interface{}) {
fs.Infof("*go-mega*", format, v...)
})
if opt.Debug {
srv.SetDebugger(func(format string, v ...any) {
srv.SetDebugger(func(format string, v ...interface{}) {
fs.Debugf("*go-mega*", format, v...)
})
}
@@ -499,8 +498,11 @@ func (f *Fs) list(ctx context.Context, dir *mega.Node, fn listFn) (found bool, e
if err != nil {
return false, fmt.Errorf("list failed: %w", err)
}
if slices.ContainsFunc(nodes, fn) {
found = true
for _, item := range nodes {
if fn(item) {
found = true
break
}
}
return
}
@@ -1154,7 +1156,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Upload the chunks
// FIXME do this in parallel
for id := range u.Chunks() {
for id := 0; id < u.Chunks(); id++ {
_, chunkSize, err := u.ChunkLocation(id)
if err != nil {
return fmt.Errorf("upload failed to read chunk location: %w", err)
+1 -1
View File
@@ -29,7 +29,7 @@ func testPurgeListDeadlock(t *testing.T) {
r.Fremote.Features().Disable("Purge") // force fallback-purge
// make a lot of files to prevent it from finishing too quickly
for i := range 100 {
for i := 0; i < 100; i++ {
dst := "file" + fmt.Sprint(i) + ".txt"
r.WriteObject(ctx, dst, "hello", t1)
}
+4 -4
View File
@@ -274,7 +274,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// Command the backend to run a named commands: du and symlink
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "du":
// No arg parsing needed, the path is passed in the fs
@@ -858,7 +858,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
// callBackend calls NetStorage API using either rest.Call or rest.CallXML function,
// depending on whether the response is required
func (f *Fs) callBackend(ctx context.Context, URL, method, actionHeader string, noResponse bool, response any, options []fs.OpenOption) (io.ReadCloser, error) {
func (f *Fs) callBackend(ctx context.Context, URL, method, actionHeader string, noResponse bool, response interface{}, options []fs.OpenOption) (io.ReadCloser, error) {
opts := rest.Opts{
Method: method,
RootURL: URL,
@@ -1080,7 +1080,7 @@ func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.Ope
}
// netStorageDuRequest performs a NetStorage du request
func (f *Fs) netStorageDuRequest(ctx context.Context) (any, error) {
func (f *Fs) netStorageDuRequest(ctx context.Context) (interface{}, error) {
URL := f.url("")
const actionHeader = "version=1&action=du&format=xml&encoding=utf-8"
duResp := &Du{}
@@ -1100,7 +1100,7 @@ func (f *Fs) netStorageDuRequest(ctx context.Context) (any, error) {
}
// netStorageDuRequest performs a NetStorage symlink request
func (f *Fs) netStorageSymlinkRequest(ctx context.Context, URL string, dst string, modTime *int64) (any, error) {
func (f *Fs) netStorageSymlinkRequest(ctx context.Context, URL string, dst string, modTime *int64) (interface{}, error) {
target := url.QueryEscape(strings.TrimSuffix(dst, "/"))
actionHeader := "version=1&action=symlink&target=" + target
if modTime != nil {
+4 -1
View File
@@ -2532,7 +2532,10 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
remaining := size
position := int64(0)
for remaining > 0 {
n := min(remaining, int64(o.fs.opt.ChunkSize))
n := int64(o.fs.opt.ChunkSize)
if remaining < n {
n = remaining
}
seg := readers.NewRepeatableReader(io.LimitReader(in, n))
fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)
@@ -86,7 +86,7 @@ func (q *quickXorHash) Write(p []byte) (n int, err error) {
// Calculate the current checksum
func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
for i := range dataSize {
for i := 0; i < dataSize; i++ {
shift := (i * 11) % 160
shiftBytes := shift / 8
shiftBits := shift % 8
@@ -130,7 +130,10 @@ func TestQuickXorHashByBlock(t *testing.T) {
require.NoError(t, err, what)
h := New()
for i := 0; i < len(in); i += blockSize {
end := min(i+blockSize, len(in))
end := i + blockSize
if end > len(in) {
end = len(in)
}
n, err := h.Write(in[i:end])
require.Equal(t, end-i, n, what)
require.NoError(t, err, what)
+6 -3
View File
@@ -491,7 +491,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
Method: "POST",
Path: "/file/move_copy.json",
}
var request any = moveCopyFileData
var request interface{} = moveCopyFileData
// use /file/rename.json if moving within the same directory
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
@@ -564,7 +564,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
Method: "POST",
Path: "/folder/move_copy.json",
}
var request any = moveFolderData
var request interface{} = moveFolderData
// use /folder/rename.json if moving within the same parent directory
if srcDirectoryID == dstDirectoryID {
@@ -1042,7 +1042,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
chunkCounter := 0
for remainingBytes > 0 {
currentChunkSize := min(int64(o.fs.opt.ChunkSize), remainingBytes)
currentChunkSize := int64(o.fs.opt.ChunkSize)
if currentChunkSize > remainingBytes {
currentChunkSize = remainingBytes
}
remainingBytes -= currentChunkSize
fs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes)
+3 -3
View File
@@ -131,7 +131,7 @@ If it is a string or a []string it will be shown to the user
otherwise it will be JSON encoded and shown to the user like that
*/
func (f *Fs) Command(ctx context.Context, commandName string, args []string,
opt map[string]string) (result any, err error) {
opt map[string]string) (result interface{}, err error) {
// fs.Debugf(f, "command %v, args: %v, opts:%v", commandName, args, opt)
switch commandName {
case operationRename:
@@ -159,7 +159,7 @@ func (f *Fs) Command(ctx context.Context, commandName string, args []string,
}
}
func (f *Fs) rename(ctx context.Context, remote, newName string) (any, error) {
func (f *Fs) rename(ctx context.Context, remote, newName string) (interface{}, error) {
if remote == "" {
return nil, fmt.Errorf("path to object file cannot be empty")
}
@@ -332,7 +332,7 @@ func (f *Fs) listMultipartUploadParts(ctx context.Context, bucketName, bucketPat
return uploadedParts, nil
}
func (f *Fs) restore(ctx context.Context, opt map[string]string) (any, error) {
func (f *Fs) restore(ctx context.Context, opt map[string]string) (interface{}, error) {
req := objectstorage.RestoreObjectsRequest{
NamespaceName: common.String(f.opt.Namespace),
RestoreObjectsDetails: objectstorage.RestoreObjectsDetails{},
+1 -1
View File
@@ -112,7 +112,7 @@ func copyObjectWaitForWorkRequest(ctx context.Context, wID *string, entityType s
string(objectstorage.WorkRequestSummaryStatusCanceled),
string(objectstorage.WorkRequestStatusFailed),
},
Refresh: func() (any, string, error) {
Refresh: func() (interface{}, string, error) {
getWorkRequestRequest := objectstorage.GetWorkRequestRequest{}
getWorkRequestRequest.WorkRequestId = wID
workRequestResponse, err := client.GetWorkRequest(context.Background(), getWorkRequestRequest)
+1 -1
View File
@@ -131,7 +131,7 @@ func (o *Object) setMetaData(
contentMd5 *string,
contentType *string,
lastModified *common.SDKTime,
storageTier any,
storageTier interface{},
meta map[string]string) error {
if contentLength != nil {
+11 -9
View File
@@ -5,7 +5,6 @@ package oracleobjectstorage
import (
"context"
"fmt"
"slices"
"strings"
"time"
@@ -24,7 +23,7 @@ var refreshGracePeriod = 30 * time.Second
//
// `state` is the latest state of that object. And `err` is any error that
// may have happened while refreshing the state.
type StateRefreshFunc func() (result any, state string, err error)
type StateRefreshFunc func() (result interface{}, state string, err error)
// StateChangeConf is the configuration struct used for `WaitForState`.
type StateChangeConf struct {
@@ -57,7 +56,7 @@ type StateChangeConf struct {
// reach the target state.
//
// Cancellation from the passed in context will cancel the refresh loop
func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (any, error) {
func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (interface{}, error) {
// fs.Debugf(entityType, "Waiting for state to become: %s", conf.Target)
notfoundTick := 0
@@ -73,7 +72,7 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
}
type Result struct {
Result any
Result interface{}
State string
Error error
Done bool
@@ -166,9 +165,12 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
}
}
if slices.Contains(conf.Pending, currentState) {
found = true
targetOccurrence = 0
for _, allowed := range conf.Pending {
if currentState == allowed {
found = true
targetOccurrence = 0
break
}
}
if !found && len(conf.Pending) > 0 {
@@ -276,8 +278,8 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
// NotFoundError resource not found error
type NotFoundError struct {
LastError error
LastRequest any
LastResponse any
LastRequest interface{}
LastResponse interface{}
Message string
Retries int
}
+5 -2
View File
@@ -990,7 +990,10 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
if err != nil {
return nil, err
}
free := max(q.Quota-q.UsedQuota, 0)
free := q.Quota - q.UsedQuota
if free < 0 {
free = 0
}
usage = &fs.Usage{
Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used
Used: fs.NewUsageValue(q.UsedQuota), // bytes in use
@@ -1321,7 +1324,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if err != nil {
// sometimes pcloud leaves a half complete file on
// error, so delete it if it exists, trying a few times
for range 5 {
for i := 0; i < 5; i++ {
delObj, delErr := o.fs.NewObject(ctx, o.remote)
if delErr == nil && delObj != nil {
_ = delObj.Remove(ctx)
+1 -1
View File
@@ -37,7 +37,7 @@ func (c *writerAt) Close() error {
}
sizeOk := false
sizeLastSeen := int64(0)
for retry := range 5 {
for retry := 0; retry < 5; retry++ {
fs.Debugf(c.remote, "checking file size: try %d/5", retry)
obj, err := c.fs.NewObject(c.ctx, c.remote)
if err != nil {
+93 -93
View File
@@ -71,14 +71,14 @@ type Error struct {
// ErrorDetails contains further details of api error
type ErrorDetails struct {
Type string `json:"@type,omitempty"`
Reason string `json:"reason,omitempty"`
Domain string `json:"domain,omitempty"`
Metadata struct{} `json:"metadata,omitempty"` // TODO: undiscovered yet
Locale string `json:"locale,omitempty"` // e.g. "en"
Message string `json:"message,omitempty"`
StackEntries []any `json:"stack_entries,omitempty"` // TODO: undiscovered yet
Detail string `json:"detail,omitempty"`
Type string `json:"@type,omitempty"`
Reason string `json:"reason,omitempty"`
Domain string `json:"domain,omitempty"`
Metadata struct{} `json:"metadata,omitempty"` // TODO: undiscovered yet
Locale string `json:"locale,omitempty"` // e.g. "en"
Message string `json:"message,omitempty"`
StackEntries []interface{} `json:"stack_entries,omitempty"` // TODO: undiscovered yet
Detail string `json:"detail,omitempty"`
}
// Error returns a string for the error and satisfies the error interface
@@ -168,44 +168,44 @@ type FileList struct {
// for a single file, i.e. supports for higher `--multi-thread-streams=N`.
// However, it is not generally applicable as it is only for media.
type File struct {
Apps []*FileApp `json:"apps,omitempty"`
Audit *FileAudit `json:"audit,omitempty"`
Collection string `json:"collection,omitempty"` // TODO
CreatedTime Time `json:"created_time,omitempty"`
DeleteTime Time `json:"delete_time,omitempty"`
FileCategory string `json:"file_category,omitempty"` // "AUDIO", "VIDEO"
FileExtension string `json:"file_extension,omitempty"`
FolderType string `json:"folder_type,omitempty"`
Hash string `json:"hash,omitempty"` // custom hash with a form of sha1sum
IconLink string `json:"icon_link,omitempty"`
ID string `json:"id,omitempty"`
Kind string `json:"kind,omitempty"` // "drive#file"
Links *FileLinks `json:"links,omitempty"`
Md5Checksum string `json:"md5_checksum,omitempty"`
Medias []*Media `json:"medias,omitempty"`
MimeType string `json:"mime_type,omitempty"`
ModifiedTime Time `json:"modified_time,omitempty"` // updated when renamed or moved
Name string `json:"name,omitempty"`
OriginalFileIndex int `json:"original_file_index,omitempty"` // TODO
OriginalURL string `json:"original_url,omitempty"`
Params *FileParams `json:"params,omitempty"`
ParentID string `json:"parent_id,omitempty"`
Phase string `json:"phase,omitempty"`
Revision int `json:"revision,omitempty,string"`
ReferenceEvents []any `json:"reference_events"`
ReferenceResource any `json:"reference_resource"`
Size int64 `json:"size,omitempty,string"`
SortName string `json:"sort_name,omitempty"`
Space string `json:"space,omitempty"`
SpellName []any `json:"spell_name,omitempty"` // TODO maybe list of something?
Starred bool `json:"starred,omitempty"`
Tags []any `json:"tags"`
ThumbnailLink string `json:"thumbnail_link,omitempty"`
Trashed bool `json:"trashed,omitempty"`
UserID string `json:"user_id,omitempty"`
UserModifiedTime Time `json:"user_modified_time,omitempty"`
WebContentLink string `json:"web_content_link,omitempty"`
Writable bool `json:"writable,omitempty"`
Apps []*FileApp `json:"apps,omitempty"`
Audit *FileAudit `json:"audit,omitempty"`
Collection string `json:"collection,omitempty"` // TODO
CreatedTime Time `json:"created_time,omitempty"`
DeleteTime Time `json:"delete_time,omitempty"`
FileCategory string `json:"file_category,omitempty"` // "AUDIO", "VIDEO"
FileExtension string `json:"file_extension,omitempty"`
FolderType string `json:"folder_type,omitempty"`
Hash string `json:"hash,omitempty"` // custom hash with a form of sha1sum
IconLink string `json:"icon_link,omitempty"`
ID string `json:"id,omitempty"`
Kind string `json:"kind,omitempty"` // "drive#file"
Links *FileLinks `json:"links,omitempty"`
Md5Checksum string `json:"md5_checksum,omitempty"`
Medias []*Media `json:"medias,omitempty"`
MimeType string `json:"mime_type,omitempty"`
ModifiedTime Time `json:"modified_time,omitempty"` // updated when renamed or moved
Name string `json:"name,omitempty"`
OriginalFileIndex int `json:"original_file_index,omitempty"` // TODO
OriginalURL string `json:"original_url,omitempty"`
Params *FileParams `json:"params,omitempty"`
ParentID string `json:"parent_id,omitempty"`
Phase string `json:"phase,omitempty"`
Revision int `json:"revision,omitempty,string"`
ReferenceEvents []interface{} `json:"reference_events"`
ReferenceResource interface{} `json:"reference_resource"`
Size int64 `json:"size,omitempty,string"`
SortName string `json:"sort_name,omitempty"`
Space string `json:"space,omitempty"`
SpellName []interface{} `json:"spell_name,omitempty"` // TODO maybe list of something?
Starred bool `json:"starred,omitempty"`
Tags []interface{} `json:"tags"`
ThumbnailLink string `json:"thumbnail_link,omitempty"`
Trashed bool `json:"trashed,omitempty"`
UserID string `json:"user_id,omitempty"`
UserModifiedTime Time `json:"user_modified_time,omitempty"`
WebContentLink string `json:"web_content_link,omitempty"`
Writable bool `json:"writable,omitempty"`
}
// FileLinks includes links to file at backend
@@ -235,18 +235,18 @@ type Media struct {
VideoType string `json:"video_type,omitempty"` // "mpegts"
HdrType string `json:"hdr_type,omitempty"`
} `json:"video,omitempty"`
Link *Link `json:"link,omitempty"`
NeedMoreQuota bool `json:"need_more_quota,omitempty"`
VipTypes []any `json:"vip_types,omitempty"` // TODO maybe list of something?
RedirectLink string `json:"redirect_link,omitempty"`
IconLink string `json:"icon_link,omitempty"`
IsDefault bool `json:"is_default,omitempty"`
Priority int `json:"priority,omitempty"`
IsOrigin bool `json:"is_origin,omitempty"`
ResolutionName string `json:"resolution_name,omitempty"`
IsVisible bool `json:"is_visible,omitempty"`
Category string `json:"category,omitempty"` // "category_origin"
Audio any `json:"audio"` // TODO: undiscovered yet
Link *Link `json:"link,omitempty"`
NeedMoreQuota bool `json:"need_more_quota,omitempty"`
VipTypes []interface{} `json:"vip_types,omitempty"` // TODO maybe list of something?
RedirectLink string `json:"redirect_link,omitempty"`
IconLink string `json:"icon_link,omitempty"`
IsDefault bool `json:"is_default,omitempty"`
Priority int `json:"priority,omitempty"`
IsOrigin bool `json:"is_origin,omitempty"`
ResolutionName string `json:"resolution_name,omitempty"`
IsVisible bool `json:"is_visible,omitempty"`
Category string `json:"category,omitempty"` // "category_origin"
Audio interface{} `json:"audio"` // TODO: undiscovered yet
}
// FileParams includes parameters for instant open
@@ -263,20 +263,20 @@ type FileParams struct {
// FileApp includes parameters for instant open
type FileApp struct {
ID string `json:"id,omitempty"` // "decompress" for rar files
Name string `json:"name,omitempty"` // decompress" for rar files
Access []any `json:"access,omitempty"`
Link string `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
RedirectLink string `json:"redirect_link,omitempty"`
VipTypes []any `json:"vip_types,omitempty"`
NeedMoreQuota bool `json:"need_more_quota,omitempty"`
IconLink string `json:"icon_link,omitempty"`
IsDefault bool `json:"is_default,omitempty"`
Params struct{} `json:"params,omitempty"` // TODO
CategoryIDs []any `json:"category_ids,omitempty"`
AdSceneType int `json:"ad_scene_type,omitempty"`
Space string `json:"space,omitempty"`
Links struct{} `json:"links,omitempty"` // TODO
ID string `json:"id,omitempty"` // "decompress" for rar files
Name string `json:"name,omitempty"` // decompress" for rar files
Access []interface{} `json:"access,omitempty"`
Link string `json:"link,omitempty"` // "https://mypikpak.com/drive/decompression/{File.Id}?gcid={File.Hash}\u0026wv-style=topbar%3Ahide"
RedirectLink string `json:"redirect_link,omitempty"`
VipTypes []interface{} `json:"vip_types,omitempty"`
NeedMoreQuota bool `json:"need_more_quota,omitempty"`
IconLink string `json:"icon_link,omitempty"`
IsDefault bool `json:"is_default,omitempty"`
Params struct{} `json:"params,omitempty"` // TODO
CategoryIDs []interface{} `json:"category_ids,omitempty"`
AdSceneType int `json:"ad_scene_type,omitempty"`
Space string `json:"space,omitempty"`
Links struct{} `json:"links,omitempty"` // TODO
}
// ------------------------------------------------------------
@@ -290,27 +290,27 @@ type TaskList struct {
// Task is a basic element representing a single task such as offline download and upload
type Task struct {
Kind string `json:"kind,omitempty"` // "drive#task"
ID string `json:"id,omitempty"` // task id?
Name string `json:"name,omitempty"` // torrent name?
Type string `json:"type,omitempty"` // "offline"
UserID string `json:"user_id,omitempty"`
Statuses []any `json:"statuses,omitempty"` // TODO
StatusSize int `json:"status_size,omitempty"` // TODO
Params *TaskParams `json:"params,omitempty"` // TODO
FileID string `json:"file_id,omitempty"`
FileName string `json:"file_name,omitempty"`
FileSize string `json:"file_size,omitempty"`
Message string `json:"message,omitempty"` // e.g. "Saving"
CreatedTime Time `json:"created_time,omitempty"`
UpdatedTime Time `json:"updated_time,omitempty"`
ThirdTaskID string `json:"third_task_id,omitempty"` // TODO
Phase string `json:"phase,omitempty"` // e.g. "PHASE_TYPE_RUNNING"
Progress int `json:"progress,omitempty"`
IconLink string `json:"icon_link,omitempty"`
Callback string `json:"callback,omitempty"`
ReferenceResource any `json:"reference_resource,omitempty"` // TODO
Space string `json:"space,omitempty"`
Kind string `json:"kind,omitempty"` // "drive#task"
ID string `json:"id,omitempty"` // task id?
Name string `json:"name,omitempty"` // torrent name?
Type string `json:"type,omitempty"` // "offline"
UserID string `json:"user_id,omitempty"`
Statuses []interface{} `json:"statuses,omitempty"` // TODO
StatusSize int `json:"status_size,omitempty"` // TODO
Params *TaskParams `json:"params,omitempty"` // TODO
FileID string `json:"file_id,omitempty"`
FileName string `json:"file_name,omitempty"`
FileSize string `json:"file_size,omitempty"`
Message string `json:"message,omitempty"` // e.g. "Saving"
CreatedTime Time `json:"created_time,omitempty"`
UpdatedTime Time `json:"updated_time,omitempty"`
ThirdTaskID string `json:"third_task_id,omitempty"` // TODO
Phase string `json:"phase,omitempty"` // e.g. "PHASE_TYPE_RUNNING"
Progress int `json:"progress,omitempty"`
IconLink string `json:"icon_link,omitempty"`
Callback string `json:"callback,omitempty"`
ReferenceResource interface{} `json:"reference_resource,omitempty"` // TODO
Space string `json:"space,omitempty"`
}
// TaskParams includes parameters informing status of Task
+1 -1
View File
@@ -638,7 +638,7 @@ func (c *pikpakClient) SetCaptchaTokener(ctx context.Context, m configmap.Mapper
return c
}
func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request any, response any) (resp *http.Response, err error) {
func (c *pikpakClient) CallJSON(ctx context.Context, opts *rest.Opts, request interface{}, response interface{}) (resp *http.Response, err error) {
if c.captcha != nil {
token, err := c.captcha.Token(opts)
if err != nil || token == "" {
+2 -2
View File
@@ -1232,7 +1232,7 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
params := url.Values{}
iVal := reflect.ValueOf(&form.MultiParts).Elem()
iTyp := iVal.Type()
for i := range iVal.NumField() {
for i := 0; i < iVal.NumField(); i++ {
params.Set(iTyp.Field(i).Tag.Get("json"), iVal.Field(i).String())
}
formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, params, "file", name)
@@ -1520,7 +1520,7 @@ Result:
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "addurl":
if len(arg) != 1 {
+4 -3
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"net/http"
"slices"
"strconv"
"time"
@@ -14,8 +13,10 @@ import (
)
func checkStatusCode(resp *http.Response, expected ...int) error {
if slices.Contains(expected, resp.StatusCode) {
return nil
for _, code := range expected {
if resp.StatusCode == code {
return nil
}
}
return &statusCodeError{response: resp}
}
+4 -1
View File
@@ -332,7 +332,10 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
var offsetMismatch bool
buf := make([]byte, defaultChunkSize)
for clientOffset < size {
chunkSize := min(size-clientOffset, int64(defaultChunkSize))
chunkSize := size - clientOffset
if chunkSize >= int64(defaultChunkSize) {
chunkSize = int64(defaultChunkSize)
}
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
chunkStart := clientOffset
reqSize := chunkSize
+1 -1
View File
@@ -358,7 +358,7 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) (err error) {
})()
ch := make(chan chunk, mu.cfg.concurrency)
for range mu.cfg.concurrency {
for i := 0; i < mu.cfg.concurrency; i++ {
mu.wg.Add(1)
go mu.readChunk(ch)
}
+4 -3
View File
@@ -15,7 +15,6 @@ import (
"net/http"
"net/url"
"path"
"slices"
"strconv"
"strings"
"time"
@@ -644,8 +643,10 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
return err
}
if slices.Contains(result.IDs, id) {
return nil
for _, removedID := range result.IDs {
if removedID == id {
return nil
}
}
return fmt.Errorf("file %s was not deleted successfully", id)
+5 -1
View File
@@ -59,7 +59,11 @@ func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed f
defer func() { u.fileUsage[fileID] = borrowed }()
effectiveChunkSize := max(int64(speed*u.effectiveTime.Seconds()), u.reserved)
effectiveChunkSize := int64(speed * u.effectiveTime.Seconds())
if effectiveChunkSize < u.reserved {
effectiveChunkSize = u.reserved
}
if neededMemory < effectiveChunkSize {
effectiveChunkSize = neededMemory
+1 -1
View File
@@ -40,7 +40,7 @@ func (signer *IbmIamSigner) SignHTTP(ctx context.Context, credentials aws.Creden
return nil
}
// NoOpCredentialsProvider is needed since S3 SDK requires having credentials, even though authentication is happening via IBM IAM.
// NoOpCredentialsProvider is needed since S3 SDK requires having credentials, eventhough authentication is happening via IBM IAM.
type NoOpCredentialsProvider struct{}
// Retrieve returns mock credentials for the NoOpCredentialsProvider.
+6 -5
View File
@@ -19,7 +19,6 @@ import (
"net/url"
"path"
"regexp"
"slices"
"sort"
"strconv"
"strings"
@@ -3098,8 +3097,10 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
return true, err
}
}
if slices.Contains(retryErrorCodes, httpStatusCode) {
return true, err
for _, e := range retryErrorCodes {
if httpStatusCode == e {
return true, err
}
}
}
// Ok, not an awserr, check for generic failure conditions
@@ -3229,7 +3230,7 @@ func fixupRequest(o *s3.Options, opt *Options) {
type s3logger struct{}
// Logf is expected to support the standard fmt package "verbs".
func (s3logger) Logf(classification logging.Classification, format string, v ...any) {
func (s3logger) Logf(classification logging.Classification, format string, v ...interface{}) {
switch classification {
default:
case logging.Debug:
@@ -5252,7 +5253,7 @@ It doesn't return anything.
// The result should be capable of being JSON encoded
// If it is a string or a []string it will be shown to the user
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
switch name {
case "restore":
req := s3.RestoreObjectInput{
+4 -4
View File
@@ -9,9 +9,9 @@ import (
// Renew allows tokens to be renewed on expiry.
type Renew struct {
ts *time.Ticker // timer indicating when it's time to renew the token
run func() error // the callback to do the renewal
done chan any // channel to end the go routine
ts *time.Ticker // timer indicating when it's time to renew the token
run func() error // the callback to do the renewal
done chan interface{} // channel to end the go routine
shutdown *sync.Once
}
@@ -22,7 +22,7 @@ func NewRenew(every time.Duration, run func() error) *Renew {
r := &Renew{
ts: time.NewTicker(every),
run: run,
done: make(chan any),
done: make(chan interface{}),
shutdown: &sync.Once{},
}
go r.renewOnExpiry()
+1 -1
View File
@@ -1313,7 +1313,7 @@ func (f *Fs) getCachedLibraries(ctx context.Context) ([]api.Library, error) {
f.librariesMutex.Lock()
defer f.librariesMutex.Unlock()
libraries, err := f.libraries.Get(librariesCacheKey, func(key string) (value any, ok bool, error error) {
libraries, err := f.libraries.Get(librariesCacheKey, func(key string) (value interface{}, ok bool, error error) {
// Load the libraries if not present in the cache
libraries, err := f.getLibraries(ctx)
if err != nil {
+1 -2
View File
@@ -8,7 +8,6 @@ import (
"fmt"
"io"
"os/exec"
"slices"
"strings"
"time"
@@ -90,7 +89,7 @@ func (f *Fs) newSSHSessionExternal() *sshSessionExternal {
// Connect to a remote host and request the sftp subsystem via
// the 'ssh' command. This assumes that passwordless login is
// correctly configured.
ssh := slices.Clone(s.f.opt.SSH)
ssh := append([]string(nil), s.f.opt.SSH...)
s.cmd = exec.CommandContext(ctx, ssh[0], ssh[1:]...)
// Allow the command a short time only to shut down
+2 -2
View File
@@ -20,13 +20,13 @@ func TestStringLock(t *testing.T) {
inner = 100
total = outer * inner
)
for range outer {
for k := 0; k < outer; k++ {
for j := range counter {
wg.Add(1)
go func(j int) {
defer wg.Done()
ID := fmt.Sprintf("%d", j)
for range inner {
for i := 0; i < inner; i++ {
lock.Lock(ID)
n := counter[j]
time.Sleep(1 * time.Millisecond)
+1 -1
View File
@@ -537,7 +537,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// Fill up (or reset) the buffer tokens
func (f *Fs) fillBufferTokens() {
f.bufferTokens = make(chan []byte, f.ci.Transfers)
for range f.ci.Transfers {
for i := 0; i < f.ci.Transfers; i++ {
f.bufferTokens <- nil
}
}
+4 -1
View File
@@ -57,7 +57,10 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
return nil, fmt.Errorf("can't use method %q with newLargeUpload", info.Method)
}
threads := min(f.ci.Transfers, info.MaxNumberOfThreads)
threads := f.ci.Transfers
if threads > info.MaxNumberOfThreads {
threads = info.MaxNumberOfThreads
}
// unwrap the accounting from the input, we use wrap to put it
// back on after the buffering
+1 -1
View File
@@ -337,7 +337,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
}
// Cleanup stray files left after failed upload
for range 5 {
for i := 0; i < 5; i++ {
cleanObj, cleanErr := f.NewObject(ctx, src.Remote())
if cleanErr == nil {
cleanErr = cleanObj.Remove(ctx)
+1 -1
View File
@@ -574,7 +574,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
RootURL: pathID,
NoResponse: true,
}
var mkdir any
var mkdir interface{}
if pathID == f.opt.RootID {
// folders at the root are syncFolders
mkdir = &api.CreateSyncFolder{
+9 -7
View File
@@ -8,10 +8,8 @@ import (
"errors"
"fmt"
"io"
"maps"
"path"
"regexp"
"slices"
"strconv"
"strings"
"sync"
@@ -419,8 +417,10 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
}
// If this is a swift.Error object extract the HTTP error code
if swiftError, ok := err.(*swift.Error); ok {
if slices.Contains(retryErrorCodes, swiftError.StatusCode) {
return true, err
for _, e := range retryErrorCodes {
if swiftError.StatusCode == e {
return true, err
}
}
}
// Check for generic failure conditions
@@ -701,7 +701,7 @@ func (f *Fs) listContainerRoot(ctx context.Context, container, directory, prefix
if !recurse {
opts.Delimiter = '/'
}
return f.c.ObjectsWalk(ctx, container, &opts, func(ctx context.Context, opts *swift.ObjectsOpts) (any, error) {
return f.c.ObjectsWalk(ctx, container, &opts, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) {
var objects []swift.Object
var err error
err = f.pacer.Call(func() (bool, error) {
@@ -1378,7 +1378,9 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
meta := o.headers.ObjectMetadata()
meta.SetModTime(modTime)
newHeaders := meta.ObjectHeaders()
maps.Copy(o.headers, newHeaders)
for k, v := range newHeaders {
o.headers[k] = v
}
// Include any other metadata from request
for k, v := range o.headers {
if strings.HasPrefix(k, "X-Object-") {
@@ -1448,7 +1450,7 @@ func (o *Object) removeSegmentsLargeObject(ctx context.Context, container string
// encoded but we need '&' encoded.
func urlEncode(str string) string {
var buf bytes.Buffer
for i := range len(str) {
for i := 0; i < len(str); i++ {
c := str[i]
if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '/' || c == '.' || c == '_' || c == '-' {
_ = buf.WriteByte(c)
+2 -2
View File
@@ -82,8 +82,8 @@ type File struct {
ContentType string `json:"content_type"`
Format struct {
} `json:"format"`
DownloadTypes []any `json:"download_types"`
ThumbnailInfo []any `json:"thumbnail_info"`
DownloadTypes []interface{} `json:"download_types"`
ThumbnailInfo []interface{} `json:"thumbnail_info"`
PreviewInfo struct {
} `json:"preview_info"`
Privacy string `json:"privacy"`
+2 -2
View File
@@ -729,7 +729,7 @@ func (o *Object) Storable() bool {
return true
}
func (o *Object) updateFileProperties(ctx context.Context, req any) (err error) {
func (o *Object) updateFileProperties(ctx context.Context, req interface{}) (err error) {
var resp *api.File
opts := rest.Opts{
@@ -887,7 +887,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Remove implements the mandatory method fs.Object.Remove
func (o *Object) Remove(ctx context.Context) error {
for range 2 {
for i := 0; i < 2; i++ {
// First call moves the item to recycle bin, second deletes it for good
var err error
opts := rest.Opts{
+2 -2
View File
@@ -902,7 +902,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// Backward compatible to old config
if len(opt.Upstreams) == 0 && len(opt.Remotes) > 0 {
for i := range len(opt.Remotes) - 1 {
for i := 0; i < len(opt.Remotes)-1; i++ {
opt.Remotes[i] += ":ro"
}
opt.Upstreams = opt.Remotes
@@ -1045,7 +1045,7 @@ func parentDir(absPath string) string {
func multithread(num int, fn func(int)) {
var wg sync.WaitGroup
for i := range num {
for i := 0; i < num; i++ {
wg.Add(1)
i := i
go func() {
+1 -1
View File
@@ -246,7 +246,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
return f, nil
}
func (f *Fs) decodeError(resp *http.Response, response any) (err error) {
func (f *Fs) decodeError(resp *http.Response, response interface{}) (err error) {
defer fs.CheckClose(resp.Body, &err)
body, err := io.ReadAll(resp.Body)
+5 -1
View File
@@ -112,8 +112,12 @@ func (o *Object) uploadChunks(ctx context.Context, in0 io.Reader, size int64, pa
return err
}
contentLength := chunkSize
// Last chunk may be smaller
contentLength := min(size-offset, chunkSize)
if size-offset < contentLength {
contentLength = size - offset
}
endOffset := offset + contentLength - 1
+1 -1
View File
@@ -185,7 +185,7 @@ func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SharepointSuccessRe
if err != nil {
return nil, err
}
reqData := map[string]any{
reqData := map[string]interface{}{
"Username": ca.user,
"Password": ca.pass,
"Address": ca.endpoint,
+15 -15
View File
@@ -23,20 +23,20 @@ type ResourceInfoRequestOptions struct {
// ResourceInfoResponse struct is returned by the API for metadata requests.
type ResourceInfoResponse struct {
PublicKey string `json:"public_key"`
Name string `json:"name"`
Created string `json:"created"`
CustomProperties map[string]any `json:"custom_properties"`
Preview string `json:"preview"`
PublicURL string `json:"public_url"`
OriginPath string `json:"origin_path"`
Modified string `json:"modified"`
Path string `json:"path"`
Md5 string `json:"md5"`
ResourceType string `json:"type"`
MimeType string `json:"mime_type"`
Size int64 `json:"size"`
Embedded *ResourceListResponse `json:"_embedded"`
PublicKey string `json:"public_key"`
Name string `json:"name"`
Created string `json:"created"`
CustomProperties map[string]interface{} `json:"custom_properties"`
Preview string `json:"preview"`
PublicURL string `json:"public_url"`
OriginPath string `json:"origin_path"`
Modified string `json:"modified"`
Path string `json:"path"`
Md5 string `json:"md5"`
ResourceType string `json:"type"`
MimeType string `json:"mime_type"`
Size int64 `json:"size"`
Embedded *ResourceListResponse `json:"_embedded"`
}
// ResourceListResponse struct
@@ -64,7 +64,7 @@ type AsyncStatus struct {
// CustomPropertyResponse struct we send and is returned by the API for CustomProperty request.
type CustomPropertyResponse struct {
CustomProperties map[string]any `json:"custom_properties"`
CustomProperties map[string]interface{} `json:"custom_properties"`
}
// SortMode struct - sort mode
+1 -1
View File
@@ -1024,7 +1024,7 @@ func (o *Object) setCustomProperty(ctx context.Context, property string, value s
}
opts.Parameters.Set("path", o.fs.opt.Enc.FromStandardPath(o.filePath()))
rcm := map[string]any{
rcm := map[string]interface{}{
property: value,
}
cpr := api.CustomPropertyResponse{CustomProperties: rcm}
+1 -2
View File
@@ -11,5 +11,4 @@
<services+github@simjo.st>
<seb•ɑƬ•chezwam•ɖɵʈ•org>
<allllaboutyou@gmail.com>
<psycho@feltzv.fr>
<afw5059@gmail.com>
<psycho@feltzv.fr>
-1
View File
@@ -15,7 +15,6 @@ import (
_ "github.com/rclone/rclone/cmd/cleanup"
_ "github.com/rclone/rclone/cmd/cmount"
_ "github.com/rclone/rclone/cmd/config"
_ "github.com/rclone/rclone/cmd/convmv"
_ "github.com/rclone/rclone/cmd/copy"
_ "github.com/rclone/rclone/cmd/copyto"
_ "github.com/rclone/rclone/cmd/copyurl"
+1 -1
View File
@@ -82,7 +82,7 @@ Note to run these commands on a running backend then see
return err
}
// Run the command
var out any
var out interface{}
switch name {
case "help":
return showHelp(fsInfo)
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
// Names comprises a set of file names
type Names map[string]any
type Names map[string]interface{}
// ToNames converts string slice to a set of names
func ToNames(list []string) Names {
+5 -5
View File
@@ -627,7 +627,7 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
testFunc := func() {
src := filepath.Join(b.dataDir, "file7.txt")
for i := range 50 {
for i := 0; i < 50; i++ {
dst := "file" + fmt.Sprint(i) + ".txt"
err := b.copyFile(ctx, src, b.replaceHex(b.path2), dst)
if err != nil {
@@ -1606,7 +1606,7 @@ func (b *bisyncTest) mangleResult(dir, file string, golden bool) string {
s = pathReplacer.Replace(strings.TrimSpace(s))
// Apply regular expression replacements
for i := range repFrom {
for i := 0; i < len(repFrom); i++ {
s = repFrom[i].ReplaceAllString(s, repTo[i])
}
s = strings.TrimSpace(s)
@@ -1621,7 +1621,7 @@ func (b *bisyncTest) mangleResult(dir, file string, golden bool) string {
// Sort consecutive groups of naturally unordered lines.
// Any such group must end before the log ends or it might be lost.
absorbed := false
for i := range dampers {
for i := 0; i < len(dampers); i++ {
match := false
if s != "" && !absorbed {
match = hoppers[i].MatchString(s)
@@ -1869,7 +1869,7 @@ func fileType(fileName string) string {
}
// logPrintf prints a message to stdout and to the test log
func (b *bisyncTest) logPrintf(text string, args ...any) {
func (b *bisyncTest) logPrintf(text string, args ...interface{}) {
line := fmt.Sprintf(text, args...)
fs.Log(nil, line)
if b.logFile != nil {
@@ -1936,7 +1936,7 @@ func ctxNoDsStore(ctx context.Context, t *testing.T) (context.Context, *filter.F
return ctxNoDsStore, fi
}
func checkError(t *testing.T, err error, msgAndArgs ...any) {
func checkError(t *testing.T, err error, msgAndArgs ...interface{}) {
if errors.Is(err, fs.ErrorCantUploadEmptyFiles) {
t.Skipf("Skip test because remote cannot upload empty files")
}
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"github.com/rclone/rclone/lib/terminal"
)
func (b *bisyncRun) indentf(tag, file, format string, args ...any) {
func (b *bisyncRun) indentf(tag, file, format string, args ...interface{}) {
b.indent(tag, file, fmt.Sprintf(format, args...))
}
+2 -2
View File
@@ -524,7 +524,7 @@ func (b *bisyncRun) testFn() {
}
}
func (b *bisyncRun) handleErr(o any, msg string, err error, critical, retryable bool) {
func (b *bisyncRun) handleErr(o interface{}, msg string, err error, critical, retryable bool) {
if err != nil {
if retryable {
b.retryable = true
@@ -624,7 +624,7 @@ func (b *bisyncRun) debugFn(nametocheck string, fn func()) {
// waitFor runs fn() until it returns true or the timeout expires
func waitFor(msg string, totalWait time.Duration, fn func() bool) (ok bool) {
const individualWait = 1 * time.Second
for i := range int(totalWait / individualWait) {
for i := 0; i < int(totalWait/individualWait); i++ {
ok = fn()
if ok {
return ok
-504
View File
@@ -1,504 +0,0 @@
// Package convmv provides the convmv command.
package convmv
import (
"context"
"encoding/base64"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"unicode/utf8"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/random"
"github.com/spf13/cobra"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/unicode/norm"
)
// Globals
var (
Opt ConvOpt
Cmaps = map[int]*charmap.Charmap{}
)
// ConvOpt sets the conversion options
type ConvOpt struct {
ctx context.Context
f fs.Fs
ConvertAlgo Convert
FindReplace []string
Prefix string
Suffix string
Max int
Enc encoder.MultiEncoder
CmapFlag fs.Enum[cmapChoices]
Cmap *charmap.Charmap
List bool
}
func init() {
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
flags.FVarP(cmdFlags, &Opt.ConvertAlgo, "conv", "t", "Conversion algorithm: "+Opt.ConvertAlgo.Help(), "")
flags.StringVarP(cmdFlags, &Opt.Prefix, "prefix", "", "", "In 'prefix' or 'trimprefix' mode, append or trim this prefix", "")
flags.StringVarP(cmdFlags, &Opt.Suffix, "suffix", "", "", "In 'suffix' or 'trimsuffix' mode, append or trim this suffix", "")
flags.IntVarP(cmdFlags, &Opt.Max, "max", "m", -1, "In 'truncate' mode, truncate all path segments longer than this many characters", "")
flags.StringArrayVarP(cmdFlags, &Opt.FindReplace, "replace", "r", nil, "In 'replace' mode, this is a pair of find,replace values (can repeat flag more than once)", "")
flags.FVarP(cmdFlags, &Opt.Enc, "encoding", "", "Custom backend encoding: (use --list to see full list)", "")
flags.FVarP(cmdFlags, &Opt.CmapFlag, "charmap", "", "Other character encoding (use --list to see full list) ", "")
flags.BoolVarP(cmdFlags, &Opt.List, "list", "", false, "Print full list of options", "")
}
// Convert describes conversion setting
type Convert = fs.Enum[convertChoices]
// Supported conversion options
const (
ConvNone Convert = iota
ConvToNFC
ConvToNFD
ConvToNFKC
ConvToNFKD
ConvFindReplace
ConvPrefix
ConvSuffix
ConvTrimPrefix
ConvTrimSuffix
ConvIndex
ConvDate
ConvTruncate
ConvBase64Encode
ConvBase64Decode
ConvEncoder
ConvDecoder
ConvISO8859_1
ConvWindows1252
ConvMacintosh
ConvCharmap
ConvLowercase
ConvUppercase
ConvTitlecase
ConvASCII
ConvURL
ConvMapper
)
type convertChoices struct{}
func (convertChoices) Choices() []string {
return []string{
ConvNone: "none",
ConvToNFC: "nfc",
ConvToNFD: "nfd",
ConvToNFKC: "nfkc",
ConvToNFKD: "nfkd",
ConvFindReplace: "replace",
ConvPrefix: "prefix",
ConvSuffix: "suffix",
ConvTrimPrefix: "trimprefix",
ConvTrimSuffix: "trimsuffix",
ConvIndex: "index",
ConvDate: "date",
ConvTruncate: "truncate",
ConvBase64Encode: "base64encode",
ConvBase64Decode: "base64decode",
ConvEncoder: "encoder",
ConvDecoder: "decoder",
ConvISO8859_1: "ISO-8859-1",
ConvWindows1252: "Windows-1252",
ConvMacintosh: "Macintosh",
ConvCharmap: "charmap",
ConvLowercase: "lowercase",
ConvUppercase: "uppercase",
ConvTitlecase: "titlecase",
ConvASCII: "ascii",
ConvURL: "url",
ConvMapper: "mapper",
}
}
func (convertChoices) Type() string {
return "string"
}
type cmapChoices struct{}
func (cmapChoices) Choices() []string {
choices := make([]string, 1)
i := 0
for _, enc := range charmap.All {
c, ok := enc.(*charmap.Charmap)
if !ok {
continue
}
name := strings.ReplaceAll(c.String(), " ", "-")
if name == "" {
name = fmt.Sprintf("unknown-%d", i)
}
Cmaps[i] = c
choices = append(choices, name)
i++
}
return choices
}
func (cmapChoices) Type() string {
return "string"
}
func charmapByID(cm fs.Enum[cmapChoices]) *charmap.Charmap {
c, ok := Cmaps[int(cm)]
if ok {
return c
}
return nil
}
var commandDefinition = &cobra.Command{
Use: "convmv source:path",
Short: `Convert file and directory names`,
// Warning! "|" will be replaced by backticks below
Long: strings.ReplaceAll(`
This command renames files and directory names according a user supplied conversion.
It is useful for renaming a lot of files in an automated way.
`+sprintList()+`
`, "|", "`"),
Annotations: map[string]string{
"versionIntroduced": "v1.70",
"groups": "Filter,Listing,Important,Copy",
},
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc, srcFileName := cmd.NewFsFile(args[0])
cmd.Run(false, true, command, func() error { // retries switched off to prevent double-encoding
return Convmv(context.Background(), fsrc, srcFileName)
})
},
}
// Convmv converts and renames files and directories
// pass srcFileName == "" to convmv every object in fsrc instead of a single object
func Convmv(ctx context.Context, f fs.Fs, srcFileName string) error {
Opt.ctx = ctx
Opt.f = f
if Opt.List {
printList()
return nil
}
err := Opt.validate()
if err != nil {
return err
}
if srcFileName == "" {
// it's a dir
return walkConv(ctx, f, "")
}
// it's a file
obj, err := f.NewObject(Opt.ctx, srcFileName)
if err != nil {
return err
}
oldName, newName, skip, err := parseEntry(obj)
if err != nil {
return err
}
if skip {
return nil
}
return operations.MoveFile(Opt.ctx, Opt.f, Opt.f, newName, oldName)
}
func (opt *ConvOpt) validate() error {
switch opt.ConvertAlgo {
case ConvNone:
return errors.New("must choose a conversion mode with -t flag")
case ConvFindReplace:
if len(opt.FindReplace) == 0 {
return errors.New("must include --replace flag in replace mode")
}
for _, set := range opt.FindReplace {
split := strings.Split(set, ",")
if len(split) != 2 {
return errors.New("--replace must include exactly two comma-separated values")
}
if split[0] == "" {
return errors.New("'find' value cannot be blank ('replace' can be)")
}
}
case ConvPrefix, ConvTrimPrefix:
if opt.Prefix == "" {
return errors.New("must include a --prefix")
}
case ConvSuffix, ConvTrimSuffix:
if opt.Suffix == "" {
return errors.New("must include a --suffix")
}
case ConvTruncate:
if opt.Max < 1 {
return errors.New("--max cannot be less than 1 in 'truncate' mode")
}
case ConvCharmap:
if opt.CmapFlag == 0 {
return errors.New("must specify a charmap with --charmap flag")
}
c := charmapByID(opt.CmapFlag)
if c == nil {
return errors.New("unknown charmap")
}
opt.Cmap = c
}
return nil
}
// keeps track of which dirs we've already renamed
func walkConv(ctx context.Context, f fs.Fs, dir string) error {
entries, err := list.DirSorted(ctx, f, false, dir)
if err != nil {
return err
}
return walkFunc(dir, entries, nil)
}
func walkFunc(path string, entries fs.DirEntries, err error) error {
fs.Debugf(path, "walking dir")
if err != nil {
return err
}
for _, entry := range entries {
switch x := entry.(type) {
case fs.Object:
oldName, newName, skip, err := parseEntry(x)
if err != nil {
return err
}
if skip {
continue
}
fs.Debugf(x, "%v %v %v %v %v", Opt.ctx, Opt.f, Opt.f, newName, oldName)
err = operations.MoveFile(Opt.ctx, Opt.f, Opt.f, newName, oldName)
if err != nil {
return err
}
case fs.Directory:
oldName, newName, skip, err := parseEntry(x)
if err != nil {
return err
}
if !skip { // still want to recurse during dry-runs to get accurate logs
err = DirMoveCaseInsensitive(Opt.ctx, Opt.f, oldName, newName)
if err != nil {
return err
}
} else {
newName = oldName // otherwise dry-runs won't be able to find it
}
// recurse, calling it by its new name
err = walkConv(Opt.ctx, Opt.f, newName)
if err != nil {
return err
}
}
}
return nil
}
// ConvertPath converts a path string according to the chosen ConvertAlgo.
// Each path segment is converted separately, to preserve path separators.
// If baseOnly is true, only the base will be converted (useful for renaming while walking a dir tree recursively.)
// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH"
// otherwise, the entire is path is converted.
func ConvertPath(s string, ConvertAlgo Convert, baseOnly bool) (string, error) {
if s == "" || s == "/" || s == "\\" || s == "." {
return "", nil
}
if baseOnly {
convertedBase, err := ConvertPathSegment(filepath.Base(s), ConvertAlgo)
return filepath.Join(filepath.Dir(s), convertedBase), err
}
segments := strings.Split(s, string(os.PathSeparator))
convertedSegments := make([]string, len(segments))
for _, seg := range segments {
convSeg, err := ConvertPathSegment(seg, ConvertAlgo)
if err != nil {
return "", err
}
convertedSegments = append(convertedSegments, convSeg)
}
return filepath.Join(convertedSegments...), nil
}
// ConvertPathSegment converts one path segment (or really any string) according to the chosen ConvertAlgo.
// It assumes path separators have already been trimmed.
func ConvertPathSegment(s string, ConvertAlgo Convert) (string, error) {
fs.Debugf(s, "converting")
switch ConvertAlgo {
case ConvNone:
return s, nil
case ConvToNFC:
return norm.NFC.String(s), nil
case ConvToNFD:
return norm.NFD.String(s), nil
case ConvToNFKC:
return norm.NFKC.String(s), nil
case ConvToNFKD:
return norm.NFKD.String(s), nil
case ConvBase64Encode:
return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes
case ConvBase64Decode:
if s == ".DS_Store" {
return s, nil
}
b, err := base64.URLEncoding.DecodeString(s)
return string(b), err
case ConvFindReplace:
oldNews := []string{}
for _, pair := range Opt.FindReplace {
split := strings.Split(pair, ",")
oldNews = append(oldNews, split...)
}
replacer := strings.NewReplacer(oldNews...)
return replacer.Replace(s), nil
case ConvPrefix:
return Opt.Prefix + s, nil
case ConvSuffix:
return s + Opt.Suffix, nil
case ConvTrimPrefix:
return strings.TrimPrefix(s, Opt.Prefix), nil
case ConvTrimSuffix:
return strings.TrimSuffix(s, Opt.Suffix), nil
case ConvTruncate:
if Opt.Max <= 0 {
return s, nil
}
if utf8.RuneCountInString(s) <= Opt.Max {
return s, nil
}
runes := []rune(s)
return string(runes[:Opt.Max]), nil
case ConvEncoder:
return Opt.Enc.Encode(s), nil
case ConvDecoder:
return Opt.Enc.Decode(s), nil
case ConvISO8859_1:
return encodeWithReplacement(s, charmap.ISO8859_1), nil
case ConvWindows1252:
return encodeWithReplacement(s, charmap.Windows1252), nil
case ConvMacintosh:
return encodeWithReplacement(s, charmap.Macintosh), nil
case ConvCharmap:
return encodeWithReplacement(s, Opt.Cmap), nil
case ConvLowercase:
return strings.ToLower(s), nil
case ConvUppercase:
return strings.ToUpper(s), nil
case ConvTitlecase:
return strings.ToTitle(s), nil
case ConvASCII:
return toASCII(s), nil
default:
return "", errors.New("this option is not yet implemented")
}
}
func parseEntry(e fs.DirEntry) (oldName, newName string, skip bool, err error) {
oldName = e.Remote()
newName, err = ConvertPath(oldName, Opt.ConvertAlgo, true)
if err != nil {
fs.Errorf(oldName, "error converting: %v", err)
return oldName, newName, true, err
}
if oldName == newName {
fs.Debugf(oldName, "name is already correct - skipping")
return oldName, newName, true, nil
}
skip = operations.SkipDestructive(Opt.ctx, oldName, "rename to "+newName)
return oldName, newName, skip, nil
}
// DirMoveCaseInsensitive does DirMove in two steps (to temp name, then real name)
// which is necessary for some case-insensitive backends
func DirMoveCaseInsensitive(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) {
tmpDstRemote := dstRemote + "-rclone-move-" + random.String(8)
err = operations.DirMove(ctx, f, srcRemote, tmpDstRemote)
if err != nil {
return err
}
return operations.DirMove(ctx, f, tmpDstRemote, dstRemote)
}
func encodeWithReplacement(s string, cmap *charmap.Charmap) string {
return strings.Map(func(r rune) rune {
b, ok := cmap.EncodeRune(r)
if !ok {
return '_'
}
return cmap.DecodeByte(b)
}, s)
}
func toASCII(s string) string {
return strings.Map(func(r rune) rune {
if r <= 127 {
return r
}
return -1
}, s)
}
func sprintList() string {
var out strings.Builder
_, _ = out.WriteString(`### Conversion modes
The conversion mode |-t| or |--conv| flag must be specified. This
defines what transformation the |convmv| command will make.
`)
for _, v := range Opt.ConvertAlgo.Choices() {
_, _ = fmt.Fprintf(&out, "- `%s`\n", v)
}
_, _ = out.WriteRune('\n')
_, _ = out.WriteString(`### Char maps
These are the choices for the |--charmap| flag.
`)
for _, v := range Opt.CmapFlag.Choices() {
_, _ = fmt.Fprintf(&out, "- `%s`\n", v)
}
_, _ = out.WriteRune('\n')
_, _ = out.WriteString(`### Encoding masks
These are the valid options for the --encoding flag.
`)
for _, v := range strings.Split(encoder.ValidStrings(), ", ") {
_, _ = fmt.Fprintf(&out, "- `%s`\n", v)
}
_, _ = out.WriteRune('\n')
sprintExamples(&out)
return out.String()
}
func printList() {
fmt.Println(sprintList())
}
Loaded 100 of 245 files, more files were not shown because too many files have changed in this diff. Show more