Compare commits

..
4 Commits
Author SHA1 Message Date
Nick Craig-Wood 503c975871 rc: fix LogLevel in _config being ignored
Before this change setting LogLevel in the _config parameter of an rc
call had no effect as both the log functions and the log handler only
looked at the global log level.

The context aware log functions now use the log level from the config
in the context, and the log handler lets a log level in the context
which differs from the global one override its own level. This means
an rc call can be made more or less verbose than the rest of rclone
with, for example

    "_config": {"LogLevel": "DEBUG"}

This applies to the logs made with the context aware log functions, so
not to those made by the backends which still use the global log level.

Note that a config copied into a context no longer follows later
changes to the global log level.
2026-10-13 17:58:32 +01:00
Nick Craig-Wood f03a766bdb operations, sync, march, walk, list, filter: attribute logs to the rc job
Use the context aware log functions throughout the code which runs
sync, copy, move, check and the other operations, and the directory
listings they do, so that the logs they make, such as the per file
"Copied (new)" logs, are attributed to the rc job and stats group of
the operation.
2026-10-13 17:58:32 +01:00
Nick Craig-Wood e9d3f2a166 fs: add context aware log functions so logs can be attributed
This adds ErrorfCtx, LogfCtx, InfofCtx, DebugfCtx and LogPrintfCtx
which take a context and pass it on to the log handler. The handler
can then attribute the log to what made it using the new
StatsGroupFromContext and JobIDFromContext function pointers, which
fs/accounting and fs/rc/jobs fill in.

The format string is no longer assigned to in logPrintf so that go
vet recognises these as printf wrappers and checks the arguments of
their callers.
2026-10-13 17:58:32 +01:00
Nick Craig-Wood a46daee827 docs: add section on performance 2026-10-13 17:58:32 +01:00
373 changed files with 64965 additions and 43685 deletions

No files matched your search

-1
View File
@@ -8,7 +8,6 @@ lib/http/templates/** text eol=lf
# Ignore generated files in GitHub language statistics and diffs
/MANUAL.* linguist-generated=true
/rclone.1 linguist-generated=true
cmd/bisync/testdata/*/golden/** linguist-generated=true
# Don't fiddle with the line endings of test data
**/testdata/** -text
+12 -15
View File
@@ -37,7 +37,7 @@ jobs:
include:
- job_name: linux
os: ubuntu-latest
go: '~1.27.3'
go: '~1.27.2'
gotags: cmount
build_flags: '-include "^linux/"'
check: true
@@ -48,14 +48,14 @@ jobs:
- job_name: linux_386
os: ubuntu-latest
go: '~1.27.3'
go: '~1.27.2'
goarch: 386
gotags: cmount
quicktest: true
- job_name: mac_amd64
os: macos-latest
go: '~1.27.3'
go: '~1.27.2'
gotags: 'cmount'
build_flags: '-include "^darwin/amd64" -cgo'
quicktest: true
@@ -64,14 +64,14 @@ jobs:
- job_name: mac_arm64
os: macos-latest
go: '~1.27.3'
go: '~1.27.2'
gotags: 'cmount'
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
deploy: true
- job_name: windows
os: windows-latest
go: '~1.27.3'
go: '~1.27.2'
gotags: cmount
cgo: '0'
build_flags: '-include "^windows/"'
@@ -81,14 +81,14 @@ jobs:
- job_name: other_os
os: ubuntu-latest
go: '~1.27.3'
go: '~1.27.2'
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
compile_all: true
deploy: true
- job_name: go1.26
os: ubuntu-latest
go: '~1.26.10'
go: '~1.26.9'
quicktest: true
racequicktest: true
@@ -177,12 +177,9 @@ jobs:
# https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
# https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
unset HOMEBREW_NO_INSTALL_FROM_API
# brew untap fails if the tap isn't installed, which varies between runner images
for tap in homebrew/core homebrew/cask aws/tap; do
if brew tap | grep -qx "$tap"; then
brew untap --force "$tap"
fi
done
brew untap --force homebrew/core
brew untap --force homebrew/cask
brew untap --force aws/tap
brew update
brew install --cask macfuse
brew install git-annex git-annex-remote-rclone
@@ -287,7 +284,7 @@ jobs:
id: setup-go
uses: actions/setup-go@v7
with:
go-version: '~1.27.3'
go-version: '~1.27.2'
check-latest: true
cache: false
@@ -402,7 +399,7 @@ jobs:
id: setup-go
uses: actions/setup-go@v7
with:
go-version: '~1.27.3'
go-version: '~1.27.2'
# Caching is handled explicitly below to share the module cache
# with the other jobs - see the build job for the rationale.
cache: false
-1
View File
@@ -20,4 +20,3 @@ __pycache__
.DS_Store
resource_windows_*.syso
.devcontainer
/cmd/bisync/testdata/bisync_vscode_debuggers_launch.json
Generated
+2678 -7694
View File
File diff suppressed because it is too large. Load diff
Generated
+618 -4604
View File
File diff suppressed because it is too large. Load diff
Generated
+716 -5112
View File
File diff suppressed because it is too large. Load diff
-6
View File
@@ -28,14 +28,12 @@ directories to and from different cloud storage providers.
- Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
- Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
- ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
- Autonomys Auto Drive [:page_facing_up:](https://rclone.org/s3/#autonomys)
- Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
- Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
- Box [:page_facing_up:](https://rclone.org/box/)
- Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
- China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
- Clever Cloud Cellar [:page_facing_up:](https://rclone.org/s3/#clever-cloud-cellar)
- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
- Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
@@ -73,11 +71,8 @@ directories to and from different cloud storage providers.
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
- Infomaniak Swiss Backup (S3) [:page_facing_up:](https://rclone.org/s3/#infomaniak)
- Infomaniak Swiss Backup (Swift) [:page_facing_up:](https://rclone.org/swift/)
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
- kDrive [:page_facing_up:](https://rclone.org/kdrive/)
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
- Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
- Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
@@ -93,7 +88,6 @@ directories to and from different cloud storage providers.
- Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
- Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
- Minio [:page_facing_up:](https://rclone.org/s3/#minio)
- Neon Object Storage [:page_facing_up:](https://rclone.org/s3/#neon)
- Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
- Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
- OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
+2 -6
View File
@@ -62,14 +62,10 @@ This file describes how to make the various kinds of releases
Early in the next release cycle update the dependencies.
- Active Pins
- `github.com/coreos/go-systemd/v22` v22.6.0 - v22.7.0 fails to compile on netbsd
- `github.com/hanwen/go-fuse/v2` v2.11.0 - v2.12.0 fails to compile on 32
bit linux, awaiting fix for
[go-fuse/issues/628](https://github.com/hanwen/go-fuse/issues/628)
- `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
- See: [#9539](https://github.com/rclone/rclone/issues/9539)
- Review Active Pins in go.mod and remove if possible
- `make updatedirect`
- The update ignores the pins so put them back, eg
`go get github.com/coreos/go-systemd/v22@v22.6.0` then `go mod tidy`
- `make GOTAGS=cmount`
- `make compiletest`
- Fix anything which doesn't compile at this point and commit changes here
-1
View File
@@ -39,7 +39,6 @@ import (
_ "github.com/rclone/rclone/backend/internetarchive"
_ "github.com/rclone/rclone/backend/internxt"
_ "github.com/rclone/rclone/backend/jottacloud"
_ "github.com/rclone/rclone/backend/kdrive"
_ "github.com/rclone/rclone/backend/koofr"
_ "github.com/rclone/rclone/backend/linkbox"
_ "github.com/rclone/rclone/backend/local"
+2 -6
View File
@@ -194,12 +194,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
}
// Path to the archive
archiveRemote := remote[:len(foundArchive.remote)]
// Remote is archive leaf name. This must understand remote
// names as the archive may be in the root, e.g. "remote:file.zip"
_, foundArchive.remote, err = fspath.Split(archiveRemote)
if err != nil {
return nil, fmt.Errorf("failed to parse archive path %q: %w", archiveRemote, err)
}
// Remote is archive leaf name
foundArchive.remote = path.Base(archiveRemote)
foundArchive.prefix = ""
// Point remote to archive file
remote = archiveRemote
-45
View File
@@ -17,7 +17,6 @@ import (
"strings"
"testing"
_ "github.com/rclone/rclone/backend/alias"
"github.com/rclone/rclone/backend/archive/archiver"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
@@ -324,50 +323,6 @@ func TestArchiveUncleanRoot(t *testing.T) {
}
}
// TestArchiveInRemoteRoot checks that an archive in the root of a
// remote, with no "/" between the remote name and the archive name, can
// be read, as can a path within it.
func TestArchiveInRemoteRoot(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, err := zw.Create("sub/a.txt")
require.NoError(t, err)
_, err = w.Write([]byte("data"))
require.NoError(t, err)
require.NoError(t, zw.Close())
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.zip"), buf.Bytes(), 0600))
t.Chdir(dir)
for _, test := range []struct {
name string
remote string
}{
{name: "plain", remote: ":local:test.zip"},
{name: "slash in remote name", remote: ":alias,remote='" + dir + "':test.zip"},
} {
for _, sub := range []struct {
path string
want string
}{
{path: "", want: "sub"},
{path: "/sub", want: "a.txt"},
} {
t.Run(test.name+sub.path, func(t *testing.T) {
f, err := cache.Get(ctx, ":archive:"+test.remote+sub.path)
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, sub.want, entries[0].Remote())
})
}
}
}
// escapingObject is an object whose remote is not where it was asked for.
type escapingObject struct {
fs.Object
+2 -2
View File
@@ -79,7 +79,7 @@ This is required and is the name of the share to access.
Help: `Upload chunk size.
Note that this is stored in memory and there may be up to
"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
in memory.`,
Default: defaultChunkSize,
Advanced: true,
@@ -95,7 +95,7 @@ links and these uploads do not fully utilize your bandwidth, then
increasing this may help to speed up the transfers.
Note that chunks are stored in memory and there may be up to
"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
in memory.`,
Default: 16,
Advanced: true,
+2 -2
View File
@@ -277,7 +277,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
Name: "sse_customer_key",
Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
Alternatively you can provide --b2-sse-customer-key-base64.`,
Alternatively you can provide --sse-customer-key-base64.`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "",
@@ -288,7 +288,7 @@ Alternatively you can provide --b2-sse-customer-key-base64.`,
Name: "sse_customer_key_base64",
Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
Alternatively you can provide --b2-sse-customer-key.`,
Alternatively you can provide --sse-customer-key.`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "",
+9 -10
View File
@@ -27,7 +27,7 @@ import (
"sync/atomic"
"time"
"github.com/golang-jwt/jwt/v5"
"github.com/golang-jwt/jwt/v4"
"github.com/rclone/rclone/backend/box/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
@@ -77,10 +77,7 @@ var (
)
type boxCustomClaims struct {
jwt.RegisteredClaims
// Audience overrides RegisteredClaims.Audience so it is sent as
// a single string rather than an array of strings
Audience string `json:"aud,omitempty"`
jwt.StandardClaims
BoxSubType string `json:"box_sub_type,omitempty"`
}
@@ -226,7 +223,7 @@ func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
if configFileCredentials == "" {
configFile, _ := m.Get("box_config_file")
configFileBytes, err = os.ReadFile(env.ShellExpand(configFile))
configFileBytes, err = os.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("box: failed to read Box config: %w", err)
}
@@ -246,13 +243,15 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
}
claims = &boxCustomClaims{
RegisteredClaims: jwt.RegisteredClaims{
ID: val,
//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
StandardClaims: jwt.StandardClaims{
Id: val,
Issuer: boxConfig.BoxAppSettings.ClientID,
Subject: boxConfig.EnterpriseID,
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
Audience: tokenURL,
ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
},
Audience: tokenURL,
BoxSubType: boxSubType,
}
return claims, nil
-118
View File
@@ -1,118 +0,0 @@
package box
import (
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/rclone/rclone/backend/box/api"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/lib/jwtutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// decodeJWTPart decodes one base64url encoded part of a JWT into a map
func decodeJWTPart(t *testing.T, part string) map[string]any {
data, err := base64.RawURLEncoding.DecodeString(part)
require.NoError(t, err)
var result map[string]any
require.NoError(t, json.Unmarshal(data, &result))
return result
}
// Test the JWT assertion sent to the token endpoint is in the format Box expects
func TestJWTAssertion(t *testing.T) {
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
boxConfig := &api.ConfigJSON{
BoxAppSettings: api.AppSettings{
ClientID: "client-id",
ClientSecret: "client-secret",
AppAuth: api.AppAuth{
PublicKeyID: "public-key-id",
},
},
EnterpriseID: "enterprise-id",
}
var form map[string][]string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "POST", r.Method)
assert.NoError(t, r.ParseForm())
form = r.PostForm
_, _ = w.Write([]byte(`{"access_token":"access-token","token_type":"bearer","expires_in":3600}`))
}))
defer ts.Close()
claims, err := getClaims(boxConfig, "enterprise")
require.NoError(t, err)
m := configmap.Simple{}
start := time.Now()
err = jwtutil.Config("box", "test", ts.URL, *claims, getSigningHeaders(boxConfig), getQueryParams(boxConfig), privateKey, m, ts.Client())
require.NoError(t, err)
// Check the token was stored
assert.Contains(t, m["token"], `"access_token":"access-token"`)
// Check the form parameters
assert.Equal(t, []string{"urn:ietf:params:oauth:grant-type:jwt-bearer"}, form["grant_type"])
assert.Equal(t, []string{"client-id"}, form["client_id"])
assert.Equal(t, []string{"client-secret"}, form["client_secret"])
require.Len(t, form["assertion"], 1)
parts := strings.Split(form["assertion"][0], ".")
require.Len(t, parts, 3)
// Check the signature independently of the JWT library
signature, err := base64.RawURLEncoding.DecodeString(parts[2])
require.NoError(t, err)
hash := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
require.NoError(t, rsa.VerifyPKCS1v15(&privateKey.PublicKey, crypto.SHA256, hash[:], signature))
// Check the header
assert.Equal(t, map[string]any{
"alg": "RS256",
"typ": "JWT",
"kid": "public-key-id",
}, decodeJWTPart(t, parts[0]))
// Check the claims - aud must be a string, not an array
payload := decodeJWTPart(t, parts[1])
exp, ok := payload["exp"].(float64)
require.True(t, ok, "exp should be a number")
assert.InDelta(t, float64(start.Add(45*time.Second).Unix()), exp, 5)
jti, ok := payload["jti"].(string)
require.True(t, ok, "jti should be a string")
assert.Len(t, jti, 40)
delete(payload, "exp")
delete(payload, "jti")
assert.Equal(t, map[string]any{
"aud": tokenURL,
"iss": "client-id",
"sub": "enterprise-id",
"box_sub_type": "enterprise",
}, payload)
}
// Test box_config_file has ~ and environment variables expanded
func TestGetBoxConfigShellExpand(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"enterpriseID":"enterprise-id"}`), 0600))
t.Setenv("RCLONE_TEST_BOX_CONFIG_DIR", dir)
m := configmap.Simple{
"box_config_file": "$RCLONE_TEST_BOX_CONFIG_DIR/config.json",
}
boxConfig, err := getBoxConfig(m)
require.NoError(t, err)
assert.Equal(t, "enterprise-id", boxConfig.EnterpriseID)
}
+5 -26
View File
@@ -1502,21 +1502,11 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {
func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
oldFsObject, err := f.NewObject(ctx, remote)
if err == nil {
removeStaleChunks(ctx, oldFsObject.(*Object), nil)
}
}
func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {
currentRemotes := make(map[string]struct{}, len(currentChunks))
for _, chunk := range currentChunks {
currentRemotes[chunk.Remote()] = struct{}{}
}
for _, chunk := range oldObject.chunks {
if _, found := currentRemotes[chunk.Remote()]; found {
continue
}
if err := chunk.Remove(ctx); err != nil {
fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
oldObject := oldFsObject.(*Object)
for _, chunk := range oldObject.chunks {
if err := chunk.Remove(ctx); err != nil {
fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
}
}
}
}
@@ -1697,20 +1687,12 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
// metadata format which might involve unsupported chunk types.
return nil, fmt.Errorf("can't %s this file: %w", opName, err)
}
// Snapshot destination chunks before its metadata is replaced.
var oldObject *Object
if oldFsObject, err := f.NewObject(ctx, remote); err == nil {
oldObject = oldFsObject.(*Object)
}
if !o.isComposite() {
fs.Debugf(o, "%s non-chunked object...", opName)
oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
if err != nil {
return nil, err
}
if oldObject != nil {
removeStaleChunks(ctx, oldObject, nil)
}
return f.newObject("", oResult, nil), nil
}
@@ -1776,9 +1758,6 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
silentlyRemove(ctx, newObj)
return nil, err
}
if oldObject != nil {
removeStaleChunks(ctx, oldObject, newChunks)
}
return newObj, nil
}
-102
View File
@@ -11,7 +11,6 @@ import (
"strings"
"testing"
_ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/fspath"
@@ -878,104 +877,6 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
_ = operations.Purge(ctx, f.base, dir)
}
func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {
ctx := context.Background()
const dir = "serveroverwrite"
remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)
fsResult, err := fs.NewFs(ctx, remote)
require.NoError(t, err)
copyFs, ok := fsResult.(*Fs)
require.True(t, ok)
oldMoveChunkSize := moveFs.opt.ChunkSize
moveFs.opt.ChunkSize = fs.SizeSuffix(3)
defer func() {
moveFs.opt.ChunkSize = oldMoveChunkSize
_ = operations.Purge(ctx, copyFs.base, "")
_ = operations.Purge(ctx, moveFs.base, dir)
}()
type operation struct {
name string
fs *Fs
enabled bool
do copyMoveFn
}
operations := []operation{
{name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},
{name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},
}
testCases := []struct {
name string
sourceContents string
targetContents string
}{
{name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},
{name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},
}
transactionModes := []struct {
name string
useNoRename bool
}{
{name: "Rename"},
{name: "NoRename", useNoRename: true},
}
for _, operation := range operations {
t.Run(operation.name, func(t *testing.T) {
if !operation.enabled {
t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))
}
for _, transactionMode := range transactionModes {
t.Run(transactionMode.name, func(t *testing.T) {
f := operation.fs
if transactionMode.useNoRename && !f.useMeta {
t.Skip("Can't test norename transactions without metadata")
}
oldUseNoRename := f.useNoRename
f.useNoRename = transactionMode.useNoRename
defer func() { f.useNoRename = oldUseNoRename }()
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))
source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)
target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)
targetObject := target.(*Object)
require.NotEmpty(t, targetObject.chunks)
oldChunks := append([]fs.Object(nil), targetObject.chunks...)
result, err := operation.do(ctx, source, target.Remote())
require.NoError(t, err)
reader, err := result.Open(ctx)
require.NoError(t, err)
contents, err := io.ReadAll(reader)
require.NoError(t, err)
require.NoError(t, reader.Close())
assert.Equal(t, testCase.sourceContents, string(contents))
resultObject := result.(*Object)
currentChunks := make(map[string]struct{}, len(resultObject.chunks))
for _, chunk := range resultObject.chunks {
currentChunks[chunk.Remote()] = struct{}{}
}
removed := 0
for _, chunk := range oldChunks {
if _, found := currentChunks[chunk.Remote()]; found {
continue
}
_, err := f.base.NewObject(ctx, chunk.Remote())
assert.Equal(t, fs.ErrorObjectNotFound, err)
removed++
}
assert.Positive(t, removed)
})
}
})
}
})
}
}
// Test that md5all creates metadata even for small files
func testMD5AllSlow(t *testing.T, f *Fs) {
ctx := context.Background()
@@ -1049,9 +950,6 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("ChunkerServerSideMove", func(t *testing.T) {
testChunkerServerSideMove(t, f)
})
t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {
testChunkerServerSideOverwrite(t, f)
})
t.Run("MD5AllSlow", func(t *testing.T) {
testMD5AllSlow(t, f)
})
+1 -1
View File
@@ -1197,7 +1197,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
return "", nil
}
// if this is wrapping a local object then we work out the hash
if srcObj != nil && srcObj.Fs().Features().IsLocal {
if srcObj.Fs().Features().IsLocal {
// Read the data and encrypt it to calculate the hash
fs.Debugf(o, "Computing %v hash of encrypted source", hash)
return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)
-10
View File
@@ -93,16 +93,6 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
}
// Test ObjectInfo.Hash doesn't panic when the source is an
// fs.OverrideRemote which wraps an fs.ObjectInfo that isn't an fs.Object
func TestObjectInfoHashOverrideRemoteNotObject(t *testing.T) {
src := object.NewStaticObjectInfo("file", time.Now(), 3, true, nil, nil)
oi := (&Fs{}).newObjectInfo(fs.NewOverrideRemote(src, "new_remote"), nonce{})
gotHash, err := oi.Hash(context.Background(), hash.MD5)
require.NoError(t, err)
assert.Equal(t, "", gotHash)
}
func testComputeHash(t *testing.T, f *Fs) {
var (
contents = random.String(100)
+1 -9
View File
@@ -321,13 +321,10 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
}
return false
})
if err != nil {
return nil, err
}
if !found {
return nil, fs.ErrorObjectNotFound
}
return info, nil
return info, err
}
// errorHandler parses a non 2xx error response into an error
@@ -549,11 +546,6 @@ OUTER:
if err != nil {
return found, fmt.Errorf("couldn't list files: %w", err)
}
// Drime won't page past a fixed number of entries (20,000 for
// rclone) and serves the last page it allows again instead
if result.CurrentPage != page {
return found, errors.New("couldn't list files: directory has more entries than drime will list")
}
for _, item := range result.Data {
if item.Type == api.ItemTypeFolder {
if filesOnly {
-158
View File
@@ -1,158 +0,0 @@
package drime
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/backend/drime/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
testPerPage = 5 // entries per page the test server returns
testLimit = 20 // entries the test server will page through
)
// listingServer serves /drive/file-entries from entries the way Drime
// does.
//
// It returns testPerPage entries per page and won't page past testLimit
// entries. Asking for a later page returns the last page it allows
// again, with last_page one beyond it.
//
// It counts the requests made in requests.
func listingServer(t *testing.T, entries []api.Item, requests *atomic.Int64) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
assert.Equal(t, "/drive/file-entries", r.URL.Path)
params := r.URL.Query()
page, err := strconv.Atoi(params.Get("page"))
require.NoError(t, err)
maxPage := testLimit / testPerPage
currentPage := min(page, maxPage)
start := min((currentPage-1)*testPerPage, len(entries))
end := min(start+testPerPage, len(entries))
lastPage := currentPage
if page > maxPage || end < len(entries) {
lastPage = currentPage + 1
}
w.Header().Set("Content-Type", "application/json")
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
"current_page": currentPage,
"last_page": lastPage,
"per_page": testPerPage,
"total": (currentPage + 1) * testPerPage,
"data": entries[start:end],
}))
}))
}
// makeEntries makes n file entries
func makeEntries(n int) []api.Item {
entries := make([]api.Item, n)
for i := range entries {
entries[i] = api.Item{
ID: json.Number(strconv.Itoa(1000 + i)),
Name: fmt.Sprintf("file%05d", i),
Type: "text",
}
}
return entries
}
// testFs makes an Fs which uses server as its API
func testFs(t *testing.T, server *httptest.Server) *Fs {
t.Helper()
f := &Fs{
opt: Options{ListChunk: 1000},
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
}
f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
f.srv.SetErrorHandler(errorHandler)
return f
}
// listNames lists a directory with listAll and returns the names found
func listNames(ctx context.Context, f *Fs) (names []string, err error) {
_, err = f.listAll(ctx, "1", false, false, "", func(item *api.Item) bool {
names = append(names, item.Name)
return false
})
return names, err
}
// TestListAll checks that listAll returns every entry exactly once
func TestListAll(t *testing.T) {
for _, test := range []struct {
name string
entries int
}{
{name: "Empty", entries: 0},
{name: "OnePage", entries: 3},
{name: "ExactPages", entries: 10},
{name: "PartialLastPage", entries: 13},
{name: "AtLimit", entries: testLimit},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var requests atomic.Int64
entries := makeEntries(test.entries)
server := listingServer(t, entries, &requests)
defer server.Close()
f := testFs(t, server)
names, err := listNames(ctx, f)
require.NoError(t, err)
var want []string
for _, entry := range entries {
want = append(want, entry.Name)
}
assert.ElementsMatch(t, want, names)
})
}
}
// TestListAllTooManyEntries checks that listAll returns an error for a
// directory with more entries than the server will list, rather than
// looping forever.
func TestListAllTooManyEntries(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var requests atomic.Int64
server := listingServer(t, makeEntries(2*testLimit), &requests)
defer server.Close()
f := testFs(t, server)
_, err := listNames(ctx, f)
require.Error(t, err)
assert.NoError(t, ctx.Err(), "listing didn't finish")
assert.LessOrEqual(t, requests.Load(), int64(testLimit/testPerPage+1))
}
// TestGetItemListError checks that getItem returns listing errors
// rather than fs.ErrorObjectNotFound.
func TestGetItemListError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"message":"access denied"}`, http.StatusForbidden)
}))
defer server.Close()
f := testFs(t, server)
_, err := f.getItem(context.Background(), "1000", "1", "file00000")
require.Error(t, err)
assert.NotErrorIs(t, err, fs.ErrorObjectNotFound)
assert.Contains(t, err.Error(), "access denied")
}
+9 -22
View File
@@ -568,7 +568,7 @@ recommended usage is using the flag form --drive-size-as-quota when
doing rclone ls/lsl/lsf/lsjson/etc only.
If you do use this flag for syncing (not recommended) then you will
need to use --ignore-size also.`,
need to use --ignore size also.`,
Advanced: true,
Hide: fs.OptionHideConfigurator,
}, {
@@ -2298,9 +2298,15 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
wg.Wait()
// if the input channel overflowed add the collected entries to the channel now
for len(overflow) > 0 {
if !refillListRInput(&mu, &in, &overflow, &wg) {
return
mu.Lock()
// only fill half of the channel to prevent entries being put into overflow again
l := min(len(overflow), listRInputBuffer/2)
wg.Add(l)
for _, d := range overflow[:l] {
in <- d
}
overflow = overflow[l:]
mu.Unlock()
// wait again for the completion of all directories
wg.Wait()
@@ -2348,25 +2354,6 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return nil
}
// refillListRInput moves queued directory jobs back onto in.
// It returns false when a listing error has already closed in.
// Sending on that nil channel would block forever while holding mu.
func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {
mu.Lock()
defer mu.Unlock()
if *in == nil {
return false
}
// only fill half of the channel to prevent entries being put into overflow again
l := min(len(*overflow), listRInputBuffer/2)
wg.Add(l)
for _, d := range (*overflow)[:l] {
*in <- d
}
*overflow = (*overflow)[l:]
return true
}
const shortcutSeparator = '\t'
// joinID adds an actual drive ID to the shortcut ID it came from
-48
View File
@@ -916,51 +916,3 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {
mu := stdsync.Mutex{}
wg := stdsync.WaitGroup{}
in := make(chan listREntry, listRInputBuffer)
overflow := []listREntry{{id: "folder", path: "folder"}}
mu.Lock()
close(in)
in = nil
mu.Unlock()
done := make(chan bool, 1)
go func() {
done <- refillListRInput(&mu, &in, &overflow, &wg)
}()
select {
case ok := <-done:
require.False(t, ok, "refill continued after the input channel was closed")
case <-time.After(time.Second):
t.Fatal("refill blocked after the input channel was closed")
}
require.Len(t, overflow, 1)
finished := make(chan struct{})
go func() {
wg.Wait()
close(finished)
}()
select {
case <-finished:
case <-time.After(time.Second):
t.Fatal("refill incremented the wait group after the input channel was closed")
}
}
func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {
mu := stdsync.Mutex{}
wg := stdsync.WaitGroup{}
in := make(chan listREntry, listRInputBuffer)
overflow := []listREntry{{id: "folder", path: "folder"}}
require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")
require.Empty(t, overflow)
got := <-in
require.Equal(t, listREntry{id: "folder", path: "folder"}, got)
wg.Done()
}
+14 -32
View File
@@ -584,9 +584,8 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
//
// It nils the pointed to connection out so it can't be reused
//
// if err is not nil then it discards the connection unless the error
// carries an FTP reply code, in which case it checks the connection
// is alive using a NOOP request
// if err is not nil then it checks the connection is alive using a
// NOOP request
func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
if f.opt.Concurrency > 0 {
defer f.tokens.Put()
@@ -600,17 +599,14 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
}
*pc = nil
if err != nil {
// An error without an FTP reply code means the control
// connection can no longer be trusted, so discard it.
if textprotoError(err) == nil {
fs.Debugf(f, "Discarding connection after non-FTP error: %v", err)
_ = c.Quit()
return
}
if nopErr := c.NoOp(); nopErr != nil {
fs.Debugf(f, "Connection failed, closing: %v", nopErr)
_ = c.Quit()
return
// If not a regular FTP error code then check the connection
if tpErr := textprotoError(err); tpErr != nil {
nopErr := c.NoOp()
if nopErr != nil {
fs.Debugf(f, "Connection failed, closing: %v", nopErr)
_ = c.Quit()
return
}
}
}
f.poolMu.Lock()
@@ -1269,18 +1265,15 @@ func (o *Object) Storable() bool {
// ftpReadCloser implements io.ReadCloser for FTP objects.
type ftpReadCloser struct {
rc io.ReadCloser
c *ftp.ServerConn
f *Fs
err error // errors found during read
closed bool // set by Close so a second call is a no-op
read int64 // bytes returned by Read
rc io.ReadCloser
c *ftp.ServerConn
f *Fs
err error // errors found during read
}
// Read bytes into p
func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
n, err = f.rc.Read(p)
f.read += int64(n)
if err != nil && err != io.EOF {
f.err = err // store any errors for Close to examine
}
@@ -1289,10 +1282,6 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
// Close the FTP reader and return the connection to the pool
func (f *ftpReadCloser) Close() error {
if f.closed {
return nil
}
f.closed = true
var err error
errchan := make(chan error, 1)
go func() {
@@ -1316,7 +1305,6 @@ func (f *ftpReadCloser) Close() error {
// if errors while reading or closing, dump the connection
if err != nil || f.err != nil {
_ = f.c.Quit()
f.c = nil // never pool a connection after Quit
f.f.putFtpConnection(nil, nil)
} else {
f.f.putFtpConnection(&f.c, nil)
@@ -1328,12 +1316,6 @@ func (f *ftpReadCloser) Close() error {
switch errX.Code {
case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend, ftp.StatusRequestedFileActionOK:
err = nil
case ftp.StatusCanNotOpenDataConnection:
// Egnyte answers 425 when a RETR is closed before the first
// read; a transfer cut after data was read stays an error
if f.read == 0 {
err = nil
}
}
}
return err
-163
View File
@@ -1,19 +1,12 @@
package ftp
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"net/textproto"
"strings"
"sync"
"testing"
"time"
"github.com/jlaffaye/ftp"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/object"
@@ -151,159 +144,3 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
// fakeFTPServer is a minimal FTP control-connection server which answers
// every command with 200 OK.
type fakeFTPServer struct {
ln net.Listener
mu sync.Mutex
conn net.Conn
}
func startFakeFTPServer(t *testing.T) *fakeFTPServer {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
s := &fakeFTPServer{ln: ln}
go s.serve()
t.Cleanup(func() { s.close() })
return s
}
func (s *fakeFTPServer) serve() {
conn, err := s.ln.Accept()
if err != nil {
return
}
s.mu.Lock()
s.conn = conn
s.mu.Unlock()
defer func() { _ = conn.Close() }()
_, _ = fmt.Fprint(conn, "220 fake FTP server\r\n")
scanner := bufio.NewScanner(conn)
for scanner.Scan() {
_, _ = fmt.Fprint(conn, "200 OK\r\n")
}
}
// killConnection closes the server side of the control connection,
// simulating a server that dropped an idle connection.
func (s *fakeFTPServer) killConnection() {
s.mu.Lock()
defer s.mu.Unlock()
if s.conn != nil {
_ = s.conn.Close()
}
}
func (s *fakeFTPServer) close() {
s.mu.Lock()
defer s.mu.Unlock()
if s.conn != nil {
_ = s.conn.Close()
}
_ = s.ln.Close()
}
// TestPutFtpConnectionDiscardsDeadConnection checks that a connection
// returned with a raw network error is closed and never re-pooled, while
// a connection returned with a plain FTP reply-code error is probed with
// NOOP and pooled while the probe succeeds.
func TestPutFtpConnectionDiscardsDeadConnection(t *testing.T) {
t.Run("network error discards", func(t *testing.T) {
server := startFakeFTPServer(t)
c, err := ftp.Dial(server.ln.Addr().String())
require.NoError(t, err)
f := &Fs{opt: Options{}}
f.putFtpConnection(&c, errors.New("unexpected EOF"))
assert.Nil(t, c)
assert.Empty(t, f.pool)
})
t.Run("FTP reply error with live connection pools", func(t *testing.T) {
server := startFakeFTPServer(t)
c, err := ftp.Dial(server.ln.Addr().String())
require.NoError(t, err)
f := &Fs{opt: Options{}}
f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
assert.Nil(t, c)
require.Len(t, f.pool, 1)
assert.NoError(t, f.pool[0].NoOp())
_ = f.pool[0].Quit()
})
t.Run("FTP reply error with dead connection discards", func(t *testing.T) {
server := startFakeFTPServer(t)
c, err := ftp.Dial(server.ln.Addr().String())
require.NoError(t, err)
server.killConnection()
// Let the FIN propagate so the NOOP probe observes the dead connection.
time.Sleep(100 * time.Millisecond)
f := &Fs{opt: Options{}}
f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
assert.Nil(t, c)
assert.Empty(t, f.pool)
})
}
// fakeDataReader stands in for the RETR data stream of ftpReadCloser.
type fakeDataReader struct {
data []byte
closeErr error
}
func (r *fakeDataReader) Read(p []byte) (int, error) {
if len(r.data) == 0 {
return 0, io.EOF
}
n := copy(p, r.data)
r.data = r.data[n:]
return n, nil
}
func (r *fakeDataReader) Close() error { return r.closeErr }
// TestFtpReadCloserClose checks that an early-closed RETR answered with 425
// is masked only when nothing was read, that a connection which got QUIT is
// never pooled, and that a second Close does not touch the pool.
func TestFtpReadCloserClose(t *testing.T) {
err425 := &textproto.Error{Code: ftp.StatusCanNotOpenDataConnection, Msg: "Can't open data connection."}
newReader := func(t *testing.T, data []byte, closeErr error) (*ftpReadCloser, *Fs) {
server := startFakeFTPServer(t)
c, err := ftp.Dial(server.ln.Addr().String())
require.NoError(t, err)
f := &Fs{opt: Options{}}
return &ftpReadCloser{rc: &fakeDataReader{data: data, closeErr: closeErr}, c: c, f: f}, f
}
t.Run("425 before any read is masked and the connection is not pooled", func(t *testing.T) {
rc, f := newReader(t, nil, err425)
assert.NoError(t, rc.Close())
assert.Empty(t, f.pool)
assert.NoError(t, rc.Close())
assert.Empty(t, f.pool)
})
t.Run("425 after data was read stays an error", func(t *testing.T) {
rc, f := newReader(t, []byte("partial"), err425)
buf := make([]byte, 4)
n, err := rc.Read(buf)
require.NoError(t, err)
require.Equal(t, 4, n)
err = rc.Close()
require.Error(t, err)
assert.Equal(t, ftp.StatusCanNotOpenDataConnection, textprotoError(err).Code)
assert.Empty(t, f.pool)
})
t.Run("clean close pools the connection once", func(t *testing.T) {
rc, f := newReader(t, []byte("data"), nil)
_, err := io.ReadAll(rc)
require.NoError(t, err)
assert.NoError(t, rc.Close())
require.Len(t, f.pool, 1)
assert.NoError(t, rc.Close())
assert.Len(t, f.pool, 1)
_ = f.pool[0].Quit()
})
}
-44
View File
@@ -122,49 +122,6 @@ func (f *Fs) testUpdateStoresHash(t *testing.T) {
_ = operations.Purge(ctx, f, dirName)
}
func (f *Fs) testSetModTimeKeepsHash(t *testing.T) {
ctx := context.Background()
const dirName = "setmodtime_hash_1"
const fileName = dirName + "/file_setmodtime_1"
const longTime = fs.ModTimeNotSupported
hashType := f.keepHashes.GetOne()
// upload a file to hasher via Put
src := putFile(ctx, t, f, fileName, "setmodtime content")
dst, err := f.NewObject(ctx, fileName)
require.NoError(t, err)
require.NotNil(t, dst)
_ = src
// verify hash was stored after Put
if f.opt.MaxAge <= 0 {
t.Skip("hash cache is disabled")
}
hash1, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
require.NoError(t, err)
require.NotEmpty(t, hash1)
// change only the modification time, as operations does when it
// syncs the modtime of an otherwise unchanged destination file
newTime := fstest.Time("2002-03-04T05:06:07.499999999Z")
require.NoError(t, dst.SetModTime(ctx, newTime))
// the cached hash must survive a pure modtime change
hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
assert.NoError(t, err)
assert.Equal(t, hash1, hash2, "hash should be kept after SetModTime")
// and it must be found via the fingerprint-checked lookup too
dst2, err := f.NewObject(ctx, fileName)
require.NoError(t, err)
hashVal, err := dst2.Hash(ctx, hashType)
assert.NoError(t, err)
assert.Equal(t, hash1, hashVal, "hash should be returned after SetModTime")
_ = operations.Purge(ctx, f, dirName)
}
// InternalTest dispatches all internal tests
func (f *Fs) InternalTest(t *testing.T) {
if !kv.Supported() {
@@ -172,7 +129,6 @@ func (f *Fs) InternalTest(t *testing.T) {
}
t.Run("UploadFromCrypt", f.testUploadFromCrypt)
t.Run("UpdateStoresHash", f.testUpdateStoresHash)
t.Run("SetModTimeKeepsHash", f.testSetModTimeKeepsHash)
}
var _ fstests.InternalTester = (*Fs)(nil)
-28
View File
@@ -206,34 +206,6 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
return err
}
// kvUpdateFingerprint: rewrite the fingerprint of an existing record
type kvUpdateFingerprint struct {
key string
oldFp string
newFp string
age time.Duration
}
func (op *kvUpdateFingerprint) Do(ctx context.Context, b kv.Bucket) error {
data := b.Get([]byte(op.key))
if len(data) == 0 {
return nil
}
var r hashRecord
if err := r.decode(op.key, data); err != nil {
return nil
}
if r.Fp != op.oldFp || time.Since(r.Created) > op.age {
return nil
}
r.Fp = op.newFp
data, err := r.encode(op.key)
if err != nil {
return err
}
return b.Put([]byte(op.key), data)
}
// kvDump: dump the database.
// Note: long dump can cause concurrent operations to fail.
type kvDump struct {
+5 -35
View File
@@ -56,23 +56,6 @@ func (o *Object) putHashes(ctx context.Context, rawHashes hashMap) error {
return o.f.putRawHashes(ctx, key, fp, hashes)
}
// updateFingerprint rewrites the fingerprint of the cached hash
// record for remote from oldFp to newFp, keeping the hashes. It is
// used when only the modification time of an object changes, so the
// cached hashes remain valid under the new fingerprint.
func (f *Fs) updateFingerprint(ctx context.Context, remote, oldFp, newFp string) error {
if oldFp == "" || newFp == "" || oldFp == newFp || f.opt.MaxAge <= 0 {
return nil
}
key := path.Join(f.Fs.Root(), remote)
return f.db.Do(true, &kvUpdateFingerprint{
key: key,
oldFp: oldFp,
newFp: newFp,
age: time.Duration(f.opt.MaxAge),
})
}
// set hashes for a path without any validation
func (f *Fs) putRawHashes(ctx context.Context, key, fp string, hashes operations.HashSums) error {
return f.db.Do(true, &kvPut{
@@ -213,27 +196,14 @@ func (o *Object) Remove(ctx context.Context) error {
}
// SetModTime sets the modification time of the file.
//
// When the fingerprint includes the modtime, the contents have not
// changed, so the cached hashes are carried over to the fingerprint
// for the new modtime. Otherwise the cache entry is pruned when the
// modtime changes so that touching a file will trigger checksum
// recalculation even on backends that don't provide modTime with
// fingerprint.
// Also prunes the cache entry when modtime changes so that
// touching a file will trigger checksum recalculation even
// on backends that don't provide modTime with fingerprint.
func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
if mtime == o.Object.ModTime(ctx) {
return o.Object.SetModTime(ctx, mtime)
}
if !o.f.fpTime {
if mtime != o.Object.ModTime(ctx) {
_ = o.f.pruneHash(o.Remote())
return o.Object.SetModTime(ctx, mtime)
}
oldFp := o.fingerprint(ctx)
if err := o.Object.SetModTime(ctx, mtime); err != nil {
return err
}
_ = o.f.updateFingerprint(ctx, o.Remote(), oldFp, o.fingerprint(ctx))
return nil
return o.Object.SetModTime(ctx, mtime)
}
// Open opens the file for read.
+10 -43
View File
@@ -63,8 +63,7 @@ The headers are only sent to the host in the configured URL. If the
server redirects to another host (including a subdomain or a different
port) the headers are not sent to it, or to any further hop in that
redirect chain. When headers are set, a redirect from https to http is
refused as it would send them in cleartext, even with
--allow-https-downgrade.`,
refused as it would send them in cleartext.`,
Default: fs.CommaSepList{},
Advanced: true,
}, {
@@ -101,30 +100,6 @@ that directory listings are much quicker, but rclone won't have the times or
sizes of any files, and some files that don't exist may be in the listing.`,
Default: false,
Advanced: true,
}, {
Name: "no_head_size",
Help: `Set the reported file size for --http-no-head.
When HEAD requests are disabled, rclone normally reports an unknown file size.
Set this option to a non-negative value to report that size for every file,
including 0. Leave it unset (the default, shown as "off") to keep reporting
an unknown size.
The configured size is authoritative. If it does not match the actual file
size, transfers may fail their size check and retry unless --ignore-size is
used. It can also make multi-thread transfers request incorrect ranges and
make mounted files unreadable past the reported size.`,
Default: fs.SizeSuffix(-1),
Advanced: true,
}, {
Name: "no_head_time",
Help: `Set a default modification time for --http-no-head.
When HEAD requests are disabled, rclone normally reports an unset modification
time. Set this option to report the specified time for every file instead.
Changing the configured time later makes every file appear modified to a sync.`,
Default: fs.Time{},
Advanced: true,
}, {
Name: "no_escape",
Help: "Do not escape URL metacharacters in path names.",
@@ -170,13 +145,11 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
// Options defines the configuration for this backend
type Options struct {
Endpoint string `config:"url"`
NoSlash bool `config:"no_slash"`
NoHead bool `config:"no_head"`
NoHeadSize fs.SizeSuffix `config:"no_head_size"`
NoHeadTime fs.Time `config:"no_head_time"`
Headers fs.CommaSepList `config:"headers"`
NoEscape bool `config:"no_escape"`
Endpoint string `config:"url"`
NoSlash bool `config:"no_slash"`
NoHead bool `config:"no_head"`
Headers fs.CommaSepList `config:"headers"`
NoEscape bool `config:"no_escape"`
}
// Fs stores the interface to the remote HTTP files
@@ -316,9 +289,8 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
}
client := fshttp.NewClient(ctx)
// Without configured headers keep the default policy so
// --allow-https-downgrade can follow public mirrors that
// redirect from https to http.
// Without configured headers keep the default policy which
// public mirrors that redirect from https to http rely on.
if len(opt.Headers) > 0 {
client.CheckRedirect = checkRedirect(opt)
}
@@ -350,9 +322,7 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
// the host specified in the config file.
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
opt := &Options{
NoHeadSize: fs.SizeSuffix(-1),
}
opt := new(Options)
err := configstruct.Set(m, opt)
if err != nil {
return nil, err
@@ -747,11 +717,8 @@ func (o *Object) url() string {
// head sends a HEAD request to update info fields in the Object
func (o *Object) head(ctx context.Context) error {
if o.fs.opt.NoHead {
o.size = int64(o.fs.opt.NoHeadSize)
o.size = -1
o.modTime = timeUnset
if o.fs.opt.NoHeadTime.IsSet() {
o.modTime = time.Time(o.fs.opt.NoHeadTime)
}
o.contentType = fs.MimeType(ctx, o)
return nil
}
+4 -65
View File
@@ -314,54 +314,6 @@ func TestOpen(t *testing.T) {
}
}
func TestNoHeadMetadataDefaults(t *testing.T) {
tests := []struct {
name string
size string
modTime string
wantSize int64
wantModTime time.Time
}{
{
name: "defaults",
wantSize: -1,
wantModTime: timeUnset,
}, {
name: "zero size",
size: "0B",
wantSize: 0,
wantModTime: timeUnset,
}, {
name: "configured metadata",
size: "100B",
modTime: "2024-08-05T07:36:07Z",
wantSize: 100,
wantModTime: time.Date(2024, 8, 5, 7, 36, 7, 0, time.UTC),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
m := prepareServer(t)
m.Set("no_head", "true")
if test.size != "" {
m.Set("no_head_size", test.size)
}
if test.modTime != "" {
m.Set("no_head_time", test.modTime)
}
f, err := NewFs(context.Background(), remoteName, "", m)
require.NoError(t, err)
o, err := f.NewObject(context.Background(), "four/under four.txt")
require.NoError(t, err)
assert.Equal(t, test.wantSize, o.Size())
assert.Equal(t, test.wantModTime, o.ModTime(context.Background()))
})
}
}
func TestMimeType(t *testing.T) {
f := prepare(t)
@@ -688,7 +640,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
// open makes a remote pointing at ts with or without headers
// and opens file.txt on it
open := func(t *testing.T, withHeaders, allowDowngrade bool) (io.ReadCloser, error) {
open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
configfile.Install()
m := configmap.Simple{
"type": "http",
@@ -700,7 +652,6 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
}
ctx, ci := fs.AddConfig(context.Background())
ci.InsecureSkipVerify = true
ci.AllowHTTPSDowngrade = allowDowngrade
f, err := NewFs(ctx, remoteName, "", m)
require.NoError(t, err)
@@ -712,25 +663,13 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
t.Run("WithHeaders", func(t *testing.T) {
// Headers could leak so the downgrade must be refused
_, err := open(t, true, false)
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
})
t.Run("WithHeadersAllowDowngrade", func(t *testing.T) {
// Headers could leak so the downgrade is refused even when allowed
_, err := open(t, true, true)
_, err := open(t, true)
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
})
t.Run("WithoutHeaders", func(t *testing.T) {
// The transport refuses downgrades by default
_, err := open(t, false, false)
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
})
t.Run("WithoutHeadersAllowDowngrade", func(t *testing.T) {
// Nothing to leak so the downgrade is followed when allowed
fd, err := open(t, false, true)
// Nothing to leak so the downgrade is followed
fd, err := open(t, false)
require.NoError(t, err)
data, err := io.ReadAll(fd)
require.NoError(t, err)
+3 -13
View File
@@ -50,10 +50,6 @@ const (
defaultChunkSize = 8 * fs.Mebi
)
// jsonContentType is sent with JSON request bodies as the API doesn't
// decode non-ASCII file names in them as UTF-8 unless told the charset.
const jsonContentType = "application/json; charset=UTF-8"
// OAuth2 configuration
var oauthConfig = &oauthutil.Config{
Scopes: []string{
@@ -551,7 +547,6 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
Parameters: url.Values{
"fields": []string{"*"},
},
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
}
var info *api.File
@@ -1020,7 +1015,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
Parameters: url.Values{
"fields": []string{"*"},
},
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
}
copyReq := api.CopyFileRequest{
@@ -1205,10 +1199,9 @@ func (f *Fs) updateFile(ctx context.Context, fileID string, parameters url.Value
}
parameters.Set("fields", "*")
opts := rest.Opts{
Method: "PATCH",
Path: "/files/" + fileID,
Parameters: parameters,
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
Method: "PATCH",
Path: "/files/" + fileID,
Parameters: parameters,
}
err = f.pacer.Call(func() (bool, error) {
resp, err := f.srv.CallJSON(ctx, &opts, req, &info)
@@ -1678,7 +1671,6 @@ func (o *Object) uploadEmpty(ctx context.Context, leaf, directoryID string, src
Parameters: url.Values{
"fields": []string{"*"},
},
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
}
var info *api.File
@@ -1854,7 +1846,6 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
"fields": []string{"*"},
},
ExtraHeaders: map[string]string{
"Content-Type": jsonContentType,
"X-Upload-Content-Length": strconv.FormatInt(size, 10),
},
RootURL: o.fs.uploadURL,
@@ -2154,7 +2145,6 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
Parameters: url.Values{
"fields": []string{"*"},
},
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
}
var info *api.File
-1
View File
@@ -694,7 +694,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
CaseInsensitive: true, // names are matched with EqualFold, see findLeafItem
CanHaveEmptyDirectories: true,
PartialUploads: false,
}).Fill(ctx, f)
-2
View File
@@ -77,8 +77,6 @@ func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadPara
if err != nil {
return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
}
// Stop the form writer reading file if the request fails before sending it all
defer func() { _ = formReader.Close() }()
opts := rest.Opts{
Method: "POST",
+4 -10
View File
@@ -86,8 +86,8 @@ func getServices() []service {
{"jottacloud", "Jottacloud", "id.jottacloud.com", "jottacloud", "desktop", []string{"openid", "jotta-default", "offline_access"}}, // Chose client id "desktop" here, will be identified as "Jottacloud for Desktop" in "My logged in devices", but could have used "jottacli" here as well.
{"elgiganten_dk", "Elgiganten Cloud (Denmark)", "cloud.elgiganten.dk", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
{"elgiganten_se", "Elgiganten Cloud (Sweden)", "cloud.elgiganten.se", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "offline_access"}},
{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "offline_access"}},
{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "jotta-default", "offline_access"}},
{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "jotta-default", "offline_access"}},
{"gigantti", "Gigantti Cloud (Finland)", "cloud.gigantti.fi", "gigantti", "desktop", []string{"openid", "jotta-default", "offline_access"}},
{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
@@ -258,7 +258,6 @@ service, for the official service on https://www.jottacloud.com/web/secure.`)
}
m.Set("configVersion", fmt.Sprint(configVersion))
m.Set(configClientID, service.clientID)
m.Set(configClientSecret, "")
m.Set(configTokenURL, wellKnown.TokenEndpoint)
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
OAuth2Config: &oauthutil.Config{
@@ -1501,9 +1500,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
if err == nil {
createTime := srcObj.createTime
var createTime time.Time
var createTimeMeta bool
modTime := srcObj.modTime
var modTime time.Time
var modTimeMeta bool
if meta != nil {
createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
@@ -1519,11 +1518,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
// The restored file keeps the name it was trashed with, which may differ in case
if err == nil && info != nil && info.Name != f.opt.Enc.FromStandardName(path.Base(remote)) {
fs.Debugf(src, "Renaming restored %q to %q", info.Name, path.Base(remote))
info, err = f.copyOrMove(ctx, "mv", f.filePath(remote), remote)
}
} else if createTimeMeta || modTimeMeta {
info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
}
-235
View File
@@ -1,235 +0,0 @@
// Package api has type definitions for kDrive
//
// Converted from the API docs with help from https://mholt.github.io/json-to-go/
package api
import (
"fmt"
"strconv"
"time"
)
// Time represents unix timestamp date and time information for the
// kDrive API
type Time time.Time
// UnmarshalJSON turns JSON into a Time
func (t *Time) UnmarshalJSON(data []byte) error {
if string(data) == "null" {
return nil
}
timestamp, err := strconv.ParseInt(string(data), 10, 64)
if err != nil {
return err
}
newT := time.Unix(timestamp, 0)
*t = Time(newT)
return nil
}
// ResultStatus return error details from kdrive when things go wrong
//
// If Status is "success" then everything is OK
type ResultStatus struct {
Status string `json:"result"`
ErrorDetail struct {
Result string `json:"code"`
ErrorString string `json:"description"`
Context struct {
Values []string `json:"values"`
} `json:"context"`
Errors []struct {
Result string `json:"code"`
ErrorString string `json:"description"`
} `json:"errors"`
} `json:"error"`
}
// Error returns a string for the error and satisfies the error interface
func (e *ResultStatus) Error() string {
var details string
for i := range e.ErrorDetail.Errors {
details += "|" + e.ErrorDetail.Errors[i].Result
}
return fmt.Sprintf("kDrive error: %s (%s: %s %s)", e.Status, e.ErrorDetail.Result, e.ErrorDetail.ErrorString, details)
}
// IsError returns true if there is an error
func (e *ResultStatus) IsError() bool {
return e.Status != "success"
}
// Update returns err directly if it was != nil, otherwise it returns
// an Error or nil if no error was detected
func (e *ResultStatus) Update(err error) error {
if err != nil {
return err
}
if e.IsError() {
return e
}
return nil
}
// Check ResultStatus satisfies the error interface
var _ error = (*ResultStatus)(nil)
// Item describes a folder or a file as returned by Get Folder Items and others
type Item struct {
ID int `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
FullPath string `json:"path"`
Status string `json:"status"`
Hash string `json:"hash"`
Size int64 `json:"size"`
Visibility string `json:"visibility"`
DriveID int `json:"drive_id"`
Depth int `json:"depth"`
CreatedBy int `json:"created_by"`
CreatedAt Time `json:"created_at"`
AddedAt int `json:"added_at"`
LastModifiedAt Time `json:"last_modified_at"`
LastModifiedBy int `json:"last_modified_by"`
RevisedAt int `json:"revised_at"`
UpdatedAt int `json:"updated_at"`
MimeType string `json:"mime_type"`
ParentID int `json:"parent_id"`
Color string `json:"color"`
}
// ItemResult is returned when an item is queried
type ItemResult struct {
ResultStatus
Data Item `json:"data"`
}
// SearchResult is returned when a list of items is requested
type SearchResult struct {
ResultStatus
Data []Item `json:"data"`
Cursor string `json:"cursor"`
HasMore bool `json:"has_more"`
ResponseAt int `json:"response_at"`
}
// ModTime returns the modification time of the item
func (i *Item) ModTime() (t time.Time) {
t = time.Time(i.LastModifiedAt)
if t.IsZero() {
t = time.Time(i.CreatedAt)
}
return t
}
// CancelResource is a kdrive resource that can be cancelled after some action
type CancelResource struct {
CancelID string `json:"cancel_id"`
ValidUntil int `json:"valid_until"`
}
// CancellableResponse is returned from kdrive when an action can be cancelled afterwards
type CancellableResponse struct {
ResultStatus
Data CancelResource `json:"data"`
}
// CreateDirResult is returned from kdrive after a call to MkDir
type CreateDirResult struct {
ResultStatus
Data Item `json:"data"`
}
// FileCopyResponse is returned from kdrive after a call to Copy
type FileCopyResponse struct {
ResultStatus
Data Item `json:"data"`
}
// UploadFileResponse is returned from kdrive after a call to Upload
type UploadFileResponse struct {
ResultStatus
Data Item `json:"data"`
}
// ChecksumFileResult is returned from kdrive after a call to Hash
type ChecksumFileResult struct {
ResultStatus
Data struct {
Hash string `json:"hash"`
} `json:"data"`
}
// PubLinkResult is returned from kdrive after a call to public link API
type PubLinkResult struct {
ResultStatus
Data struct {
URL string `json:"url"`
FileID int `json:"file_id"`
Right string `json:"right"`
ValidUntil int `json:"valid_until"`
CreatedBy int `json:"created_by"`
CreatedAt int `json:"created_at"`
UpdatedAt int `json:"updated_at"`
Capabilities struct {
CanEdit bool `json:"can_edit"`
CanSeeStats bool `json:"can_see_stats"`
CanSeeInfo bool `json:"can_see_info"`
CanDownload bool `json:"can_download"`
CanComment bool `json:"can_comment"`
CanRequestAccess bool `json:"can_request_access"`
} `json:"capabilities"`
AccessBlocked bool `json:"access_blocked"`
} `json:"data"`
}
// PubLinkUpdateResult is returned from kdrive after a updating a public link
type PubLinkUpdateResult struct {
ResultStatus
Data bool `json:"data"`
}
// QuotaInfo is return from kdrive after a call get drive info
type QuotaInfo struct {
ResultStatus
Data struct {
Size int64 `json:"size"`
UsedSize int64 `json:"used_size"`
} `json:"data"`
}
// SessionStartResponse is used to start a multichunk upload session
type SessionStartResponse struct {
ResultStatus
Data struct {
Token string `json:"token"`
UploadURL string `json:"upload_url"` // URL pour uploader les chunks
} `json:"data"`
}
// ChunkUploadResponse is returned by kdrive after a chunk upload
type ChunkUploadResponse struct {
ResultStatus
Data struct {
Hash string `json:"hash,omitempty"` // Only present when requested with with:'hash'
} `json:"data"`
}
// SessionFinishResponse is used to end a multichunk upload session
type SessionFinishResponse struct {
ResultStatus
Data struct {
Token string `json:"token"`
File Item `json:"file"`
Result bool `json:"result"`
Message string `json:"message"`
} `json:"data"`
}
// SessionCancelResponse is used to cancel a multichunk upload session
type SessionCancelResponse struct {
ResultStatus
Data bool `json:"data"`
}
File diff suppressed because it is too large. Load diff
-551
View File
@@ -1,551 +0,0 @@
// Unit tests for the kdrive backend
package kdrive
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/backend/kdrive/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestCheckChunkSize checks chunk_size validation against the API limits
func TestCheckChunkSize(t *testing.T) {
for _, test := range []struct {
in fs.SizeSuffix
wantErr string
}{
{in: 20 * fs.Mebi, wantErr: ""}, // default
{in: 1000 * 1000, wantErr: ""}, // API minimum
{in: 1000 * 1000 * 1000, wantErr: ""}, // API maximum
{in: 0, wantErr: "too small"}, // panics without validation
{in: 999 * 1000, wantErr: "too small"}, // below API minimum
{in: 1000*1000*1000 + 1, wantErr: "too large"}, // above API maximum
{in: 2 * 1000 * 1000 * 1000, wantErr: "too large"}, // way above API maximum
} {
t.Run(test.in.String(), func(t *testing.T) {
err := checkChunkSize(test.in)
if test.wantErr == "" {
assert.NoError(t, err)
} else {
require.Error(t, err)
assert.Contains(t, err.Error(), test.wantErr)
}
})
}
}
// TestOpenChunkWriterClampsCallerChunkSize checks that a caller
// supplied chunk size hint below the API minimum is raised to it
func TestOpenChunkWriterClampsCallerChunkSize(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(io.Discard, r.Body)
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"token":"tok","upload_url":"http://unused"}}`)
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
f.dirCache = dircache.New("", "1", f)
src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0)}
info, _, err := f.OpenChunkWriter(context.Background(), "file.txt", src, &fs.ChunkOption{ChunkSize: 512 * 1024})
require.NoError(t, err)
assert.Equal(t, int64(minChunkSize), info.ChunkSize, "a chunk size hint below the API minimum must be raised to it")
}
// TestObjectSetMetaDataResetsStaleHash checks that setMetaData always
// replaces the hash (a chunked upload finish response carries none)
func TestObjectSetMetaDataResetsStaleHash(t *testing.T) {
for _, test := range []struct {
name string
staleHash string
staleNested bool
newHash string
wantHash string
wantIsNested bool
wantHashKnown bool
}{
{
name: "empty hash clears stale plain hash",
staleHash: "deadbeef",
staleNested: false,
newHash: "",
wantHash: "",
wantIsNested: false,
wantHashKnown: false,
},
{
name: "empty hash clears stale nested hash",
staleHash: "deadbeef",
staleNested: true,
newHash: "",
wantHash: "",
wantIsNested: false,
wantHashKnown: false,
},
{
name: "plain hash is set",
staleHash: "deadbeef",
staleNested: false,
newHash: "xxh3:12345678",
wantHash: "12345678",
wantIsNested: false,
wantHashKnown: true,
},
{
name: "nested hash is set",
staleHash: "deadbeef",
staleNested: false,
newHash: "N:xxh3:87654321",
wantHash: "87654321",
wantIsNested: true,
wantHashKnown: true,
},
} {
t.Run(test.name, func(t *testing.T) {
o := &Object{
remote: "file.txt",
hasMetaData: true,
xxh3: test.staleHash,
nestedHash: test.staleNested,
}
info := &api.Item{
ID: 42,
Name: "file.txt",
Type: "file",
Hash: test.newHash,
}
err := o.setMetaData(info)
require.NoError(t, err)
assert.Equal(t, test.wantHash, o.xxh3, "hash after setMetaData")
assert.Equal(t, test.wantIsNested, o.nestedHash, "nested flag after setMetaData")
assert.True(t, o.hasMetaData)
// Hash() must reflect the new state. Only checked when a
// hash is known: with no hash Hash() would query the API.
// Nested hashes are not comparable with plain XXH3, so
// Hash() returns "" for them.
if test.wantHashKnown {
got, gotErr := o.Hash(context.Background(), hash.XXH3)
require.NoError(t, gotErr)
if test.wantIsNested {
assert.Equal(t, "", got, "nested hash must not be returned as XXH3")
} else {
assert.Equal(t, test.wantHash, got)
}
}
})
}
}
// newTestFs creates an Fs wired to a test server with a plain client
// (no OAuth) and a fast pacer
func newTestFs(t *testing.T, endpoint string) *Fs {
t.Helper()
return &Fs{
name: "TestKdrive",
root: "",
opt: Options{DriveID: "1234", Endpoint: endpoint},
srv: rest.NewClient(&http.Client{}).SetRoot(endpoint),
pacer: fs.NewPacer(context.Background(),
pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond), pacer.DecayConstant(2))),
}
}
// TestWriteChunkDetectsResultError checks that a 200 with result:error
// is reported as an error
func TestWriteChunkDetectsResultError(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(io.Discard, r.Body)
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"chunk_error","description":"chunk upload failed"}}`)
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
u := &uploadSession{
f: f,
token: "tok",
uploadURL: ts.URL,
fileSize: 4,
chunkSize: 4,
chunkHashes: make(map[int64]string),
}
_, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
require.Error(t, err, "result:error with HTTP 200 must not be treated as success")
assert.Contains(t, err.Error(), "chunk upload failed")
}
// TestWriteChunkRetriesAndRewinds checks that transient failures are
// retried with the chunk re-sent from the start of the reader
func TestWriteChunkRetriesAndRewinds(t *testing.T) {
var mu sync.Mutex
var attempts int
var bodies []string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
mu.Lock()
attempts++
bodies = append(bodies, string(body))
current := attempts
mu.Unlock()
w.Header().Set("Content-Type", "application/json")
if current == 1 {
w.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
return
}
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"chunk_number":1,"status":"OK"}}`)
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
u := &uploadSession{
f: f,
token: "tok",
uploadURL: ts.URL,
fileSize: 4,
chunkSize: 4,
chunkHashes: make(map[int64]string),
}
written, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
require.NoError(t, err)
assert.Equal(t, int64(4), written)
mu.Lock()
defer mu.Unlock()
require.Len(t, bodies, 2, "chunk upload must be retried once after a 5xx")
assert.Equal(t, "test", bodies[0], "first attempt sends the chunk")
assert.Equal(t, "test", bodies[1], "retry must re-send the chunk from the start of the reader")
}
// kdriveMockItem returns a JSON body for a single item result
func kdriveMockItem(id int, name string) string {
return fmt.Sprintf(`{"result":"success","data":{"id":%d,"name":%q,"type":"file","path":%q,"last_modified_at":1700000000}}`, id, name, "/"+name)
}
// mockObjectInfo is a minimal fs.ObjectInfo for testing
type mockObjectInfo struct {
remote string
size int64
modTime time.Time
hash string
}
func (m *mockObjectInfo) Fs() fs.Info { return nil }
func (m *mockObjectInfo) String() string { return m.remote }
func (m *mockObjectInfo) Remote() string { return m.remote }
func (m *mockObjectInfo) Size() int64 { return m.size }
func (m *mockObjectInfo) ModTime(_ context.Context) time.Time { return m.modTime }
func (m *mockObjectInfo) Storable() bool { return true }
func (m *mockObjectInfo) MimeType(_ context.Context) string { return "application/octet-stream" }
func (m *mockObjectInfo) Hash(_ context.Context, t hash.Type) (string, error) {
if m.hash == "" || t != hash.XXH3 {
return "", hash.ErrUnsupported
}
return m.hash, nil
}
// TestUpdateDirectNoRetryOnServerError checks that a failed single
// part upload is not retried here: the input must not be buffered, the
// higher layers retry the whole transfer
func TestUpdateDirectNoRetryOnServerError(t *testing.T) {
var attempts int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&attempts, 1)
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(io.Discard, r.Body)
w.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
o := &Object{fs: f, remote: "file.txt"}
src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: "12345678"}
err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
require.Error(t, err)
assert.Equal(t, int32(1), atomic.LoadInt32(&attempts), "single part uploads must not be retried here")
}
// TestUpdateDirect checks the upload parameters and metadata of a
// successful single part upload, and that total_chunk_hash is only
// sent when the source hash is known
func TestUpdateDirect(t *testing.T) {
for _, test := range []struct {
name string
srcHash string
wantTotalHash string
}{
{name: "hash known", srcHash: "12345678", wantTotalHash: "xxh3:12345678"},
{name: "hash unknown", srcHash: "", wantTotalHash: ""},
} {
t.Run(test.name, func(t *testing.T) {
var gotTotalHash, gotBody string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
gotTotalHash = r.URL.Query().Get("total_chunk_hash")
body, _ := io.ReadAll(r.Body)
gotBody = string(body)
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"id":55,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:99998888","last_modified_at":1700000000}}`)
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
o := &Object{fs: f, remote: "file.txt"}
src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: test.srcHash}
err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
require.NoError(t, err)
assert.Equal(t, "test", gotBody, "the content must be streamed as-is")
assert.Equal(t, test.wantTotalHash, gotTotalHash, "total_chunk_hash")
assert.Equal(t, "55", o.id)
gotHash, hashErr := o.Hash(context.Background(), hash.XXH3)
require.NoError(t, hashErr)
assert.Equal(t, "99998888", gotHash)
})
}
}
// TestNewFsRequiresDriveID checks that NewFs fails fast when drive_id
// is missing
func TestNewFsRequiresDriveID(t *testing.T) {
m := configmap.Simple{"chunk_size": "20Mi"}
_, err := NewFs(context.Background(), "kdrive", "", m)
require.Error(t, err)
assert.Contains(t, err.Error(), "drive_id")
}
// TestUploadSessionStringMasksToken checks that the upload session
// String does not leak the session token in debug logs
func TestUploadSessionStringMasksToken(t *testing.T) {
u := &uploadSession{token: "very-secret-uuid"}
assert.NotContains(t, u.String(), "very-secret-uuid")
}
// TestListNonRecursiveSingleRoundTrip checks that a non recursive
// listing needs a single API request: without depth=unlimited the API
// returns direct children only, so neither the parent item lookup nor
// a child filter is needed
func TestListNonRecursiveSingleRoundTrip(t *testing.T) {
var requests int32
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
atomic.AddInt32(&requests, 1)
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(io.Discard, r.Body)
switch {
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/files"):
// Direct children only, as returned without depth=unlimited
_, _ = fmt.Fprintln(w, `{"result":"success","data":[`+
`{"id":10,"name":"sub","type":"dir","path":"/sub"},`+
`{"id":11,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:12345678","last_modified_at":1700000000}],"has_more":false}`)
default:
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
}
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
f.dirCache = dircache.New("", "1", f)
entries, err := f.List(context.Background(), "")
require.NoError(t, err)
assert.Equal(t, int32(1), atomic.LoadInt32(&requests), "non recursive listing must make a single API request")
names := make([]string, 0, len(entries))
for _, entry := range entries {
names = append(names, entry.Remote())
}
assert.ElementsMatch(t, []string{"sub", "file.txt"}, names)
}
// TestMoveConflictReturnsError checks that moving onto an occupied
// destination reports the API conflict instead of resolving it: rclone
// removes the destination itself before calling Move
func TestMoveConflictReturnsError(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(io.Discard, r.Body)
switch {
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"conflict_error","description":"destination already exists"}}`)
default:
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
}
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
f.dirCache = dircache.New("", "1", f)
srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
_, err := f.Move(context.Background(), srcObj, "file2")
require.Error(t, err, "a move onto an occupied destination must fail")
assert.Contains(t, err.Error(), "conflict", "the API conflict must be reported")
}
// TestMoveDetectsForeignObjectAtDestination checks that Move does not
// return a file re-created by another writer at the destination
func TestMoveDetectsForeignObjectAtDestination(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(io.Discard, r.Body)
switch {
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
// The move API reports success...
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
// ...but the file at the destination is not the moved
// file: another writer re-created it
_, _ = fmt.Fprintln(w, kdriveMockItem(999, "file2"))
default:
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
}
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
f.dirCache = dircache.New("", "1", f)
srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
_, err := f.Move(context.Background(), srcObj, "file2")
require.Error(t, err, "Move must not report success when the destination is not the moved file")
}
// TestMoveSuccess checks the nominal move: the object returned is the
// moved file (same ID as the source).
func TestMoveSuccess(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.Copy(io.Discard, r.Body)
switch {
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
// After the move, the moved file is at the destination
_, _ = fmt.Fprintln(w, kdriveMockItem(111, "file2"))
default:
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
}
}))
defer ts.Close()
f := newTestFs(t, ts.URL)
f.dirCache = dircache.New("", "1", f)
srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
dstObj, err := f.Move(context.Background(), srcObj, "file2")
require.NoError(t, err)
moved, ok := dstObj.(*Object)
require.True(t, ok)
assert.Equal(t, "111", moved.id, "the returned object must be the moved file")
}
// TestFormatModTime checks that pre-1970 times format as negative
// unix timestamps instead of wrapping through a uint64 cast
func TestFormatModTime(t *testing.T) {
for _, test := range []struct {
in time.Time
want string
}{
{in: time.Unix(1700000000, 0), want: "1700000000"},
{in: time.Unix(0, 0), want: "0"},
{in: time.Unix(-1577923200, 0), want: "-1577923200"}, // Sat Jan 2 1965
} {
t.Run(test.want, func(t *testing.T) {
assert.Equal(t, test.want, formatModTime(test.in))
})
}
}
// TestObjectSetMetaDataIsDir checks that directories are rejected
func TestObjectSetMetaDataIsDir(t *testing.T) {
o := &Object{remote: "dir"}
info := &api.Item{ID: 42, Name: "dir", Type: "dir"}
err := o.setMetaData(info)
assert.Equal(t, fs.ErrorIsDir, err)
}
// TestShouldRetryLockError checks that transient locks held by server
// side copies are retried while locks held by applications and other
// errors are not
func TestShouldRetryLockError(t *testing.T) {
for _, test := range []struct {
name string
body string
status int
wantRetry bool
}{
{
name: "lock held by internal copy",
body: `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy","context":{"values":["internal_copy"]}}}`,
status: http.StatusForbidden,
wantRetry: true,
},
{
name: "lock held by an application",
body: `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by onlyoffice","context":{"values":["onlyoffice"]}}}`,
status: http.StatusForbidden,
wantRetry: false,
},
{
name: "lock without context",
body: `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy"}}`,
status: http.StatusForbidden,
wantRetry: false,
},
{
name: "other api error",
body: `{"result":"error","error":{"code":"object_not_found","description":"File not found"}}`,
status: http.StatusNotFound,
wantRetry: false,
},
{
name: "server error",
status: http.StatusServiceUnavailable,
wantRetry: true,
},
} {
t.Run(test.name, func(t *testing.T) {
var apiErr error
if test.body != "" {
e := new(api.ResultStatus)
require.NoError(t, json.Unmarshal([]byte(test.body), e))
apiErr = e
}
resp := &http.Response{StatusCode: test.status, Header: http.Header{}}
got, gotErr := shouldRetry(context.Background(), resp, apiErr)
assert.Equal(t, test.wantRetry, got)
assert.Equal(t, apiErr, gotErr)
})
}
}
-25
View File
@@ -1,25 +0,0 @@
// Test kdrive filesystem interface
package kdrive
import (
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestKdrive:",
NilObject: (*Object)(nil),
ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: minChunkSize,
MaxChunkSize: maxChunkSize,
},
})
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
return f.setChunkSize(cs)
}
-67
View File
@@ -1,67 +0,0 @@
// Package khash provides the kDrive specific hash handling.
//
// kDrive uses XXH3-64 for file hashing. Files uploaded in chunks (by
// rclone, or by the kDrive apps which each use their own chunk size)
// get a "nested" hash: the XXH3 of the concatenated lowercase hex
// strings of the individual chunk hashes. Such a nested hash can't be
// compared with a plain XXH3 of the file contents.
//
// Hash formats returned by the kDrive API:
// - "xxh3:HASH" for a plain XXH3 of the file contents
// - "N:xxh3:HASH" for the nested hash of a chunked upload
package khash
import (
"encoding/hex"
"errors"
"fmt"
"strings"
"github.com/zeebo/xxh3"
)
// ParseHash parses a kDrive hash string from the API and returns the hex hash value
// along with whether it is a nested (chunked) hash.
func ParseHash(hashStr string) (hexHash string, isNested bool, err error) {
if hashStr == "" {
return "", false, nil
}
// Check for nested hash format: "N:xxh3:HASH"
if strings.HasPrefix(hashStr, "N:xxh3:") {
hashStr = strings.TrimPrefix(hashStr, "N:xxh3:")
isNested = true
} else {
hashStr = strings.TrimPrefix(hashStr, "xxh3:")
}
// Validate hex string
if _, decodeErr := hex.DecodeString(hashStr); decodeErr != nil {
return "", false, fmt.Errorf("invalid hash format: %w", decodeErr)
}
return hashStr, isNested, nil
}
// NestedChunkHash computes the kDrive nested hash from the hashes of the
// file chunks, in order.
//
// kDrive stores the nested hash of a chunked file as the XXH3 of the
// concatenated lowercase hex strings of the individual chunk hashes.
// Each chunk hash may be passed with or without the "xxh3:" prefix.
func NestedChunkHash(chunkHashes []string) (string, error) {
if len(chunkHashes) == 0 {
return "", errors.New("no chunk hashes to nest")
}
hexHashes := make([]string, len(chunkHashes))
for i, chunkHash := range chunkHashes {
hexHash, _, err := ParseHash(chunkHash)
if err != nil {
return "", fmt.Errorf("invalid chunk hash %q: %w", chunkHash, err)
}
hexHashes[i] = hexHash
}
hasher := xxh3.New()
_, _ = hasher.Write([]byte(strings.Join(hexHashes, "")))
return hex.EncodeToString(hasher.Sum(nil)), nil
}
-150
View File
@@ -1,150 +0,0 @@
package khash
import "testing"
func TestParseHash(t *testing.T) {
tests := []struct {
name string
input string
expectedHash string
isNested bool
expectError bool
}{
{
name: "simple hash with prefix",
input: "xxh3:1db797b96febd334",
expectedHash: "1db797b96febd334",
isNested: false,
},
{
name: "nested hash",
input: "N:xxh3:877abf3579f0a5c0",
expectedHash: "877abf3579f0a5c0",
isNested: true,
},
{
name: "plain hex",
input: "1db797b96febd334",
expectedHash: "1db797b96febd334",
isNested: false,
},
{
name: "empty string",
input: "",
expectedHash: "",
isNested: false,
},
{
name: "invalid hex",
input: "xxh3:notahexvalue",
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
hash, isNested, err := ParseHash(tt.input)
if tt.expectError {
if err == nil {
t.Error("Expected error but got none")
}
return
}
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if hash != tt.expectedHash {
t.Errorf("Hash: got %q, expected %q", hash, tt.expectedHash)
}
if isNested != tt.isNested {
t.Errorf("IsNested: got %t, expected %t", isNested, tt.isNested)
}
})
}
}
func TestNestedChunkHash(t *testing.T) {
// Golden values: the nested hash is the XXH3 of the concatenated
// lowercase hex strings of the chunk hashes, in order
chunk1 := "xxh3:d447b1ea40e6988b" // XXH3 of "hello world"
chunk2 := "2d06800538d394c2" // XXH3 of ""
tests := []struct {
name string
chunkHashes []string
expected string
expectError bool
}{
{
name: "two chunks in order",
chunkHashes: []string{chunk1, chunk2},
expected: "6d51edccc9c05a8b",
},
{
name: "order matters",
chunkHashes: []string{chunk2, chunk1},
expected: "596c127c7bcc8820",
},
{
name: "single chunk",
chunkHashes: []string{chunk1},
expected: "b73081dff8fe443c",
},
{
name: "mixed prefixes",
chunkHashes: []string{chunk1, "xxh3:" + chunk2, chunk1},
expected: "ca0b13abcdcd1661",
},
{
name: "no chunks",
chunkHashes: []string{},
expectError: true,
},
{
name: "invalid chunk hash",
chunkHashes: []string{chunk1, "notahexvalue"},
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
nested, err := NestedChunkHash(tt.chunkHashes)
if tt.expectError {
if err == nil {
t.Error("Expected error but got none")
}
return
}
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if nested != tt.expected {
t.Errorf("NestedChunkHash: got %q, expected %q", nested, tt.expected)
}
})
}
}
func TestNestedChunkHashAcceptsPrefixedHashes(t *testing.T) {
plain := []string{"d447b1ea40e6988b", "2d06800538d394c2"}
prefixed := []string{"xxh3:d447b1ea40e6988b", "N:xxh3:2d06800538d394c2"}
gotPlain, err := NestedChunkHash(plain)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
gotPrefixed, err := NestedChunkHash(prefixed)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if gotPlain != gotPrefixed {
t.Errorf("Prefixed hashes should give the same nested hash: %q != %q", gotPlain, gotPrefixed)
}
}
-383
View File
@@ -1,383 +0,0 @@
package kdrive
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/rclone/rclone/backend/kdrive/api"
"github.com/rclone/rclone/backend/kdrive/khash"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/lib/rest"
"github.com/zeebo/xxh3"
"golang.org/x/text/unicode/norm"
)
const (
// maxUploadParts is the maximum number of chunks the kDrive API accepts per file
maxUploadParts = 10000
// maxChunkSize is the largest chunk size the kDrive API accepts
maxChunkSize = 1 * 1000 * 1000 * 1000
// minChunkSize is the smallest chunk size the kDrive API accepts
minChunkSize = 1 * 1000 * 1000
)
// checkChunkSize checks the user supplied chunk_size against the
// limits the kDrive API accepts
func checkChunkSize(chunkSize fs.SizeSuffix) error {
if chunkSize < minChunkSize {
return fmt.Errorf("chunk_size %s is too small: the kDrive API requires chunks of at least 1 MB", chunkSize)
}
if chunkSize > maxChunkSize {
return fmt.Errorf("chunk_size %s is too large: the kDrive API accepts chunks of at most 1 GB (1000000000 bytes)", chunkSize)
}
return nil
}
// setChunkSize sets the chunk size, returning the old value
func (f *Fs) setChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
err = checkChunkSize(cs)
if err == nil {
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
}
return
}
// formatModTime formats the modification time of an upload as the
// unix timestamp string the kDrive API expects
func formatModTime(modTime time.Time) string {
return fmt.Sprintf("%d", modTime.Unix())
}
// uploadSession implements fs.ChunkWriter for kdrive multipart uploads
type uploadSession struct {
f *Fs
token string
uploadURL string
fileInfo *api.Item
fileSize int64 // total size of the file being uploaded
chunkSize int64 // size of the chunks of this session
mu sync.Mutex
chunkHashes map[int64]string // chunk number -> hash of the chunk contents
}
// OpenChunkWriter returns the chunk size and a ChunkWriter
//
// Pass in the remote and the src object
// You can also use options to hint at the desired chunk size
// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
fileSize := src.Size()
if fileSize < 0 {
return info, nil, errors.New("kdrive can't upload files with unknown size")
}
dir, leaf := path.Split(remote)
dir = strings.TrimSuffix(dir, "/")
// Create parent directories if they don't exist
parentID, err := f.dirCache.FindDir(ctx, dir, true)
if err != nil {
return info, nil, fmt.Errorf("failed to find parent directory: %w", err)
}
// Calculate the chunk size, honouring any fs.ChunkOption hint
chunkSize := f.opt.ChunkSize
for _, opt := range options {
if chunkOpt, ok := opt.(*fs.ChunkOption); ok && chunkOpt.ChunkSize > 0 {
chunkSize = fs.SizeSuffix(chunkOpt.ChunkSize)
}
}
chunkSize = chunksize.Calculator(src, fileSize, maxUploadParts, chunkSize)
if chunkSize < minChunkSize {
chunkSize = minChunkSize
}
if chunkSize > maxChunkSize {
chunkSize = maxChunkSize
}
// Calculate the total number of chunks
totalChunks := max((fileSize+int64(chunkSize)-1)/int64(chunkSize), 1)
sessionReq := struct {
Conflict string `json:"conflict"`
DirectoryID string `json:"directory_id"`
FileName string `json:"file_name"`
LastModifiedAt string `json:"last_modified_at"`
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
}{
Conflict: "version",
DirectoryID: parentID,
FileName: f.opt.Enc.FromStandardName(norm.NFC.String(leaf)),
LastModifiedAt: formatModTime(src.ModTime(ctx)),
TotalChunks: totalChunks,
TotalSize: fileSize,
}
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/upload/session/start", f.opt.DriveID),
}
var sessionResp api.SessionStartResponse
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
var callErr error
resp, callErr = f.srv.CallJSON(ctx, &opts, &sessionReq, &sessionResp)
callErr = sessionResp.ResultStatus.Update(callErr)
return shouldRetry(ctx, resp, callErr)
})
if err != nil {
return info, nil, fmt.Errorf("failed to start upload session: %w", err)
}
info = fs.ChunkWriterInfo{
ChunkSize: int64(chunkSize),
Concurrency: f.opt.UploadConcurrency,
}
writer = &uploadSession{
f: f,
token: sessionResp.Data.Token,
uploadURL: sessionResp.Data.UploadURL,
fileSize: fileSize,
chunkSize: int64(chunkSize),
chunkHashes: make(map[int64]string),
}
fs.Debugf(f, "open chunk writer: started upload session for %d chunks of %v", totalChunks, chunkSize)
return info, writer, nil
}
// WriteChunk uploads a single chunk
// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
func (u *uploadSession) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
if chunkNumber < 0 {
return -1, fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
}
// Calculate the size of this chunk from the session parameters so
// the chunk can be streamed to the API in a single pass
chunkLen := u.chunkSize
if remaining := u.fileSize - int64(chunkNumber)*u.chunkSize; remaining < chunkLen {
chunkLen = remaining
}
if chunkLen <= 0 {
return 0, nil
}
sourceChunkNumber := chunkNumber + 1 // kDrive API uses 1-based numbering
// The chunk_hash parameter is optional, so the chunk is streamed
// in a single pass, hashing the contents on the way with a
// TeeReader. The server returns the hash of the chunk it received,
// which is checked below. Retries seek back to the chunk start
// and hash with a fresh hasher.
var chunkHash string
err = u.f.pacer.Call(func() (bool, error) {
if _, seekErr := reader.Seek(0, io.SeekStart); seekErr != nil {
return false, fmt.Errorf("failed to rewind chunk %d: %w", sourceChunkNumber, seekErr)
}
chunkHasher := xxh3.New()
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
chunkOpts := rest.Opts{
Method: "POST",
RootURL: u.uploadURL,
Path: fmt.Sprintf("/3/drive/%s/upload/session/%s/chunk", u.f.opt.DriveID, u.token),
Parameters: url.Values{
"chunk_number": {fmt.Sprintf("%d", sourceChunkNumber)},
"chunk_size": {fmt.Sprintf("%d", chunkLen)},
"with": {"hash"},
},
Body: io.TeeReader(reader, chunkHasher),
ContentLength: &chunkLen,
}
var chunkResp api.ChunkUploadResponse
resp, callErr := u.f.srv.CallJSON(ctx, &chunkOpts, nil, &chunkResp)
callErr = chunkResp.ResultStatus.Update(callErr)
if callErr != nil {
return shouldRetry(ctx, resp, callErr)
}
chunkHash = hex.EncodeToString(chunkHasher.Sum(nil))
// The server hashed the chunk it received, so compare with what we
// sent to detect any corruption immediately
if chunkResp.Data.Hash != "" {
if serverHash, _, parseErr := khash.ParseHash(chunkResp.Data.Hash); parseErr == nil {
if !strings.EqualFold(chunkHash, serverHash) {
return false, fmt.Errorf("chunk %d hash mismatch: client=%s, server=%s", sourceChunkNumber, chunkHash, serverHash)
}
} else {
fs.Debugf(u, "Failed to parse server chunk hash %q: %v", chunkResp.Data.Hash, parseErr)
}
}
return false, nil
})
if err != nil {
return -1, err
}
u.mu.Lock()
u.chunkHashes[int64(chunkNumber)] = chunkHash
u.mu.Unlock()
fs.Debugf(u, "uploaded chunk %d (size: %d, hash: %s)", sourceChunkNumber, chunkLen, chunkHash)
return chunkLen, nil
}
// Close finalizes the upload session, verifies the hash of the uploaded
// file and returns the created file info
// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
func (u *uploadSession) Close(ctx context.Context) error {
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
opts := rest.Opts{
Method: "POST",
Path: fmt.Sprintf("/3/drive/%s/upload/session/%s/finish", u.f.opt.DriveID, u.token),
}
var resp *http.Response
var finishResp api.SessionFinishResponse
err := u.f.pacer.Call(func() (bool, error) {
var callErr error
resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &finishResp)
callErr = finishResp.ResultStatus.Update(callErr)
return shouldRetry(ctx, resp, callErr)
})
if err != nil {
return fmt.Errorf("failed to finish upload session: %w", err)
}
u.fileInfo = &finishResp.Data.File
fs.Debugf(u, "multipart upload completed: file id %d", finishResp.Data.File.ID)
return u.verifyHash(ctx)
}
// verifyHash checks the hash of the uploaded file against the hashes of
// the chunks we uploaded, deleting the file if it is corrupted
func (u *uploadSession) verifyHash(ctx context.Context) (err error) {
remoteHash := u.fileInfo.Hash
if remoteHash == "" {
// Hash might be missing in the finish response, try to fetch it
obj := &Object{
fs: u.f,
id: strconv.Itoa(u.fileInfo.ID),
}
var hashErr error
if remoteHash, hashErr = obj.retrieveHash(ctx); hashErr != nil {
// skip verification
fs.Debugf(u, "Failed to retrieve hash for verification: %v", hashErr)
return nil
}
}
parsedRemoteHash, isNested, err := khash.ParseHash(remoteHash)
if err != nil {
fs.Debugf(u, "Failed to parse server hash %q: %v", remoteHash, err)
return nil
}
u.mu.Lock()
defer u.mu.Unlock()
// Rebuild the hash the server should have stored from the chunk
// hashes we uploaded, in order
chunkCount := int64(len(u.chunkHashes))
if chunkCount == 0 {
return nil
}
chunkHashes := make([]string, 0, chunkCount)
for i := int64(0); i < chunkCount; i++ {
chunkHash, ok := u.chunkHashes[i]
if !ok {
fs.Debugf(u, "Chunk %d hash missing, skipping hash verification", i)
return nil
}
chunkHashes = append(chunkHashes, chunkHash)
}
var localHash string
if isNested {
if localHash, err = khash.NestedChunkHash(chunkHashes); err != nil {
fs.Debugf(u, "Failed to compute nested hash: %v", err)
return nil
}
} else if chunkCount == 1 {
localHash, _, _ = khash.ParseHash(chunkHashes[0])
} else {
// The server returned a simple hash for a multi chunk upload,
// nothing to compare it against
fs.Debugf(u, "Server returned a simple hash for a %d chunk upload, skipping hash verification", chunkCount)
return nil
}
if !strings.EqualFold(localHash, parsedRemoteHash) {
err = fmt.Errorf(
"multipart upload hash mismatch: local=%s, remote=%s",
localHash, parsedRemoteHash,
)
// Remove the corrupted file
obj := &Object{
fs: u.f,
id: strconv.Itoa(u.fileInfo.ID),
}
if delErr := obj.Remove(ctx); delErr != nil {
fs.Errorf(nil, "Failed to remove corrupted file after hash mismatch: %v", delErr)
} else {
fs.Debugf(nil, "Removed corrupted file after hash mismatch")
}
return err
}
fs.Debugf(u, "Multipart upload hash verified: %s", localHash)
return nil
}
// Abort the upload session
// @see https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
func (u *uploadSession) Abort(ctx context.Context) error {
// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
opts := rest.Opts{
Method: "DELETE",
Path: fmt.Sprintf("/2/drive/%s/upload/session/%s", u.f.opt.DriveID, u.token),
}
var resp *http.Response
var cancelResp api.SessionCancelResponse
err := u.f.pacer.Call(func() (bool, error) {
var callErr error
resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &cancelResp)
callErr = cancelResp.ResultStatus.Update(callErr)
return shouldRetry(ctx, resp, callErr)
})
if err != nil {
return fmt.Errorf("failed to cancel upload session: %w", err)
}
fs.Debugf(u, "upload session cancelled")
return nil
}
// String implements fmt.Stringer
func (u *uploadSession) String() string {
return "kdrive upload session"
}
// Verify interface compliance
var _ fs.ChunkWriter = (*uploadSession)(nil)
+1 -24
View File
@@ -6,13 +6,11 @@ package local
import (
"context"
"fmt"
"os"
"path/filepath"
"runtime"
"github.com/go-darwin/apfs"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/random"
)
// Copy src to this remote using server-side copy operations.
@@ -65,7 +63,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil {
return nil, err
}
err = cloneOver(srcPath, dstPath)
err = Clone(srcPath, dstPath)
if err != nil {
return nil, err
}
@@ -81,27 +79,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
return f.NewObject(ctx, remote)
}
// cloneOver clones src to dst, replacing dst if it already exists.
//
// COPYFILE_CLONE implies COPYFILE_EXCL so Clone fails if dst exists.
// Cloning to a temporary name in the same directory and renaming it
// over dst avoids this and leaves dst untouched if the clone fails.
func cloneOver(src, dst string) error {
if _, err := os.Lstat(dst); os.IsNotExist(err) {
return Clone(src, dst)
}
tmp := filepath.Join(filepath.Dir(dst), "."+random.String(16)+".rclone-clone")
err := Clone(src, tmp)
if err == nil {
err = os.Rename(tmp, dst)
}
if err != nil {
_ = os.Remove(tmp)
return fmt.Errorf("failed to clone %q to %q: %w", src, dst, err)
}
return nil
}
// Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)
// note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical
// https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html
+17 -76
View File
@@ -87,10 +87,8 @@ Metadata is supported on files and directories.
When restoring metadata with ` + "`--metadata`" + ` rclone applies the
"mode", "uid" and "gid" from the source. These come from the source
remote which may not be trusted, so restoring metadata as root from an
untrusted source can change file ownership and is not recommended. A
user other than root keeps its own uid and sets the gid only to a group
it is a member of, as rsync does. The setuid, setgid and sticky bits are
not restored by default - see the
untrusted source can change file ownership and is not recommended. The
setuid, setgid and sticky bits are not restored by default - see the
` + "`--local-metadata-restore-special-bits`" + ` flag.
`,
},
@@ -254,13 +252,6 @@ deduplicated by default (i.e. having two identical files does not consume more
storage than having just one.) However, for use cases where data redundancy is
preferable, --local-no-clone can be used to disable cloning and force "deep" copies.
Cloning also preserves macOS-specific metadata such as extended attributes,
which a normal copy does not, so this metadata is lost when cloning is disabled.
If the destination file already exists, cloning replaces it with a new file,
even with --inplace, so hard links to it are broken. Use --local-no-clone to
update existing files in place.
Currently, cloning is only supported when using APFS on macOS (support for other
platforms may be added in the future.)`,
Default: false,
@@ -960,13 +951,13 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
if err != nil {
return nil, err
}
fi, err := f.lstatRoot(localPath)
fi, err := f.lstat(localPath)
if errors.Is(err, os.ErrNotExist) {
err := f.Mkdir(ctx, dir)
if err != nil {
return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err)
}
fi, err = f.lstatRoot(localPath)
fi, err = f.lstat(localPath)
if err != nil {
return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err)
}
@@ -998,34 +989,20 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
// Rmdir removes the directory
//
// If it isn't empty it will return an error
func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
localPath, err := f.localPath(dir)
if err != nil {
return err
}
// When translating symlinks go through os.Root so a symlink planted at
// any parent can't redirect the removal out of the destination, and
// don't follow the leaf as a symlink is a file in this mode. The root
// itself is the trusted destination and is removed directly.
stat, removeFn, chmod := os.Stat, os.Remove, os.Chmod
if f.opt.TranslateSymlinks && localPath != f.root {
var root *os.Root
root, localPath, err = f.osRoot(localPath)
if err != nil {
return err
}
defer fs.CheckClose(root, &err)
stat, removeFn, chmod = root.Lstat, root.Remove, root.Chmod
}
if fi, err := stat(localPath); err != nil {
if fi, err := os.Stat(localPath); err != nil {
return err
} else if !fi.IsDir() {
return fs.ErrorIsFile
}
err = removeFn(localPath)
err = os.Remove(localPath)
if runtime.GOOS == "windows" && errors.Is(err, iofs.ErrPermission) { // https://github.com/golang/go/issues/26295
if chmod(localPath, 0o600) == nil {
err = removeFn(localPath)
if os.Chmod(localPath, 0o600) == nil {
err = os.Remove(localPath)
}
}
return err
@@ -1202,14 +1179,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return fs.ErrorDirExists
}
// Create parent of destination. The root itself is the trusted
// destination so its parent, which lies outside it, is created directly.
// Create parent of destination
dstParentPath := filepath.Dir(dstPath)
if dstPath == f.root {
err = file.MkdirAll(dstParentPath, 0777)
} else {
err = f.mkdirAll(dstParentPath)
}
err = file.MkdirAll(dstParentPath, 0777)
if err != nil {
return err
}
@@ -1626,26 +1598,10 @@ func (f *Fs) mkdirAll(localPath string) (err error) {
return root.MkdirAll(rel, 0777)
}
// lstatRoot returns the info for localPath without following a symlink at
// the leaf. When translating symlinks it goes through os.Root so a symlink
// planted at any parent can't make a path outside the destination look like
// it exists, which would let its metadata be written there.
func (f *Fs) lstatRoot(localPath string) (fi os.FileInfo, err error) {
if !f.opt.TranslateSymlinks {
return f.lstat(localPath)
}
root, rel, err := f.osRoot(localPath)
if err != nil {
return nil, err
}
defer fs.CheckClose(root, &err)
return root.Lstat(rel)
}
// openFile opens localPath with the given flags and perm. When translating
// symlinks it goes through os.Root so a symlink planted at the path, or at any
// parent, is never followed out of the destination. The returned file is
// independent of the root, which is closed before returning.
// openFile opens localPath for writing. When translating symlinks it goes
// through os.Root so a symlink planted at the path, or at any parent, is never
// followed out of the destination. The returned file is independent of the
// root, which is closed before returning.
func (f *Fs) openFile(localPath string, flags int, perm os.FileMode) (fi *os.File, err error) {
if !f.opt.TranslateSymlinks {
return file.OpenFile(localPath, flags, perm)
@@ -1714,21 +1670,6 @@ func (f *Fs) chtimes(localPath string, atime, mtime time.Time) (err error) {
return root.Chtimes(rel, atime, mtime)
}
// remove removes the file, symlink or empty directory at localPath. When
// translating symlinks it goes through os.Root so a symlink planted at any
// parent can't redirect the removal out of the destination.
func (f *Fs) remove(localPath string) (err error) {
if !f.opt.TranslateSymlinks {
return remove(localPath, os.Remove)
}
root, rel, err := f.osRoot(localPath)
if err != nil {
return err
}
defer fs.CheckClose(root, &err)
return remove(rel, root.Remove)
}
// mkdirAll makes all the directories needed to store the object
func (o *Object) mkdirAll() error {
return o.fs.mkdirAll(filepath.Dir(o.path))
@@ -1853,7 +1794,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if err != nil {
fs.Logf(o, "Removing partially written file on error: %v", err)
if removeErr := o.fs.remove(o.path); removeErr != nil {
if removeErr := os.Remove(o.path); removeErr != nil {
fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
}
return err
@@ -2001,7 +1942,7 @@ func (o *Object) lstat() error {
// Remove an object
func (o *Object) Remove(ctx context.Context) error {
o.clearHashCache()
return o.fs.remove(o.path)
return remove(o.path)
}
// Metadata returns metadata for an object
-128
View File
@@ -99,31 +99,6 @@ func TestVerifyCopy(t *testing.T) {
assert.Error(t, err)
}
// Test server-side copy over an existing file
func TestCopyOverExisting(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
doCopy := r.Flocal.Features().Copy
if doCopy == nil {
t.Skip("local backend has no server-side copy on this platform")
}
t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
t2 := fstest.Time("2002-02-03T04:05:06.499999999Z")
file1 := r.WriteFile("a.txt", "hello1", t1)
file2 := r.WriteFile("b.txt", "HELLO2 longer", t2)
src, err := r.Flocal.NewObject(ctx, file1.Path)
require.NoError(t, err)
dst, err := doCopy(ctx, src, file2.Path)
require.NoError(t, err)
assert.Equal(t, file2.Path, dst.Remote())
file2.Size = file1.Size
file2.Hashes = file1.Hashes
file2.ModTime = file1.ModTime
r.CheckLocalItems(t, file1, file2)
}
// skipIfNoSymlinks skips the test if this process can't create symlinks.
//
// Windows grants the privilege only to an elevated process or one running with
@@ -560,109 +535,6 @@ func TestDirBTimeThroughPlantedSymlinkBlocked(t *testing.T) {
require.Equal(t, before, readBTime(), "btime escaped through planted symlink to %q", evilDir)
}
// TestMkdirMetadataThroughPlantedSymlinkBlocked checks MkdirMetadata can't
// be fooled into treating a directory outside the root, reached through a
// planted symlink, as the directory to apply metadata to.
func TestMkdirMetadataThroughPlantedSymlinkBlocked(t *testing.T) {
skipIfNoSymlinks(t)
ctx := context.Background()
evil := t.TempDir()
evilDir := filepath.Join(evil, "secret.d")
require.NoError(t, os.Mkdir(evilDir, 0700))
require.NoError(t, os.Mkdir(filepath.Join(evilDir, "sub"), 0700))
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
// Control: an existing in-tree directory is found as normal.
require.NoError(t, f.Mkdir(ctx, "sub"))
_, err := f.MkdirMetadata(ctx, "sub", nil)
require.NoError(t, err)
// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
require.NoError(t, putLink(ctx, f, "pwn", evilDir))
// "pwn/sub" exists outside the root but must not be found through the
// planted symlink, and must not be created there either.
_, err = f.MkdirMetadata(ctx, "pwn/sub", nil)
require.Error(t, err, "finding a directory through a planted symlink should be refused")
_, err = f.MkdirMetadata(ctx, "pwn/new", nil)
require.Error(t, err, "making a directory through a planted symlink should be refused")
_, err = os.Lstat(filepath.Join(evilDir, "new"))
require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evilDir)
}
// TestRemoveThroughPlantedSymlinkBlocked checks Remove and Rmdir can't
// delete anything outside the root through a planted symlink.
func TestRemoveThroughPlantedSymlinkBlocked(t *testing.T) {
skipIfNoSymlinks(t)
ctx := context.Background()
evil := t.TempDir()
evilFile := filepath.Join(evil, "secret.txt")
require.NoError(t, os.WriteFile(evilFile, []byte("keep me"), 0600))
evilDir := filepath.Join(evil, "secret.d")
require.NoError(t, os.Mkdir(evilDir, 0700))
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
// Control: in-tree removals work as normal.
require.NoError(t, putFile(ctx, f, "file.txt", "data"))
o, err := f.NewObject(ctx, "file.txt")
require.NoError(t, err)
require.NoError(t, o.Remove(ctx))
require.NoError(t, f.Mkdir(ctx, "dir"))
require.NoError(t, f.Rmdir(ctx, "dir"))
// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
require.NoError(t, putLink(ctx, f, "pwn", evil))
// Removing the outside file and directory named through the planted
// symlink must be refused.
o, err = f.NewObject(ctx, "pwn/secret.txt")
require.NoError(t, err)
err = o.Remove(ctx)
require.Error(t, err, "removing through a planted symlink should be refused")
_, err = os.Lstat(evilFile)
require.NoError(t, err, "a file was removed outside the destination at %q", evilFile)
err = f.Rmdir(ctx, "pwn/secret.d")
require.Error(t, err, "removing a directory through a planted symlink should be refused")
_, err = os.Lstat(evilDir)
require.NoError(t, err, "a directory was removed outside the destination at %q", evilDir)
}
// TestDirMoveThroughPlantedSymlinkBlocked checks DirMove can't move a
// directory out of the root through a planted symlink.
func TestDirMoveThroughPlantedSymlinkBlocked(t *testing.T) {
skipIfNoSymlinks(t)
ctx := context.Background()
evil := t.TempDir()
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
// Control: an in-tree directory move works as normal.
require.NoError(t, f.Mkdir(ctx, "dir"))
require.NoError(t, f.DirMove(ctx, f, "dir", "sub/moved"))
// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
require.NoError(t, putLink(ctx, f, "pwn", evil))
err := f.DirMove(ctx, f, "sub/moved", "pwn/moved")
require.Error(t, err, "moving a directory through a planted symlink should be refused")
_, err = os.Lstat(filepath.Join(evil, "moved"))
require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evil)
_, err = os.Lstat(filepath.Join(f.root, "sub", "moved"))
require.NoError(t, err, "the directory should still be in the destination")
}
// TestEncodingEscapeBlocked checks that a name from a malicious source can't
// be decoded into path syntax which writes outside the destination.
func TestEncodingEscapeBlocked(t *testing.T) {
+1 -29
View File
@@ -5,9 +5,7 @@ import (
"math"
"os"
"runtime"
"slices"
"strconv"
"sync"
"time"
"github.com/rclone/rclone/fs"
@@ -123,15 +121,13 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
uid, hasUID := o.parseMetadataInt(m, "uid", 10)
gid, hasGID := o.parseMetadataInt(m, "gid", 10)
if hasUID {
// FIXME should read UID and GID of current user and only attempt to set it if different
if !hasGID {
gid = uid
}
if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
} else if setUID, setGID := ownerToSet(uid, gid); setUID < 0 && setGID < 0 {
fs.Debugf(o, "Not setting ownership %d.%d: only root can", uid, gid)
} else {
uid, gid = setUID, setGID
if o.translatedLink {
err = os.Lchown(o.path, uid, gid)
} else {
@@ -173,27 +169,3 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
// FIXME not parsing rdev yet
return outErr
}
// processGroups are the supplementary groups of this process
var processGroups = sync.OnceValue(func() []int {
groups, _ := os.Getgroups()
return groups
})
// ownerToSet returns the uid and gid to pass to chown for the wanted
// ownership: unchanged for root, else -1 (leave it as it is) for a uid
// other than our own and for a group we are not a member of, which
// only root may set.
func ownerToSet(uid, gid int) (setUID, setGID int) {
euid := os.Geteuid()
if euid <= 0 {
return uid, gid
}
if uid != euid {
uid = -1
}
if gid != os.Getegid() && !slices.Contains(processGroups(), gid) {
gid = -1
}
return uid, gid
}
-50
View File
@@ -1,50 +0,0 @@
//go:build !windows && !plan9 && !js
package local
import (
"context"
"os"
"path/filepath"
"slices"
"syscall"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMetadataOwnershipAsNonRoot(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("needs a user other than root")
}
uid, gid := os.Geteuid(), os.Getegid()
other := uid + 1
notMember := gid + 1
for slices.Contains(processGroups(), notMember) {
notMember++
}
// only what a non-root user may set: its own uid, its groups
setUID, setGID := ownerToSet(other, notMember)
assert.Equal(t, [2]int{-1, -1}, [2]int{setUID, setGID})
setUID, setGID = ownerToSet(other, gid)
assert.Equal(t, [2]int{-1, gid}, [2]int{setUID, setGID})
setUID, setGID = ownerToSet(uid, notMember)
assert.Equal(t, [2]int{uid, -1}, [2]int{setUID, setGID})
// a file of another user (e.g. root:root) is copied with our ownership
// instead of failing with "operation not permitted"
r := fstest.NewRun(t)
r.WriteFile("owned.txt", "payload", time.Now())
f := r.Flocal.(*Fs)
obj, err := f.NewObject(context.Background(), "owned.txt")
require.NoError(t, err)
require.NoError(t, obj.(*Object).writeMetadataToFile(fs.Metadata{"uid": "0", "gid": "0"}))
fi, err := os.Stat(filepath.Join(f.root, "owned.txt"))
require.NoError(t, err)
assert.Equal(t, uint32(uid), fi.Sys().(*syscall.Stat_t).Uid)
}
+5 -3
View File
@@ -2,7 +2,9 @@
package local
// remove removes name with removeFn, retrying on a sharing violation
func remove(name string, removeFn func(string) error) error {
return removeFn(name)
import "os"
// Removes name, retrying on a sharing violation
func remove(name string) error {
return os.Remove(name)
}
+1 -1
View File
@@ -38,7 +38,7 @@ func TestRemove(t *testing.T) {
require.NoError(t, fd.Close())
})
// delete the open file
err = remove(name, os.Remove)
err = remove(name)
require.NoError(t, err)
// check it no longer exists
assert.False(t, exists())
+3 -3
View File
@@ -10,12 +10,12 @@ import (
"golang.org/x/sys/windows"
)
// remove removes name with removeFn, retrying on a sharing violation
func remove(name string, removeFn func(string) error) (err error) {
// Removes name, retrying on a sharing violation
func remove(name string) (err error) {
const maxTries = 10
var sleepTime = 1 * time.Millisecond
for i := range maxTries {
err = removeFn(name)
err = os.Remove(name)
if err == nil {
break
}
+2 -26
View File
@@ -4,7 +4,6 @@ package local
import (
"fmt"
"os"
"strings"
"syscall"
@@ -96,11 +95,6 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
if !xattrSupported || o.fs.xattrSupported.Load() == 0 {
return nil
}
// When translating symlinks (-l/--links) write the xattrs through a
// descriptor opened under os.Root so a symlink planted in the destination
// can't redirect the write outside it (CWE-59). A translated link is
// itself the leaf, so it keeps the non-following path based write.
var fd *os.File
for k, value := range metadata {
k = strings.ToLower(k)
if _, found := systemMetadataInfo[k]; found {
@@ -108,27 +102,9 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
}
k = xattrPrefix + k
v := []byte(value)
switch {
case o.fs.opt.TranslateSymlinks && !o.translatedLink:
if fd == nil {
fd, err = o.fs.openFile(o.path, os.O_RDONLY, 0)
if os.IsPermission(err) {
// A write only file can't be opened for reading but its
// owner may still set xattrs on it, so try opening it for
// writing instead (which can't succeed for a directory).
if wfd, werr := o.fs.openFile(o.path, os.O_WRONLY, 0); werr == nil {
fd, err = wfd, nil
}
}
if err != nil {
return fmt.Errorf("failed to open for xattr: %w", err)
}
defer fs.CheckClose(fd, &err)
}
err = xattr.FSet(fd, k, v)
case o.fs.opt.FollowSymlinks:
if o.fs.opt.FollowSymlinks {
err = xattr.Set(o.path, k, v)
default:
} else {
err = xattr.LSet(o.path, k, v)
}
if err != nil {
-96
View File
@@ -1,96 +0,0 @@
//go:build !openbsd && !plan9
package local
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/pkg/xattr"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestXattrThroughPlantedSymlinkBlocked checks user metadata (xattrs)
// can't be written outside the root through a planted symlink with
// --links, while in-tree writes still work.
func TestXattrThroughPlantedSymlinkBlocked(t *testing.T) {
if !xattrSupported {
t.Skip("xattrs not supported on this OS")
}
skipIfNoSymlinks(t)
ctx := context.Background()
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
metadata := fs.Metadata{
"pwned": "attacker-controlled-value",
}
// Control: an in-tree directory gets the xattr as normal.
_, err := f.MkdirMetadata(ctx, "sub", metadata)
require.NoError(t, err)
if f.xattrSupported.Load() == 0 {
t.Skip("xattrs not supported on this filesystem")
}
v, err := xattr.LGet(filepath.Join(f.root, "sub"), "user.pwned")
require.NoError(t, err)
assert.Equal(t, "attacker-controlled-value", string(v))
// A directory outside the destination, with a pre-existing child,
// whose xattrs the attacker targets.
evil := t.TempDir()
evilDir := filepath.Join(evil, "secret.d")
require.NoError(t, os.Mkdir(evilDir, 0700))
evilSub := filepath.Join(evilDir, "sub")
require.NoError(t, os.Mkdir(evilSub, 0700))
// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
require.NoError(t, putLink(ctx, f, "pwn", evilDir))
// The source now presents "pwn/sub" as a directory with attacker-chosen
// user metadata. Applying it must not reach through the planted symlink.
_, err = f.MkdirMetadata(ctx, "pwn/sub", metadata)
require.Error(t, err, "applying metadata through a planted symlink should be refused")
_, err = xattr.LGet(evilSub, "user.pwned")
require.Error(t, err, "xattr escaped through planted symlink to %q", evilSub)
}
// TestXattrWriteOnlyFile checks user metadata can still be set on a
// write-only file with --links, where the confined open can't read it.
func TestXattrWriteOnlyFile(t *testing.T) {
if !xattrSupported {
t.Skip("xattrs not supported on this OS")
}
skipIfNoSymlinks(t)
ctx := context.Background()
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
require.NoError(t, putFile(ctx, f, "wo.txt", "write only"))
path := filepath.Join(f.root, "wo.txt")
require.NoError(t, os.Chmod(path, 0200))
obj, err := f.NewObject(ctx, "wo.txt")
require.NoError(t, err)
o := obj.(*Object)
err = o.setXattr(fs.Metadata{"potato": "chips"})
require.NoError(t, err)
if f.xattrSupported.Load() == 0 {
t.Skip("xattrs not supported on this filesystem")
}
// Reading xattrs needs read permission
require.NoError(t, os.Chmod(path, 0600))
v, err := xattr.LGet(path, "user.potato")
require.NoError(t, err)
assert.Equal(t, "chips", string(v))
}
+1 -19
View File
@@ -208,24 +208,6 @@ func (d Du) GetName() xml.Name {
//------------------------------------------------------------------------------
// apiHeaders are the NetStorage API request headers which carry the
// account name, the request signature and the signed action. They must
// not be forwarded when a redirect leaves the configured host.
//
// The action header is sent uncanonicalised, hence the "*" prefix.
var apiHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "*X-Akamai-ACS-Action"}
// newClient makes the http client used for all NetStorage requests.
//
// The signer adds the API headers to every request, so the client
// strips them on cross host redirects and refuses an HTTPS to HTTP
// downgrade.
func newClient(ctx context.Context) *http.Client {
client := fshttp.NewClient(ctx)
client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(apiHeaders...)
return client
}
// NewFs creates a new Fs object from the name and root. It connects to
// the host specified in the config file.
//
@@ -258,7 +240,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, err
}
client := newClient(ctx)
client := fshttp.NewClient(ctx)
f := &Fs{
name: name,
@@ -4,17 +4,12 @@ import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -64,142 +59,3 @@ func TestListResumeCannotChangeHost(t *testing.T) {
assert.Equal(t, 1, listRequests, "list request sent to the configured host")
assert.Equal(t, 0, requests, "request sent to the other host")
}
// apiTestHeaders is a deliberately literal copy of apiHeaders, without
// the "*" prefix, so that a header dropped from the production list
// shows up as a leak in the cross host tests rather than silently
// losing coverage.
var apiTestHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "X-Akamai-ACS-Action"}
// statXML is a minimal stat response saying the root is a directory
const statXML = `<?xml version="1.0" encoding="UTF-8"?>
<stat directory="/dir"><file type="dir" name="dir" mtime="1"/></stat>`
// newTestFs makes an Fs via NewFs pointing at serverURL.
//
// NewFs makes a stat request for the root and treats any failure as
// the root not existing, so callers check what the servers received
// or make a further request rather than relying on its error.
func newTestFs(ctx context.Context, t *testing.T, serverURL string) *Fs {
u, err := url.Parse(serverURL)
require.NoError(t, err)
m := configmap.Simple{
"type": "netstorage",
"protocol": u.Scheme,
"host": u.Host + "/dir",
"account": "test-account",
"secret": obscure.MustObscure("test-secret"),
}
f, err := NewFs(ctx, "netstorage-test", "", m)
require.NoError(t, err)
return f.(*Fs)
}
// assertAPIHeaders checks all the API headers are present (want) or
// absent (!want) in r.
func assertAPIHeaders(t *testing.T, r *http.Request, want bool) {
for _, header := range apiTestHeaders {
if want {
assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)
} else {
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
}
}
}
// TestRedirectStripsAPIHeadersOnCrossHost checks the API headers are
// sent to the configured host but not to another host it redirects to.
func TestRedirectStripsAPIHeadersOnCrossHost(t *testing.T) {
var otherHits atomic.Int32
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
otherHits.Add(1)
assertAPIHeaders(t, r, false)
_, _ = w.Write([]byte(statXML))
}))
defer other.Close()
var originHits atomic.Int32
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
originHits.Add(1)
assertAPIHeaders(t, r, true)
http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
}))
defer origin.Close()
newTestFs(context.Background(), t, origin.URL)
assert.Equal(t, int32(1), originHits.Load())
assert.Equal(t, int32(1), otherHits.Load())
}
// TestUploadStripsAPIHeadersOnCrossHost checks an upload, which
// carries the API headers on the PUT as well as in the trailers, does
// not send them to another host the configured host redirects to.
func TestUploadStripsAPIHeadersOnCrossHost(t *testing.T) {
var otherHits atomic.Int32
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
otherHits.Add(1)
assertAPIHeaders(t, r, false)
_, _ = w.Write([]byte(statXML))
}))
defer other.Close()
var originHits atomic.Int32
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
originHits.Add(1)
assertAPIHeaders(t, r, true)
http.Redirect(w, r, other.URL+r.URL.Path, http.StatusMovedPermanently)
}))
defer origin.Close()
ctx := context.Background()
f := newTestFs(ctx, t, origin.URL)
o := &Object{fs: f, remote: "file"}
src := object.NewStaticObjectInfo("file", time.Now(), 4, true, nil, f)
err := o.netStorageUploadRequest(ctx, strings.NewReader("data"), src)
require.NoError(t, err)
// NewFs stat, PUT and the stat after the upload
assert.Equal(t, int32(3), originHits.Load())
assert.Equal(t, int32(3), otherHits.Load())
}
// TestRedirectKeepsAPIHeadersOnSameHost checks the API headers are
// still sent after a redirect which stays on the configured host.
func TestRedirectKeepsAPIHeadersOnSameHost(t *testing.T) {
var hits atomic.Int32
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
assertAPIHeaders(t, r, true)
if !strings.HasSuffix(r.URL.Path, "/moved") {
http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)
return
}
_, _ = w.Write([]byte(statXML))
}))
defer origin.Close()
newTestFs(context.Background(), t, origin.URL)
assert.Equal(t, int32(2), hits.Load())
}
// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to
// plaintext HTTP is not followed.
func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
var targetHits atomic.Int32
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
targetHits.Add(1)
_, _ = w.Write([]byte(statXML))
}))
defer target.Close()
origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)
}))
defer origin.Close()
ctx, ci := fs.AddConfig(context.Background())
ci.InsecureSkipVerify = true
f := newTestFs(ctx, t, origin.URL)
_, err := f.netStorageStatRequest(ctx, f.url(""), true)
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
assert.Equal(t, int32(0), targetHits.Load())
}
+38 -277
View File
@@ -41,7 +41,6 @@ import (
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"golang.org/x/oauth2"
)
const (
@@ -845,33 +844,19 @@ type Options struct {
// Fs represents a remote OneDrive
type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv *rest.Client // the connection to the OneDrive server
unAuth *rest.Client // no authentication connection to the OneDrive server
dirCache *dircache.DirCache // Map of directory path to directory id
pacer *fs.Pacer // pacer for API calls
tokenRenewer *oauthutil.Renew // renew the token on expiry
tokenSource *oauthutil.TokenSource // source of the API token
sharePoint *sharePointClients // authenticated clients for SharePoint hosts
driveID string // ID to use for querying Microsoft Graph
driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
hashType hash.Type // type of the hash we are using
}
// sharePointClients holds the clients authenticated for SharePoint hosts
type sharePointClients struct {
mu sync.Mutex
hosts map[string]sharePointAuth
}
// sharePointAuth is a client authenticated for a SharePoint host
type sharePointAuth struct {
client *rest.Client
tokenSource oauth2.TokenSource
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv *rest.Client // the connection to the OneDrive server
unAuth *rest.Client // no authentication connection to the OneDrive server
dirCache *dircache.DirCache // Map of directory path to directory id
pacer *fs.Pacer // pacer for API calls
tokenRenewer *oauthutil.Renew // renew the token on expiry
driveID string // ID to use for querying Microsoft Graph
driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
hashType hash.Type // type of the hash we are using
}
// Object describes a OneDrive object
@@ -1091,154 +1076,6 @@ func errorHandler(resp *http.Response) error {
return errResponse
}
// errPreauthRetired explains an authentication failure on a URL which
// SharePoint used to return pre-authenticated.
var errPreauthRetired = errors.New("SharePoint refused to authenticate a request - Microsoft is retiring pre-authenticated (tempauth) URLs, see https://rclone.org/onedrive/#pre-authenticated-urls")
// sharePointDomains are the domains of SharePoint Online in the
// different Microsoft clouds
var sharePointDomains = []string{".sharepoint.com", ".sharepoint.us", ".sharepoint-mil.us", ".sharepoint.cn", ".sharepoint.de"}
// stripTempauth removes the tempauth parameter from the URLs the API
// returns, which simulates the retirement of pre-authenticated URLs
// in tests.
var stripTempauth = false
// returnedURL processes a URL returned by the API before use
func returnedURL(rawURL string) string {
if !stripTempauth {
return rawURL
}
u, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
q := u.Query()
q.Del("tempauth")
u.RawQuery = q.Encode()
return u.String()
}
// urlAuth says how a URL returned by the API is authenticated
type urlAuth int
const (
urlAuthNone urlAuth = iota // call without authentication
urlAuthAPI // call with the API token
urlAuthSharePoint // call with a token for the SharePoint host
)
// apiHost returns the host the API calls made with f.srv go to.
func (f *Fs) apiHost() string {
endpoint := graphAPIEndpoint[f.opt.Region]
if f.opt.TenantURL != "" {
endpoint = f.opt.TenantURL
}
u, err := url.Parse(endpoint)
if err != nil || u.Scheme != "https" {
return ""
}
return u.Host
}
// urlAuthFor returns how to authenticate a URL returned by the API,
// such as an upload session or an async job monitor.
//
// Pre-authenticated URLs fail if they are sent an Authorization
// header and tokens must not be sent to other hosts, so only HTTPS
// URLs without tempauth on the API host or a SharePoint host are
// authenticated.
func (f *Fs) urlAuthFor(rawURL string) urlAuth {
u, err := url.Parse(rawURL)
if err != nil || u.Scheme != "https" || u.Query().Has("tempauth") {
return urlAuthNone
}
if apiHost := f.apiHost(); apiHost != "" && strings.EqualFold(u.Host, apiHost) {
return urlAuthAPI
}
if isSharePointHost(u.Hostname()) {
return urlAuthSharePoint
}
return urlAuthNone
}
// isSharePointHost returns true if host is in a SharePoint Online domain.
func isSharePointHost(host string) bool {
host = strings.ToLower(host)
for _, domain := range sharePointDomains {
if strings.HasSuffix(host, domain) {
return true
}
}
return false
}
// checkSharePointRedirect stops the SharePoint token being sent to a
// host which isn't SharePoint, as the token is added to every request
// the client makes.
func checkSharePointRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
if req.URL.Scheme != "https" || !isSharePointHost(req.URL.Hostname()) {
return fmt.Errorf("refusing to send a SharePoint token on a redirect to %s://%s", req.URL.Scheme, req.URL.Host)
}
return nil
}
// clientForURL returns the client to call a URL returned by the API with.
func (f *Fs) clientForURL(ctx context.Context, rawURL string) (*rest.Client, error) {
switch f.urlAuthFor(rawURL) {
case urlAuthAPI:
return f.srv, nil
case urlAuthSharePoint:
u, _ := url.Parse(rawURL)
return f.sharePointClient(ctx, u.Host)
}
return f.unAuth, nil
}
// sharePointClient returns a client authenticated for the SharePoint
// host, which upload session and async job monitor URLs need once
// they are no longer pre-authenticated.
//
// The token is got from the existing credentials, which Entra ID
// allows for SharePoint as the user has consented to the Graph file
// scopes.
func (f *Fs) sharePointClient(ctx context.Context, host string) (*rest.Client, error) {
if f.tokenSource == nil || f.sharePoint == nil {
return nil, fmt.Errorf("%w: no credentials to get a SharePoint token with", errPreauthRetired)
}
host = strings.ToLower(host)
f.sharePoint.mu.Lock()
defer f.sharePoint.mu.Unlock()
sp, ok := f.sharePoint.hosts[host]
if !ok {
tokenSource := f.tokenSource.ScopedTokenSource([]string{"https://" + host + "/.default"})
client := oauth2.NewClient(oauthutil.Context(ctx, fshttp.NewClient(ctx)), tokenSource)
client.CheckRedirect = checkSharePointRedirect
sp = sharePointAuth{client: rest.NewClient(client), tokenSource: tokenSource}
f.sharePoint.hosts[host] = sp
fs.Debugf(f, "Using a SharePoint token for %s", host)
}
// Get the token now so failures are reported before the client is used
if _, err := sp.tokenSource.Token(); err != nil {
return nil, fmt.Errorf("%w: failed to get a SharePoint token: %w", errPreauthRetired, err)
}
return sp.client, nil
}
// checkPreauthRetired adds an explanation to a 401 error from a URL
// returned by the API which needs a SharePoint token, as these were
// pre-authenticated until Microsoft retired them.
func (f *Fs) checkPreauthRetired(rawURL string, resp *http.Response, err error) error {
if err == nil || resp == nil || resp.StatusCode != http.StatusUnauthorized ||
f.driveType == driveTypePersonal || errors.Is(err, errPreauthRetired) || f.urlAuthFor(rawURL) != urlAuthSharePoint {
return err
}
return fmt.Errorf("%w: %w", errPreauthRetired, err)
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
const minChunkSize = fs.SizeSuffixBase
if cs%chunkSizeMultiple != 0 {
@@ -1307,18 +1144,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
ci: ci,
driveID: opt.DriveID,
driveType: opt.DriveType,
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
tokenSource: ts,
sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)},
unAuth: rest.NewClient(client).SetRoot(rootURL),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
hashType: QuickXorHashType,
name: name,
root: root,
opt: *opt,
ci: ci,
driveID: opt.DriveID,
driveType: opt.DriveType,
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
unAuth: rest.NewClient(client).SetRoot(rootURL),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
hashType: QuickXorHashType,
}
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -1890,25 +1725,11 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
var resp *http.Response
var err error
var body []byte
var redirected bool
err = f.pacer.Call(func() (bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
if err != nil {
return false, err
}
client, err := f.clientForURL(ctx, location)
if err != nil {
return false, err
}
resp, err = client.Do(req)
resp, err = http.Get(location)
if err != nil {
return fserrors.ShouldRetry(err), err
}
redirected = resp.Request.URL.String() != req.URL.String()
if resp.StatusCode == http.StatusUnauthorized {
_ = resp.Body.Close()
return false, f.checkPreauthRetired(location, resp, fmt.Errorf("async job monitor: HTTP error %s", resp.Status))
}
body, err = rest.ReadBody(resp)
return fserrors.ShouldRetry(err), err
})
@@ -1921,11 +1742,6 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
if err != nil {
return fmt.Errorf("async status result not JSON: %q: %w", body, err)
}
// An authenticated monitor redirects to the new item when the
// job has finished instead of returning its status
if status.Status == "" && redirected {
status.Status = "completed"
}
switch status.Status {
case "failed":
@@ -2025,7 +1841,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
}
// read location header
location := returnedURL(resp.Header.Get("Location"))
location := resp.Header.Get("Location")
if location == "" {
return nil, errors.New("didn't receive location header in copy response")
}
@@ -2036,10 +1852,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
fs.Debugf(src, "Server-side copy failed - file not shared between drives")
return nil, fs.ErrorCantCopy
}
if errors.Is(err, errPreauthRetired) {
fs.Debugf(src, "Server-side copy can't be monitored - copying another way: %v", err)
return nil, fs.ErrorCantCopy
}
if err != nil {
return nil, err
}
@@ -2714,23 +2526,16 @@ func (o *Object) openContentPrefer(ctx context.Context, options ...fs.OpenOption
return o.openWithRedirect(ctx, &opts)
}
// openWithRedirect downloads via /content style endpoints that 302 to
// a download URL, which is pre-authenticated or needs a SharePoint
// token.
// openWithRedirect downloads via /content style endpoints that 302 to a preauthenticated URL.
func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.ReadCloser, err error) {
var resp *http.Response
// Make a note of the redirect target as it needs different auth
// Make a note of the redirect target as we need to call it without Auth
var redirectReq *http.Request
opts.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
if stripTempauth {
if u, err := url.Parse(returnedURL(req.URL.String())); err == nil {
req.URL = u
}
}
// Download URLs must not carry the Graph Authorization header.
// Preauthenticated download URLs must not carry the Graph Authorization header.
req.Header.Del("Authorization")
// Keep Prefer on the redirect when forcing infected download; some SharePoint
// endpoints honor it only on the final download request.
@@ -2769,24 +2574,17 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
return nil, err
}
if redirectReq != nil {
redirectURL := redirectReq.URL.String()
var client *rest.Client
client, err = o.fs.clientForURL(ctx, redirectURL)
if err != nil {
return nil, err
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = client.Do(redirectReq)
resp, err = o.fs.unAuth.Do(redirectReq)
if err != nil {
return shouldRetry(ctx, resp, err)
}
// Do does not check status; treat non-2xx as failure so a malware
// unAuth.Do does not check status; treat non-2xx as failure so a malware
// JSON body is not written out as file content.
if resp.StatusCode < 200 || resp.StatusCode > 299 {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
_ = resp.Body.Close()
err = fmt.Errorf("HTTP error %d (%s) %s", resp.StatusCode, resp.Status, strings.TrimSpace(string(body)))
err = o.fs.checkPreauthRetired(redirectURL, resp, err)
if strings.Contains(string(body), "malwareDetected") || resp.Header.Get("X-Virus-Infected") != "" {
err = malwareDownloadError(o.fs.opt.AVOverride, err)
}
@@ -2808,19 +2606,7 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
// Overwrite size with actual size since size readings from Onedrive is unreliable.
o.size = resp.ContentLength
}
return o.checkRange(resp, opts.Options)
}
// checkRange returns the body of a download, checking it is the range
// requested in options.
func (o *Object) checkRange(resp *http.Response, options []fs.OpenOption) (in io.ReadCloser, err error) {
// The size isn't passed as it is unreliable on OneDrive
err = rest.CheckContentRange(resp, options, -1)
if err != nil {
_ = resp.Body.Close()
return nil, err
}
return resp.Body, nil
return resp.Body, err
}
// malwareDownloadError formats an error when the server blocks a malware-flagged file.
@@ -2860,16 +2646,12 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
}
var info api.UploadFragmentResponse
var resp *http.Response
client, err := o.fs.clientForURL(ctx, url)
if err != nil {
return 0, err
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = client.CallJSON(ctx, &opts, nil, &info)
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return 0, o.fs.checkPreauthRetired(url, resp, err)
return 0, err
}
if len(info.NextExpectedRanges) != 1 {
return 0, fmt.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)
@@ -2893,10 +2675,6 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
var resp *http.Response
var body []byte
skip := int64(0)
client, err := o.fs.clientForURL(ctx, url)
if err != nil {
return nil, err
}
err = o.fs.pacer.Call(func() (bool, error) {
toSend := chunkSize - skip
opts := rest.Opts{
@@ -2908,7 +2686,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
Options: options,
}
_, _ = chunk.Seek(skip, io.SeekStart)
resp, err = client.Call(ctx, &opts)
resp, err = o.fs.unAuth.Call(ctx, &opts)
if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
pos, posErr := o.getPosition(ctx, url)
@@ -2947,7 +2725,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
}
return false, nil
})
return info, o.fs.checkPreauthRetired(url, resp, err)
return info, err
}
// cancelUploadSession cancels an upload session
@@ -2958,15 +2736,11 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
NoResponse: true,
}
var resp *http.Response
client, err := o.fs.clientForURL(ctx, url)
if err != nil {
return err
}
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = client.Call(ctx, &opts)
resp, err = o.fs.unAuth.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
return o.fs.checkPreauthRetired(url, resp, err)
return
}
// uploadMultipart uploads a file using multipart upload
@@ -2984,20 +2758,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
if err != nil {
return nil, err
}
uploadURL := returnedURL(session.UploadURL)
// Check a session which needs SharePoint authentication can get it
// before reading any data, so it can be uploaded in a single part
// instead if it can't.
if o.fs.urlAuthFor(uploadURL) == urlAuthSharePoint {
if _, err = o.getPosition(ctx, uploadURL); err != nil {
if !errors.Is(err, errPreauthRetired) || size >= int64(maxSinglePartSize) {
return nil, err
}
fs.Debugf(o, "Uploading in a single part as the upload session can't be used: %v", err)
return o.uploadSinglepart(ctx, in, src, options...)
}
}
uploadURL := session.UploadURL
// Cancel the session if something went wrong
defer atexit.OnError(&err, func() {
@@ -3,7 +3,6 @@ package onedrive
import (
"context"
"encoding/json"
"flag"
"fmt"
"net/http"
"net/http/httptest"
@@ -16,7 +15,6 @@ import (
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/backend/onedrive/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
@@ -45,10 +43,6 @@ const (
// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_invite?view=odsp-graph-online#http-request-1
)
func init() {
flag.BoolVar(&stripTempauth, "strip-tempauth", false, "Remove tempauth from URLs the API returns to simulate the retirement of pre-authenticated URLs")
}
// TestMain drives the tests
func TestMain(m *testing.M) {
fstest.TestMain(m)
@@ -776,96 +770,3 @@ func TestListDirectorySize(t *testing.T) {
assert.Equal(t, int64(2646204760), dir.Size())
assert.Equal(t, int64(3), dir.Items())
}
// TestWaitForJobUsesConfiguredClient checks that the async job monitor is
// polled with the configured HTTP client and request context rather than
// http.DefaultClient, so proxy, TLS and timeout options and cancellation apply.
func TestWaitForJobUsesConfiguredClient(t *testing.T) {
ctx, ci := fs.AddConfig(context.Background())
ci.UserAgent = "rclone-waitforjob-test"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "rclone-waitforjob-test", r.UserAgent())
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(api.AsyncOperationStatus{Status: "deleteFailed"})
}))
t.Cleanup(server.Close)
f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
f.unAuth = rest.NewClient(fshttp.NewClient(ctx))
err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"})
require.Error(t, err)
assert.Contains(t, err.Error(), "deleteFailed")
}
// TestWaitForJobCancel checks that cancelling the context stops a poll of
// the async job monitor which is not answering.
func TestWaitForJobCancel(t *testing.T) {
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-release:
}
}))
t.Cleanup(server.Close)
t.Cleanup(func() { close(release) })
f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
done := make(chan error, 1)
go func() { done <- f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"}) }()
select {
case err := <-done:
assert.ErrorIs(t, err, context.DeadlineExceeded)
case <-time.After(5 * time.Second):
t.Fatal("waitForJob did not return after the context was cancelled")
}
}
// TestWaitForJobUnauthorized checks that a monitor URL which refuses
// the request fails straight away instead of polling until the timeout.
func TestWaitForJobUnauthorized(t *testing.T) {
var polls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
polls.Add(1)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "unauthenticated", "message": "no token"}})
}))
t.Cleanup(server.Close)
f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
f.driveType = driveTypeBusiness
err := f.waitForJob(context.Background(), server.URL+"/monitor", &Object{remote: "file.txt"})
require.Error(t, err)
assert.Contains(t, err.Error(), "401")
assert.Equal(t, int32(1), polls.Load())
}
// TestWaitForJobRedirectCompleted checks that a monitor which
// redirects to the new item, as it does when authenticated, is taken
// to mean the job has finished.
func TestWaitForJobRedirectCompleted(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/monitor":
http.Redirect(w, r, "/item", http.StatusSeeOther)
case "/item":
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprint(w, `{"id":"newid","name":"file.txt","size":5,"file":{}}`)
default:
t.Errorf("unexpected request %s", r.URL)
}
}))
t.Cleanup(server.Close)
f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt", hasMetaData: true})
require.NoError(t, err)
}
-330
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -14,14 +13,11 @@ import (
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/oauth2"
)
// fragmentServer is a fake OneDrive upload session which records the
@@ -70,8 +66,6 @@ func (s *fragmentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusAccepted)
_, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)
case r.Method == "DELETE" && r.URL.Path == "/upload":
w.WriteHeader(http.StatusNoContent)
default:
s.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
@@ -124,327 +118,3 @@ func TestUploadMultipartSkip(t *testing.T) {
assert.Equal(t, 2, s.fragments)
assert.Equal(t, 3, s.attempts, "expected one 416, a partial re-send and the final fragment")
}
// TestURLAuthFor checks that tokens are only sent to HTTPS URLs on
// the API host or a SharePoint host which aren't pre-authenticated.
func TestURLAuthFor(t *testing.T) {
const tenant = "https://contoso-my.sharepoint.com/_api"
for _, tc := range []struct {
name string
region string
tenantURL string
url string
want urlAuth
}{
{"graph", "global", "", "https://graph.microsoft.com/v1.0/drives/x/items/y/uploadSession", urlAuthAPI},
{"graph case", "global", "", "https://Graph.Microsoft.com/v1.0/monitor/abc", urlAuthAPI},
{"graph tempauth", "global", "", "https://graph.microsoft.com/v1.0/upload?tempauth=abc", urlAuthNone},
{"graph http", "global", "", "http://graph.microsoft.com/v1.0/upload", urlAuthNone},
{"graph other region", "global", "", "https://graph.microsoft.us/v1.0/upload", urlAuthNone},
{"graph us", "us", "", "https://graph.microsoft.us/v1.0/upload", urlAuthAPI},
{"graph lookalike", "global", "", "https://graph.microsoft.com.example.com/upload", urlAuthNone},
{"sharepoint tempauth", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x&tempauth=abc", urlAuthNone},
{"sharepoint", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x", urlAuthSharePoint},
{"sharepoint case", "global", "", "https://Contoso.SharePoint.com/_api/v2.0/monitor/abc", urlAuthSharePoint},
{"sharepoint us", "us", "", "https://contoso.sharepoint.us/_api/v2.0/upload", urlAuthSharePoint},
{"sharepoint cn", "cn", "", "https://contoso.sharepoint.cn/_api/v2.0/upload", urlAuthSharePoint},
{"sharepoint http", "global", "", "http://contoso-my.sharepoint.com/_api/v2.0/upload", urlAuthNone},
{"sharepoint lookalike", "global", "", "https://contoso.sharepoint.com.example.com/upload", urlAuthNone},
{"sharepoint suffix only", "global", "", "https://evilsharepoint.com/upload", urlAuthNone},
{"personal", "global", "", "https://api.onedrive.com/rup/abc", urlAuthNone},
{"personal upload", "global", "", "https://my.microsoftpersonalcontent.com/personal/abc/_api/v2.0/drive/items/x/uploadSession", urlAuthNone},
{"unparseable", "global", "", "https://graph.microsoft.com/%zz", urlAuthNone},
{"tenant", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/monitor/abc", urlAuthAPI},
{"tenant tempauth", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/upload?tempauth=abc", urlAuthNone},
{"tenant graph", "global", tenant, "https://graph.microsoft.com/v1.0/upload", urlAuthNone},
{"tenant other sharepoint", "global", tenant, "https://contoso.sharepoint.com/_api/v2.0/upload", urlAuthSharePoint},
} {
t.Run(tc.name, func(t *testing.T) {
f := &Fs{}
f.opt.Region = tc.region
f.opt.TenantURL = tc.tenantURL
assert.Equal(t, tc.want, f.urlAuthFor(tc.url))
})
}
}
// TestCheckSharePointRedirect checks the SharePoint token is only
// sent on redirects to SharePoint over HTTPS.
func TestCheckSharePointRedirect(t *testing.T) {
for _, tc := range []struct {
url string
ok bool
}{
{"https://contoso-my.sharepoint.com/download.aspx", true},
{"https://205807-ipv4v6.gr.global.aa-rt.sharepoint.com/download.aspx?tempauth=x", true},
{"https://contoso.sharepoint.us/download.aspx", true},
{"http://contoso-my.sharepoint.com/download.aspx", false},
{"https://contoso.sharepoint.com.example.com/download.aspx", false},
{"https://graph.microsoft.com/v1.0/me", false},
{"https://example.com/", false},
} {
req, err := http.NewRequest("GET", tc.url, nil)
require.NoError(t, err)
err = checkSharePointRedirect(req, nil)
if tc.ok {
assert.NoError(t, err, tc.url)
} else {
assert.Error(t, err, tc.url)
}
}
req, err := http.NewRequest("GET", "https://contoso-my.sharepoint.com/", nil)
require.NoError(t, err)
assert.Error(t, checkSharePointRedirect(req, make([]*http.Request, 10)), "too many redirects")
}
// TestCheckPreauthRetired checks which 401 errors get the explanation.
func TestCheckPreauthRetired(t *testing.T) {
const sharePointURL = "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x"
errBase := errors.New("HTTP error 401")
for _, tc := range []struct {
name string
driveType string
url string
status int
want bool
}{
{"sharepoint 401", driveTypeBusiness, sharePointURL, http.StatusUnauthorized, true},
{"library 401", driveTypeSharepoint, sharePointURL, http.StatusUnauthorized, true},
{"personal 401", driveTypePersonal, sharePointURL, http.StatusUnauthorized, false},
{"sharepoint 403", driveTypeBusiness, sharePointURL, http.StatusForbidden, false},
{"tempauth 401", driveTypeBusiness, sharePointURL + "&tempauth=abc", http.StatusUnauthorized, false},
{"graph 401", driveTypeBusiness, "https://graph.microsoft.com/v1.0/upload", http.StatusUnauthorized, false},
} {
t.Run(tc.name, func(t *testing.T) {
f := &Fs{driveType: tc.driveType}
f.opt.Region = "global"
err := f.checkPreauthRetired(tc.url, &http.Response{StatusCode: tc.status}, errBase)
assert.ErrorIs(t, err, errBase)
assert.Equal(t, tc.want, errors.Is(err, errPreauthRetired), "err = %v", err)
})
}
f := &Fs{driveType: driveTypeBusiness}
assert.NoError(t, f.checkPreauthRetired(sharePointURL, &http.Response{StatusCode: http.StatusUnauthorized}, nil))
assert.ErrorIs(t, f.checkPreauthRetired(sharePointURL, nil, errBase), errBase)
}
// TestSharePointClient checks that the client for a SharePoint host
// is authenticated with a token for that host got from the existing
// refresh token, which is fetched once and not stored.
func TestSharePointClient(t *testing.T) {
var scopes []string
var auths []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/token":
_ = r.ParseForm()
assert.Equal(t, "refresh_token", r.PostForm.Get("grant_type"))
assert.Equal(t, "refresh", r.PostForm.Get("refresh_token"))
scopes = append(scopes, r.PostForm.Get("scope"))
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprint(w, `{"access_token":"sharepoint_token","token_type":"Bearer","expires_in":3600}`)
case "/upload":
auths = append(auths, r.Header.Get("Authorization"))
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprint(w, `{"nextExpectedRanges":["0-"]}`)
case "/redirect":
http.Redirect(w, r, "/upload", http.StatusFound)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
}
}))
t.Cleanup(server.Close)
storedToken := `{"access_token":"graph_token","token_type":"Bearer","refresh_token":"refresh","expiry":"2099-01-01T00:00:00Z"}`
m := configmap.Simple{"token": storedToken}
ctx := context.Background()
_, ts, err := oauthutil.NewClientWithBaseClient(ctx, "remote", m, &oauthutil.Config{
ClientID: "client_id",
ClientSecret: "client_secret",
TokenURL: server.URL + "/token",
}, server.Client())
require.NoError(t, err)
f := &Fs{tokenSource: ts, sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)}}
for range 2 {
client, err := f.sharePointClient(ctx, "Contoso-my.sharepoint.com")
require.NoError(t, err)
_, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/upload"})
require.NoError(t, err)
}
assert.Equal(t, []string{"https://contoso-my.sharepoint.com/.default"}, scopes, "token should be fetched once for the host")
assert.Equal(t, []string{"Bearer sharepoint_token", "Bearer sharepoint_token"}, auths)
assert.Equal(t, storedToken, m["token"], "the stored token must not change")
// The token isn't sent on a redirect to a host which isn't SharePoint
client, err := f.sharePointClient(ctx, "contoso-my.sharepoint.com")
require.NoError(t, err)
_, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/redirect"})
assert.ErrorContains(t, err, "refusing to send a SharePoint token")
assert.Len(t, auths, 2)
// Without credentials there is no client and the error explains why
_, err = (&Fs{}).sharePointClient(ctx, "contoso-my.sharepoint.com")
assert.ErrorIs(t, err, errPreauthRetired)
}
// TestUploadSessionFallback checks that an upload session which needs
// SharePoint authentication which can't be got is replaced by a single
// part upload before any data is read.
func TestUploadSessionFallback(t *testing.T) {
content := bytes.Repeat([]byte("onedrive"), 1024)
var requests []string
var uploaded []byte
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests = append(requests, r.Method+" "+r.URL.Path)
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == "POST" && r.URL.Path == "/root/createUploadSession":
_, _ = fmt.Fprint(w, `{"uploadUrl":"https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x","nextExpectedRanges":["0-"]}`)
case r.Method == "PUT" && r.URL.Path == "/root/content":
var err error
uploaded, err = io.ReadAll(r.Body)
require.NoError(t, err)
_, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
case r.Method == "PATCH":
_, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
client := rest.NewClient(server.Client()).SetRoot(server.URL)
f := &Fs{
ci: fs.GetConfig(context.Background()),
srv: client,
unAuth: client,
driveType: driveTypeBusiness,
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
}
f.opt.Region = "global"
f.opt.ChunkSize = fs.SizeSuffix(len(content) / 2)
o := &Object{fs: f}
src := object.NewStaticObjectInfo("remote", time.Now(), int64(len(content)), true, nil, nil)
_, err := o.uploadMultipart(context.Background(), bytes.NewReader(content), src)
require.NoError(t, err)
assert.Equal(t, content, uploaded)
assert.Equal(t, []string{"POST /root/createUploadSession", "PUT /root/content"}, requests[:2])
}
// redirectTransport sends every request to the test server, so URLs
// on other hosts can be tested.
type redirectTransport struct {
server *httptest.Server
}
func (t redirectTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = req.Clone(req.Context())
req.URL.Scheme = "http"
req.URL.Host = t.server.Listener.Addr().String()
return t.server.Client().Transport.RoundTrip(req)
}
// newDownloadTest returns an Object which downloads content from a
// server which handles the /content call with contentHandler and
// serves download.aspx with Range support, recording the
// Authorization header of each download.
func newDownloadTest(t *testing.T, content []byte, contentHandler http.HandlerFunc) (o *Object, auths *[]string) {
auths = new([]string)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/items/fake/content":
contentHandler(w, r)
case "/download.aspx":
*auths = append(*auths, r.Header.Get("Authorization"))
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(content))
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL)
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
transport := redirectTransport{server: server}
sharePointToken := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "sharepoint_token"})
sharePointHTTP := &http.Client{Transport: &oauth2.Transport{Source: sharePointToken, Base: transport}}
f := &Fs{
ci: fs.GetConfig(context.Background()),
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
unAuth: rest.NewClient(&http.Client{Transport: transport}),
driveType: driveTypeBusiness,
tokenSource: &oauthutil.TokenSource{},
sharePoint: &sharePointClients{hosts: map[string]sharePointAuth{
"contoso-my.sharepoint.com": {client: rest.NewClient(sharePointHTTP), tokenSource: sharePointToken},
}},
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
}
f.opt.Region = "global"
return &Object{fs: f, id: "fake", size: int64(len(content))}, auths
}
// TestOpenRedirectAuth checks that downloads are redirected to a
// pre-authenticated URL without authentication and to a SharePoint
// URL with a SharePoint token.
func TestOpenRedirectAuth(t *testing.T) {
content := []byte("0123456789abcdefghij")
for _, test := range []struct {
name string
location string
wantAuth string
}{
{"tempauth", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x&tempauth=v1.secret", ""},
{"sharepoint", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", "Bearer sharepoint_token"},
} {
t.Run(test.name, func(t *testing.T) {
o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, test.location, http.StatusFound)
})
in, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})
require.NoError(t, err)
got, err := io.ReadAll(in)
require.NoError(t, err)
require.NoError(t, in.Close())
assert.Equal(t, "56789", string(got))
assert.Equal(t, []string{test.wantAuth}, *auths)
})
}
// Without credentials the error explains why
o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", http.StatusFound)
})
o.fs.tokenSource = nil
_, err := o.Open(context.Background())
assert.ErrorIs(t, err, errPreauthRetired)
assert.Empty(t, *auths)
}
// TestOpenRangeCheck checks that a download which doesn't return the
// requested range is an error rather than wrong data.
func TestOpenRangeCheck(t *testing.T) {
content := []byte("0123456789abcdefghij")
for _, test := range []struct {
name string
handler http.HandlerFunc
wantErr error
}{
{"ignored", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(content)
}, fs.ErrorRangeIgnored},
{"wrong range", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Range", "bytes 0-4/20")
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(content[:5])
}, nil},
} {
t.Run(test.name, func(t *testing.T) {
o, _ := newDownloadTest(t, content, test.handler)
_, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})
require.Error(t, err)
if test.wantErr != nil {
assert.ErrorIs(t, err, test.wantErr)
}
})
}
}
-4
View File
@@ -126,10 +126,6 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
if ociError.GetCode() == "RequestTimeout" {
return true, err
}
// Parallel part uploads can collide on the same object
if ociError.GetHTTPStatusCode() == http.StatusConflict && ociError.GetCode() == "ConcurrentObjectUpdate" {
return true, err
}
}
// Ok, not an oci error, check for generic failure conditions
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
@@ -102,34 +102,3 @@ func TestGzipEncoding(t *testing.T) {
assert.Equal(t, "", md5sum)
})
}
// serviceError is a minimal common.ServiceError for shouldRetry
type serviceError struct {
status int
code string
}
func (e serviceError) Error() string { return e.code }
func (e serviceError) GetHTTPStatusCode() int { return e.status }
func (e serviceError) GetMessage() string { return e.code }
func (e serviceError) GetCode() string { return e.code }
func (e serviceError) GetOpcRequestID() string { return "" }
func TestShouldRetryServiceError(t *testing.T) {
ctx := context.Background()
for _, test := range []struct {
status int
code string
want bool
}{
{http.StatusConflict, "ConcurrentObjectUpdate", true},
{http.StatusConflict, "BucketAlreadyExists", false},
{http.StatusRequestTimeout, "RequestTimeout", true},
{http.StatusNotFound, "ObjectNotFound", false},
} {
err := serviceError{status: test.status, code: test.code}
retry, gotErr := shouldRetry(ctx, nil, err)
assert.Equal(t, test.want, retry, test.code)
assert.Equal(t, error(err), gotErr)
}
}
-2
View File
@@ -1537,8 +1537,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if err != nil {
return fmt.Errorf("failed to make multipart upload for 0 length file: %w", err)
}
// Stop the form writer reading in if the request fails before sending it all
defer func() { _ = formReader.Close() }()
contentLength := overhead + size
+14 -35
View File
@@ -184,7 +184,7 @@ func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *api
}
// getTask gets api.Task from API for the ID passed
func (f *Fs) getTask(ctx context.Context, ID string) (info *api.Task, err error) {
func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api.Task, err error) {
opts := rest.Opts{
Method: "GET",
Path: "/drive/v1/tasks/" + ID,
@@ -192,48 +192,27 @@ func (f *Fs) getTask(ctx context.Context, ID string) (info *api.Task, err error)
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
if checkPhase {
if err == nil && info.Phase != api.PhaseTypeComplete {
// could be pending right after the task is created
return true, fmt.Errorf("%s (%s) is still in %s", info.Name, info.Type, info.Phase)
}
}
return f.shouldRetry(ctx, resp, err)
})
return
}
// waitTask waits up to taskTimeout for the async task ID to complete
// waitTask waits for async tasks to be completed
func (f *Fs) waitTask(ctx context.Context, ID string) (err error) {
start := time.Now()
sleep := taskWaitTime
for {
if err := sleepCtx(ctx, sleep); err != nil {
return err
time.Sleep(taskWaitTime)
if info, err := f.getTask(ctx, ID, true); err != nil {
if info == nil {
return fmt.Errorf("can't verify the task is completed: %q", ID)
}
info, err := f.getTask(ctx, ID)
if err != nil {
return fmt.Errorf("can't verify the task %q is completed: %w", ID, err)
}
switch info.Phase {
case api.PhaseTypeComplete:
fs.Debugf(f, "task %q (%s) completed after %v", ID, info.Type, time.Since(start))
return nil
case api.PhaseTypeError:
return fmt.Errorf("task failed: %#v", info)
}
// Tasks can stay pending for tens of seconds when the server is busy
if time.Since(start) > taskTimeout {
return fmt.Errorf("can't verify the task is completed after %v: %#v", time.Since(start), info)
}
sleep = min(2*sleep, taskMaxSleep)
}
}
// sleepCtx sleeps for d or until ctx is cancelled
func sleepCtx(ctx context.Context, d time.Duration) error {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-timer.C:
return nil
case <-ctx.Done():
return ctx.Err()
return fmt.Errorf("can't verify the task is completed: %#v", info)
}
return
}
// deleteTask remove a task having the specified ID
-10
View File
@@ -72,8 +72,6 @@ const (
minSleep = 100 * time.Millisecond
maxSleep = 2 * time.Second
taskWaitTime = 500 * time.Millisecond
taskMaxSleep = 5 * time.Second
taskTimeout = 5 * time.Minute
decayConstant = 2 // bigger for slower decay, exponential
rootURL = "https://api-drive.mypikpak.com"
@@ -1390,8 +1388,6 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
if err != nil {
return fmt.Errorf("failed to make multipart upload: %w", err)
}
// Stop the form writer reading in if the request fails before sending it all
defer func() { _ = formReader.Close() }()
contentLength := overhead + size
opts := rest.Opts{
@@ -1950,12 +1946,6 @@ func (o *Object) httpResponse(ctx context.Context, url, method string, options [
}
err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.client.Do(req)
// client.Do does not fail a completed non-2xx response, so the
// HTML error page would be returned as the file. errorHandler
// closes the body; shouldRetry then retries retryErrorCodes.
if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
err = errorHandler(res)
}
return o.fs.shouldRetry(ctx, res, err)
})
if err != nil {
-185
View File
@@ -1,185 +0,0 @@
package pikpak
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// closeTracker counts Close calls on a response body.
type closeTracker struct {
io.ReadCloser
closed *atomic.Int32
}
func (b closeTracker) Close() error {
b.closed.Add(1)
return b.ReadCloser.Close()
}
type roundTripperFunc func(*http.Request) (*http.Response, error)
func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
// newDownloadTestObject returns an object whose downloads are served by srv.
// closed counts response bodies closed through the client.
func newDownloadTestObject(t *testing.T, srv *httptest.Server, closed *atomic.Int32) *Object {
t.Helper()
ctx, ci := fs.AddConfig(context.Background())
ci.LowLevelRetries = 3
client := *srv.Client()
base := client.Transport
if base == nil {
base = http.DefaultTransport
}
client.Transport = roundTripperFunc(func(req *http.Request) (*http.Response, error) {
res, err := base.RoundTrip(req)
if err == nil && res != nil && res.Body != nil {
res.Body = closeTracker{ReadCloser: res.Body, closed: closed}
}
return res, err
})
return &Object{
fs: &Fs{
client: &client,
pacer: fs.NewPacer(ctx, &pacer.ZeroDelayCalculator{}),
},
size: 1,
}
}
// TestOpenHTTPStatus checks that a non-2xx download response is an error
// (body closed, retriable statuses retried by the pacer) and that 200 and
// 206 responses are still returned as file content.
func TestOpenHTTPStatus(t *testing.T) {
const htmlBody = "<html><head><title>502 Bad Gateway</title></head><body>502</body></html>"
fileBody := []byte("pikpak-file-bytes-0123456789")
t.Run("502", func(t *testing.T) {
var closed atomic.Int32
var hits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusBadGateway)
_, _ = io.WriteString(w, htmlBody)
}))
t.Cleanup(srv.Close)
o := newDownloadTestObject(t, srv, &closed)
in, err := o.open(context.Background(), srv.URL)
if err == nil {
got, readErr := io.ReadAll(in)
require.NoError(t, readErr)
require.NoError(t, in.Close())
t.Fatalf("open returned the error page as file data (%d bytes): %q", len(got), got)
}
assert.Nil(t, in)
assert.ErrorContains(t, err, "502")
assert.NotContains(t, err.Error(), htmlBody)
assert.Equal(t, int32(3), hits.Load(), "502 should be retried by the pacer")
assert.Equal(t, hits.Load(), closed.Load(), "error response bodies should be closed")
})
t.Run("404", func(t *testing.T) {
var closed atomic.Int32
var hits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
http.Error(w, "missing", http.StatusNotFound)
}))
t.Cleanup(srv.Close)
o := newDownloadTestObject(t, srv, &closed)
in, err := o.open(context.Background(), srv.URL)
require.Error(t, err)
assert.Nil(t, in)
assert.Equal(t, int32(1), hits.Load(), "404 is not a retry status")
assert.Equal(t, hits.Load(), closed.Load())
})
t.Run("502 then 200", func(t *testing.T) {
var closed atomic.Int32
var hits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if hits.Add(1) == 1 {
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusBadGateway)
_, _ = io.WriteString(w, htmlBody)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(fileBody)
}))
t.Cleanup(srv.Close)
o := newDownloadTestObject(t, srv, &closed)
in, err := o.open(context.Background(), srv.URL)
require.NoError(t, err)
got, err := io.ReadAll(in)
require.NoError(t, err)
require.NoError(t, in.Close())
assert.Equal(t, fileBody, got)
assert.Equal(t, int32(2), hits.Load())
assert.Equal(t, int32(2), closed.Load())
})
t.Run("200", func(t *testing.T) {
var closed atomic.Int32
var hits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
if r.Header.Get("Range") != "" {
t.Errorf("unexpected Range header %q", r.Header.Get("Range"))
}
w.Header().Set("Content-Type", "application/octet-stream")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(fileBody)
}))
t.Cleanup(srv.Close)
o := newDownloadTestObject(t, srv, &closed)
o.size = int64(len(fileBody))
in, err := o.open(context.Background(), srv.URL)
require.NoError(t, err)
got, err := io.ReadAll(in)
require.NoError(t, err)
assert.Equal(t, int32(0), closed.Load(), "successful body stays open for the caller")
require.NoError(t, in.Close())
assert.Equal(t, fileBody, got)
assert.Equal(t, int32(1), hits.Load())
assert.Equal(t, int32(1), closed.Load())
})
t.Run("206", func(t *testing.T) {
var closed atomic.Int32
var hits atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
assert.Equal(t, "bytes=0-3", r.Header.Get("Range"))
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Range", "bytes 0-3/"+strconv.Itoa(len(fileBody)))
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write(fileBody[:4])
}))
t.Cleanup(srv.Close)
o := newDownloadTestObject(t, srv, &closed)
o.size = int64(len(fileBody))
in, err := o.open(context.Background(), srv.URL, &fs.RangeOption{Start: 0, End: 3})
require.NoError(t, err)
got, err := io.ReadAll(in)
require.NoError(t, err)
require.NoError(t, in.Close())
assert.Equal(t, fileBody[:4], got)
assert.Equal(t, int32(1), hits.Load())
assert.Equal(t, int32(1), closed.Load())
})
}
-10
View File
@@ -1,10 +0,0 @@
name: Autonomys
description: Autonomys Auto Drive
endpoint:
s3.auto-drive.autonomys.xyz: Auto Drive mainnet S3 endpoint
quirks:
force_path_style: true
# UploadPartCopy is not supported, so copy in a single request
copy_cutoff: 9223372036854775807
# Object Lock is always on in COMPLIANCE mode and can't be set by clients
object_lock_supported: false
-15
View File
@@ -1,15 +0,0 @@
name: Cellar
description: Clever Cloud Cellar Object Storage
endpoint:
cellar-c2.services.clever-cloud.com: Paris, France
acl: {}
bucket_acl: true
sse_customer_algorithm: true
sse_customer_key: true
sse_customer_key_base64: true
sse_customer_key_md5: true
quirks:
# Object Lock headers are not applied on CreateMultipartUpload, so the
# Object Lock tests are skipped. Users should set
# --s3-object-lock-set-after-upload when using Object Lock.
object_lock_supported: false
+12 -48
View File
@@ -1,73 +1,37 @@
name: HuaweiOBS
description: Huawei Object Storage Service
region:
af-north-1: AF-Cairo
af-south-1: AF-Johannesburg
ap-southeast-1: CN-Hong Kong
ap-southeast-2: AP-Bangkok
ap-southeast-3: AP-Singapore
ap-southeast-4: AP-Jakarta
ap-southeast-5: AP-Manila
cn-east-2: CN East-Shanghai2
cn-east-3: CN East-Shanghai1
cn-east-4: CN East-Qingdao
cn-east-5: CN East-Suzhou
cn-east-2: CN East-Shanghai2
cn-north-1: CN North-Beijing1
cn-north-2: CN North-Beijing2
cn-north-4: CN North-Beijing4
cn-north-9: CN North-Ulanqab
cn-north-11: CN North-Hebei
cn-north-12: CN North-Hohhot
cn-south-1: CN South-Guangzhou
cn-south-2: CN South-Shenzhen
cn-south-4: CN South-Guangzhou-InvitationOnly
cn-southwest-2: CN Southwest-Guiyang1
cn-southwest-3: CN Southwest-Chongqing
eu-west-101: EU-Dublin
la-north-2: LA-Mexico City2
la-south-2: LA-Santiago
me-east-1: ME-Riyadh
na-mexico-1: LA-Mexico City1
ru-northwest-2: RU-Moscow2
ap-southeast-1: CN-Hong Kong
sa-argentina-1: LA-Buenos Aires1
sa-brazil-1: LA-Sao Paulo1
sa-chile-1: LA-Santiago2
sa-peru-1: LA-Lima1
tr-west-1: TR-Istanbul
na-mexico-1: LA-Mexico City1
sa-chile-1: LA-Santiago2
sa-brazil-1: LA-Sao Paulo1
ru-northwest-2: RU-Moscow2
endpoint:
obs.af-north-1.myhuaweicloud.com: AF-Cairo
obs.af-south-1.myhuaweicloud.com: AF-Johannesburg
obs.ap-southeast-1.myhuaweicloud.com: CN-Hong Kong
obs.ap-southeast-2.myhuaweicloud.com: AP-Bangkok
obs.ap-southeast-3.myhuaweicloud.com: AP-Singapore
obs.ap-southeast-4.myhuaweicloud.com: AP-Jakarta
obs.ap-southeast-5.myhuaweicloud.com: AP-Manila
obs.cn-east-2.myhuaweicloud.com: CN East-Shanghai2
obs.cn-east-3.myhuaweicloud.com: CN East-Shanghai1
obs.cn-east-4.myhuaweicloud.com: CN East-Qingdao
obs.cn-east-5.myhuaweicloud.com: CN East-Suzhou
obs.cn-east-2.myhuaweicloud.com: CN East-Shanghai2
obs.cn-north-1.myhuaweicloud.com: CN North-Beijing1
obs.cn-north-2.myhuaweicloud.com: CN North-Beijing2
obs.cn-north-4.myhuaweicloud.com: CN North-Beijing4
obs.cn-north-9.myhuaweicloud.com: CN North-Ulanqab
obs.cn-north-11.myhuaweicloud.com: CN North-Hebei
obs.cn-north-12.myhuaweicloud.com: CN North-Hohhot
obs.cn-south-1.myhuaweicloud.com: CN South-Guangzhou
obs.cn-south-2.myhuaweicloud.com: CN South-Shenzhen
obs.cn-south-4.myhuaweicloud.com: CN South-Guangzhou-InvitationOnly
obs.cn-southwest-2.myhuaweicloud.com: CN Southwest-Guiyang1
obs.cn-southwest-3.myhuaweicloud.com: CN Southwest-Chongqing
obs.eu-west-101.myhuaweicloud.eu: EU-Dublin
obs.la-north-2.myhuaweicloud.com: LA-Mexico City2
obs.la-south-2.myhuaweicloud.com: LA-Santiago
obs.me-east-1.myhuaweicloud.com: ME-Riyadh
obs.na-mexico-1.myhuaweicloud.com: LA-Mexico City1
obs.ru-northwest-2.myhuaweicloud.com: RU-Moscow2
obs.ap-southeast-1.myhuaweicloud.com: CN-Hong Kong
obs.sa-argentina-1.myhuaweicloud.com: LA-Buenos Aires1
obs.sa-brazil-1.myhuaweicloud.com: LA-Sao Paulo1
obs.sa-chile-1.myhuaweicloud.com: LA-Santiago2
obs.sa-peru-1.myhuaweicloud.com: LA-Lima1
obs.tr-west-1.myhuaweicloud.com: TR-Istanbul
obs.na-mexico-1.myhuaweicloud.com: LA-Mexico City1
obs.sa-chile-1.myhuaweicloud.com: LA-Santiago2
obs.sa-brazil-1.myhuaweicloud.com: LA-Sao Paulo1
obs.ru-northwest-2.myhuaweicloud.com: RU-Moscow2
acl: {}
bucket_acl: true
quirks:
-18
View File
@@ -1,18 +0,0 @@
name: Infomaniak
description: Infomaniak Swiss Backup
region:
us-east-1: |-
Default region for compatibility.
All data is stored in Switzerland.
endpoint:
s3.swiss-backup01.infomaniak.com: Swiss Backup Location 01
s3.swiss-backup02.infomaniak.com: Swiss Backup Location 02
s3.swiss-backup03.infomaniak.com: Swiss Backup Location 03
s3.swiss-backup04.infomaniak.com: Swiss Backup Location 04
location_constraint: {}
acl: {}
bucket_acl: true
quirks:
force_path_style: true
use_multipart_etag: false
-11
View File
@@ -1,11 +0,0 @@
name: Neon
description: Neon Object Storage
region:
us-east-1: US East (N. Virginia)
us-east-2: US East (Ohio)
eu-central-1: Europe (Frankfurt)
ap-southeast-1: Asia Pacific (Singapore)
endpoint: {}
quirks:
force_path_style: true # virtual-hosted style requests are not supported
object_lock_supported: false # object lock config is stored but enforcement is limited
+6 -18
View File
@@ -190,7 +190,7 @@ header is added and the default (private) will be used.
Name: "sse_customer_key",
Help: `To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
Alternatively you can provide --s3-sse-customer-key-base64.`,
Alternatively you can provide --sse-customer-key-base64.`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "",
@@ -201,7 +201,7 @@ Alternatively you can provide --s3-sse-customer-key-base64.`,
Name: "sse_customer_key_base64",
Help: `If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
Alternatively you can provide --s3-sse-customer-key.`,
Alternatively you can provide --sse-customer-key.`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "",
@@ -1446,15 +1446,10 @@ func fixupRequest(o *s3.Options, opt *Options) {
)
o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {
// Presigning replaces the "Signing" middleware before APIOptions run
signing := "Signing"
if _, ok := stack.Finalize.Get(signing); !ok {
signing = "PresignHTTPRequest"
}
if err := stack.Finalize.Insert(fixup, signing, middleware.Before); err != nil {
if err := stack.Finalize.Insert(fixup, "Signing", middleware.Before); err != nil {
return err
}
if err := stack.Finalize.Insert(restore, signing, middleware.After); err != nil {
if err := stack.Finalize.Insert(restore, "Signing", middleware.After); err != nil {
return err
}
return nil
@@ -1910,12 +1905,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if opt.Provider == "Rabata" {
f.features.Copy = nil
}
if opt.Provider == "Autonomys" {
// Versions are permanent, so Purge's version deletes are always refused.
f.features.Purge = nil
// There are no bucket endpoints: CreateBucket would be stored as an object.
f.opt.NoCheckBucket = true
}
if opt.Provider == "TencentCOS" && strings.Contains(opt.Endpoint, "cos.accelerate.myqcloud.com") {
// Global Acceleration endpoint does not support bucket creation.
f.opt.NoCheckBucket = true
@@ -2983,9 +2972,8 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
bucket, directory := f.split(dir)
// Remove directory marker file, including the one of the root
// when it is a directory in the bucket
if f.opt.DirectoryMarkers && bucket != "" && directory != "" {
// Remove directory marker file
if f.opt.DirectoryMarkers && bucket != "" && dir != "" {
o := &Object{
fs: f,
remote: dir + "/",
-27
View File
@@ -12,7 +12,6 @@ import (
"net/http"
"net/http/httptest"
"path"
"slices"
"strings"
"testing"
"time"
@@ -892,31 +891,6 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
})
}
// InternalTestRmdirRoot checks the directory marker of the root of a
// remote is removed by Rmdir.
func (f *Fs) InternalTestRmdirRoot(t *testing.T) {
if !f.opt.DirectoryMarkers {
t.Skip("needs --s3-directory-markers")
}
ctx := context.Background()
dir := "rmdir-root-" + random.String(8)
hasDir := func() bool {
entries, err := f.List(ctx, "")
require.NoError(t, err)
return slices.ContainsFunc(entries, func(entry fs.DirEntry) bool {
_, isDir := entry.(fs.Directory)
return isDir && entry.Remote() == dir
})
}
require.NoError(t, f.Mkdir(ctx, dir))
require.True(t, hasDir(), "directory marker not made")
fDir, err := cache.Get(ctx, bucket.Join(fs.ConfigStringFull(f), dir))
require.NoError(t, err)
require.NoError(t, fDir.Rmdir(ctx, ""))
assert.False(t, hasDir(), "directory marker not removed")
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("Metadata", f.InternalTestMetadata)
t.Run("NoHead", f.InternalTestNoHead)
@@ -924,7 +898,6 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("HasChildren", f.InternalTestHasChildren)
t.Run("Versions", f.InternalTestVersions)
t.Run("ObjectLock", f.InternalTestObjectLock)
t.Run("RmdirRoot", f.InternalTestRmdirRoot)
}
var _ fstests.InternalTester = (*Fs)(nil)
-28
View File
@@ -225,31 +225,3 @@ func TestPublicLink(t *testing.T) {
_, err = f.PublicLink(context.Background(), "missing", fs.DurationOff, false)
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}
func TestPublicLinkRequestFixups(t *testing.T) {
for _, tt := range []struct {
name string
useXID bool
signAcceptEncoding bool
}{
{"no x-id", false, true},
{"unsigned Accept-Encoding", true, false},
{"no x-id and unsigned Accept-Encoding", false, false},
} {
t.Run(tt.name, func(t *testing.T) {
f := newLinkTestFs(t)
f.opt.UseXID = fs.Tristate{Value: tt.useXID, Valid: true}
f.opt.SignAcceptEncoding = fs.Tristate{Value: tt.signAcceptEncoding, Valid: true}
c, _, err := s3Connection(f.ctx, &f.opt, getClient(f.ctx, &f.opt))
require.NoError(t, err)
f.c = c
link, err := f.PublicLink(context.Background(), "file", fs.Duration(time.Hour), false)
require.NoError(t, err)
u, err := url.Parse(link)
require.NoError(t, err)
assert.Equal(t, tt.useXID, u.Query().Has("x-id"))
assert.Equal(t, "host", u.Query().Get("X-Amz-SignedHeaders"))
assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
})
}
}
+4 -21
View File
@@ -30,7 +30,6 @@ import (
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/rest"
"golang.org/x/text/unicode/norm"
)
const (
@@ -206,7 +205,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt: *opt,
endpoint: u,
endpointURL: u.String(),
srv: rest.NewClient(newClient(ctx)).SetRoot(u.String()),
srv: rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
pacer: getPacer(ctx, opt.URL),
}
f.features = (&fs.Features{
@@ -370,10 +369,10 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
// Create rest client for getAuthorizationToken
url := u.String()
if !strings.HasSuffix(url, "/") {
if !strings.HasPrefix(url, "/") {
url += "/"
}
srv := rest.NewClient(newClient(ctx)).SetRoot(url)
srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(url)
// We loop asking for a 2FA code
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -401,17 +400,6 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
return nil, fmt.Errorf("unknown state %q", config.State)
}
// newClient returns the HTTP client used to talk to the Seafile API.
//
// It refuses redirects which downgrade HTTPS to plaintext HTTP as
// following one would send the authorization token, or the password
// during configuration, in cleartext.
func newClient(ctx context.Context) *http.Client {
client := fshttp.NewClient(ctx)
client.CheckRedirect = rest.RefuseHTTPSDowngradeRedirectFn
return client
}
// Shutdown the Fs
func (f *Fs) Shutdown(ctx context.Context) error {
if f.renew == nil {
@@ -1181,11 +1169,7 @@ func (f *Fs) mkLibrary(ctx context.Context, libraryName, password string) error
return nil
}
// splitPath returns the library name and the full path inside the library.
//
// The path is returned in NFC: Seafile stores names in NFC whatever form it
// is given but looks them up byte for byte. Library names are stored verbatim
// so they are left untouched.
// splitPath returns the library name and the full path inside the library
func (f *Fs) splitPath(dir string) (library, folder string) {
library = f.libraryName
folder = dir
@@ -1196,7 +1180,6 @@ func (f *Fs) splitPath(dir string) (library, folder string) {
// Adds the root folder to the path to get a full path
folder = path.Join(f.rootDirectory, folder)
}
folder = norm.NFC.String(folder)
return
}
-168
View File
@@ -2,25 +2,14 @@ package seafile
import (
"context"
"io"
"net/http"
"net/http/httptest"
"path"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/asyncreader"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/unicode/norm"
)
type pathData struct {
@@ -112,28 +101,6 @@ func TestSplitPath(t *testing.T) {
expectedLibrary: "Library",
expectedPath: path.Join("root", "path", "subpath", "to", "file"),
},
// paths inside a library are returned in NFC, library names are kept as they are
{
configLibrary: "Library",
configRoot: "",
argumentPath: norm.NFD.String("path/to/équivalent.txt"),
expectedLibrary: "Library",
expectedPath: norm.NFC.String("path/to/équivalent.txt"),
},
{
configLibrary: "Library",
configRoot: norm.NFD.String("répertoire"),
argumentPath: norm.NFD.String("équivalent.txt"),
expectedLibrary: "Library",
expectedPath: norm.NFC.String("répertoire/équivalent.txt"),
},
{
configLibrary: "",
configRoot: "",
argumentPath: norm.NFD.String("Bibliothèque/équivalent.txt"),
expectedLibrary: norm.NFD.String("Bibliothèque"),
expectedPath: norm.NFC.String("équivalent.txt"),
},
}
for _, test := range testData {
fs := &Fs{
@@ -259,138 +226,3 @@ func Test2FAStateMachine(t *testing.T) {
})
}
}
// Test that a failed upload stops reading the source so it can be closed
func TestUploadErrorReleasesSource(t *testing.T) {
ctx := context.Background()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Reply without reading the body, like a server rejecting the upload early
http.Error(w, "rejected", http.StatusBadRequest)
}))
defer server.Close()
f := &Fs{
srv: rest.NewClient(server.Client()),
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
}
// An AsyncReader holds its lock while writing to the upload
in, err := asyncreader.New(ctx, io.NopCloser(readers.NewPatternReader(64<<20)), 4)
require.NoError(t, err)
_, err = f.upload(ctx, in, server.URL+"/upload", "/file.bin")
require.Error(t, err)
closed := make(chan struct{})
go func() {
_ = in.Close()
close(closed)
}()
select {
case <-closed:
case <-time.After(10 * time.Second):
t.Fatal("source still in use after upload returned")
}
}
// testPassword is the password the downgrade tests look for on the
// plaintext hop. It is distinctive so that it can't match the JSON
// field names of the authentication request.
const testPassword = "s3cret-pass-4242"
// Test2FAGetToken checks the 2FA config flow posts the credentials to
// the auth token endpoint without doubling the slash when the
// configured URL ends in one, and saves the token it gets back.
func Test2FAGetToken(t *testing.T) {
var gotPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"token":"token123"}`))
}))
defer srv.Close()
m := configmap.Simple{"url": srv.URL + "/", "2fa": "true", "user": "username", "pass": obscure.MustObscure(testPassword)}
out, err := Config(context.Background(), "test", m, fs.ConfigIn{State: "2fa_do", Result: "123456"})
require.NoError(t, err)
assert.Nil(t, out)
assert.Equal(t, "/api2/auth-token/", gotPath)
assert.Equal(t, "token123", m["auth_token"])
assert.Equal(t, "", m["pass"])
}
// newDowngradeServers returns an HTTPS server which answers the server
// info request itself and redirects every other request to a plaintext
// HTTP server on the same machine. The returned flags record whether
// the plaintext server ever received an Authorization header or
// testPassword in a request body.
func newDowngradeServers(t *testing.T) (tlsSrv *httptest.Server, sawAuth, sawPassword *atomic.Bool) {
t.Helper()
sawAuth, sawPassword = new(atomic.Bool), new(atomic.Bool)
plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "" {
sawAuth.Store(true)
}
body, _ := io.ReadAll(r.Body)
if strings.Contains(string(body), testPassword) {
sawPassword.Store(true)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte("[]"))
}))
t.Cleanup(plain.Close)
tlsSrv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api2/server-info/" {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"version":"9.0.0"}`))
return
}
http.Redirect(w, r, plain.URL+r.URL.Path, http.StatusTemporaryRedirect)
}))
t.Cleanup(tlsSrv.Close)
return tlsSrv, sawAuth, sawPassword
}
// TestRefuseHTTPSDowngradeRedirect checks that neither the
// authorization token nor the password is sent over plaintext HTTP
// when the server redirects an HTTPS request to an http:// URL.
func TestRefuseHTTPSDowngradeRedirect(t *testing.T) {
ctx, ci := fs.AddConfig(context.Background())
ci.InsecureSkipVerify = true // trust the httptest certificate
// Every API call made through the Fs carries the token.
t.Run("Token", func(t *testing.T) {
tlsSrv, sawAuth, _ := newDowngradeServers(t)
m := configmap.Simple{"url": tlsSrv.URL, "user": "username", "auth_token": "secret"}
fsys, err := NewFs(ctx, "test", "", m)
require.NoError(t, err)
f := fsys.(*Fs)
_, err = f.getLibraries(ctx)
assert.False(t, sawAuth.Load(), "authorization token was sent over plaintext HTTP")
assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
})
// Without a token NewFs posts the user's password to get one.
t.Run("Login", func(t *testing.T) {
tlsSrv, _, sawPassword := newDowngradeServers(t)
m := configmap.Simple{"url": tlsSrv.URL, "user": "username", "pass": obscure.MustObscure(testPassword)}
_, err := NewFs(ctx, "test", "", m)
assert.False(t, sawPassword.Load(), "password was sent over plaintext HTTP")
assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
})
// The 2FA config flow posts the user's password to get a token.
t.Run("Password", func(t *testing.T) {
tlsSrv, _, sawPassword := newDowngradeServers(t)
m := configmap.Simple{"url": tlsSrv.URL, "2fa": "true", "user": "username", "pass": obscure.MustObscure(testPassword)}
out, err := Config(ctx, "test", m, fs.ConfigIn{State: "2fa_do", Result: "123456"})
require.NoError(t, err)
require.NotNil(t, out)
assert.False(t, sawPassword.Load(), "password was sent over plaintext HTTP")
assert.Equal(t, "2fa_error", out.State)
require.NotNil(t, out.Option)
assert.Contains(t, out.Option.Help, rest.ErrHTTPSDowngrade.Error())
})
}
-2
View File
@@ -693,8 +693,6 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
if err != nil {
return nil, fmt.Errorf("failed to make multipart upload: %w", err)
}
// Stop the form writer reading in if the request fails before sending it all
defer func() { _ = formReader.Close() }()
opts := rest.Opts{
Method: "POST",
+4 -14
View File
@@ -250,7 +250,7 @@ E.g. the second example above should be rewritten as:
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: encoder.Base,
Default: encoder.Display,
}, {
Name: "set_modtime",
Default: true,
@@ -1883,7 +1883,8 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
// mkParentDir makes the parent of remote if necessary and any
// directories above that
func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
return f.mkdir(ctx, path.Dir(f.remotePath(remote)))
parent := path.Dir(remote)
return f.mkdir(ctx, f.remotePath(parent))
}
// mkdir makes the directory and parents using native paths
@@ -2458,14 +2459,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
shellPath := o.shellPath()
// Some restricted shells, such as rsync.net's, end the command at a
// newline even when it is quoted, which would hash a different file.
if o.fs.shellType == "unix" && strings.Contains(shellPath, "\n") {
fs.Debugf(o, "Can't calculate %v hash of a name containing a newline", r)
return "", nil
}
shellPathArg, err := o.fs.quoteOrEscapeShellPath(shellPath)
shellPathArg, err := o.fs.quoteOrEscapeShellPath(o.shellPath())
if err != nil {
return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
}
@@ -2553,10 +2547,6 @@ func (f *Fs) remoteShellPath(remote string) string {
if f.opt.PathOverride[0] == '@' {
shellPath = path.Join(strings.TrimPrefix(f.opt.PathOverride, "@"), f.absRoot, encodedRemote)
}
// A relative path starting with "-" would be parsed as an option by the hash command
if strings.HasPrefix(shellPath, "-") {
shellPath = "./" + shellPath
}
fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath)
return shellPath
}
-26
View File
@@ -18,7 +18,6 @@ import (
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fstest/fstests"
@@ -147,31 +146,6 @@ func TestRemoteShellPathEncodesPathOverrideNames(t *testing.T) {
assert.Equal(t, "/volume/srv/root/dir/file\uFF1Aname", f.remoteShellPath("dir/file:name"))
}
func TestRemoteShellPathRelativePathOverrideLeadingDash(t *testing.T) {
f := &Fs{
absRoot: "/",
opt: Options{
Enc: encoder.Display,
PathOverride: "@.",
},
}
assert.Equal(t, "./-file", f.remoteShellPath("-file"))
assert.Equal(t, "dir/-file", f.remoteShellPath("dir/-file"))
assert.Equal(t, "file", f.remoteShellPath("file"))
f.opt.PathOverride = "."
assert.Equal(t, "./-file", f.remoteShellPath("-file"))
}
func TestDefaultEncodingStandardizesControlCharacters(t *testing.T) {
ri, err := fs.Find("sftp")
require.NoError(t, err)
enc := ri.Options.Get(config.ConfigEncoding).Default.(encoder.MultiEncoder)
assert.Equal(t, "a␊b", enc.ToStandardName("a\nb"))
assert.Equal(t, "a\nb", enc.FromStandardName("a␊b"))
}
func TestParseHash(t *testing.T) {
for i, test := range []struct {
sshOutput, checksum string
+3 -7
View File
@@ -11,7 +11,6 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/rclone/rclone/fs"
@@ -81,9 +80,8 @@ type sshSessionExternal struct {
cancel func()
startCalled bool
runningSFTP bool
waitOnce sync.Once // ensure Wait() is only called once
waitErr error // result of the Wait() call
hasExited atomic.Bool // set once Wait() has returned
waitOnce sync.Once // ensure Wait() is only called once
waitErr error // result of the Wait() call
}
func (f *Fs) newSSHSessionExternal() *sshSessionExternal {
@@ -178,8 +176,7 @@ func (s *sshSessionExternal) StdoutPipe() (io.Reader, error) {
// Return whether the command has finished or not
func (s *sshSessionExternal) exited() bool {
// Don't read s.cmd.ProcessState as Wait() writes it from another goroutine
return s.hasExited.Load()
return s.cmd.ProcessState != nil
}
// Wait for the command to exit
@@ -188,7 +185,6 @@ func (s *sshSessionExternal) Wait() error {
// This is safe even if Wait() is called from multiple goroutines.
s.waitOnce.Do(func() {
s.waitErr = s.cmd.Wait()
s.hasExited.Store(true)
if s.waitErr == nil {
fs.Debugf(s.f, "ssh external: command exited OK")
} else {
-18
View File
@@ -82,21 +82,3 @@ func TestSSHExternalCloseMultipleCalls(t *testing.T) {
// Verify the process has exited
assert.True(t, session.exited())
}
// TestSSHExternalExitedWhileWaiting verifies that exited() can be called
// while another goroutine is waiting for the process. Run with -race.
func TestSSHExternalExitedWhileWaiting(t *testing.T) {
f := &Fs{
opt: Options{
SSH: fs.SpaceSepList{"echo", "test"},
},
}
session := f.newSSHSessionExternal()
assert.NoError(t, session.Start("exit 0"))
go func() {
_ = session.Wait()
}()
assert.Eventually(t, session.exited, 10*time.Second, time.Millisecond)
}
+1 -99
View File
@@ -9,7 +9,6 @@ import (
"fmt"
"io"
"maps"
"net/http"
"path"
"regexp"
"slices"
@@ -34,7 +33,6 @@ import (
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
)
// Constants
@@ -195,18 +193,6 @@ func init() {
}, {
Value: "https://authenticate.ain.net",
Help: "Blomp Cloud Storage",
}, {
Value: "https://swiss-backup01.infomaniak.com/identity/v3",
Help: "Infomaniak Swiss Backup Location 01",
}, {
Value: "https://swiss-backup02.infomaniak.com/identity/v3",
Help: "Infomaniak Swiss Backup Location 02",
}, {
Value: "https://swiss-backup03.infomaniak.com/identity/v3",
Help: "Infomaniak Swiss Backup Location 03",
}, {
Value: "https://swiss-backup04.infomaniak.com/identity/v3",
Help: "Infomaniak Swiss Backup Location 04",
}},
}, {
Name: "user_id",
@@ -484,90 +470,6 @@ func (o *Object) split() (container, containerPath string) {
return o.fs.split(o.remote)
}
// authHeaders are the request headers which carry the Swift
// credentials: the token sent on every storage request and the user
// name and API key sent on a v1 auth request. They must not be
// forwarded when a redirect leaves the host of the original request.
var authHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}
// errRedirectResendsBody is returned when following a redirect which
// leaves the host of the original request would resend that request's
// body, such as the password in a v2 or v3 auth request, to the new
// host.
var errRedirectResendsBody = errors.New("refusing to follow redirect which leaves the original host: would resend the request body")
// transport is the http.RoundTripper for the swift connection.
//
// The swift library makes its own http.Client with the default
// redirect policy and does not let the caller replace it, so the
// redirect policy is applied here at the transport level instead. A
// redirect which downgrades HTTPS to plaintext HTTP is refused with
// rest.ErrHTTPSDowngrade. Once the redirect chain has left the host
// and port of the original request the auth headers are removed from
// every further hop, and a hop which would resend the request body is
// refused with errRedirectResendsBody.
type transport struct {
*fshttp.Transport
}
// newTransport returns the transport for the swift connection
func newTransport(ctx context.Context) *transport {
return &transport{Transport: fshttp.NewTransport(ctx)}
}
// RoundTrip implements http.RoundTripper
func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
// net/http fills in req.Response on the requests it makes to
// follow a redirect, so a request without one is not a redirect.
if req.Response != nil {
origin, leftHost := redirectOrigin(req)
if origin.URL.Scheme == "https" && req.URL.Scheme == "http" {
if req.Body != nil {
_ = req.Body.Close()
}
return nil, fmt.Errorf("redirect to %s: %w", req.URL.Redacted(), rest.ErrHTTPSDowngrade)
}
if leftHost {
// net/http resends the body on a 307 or 308 if it
// can be rewound, which for an auth request means
// the password, so those redirects stop here.
if req.Body != nil && req.Body != http.NoBody {
_ = req.Body.Close()
return nil, fmt.Errorf("redirect to %s leaves %s: %w", req.URL.Redacted(), origin.URL.Redacted(), errRedirectResendsBody)
}
for _, header := range authHeaders {
if req.Header.Get(header) == "" {
continue
}
fs.Debugf(nil, "redirect to %s leaves %s: not sending %s", req.URL.Redacted(), origin.URL.Redacted(), header)
req.Header.Del(header)
}
}
}
return t.Transport.RoundTrip(req)
}
// redirectOrigin walks back along the chain of redirects which led to
// req and returns the original request and whether any hop, including
// req, was to a different host or port from it.
//
// net/http copies the headers afresh from the original request for
// every hop, so once the chain has visited another host the headers
// must be stripped from every subsequent hop, even one back to the
// original host, as the other host chose the URL.
func redirectOrigin(req *http.Request) (origin *http.Request, leftHost bool) {
origin = req
for origin.Response != nil && origin.Response.Request != nil {
origin = origin.Response.Request
}
for hop := req; hop != origin; hop = hop.Response.Request {
if !rest.SameHost(hop.URL, origin.URL) {
leftHost = true
}
}
return origin, leftHost
}
// swiftConnection makes a connection to swift
func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Connection, error) {
ci := fs.GetConfig(ctx)
@@ -591,7 +493,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
EndpointType: swift.EndpointType(opt.EndpointType),
ConnectTimeout: time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transport
Timeout: time.Duration(10 * ci.Timeout), // Use the timeouts in the transport
Transport: newTransport(ctx),
Transport: fshttp.NewTransport(ctx),
FetchUntilEmptyPage: opt.FetchUntilEmptyPage,
PartialPageFetchThreshold: opt.PartialPageFetchThreshold,
}
-208
View File
@@ -2,22 +2,12 @@ package swift
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/ncw/swift/v2"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestInternalUrlEncode(t *testing.T) {
@@ -76,201 +66,3 @@ func TestInternalShouldRetryHeaders(t *testing.T) {
assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
}
// testToken is the auth token the redirect tests look for. It is
// distinctive so that it can't match anything else in a request.
const testToken = "AUTH_tk_secret_4242"
// authTestHeaders is a deliberately literal copy of authHeaders so
// that a header dropped from the production list shows up as a leak
// in the cross host tests rather than silently losing coverage.
var authTestHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}
// listHandler answers any request with an empty JSON listing.
func listHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte("[]"))
}
// newTestFs makes an Fs via NewFs for a container on storageURL,
// using a pre-issued token so that no auth request is made.
func newTestFs(ctx context.Context, t *testing.T, storageURL string) *Fs {
m := configmap.Simple{
"storage_url": storageURL + "/v1/AUTH_test",
"auth_token": testToken,
"chunk_size": "5Gi",
}
f, err := NewFs(ctx, "swift-test", "container", m)
require.NoError(t, err)
return f.(*Fs)
}
// assertAuthHeaders checks that every auth header is absent from r
// (!want) or that the ones named in present are there (want).
func assertAuthHeaders(t *testing.T, r *http.Request, want bool, present ...string) {
if want {
for _, header := range present {
assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)
}
return
}
for _, header := range authTestHeaders {
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
}
}
// TestRedirectStripsAuthTokenOnCrossHost checks the token is sent to
// the storage host but not to another host it redirects to, nor to
// the storage host again once the chain has been elsewhere.
func TestRedirectStripsAuthTokenOnCrossHost(t *testing.T) {
var originHits, otherHits, backHits atomic.Int32
var origin *httptest.Server
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
otherHits.Add(1)
assertAuthHeaders(t, r, false)
http.Redirect(w, r, origin.URL+"/back", http.StatusFound)
}))
defer other.Close()
origin = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/back" {
backHits.Add(1)
assertAuthHeaders(t, r, false)
listHandler(w, r)
return
}
originHits.Add(1)
assertAuthHeaders(t, r, true, "X-Auth-Token")
http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)
}))
defer origin.Close()
ctx := context.Background()
f := newTestFs(ctx, t, origin.URL)
_, err := f.List(ctx, "")
require.NoError(t, err)
assert.Equal(t, int32(1), originHits.Load())
assert.Equal(t, int32(1), otherHits.Load())
assert.Equal(t, int32(1), backHits.Load())
}
// TestRedirectKeepsAuthTokenOnSameHost checks the token is still sent
// after a redirect which stays on the storage host.
func TestRedirectKeepsAuthTokenOnSameHost(t *testing.T) {
var hits atomic.Int32
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
assertAuthHeaders(t, r, true, "X-Auth-Token")
if !strings.HasSuffix(r.URL.Path, "/moved") {
http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)
return
}
listHandler(w, r)
}))
defer origin.Close()
ctx := context.Background()
f := newTestFs(ctx, t, origin.URL)
_, err := f.List(ctx, "")
require.NoError(t, err)
assert.Equal(t, int32(2), hits.Load())
}
// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to
// plaintext HTTP is not followed, so the token never goes over the
// network in the clear.
func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
var targetHits atomic.Int32
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
targetHits.Add(1)
listHandler(w, r)
}))
defer target.Close()
origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)
}))
defer origin.Close()
// The transport is a process wide singleton so it has to be
// rebuilt to trust the httptest certificate, and rebuilt again
// afterwards so later tests don't inherit that.
fshttp.ResetTransport()
t.Cleanup(fshttp.ResetTransport)
ctx, ci := fs.AddConfig(context.Background())
ci.InsecureSkipVerify = true
f := newTestFs(ctx, t, origin.URL)
_, err := f.List(ctx, "")
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
assert.Equal(t, int32(0), targetHits.Load())
}
// TestAuthRedirectStripsCredentialsOnCrossHost checks the user name
// and API key sent on a v1 auth request go to the auth host but not
// to another host it redirects to.
func TestAuthRedirectStripsCredentialsOnCrossHost(t *testing.T) {
var originHits, otherHits atomic.Int32
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
otherHits.Add(1)
assertAuthHeaders(t, r, false)
w.Header().Set("X-Storage-Url", "http://"+r.Host+"/v1/AUTH_test")
w.Header().Set("X-Auth-Token", testToken)
}))
defer other.Close()
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
originHits.Add(1)
assertAuthHeaders(t, r, true, "X-Auth-Key", "X-Auth-User")
http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)
}))
defer origin.Close()
m := configmap.Simple{
"auth": origin.URL + "/v1.0",
"auth_version": "1",
"user": "test-user",
"key": "test-key",
"chunk_size": "5Gi",
}
_, err := NewFs(context.Background(), "swift-test", "container", m)
require.NoError(t, err)
assert.Equal(t, int32(1), originHits.Load())
assert.Equal(t, int32(1), otherHits.Load())
}
// TestAuthRedirectRefusesToResendPassword checks a v2 or v3 auth
// request, which carries the password in its body, is not resent to
// another host the auth server redirects it to.
func TestAuthRedirectRefusesToResendPassword(t *testing.T) {
for _, authVersion := range []string{"2", "3"} {
t.Run("v"+authVersion, func(t *testing.T) {
var originHits, otherHits atomic.Int32
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
otherHits.Add(1)
body, _ := io.ReadAll(r.Body)
assert.NotContains(t, string(body), "test-key", "password was resent to another host")
}))
defer other.Close()
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
originHits.Add(1)
body, _ := io.ReadAll(r.Body)
assert.Contains(t, string(body), "test-key")
http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
}))
defer origin.Close()
m := configmap.Simple{
"auth": origin.URL + "/v" + authVersion,
"auth_version": authVersion,
"user": "test-user",
"key": "test-key",
"chunk_size": "5Gi",
}
_, err := NewFs(context.Background(), "swift-test", "container", m)
require.ErrorIs(t, err, errRedirectResendsBody)
assert.Equal(t, int32(1), originHits.Load())
assert.Equal(t, int32(0), otherHits.Load())
})
}
}
+1 -17
View File
@@ -113,22 +113,6 @@ type Fs struct {
pacer *fs.Pacer // pacer for API calls
}
// authHeaders are the request headers carrying the app and session
// tokens which must not be forwarded when a redirect leaves the
// original host.
var authHeaders = []string{"X-Auth-Token", "X-User-Token"}
// newClient makes the http client used for all uloz.to requests.
//
// Downloads carry the auth headers to whatever URL the API hands out,
// so the client strips them on cross host redirects and refuses an
// HTTPS to HTTP downgrade.
func newClient(ctx context.Context) *http.Client {
client := fshttp.NewClient(ctx)
client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(authHeaders...)
return client
}
// NewFs constructs a Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -141,7 +125,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// Strip leading and trailing slashes, see https://github.com/rclone/rclone/issues/7796 for details.
root = strings.Trim(root, "/")
client := newClient(ctx)
client := fshttp.NewClient(ctx)
f := &Fs{
name: name,
-95
View File
@@ -1,95 +0,0 @@
package ulozto
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/rclone/rclone/backend/ulozto/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// authTestHeaders is a deliberately literal copy of authHeaders so
// that a header dropped from the production list fails
// TestAuthHeaders rather than silently losing test coverage.
var authTestHeaders = map[string]string{
"X-Auth-Token": "secret-app-token",
"X-User-Token": "secret-user-token",
}
func TestAuthHeaders(t *testing.T) {
assert.ElementsMatch(t, []string{"X-Auth-Token", "X-User-Token"}, authHeaders)
}
// newTestFs makes an Fs talking to apiURL with the auth headers set,
// without going through NewFs which would authenticate.
func newTestFs(ctx context.Context, apiURL string) *Fs {
client := newClient(ctx)
f := &Fs{
name: "ulozto-test",
cdn: rest.NewClient(client),
rest: rest.NewClient(client).SetRoot(apiURL),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.rest.SetErrorHandler(errorHandler)
for header, value := range authTestHeaders {
f.rest.SetHeader(header, value)
}
return f
}
// TestOpenStripsAuthHeadersOnCrossHostRedirect checks that a download
// sends the auth headers to the host the API named but not to a host
// that download redirects to.
func TestOpenStripsAuthHeadersOnCrossHostRedirect(t *testing.T) {
ctx := context.Background()
var otherHits atomic.Int32
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
otherHits.Add(1)
for header := range authTestHeaders {
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
}
_, _ = w.Write([]byte("payload"))
}))
defer other.Close()
var cdnHits atomic.Int32
cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
cdnHits.Add(1)
assert.Equal(t, "/file?hash=h", r.URL.RequestURI())
for header, value := range authTestHeaders {
assert.Equal(t, value, r.Header.Get(header), "%s should be sent to the download host", header)
}
http.Redirect(w, r, other.URL+"/payload", http.StatusTemporaryRedirect)
}))
defer cdn.Close()
apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/v5/file/download-link/vipdata", r.URL.Path)
w.Header().Set("Content-Type", "application/json")
require.NoError(t, json.NewEncoder(w).Encode(api.GetDownloadLinkResponse{
Link: cdn.URL + "/file",
Hash: "h",
}))
}))
defer apiServer.Close()
o := &Object{fs: newTestFs(ctx, apiServer.URL), slug: "slug"}
rc, err := o.Open(ctx)
require.NoError(t, err)
body, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, "payload", string(body))
assert.Equal(t, int32(1), cdnHits.Load())
assert.Equal(t, int32(1), otherHits.Load())
}
+33 -5
View File
@@ -20,9 +20,16 @@ type Uploader struct {
upload *Upload
offset int64
aborted bool
uploadSubs []chan Upload
notifyChan chan bool
overridePatchMethod bool
}
// NotifyUploadProgress subscribes to progress updates.
func (u *Uploader) NotifyUploadProgress(c chan Upload) {
u.uploadSubs = append(u.uploadSubs, c)
}
func (f *Fs) shouldRetryChunk(ctx context.Context, resp *http.Response, err error, newOff *int64) (bool, error) {
if resp == nil {
return true, err
@@ -149,15 +156,36 @@ func (u *Uploader) UploadChunk(ctx context.Context, cnt int, options ...fs.OpenO
u.upload.updateProgress(u.offset)
u.notifyChan <- true
return nil
}
// Waits for a signal to broadcast to all subscribers
func (u *Uploader) broadcastProgress() {
for range u.notifyChan {
for _, c := range u.uploadSubs {
c <- *u.upload
}
}
}
// NewUploader creates a new Uploader.
func NewUploader(f *Fs, url string, upload *Upload, offset int64) *Uploader {
return &Uploader{
fs: f,
url: url,
upload: upload,
offset: offset,
notifyChan := make(chan bool)
uploader := &Uploader{
f,
url,
upload,
offset,
false,
nil,
notifyChan,
false,
}
go uploader.broadcastProgress()
return uploader
}
-113
View File
@@ -1,113 +0,0 @@
package webdav
import (
"context"
"io"
"net/http"
"net/http/httptest"
"runtime"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"weak"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newTusTestUploader(t *testing.T, server *httptest.Server, contents string) *Uploader {
t.Helper()
f := &Fs{
opt: Options{ChunkSize: 4},
srv: rest.NewClient(server.Client()),
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(0))),
}
// Wrap the reader so NewUpload buffers it, as it does for transfer readers.
upload := NewUpload(io.LimitReader(strings.NewReader(contents), int64(len(contents))), int64(len(contents)), nil, "")
require.NotNil(t, upload)
return NewUploader(f, server.URL, upload, 0)
}
func tusTestHandler(t *testing.T, contents string, fail *atomic.Bool) http.HandlerFunc {
t.Helper()
return func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "PATCH", r.Method)
assert.Equal(t, "1.0.0", r.Header.Get("Tus-Resumable"))
if fail != nil && fail.Load() {
w.WriteHeader(http.StatusConflict)
return
}
offset, err := strconv.Atoi(r.Header.Get("Upload-Offset"))
if !assert.NoError(t, err) {
w.WriteHeader(http.StatusBadRequest)
return
}
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
end := min(offset+4, len(contents))
assert.Equal(t, contents[offset:end], string(body))
w.Header().Set("Upload-Offset", strconv.Itoa(end))
w.WriteHeader(http.StatusNoContent)
}
}
func TestTusUploadReleasesBuffer(t *testing.T) {
for _, tc := range []struct {
name string
contents string
fail bool
}{
{name: "Success", contents: "abcdefghij"},
{name: "Error", contents: "abcdefghij", fail: true},
{name: "Empty"},
} {
t.Run(tc.name, func(t *testing.T) {
var fail atomic.Bool
fail.Store(tc.fail)
server := httptest.NewServer(tusTestHandler(t, tc.contents, &fail))
defer server.Close()
released := func() weak.Pointer[Upload] {
uploader := newTusTestUploader(t, server, tc.contents)
err := uploader.Upload(context.Background())
if tc.fail {
require.ErrorIs(t, err, ErrOffsetMismatch)
} else {
require.NoError(t, err)
assert.True(t, uploader.upload.Finished())
require.NoError(t, uploader.Upload(context.Background()))
}
ref := weak.Make(uploader.upload)
runtime.KeepAlive(uploader)
return ref
}()
require.Eventually(t, func() bool {
runtime.GC()
return released.Value() == nil
}, 5*time.Second, 10*time.Millisecond, "completed upload still retains its buffered contents")
})
}
}
func TestTusUploadRetry(t *testing.T) {
const contents = "abcdefghij"
var fail atomic.Bool
fail.Store(true)
server := httptest.NewServer(tusTestHandler(t, contents, &fail))
defer server.Close()
uploader := newTusTestUploader(t, server, contents)
require.ErrorIs(t, uploader.Upload(context.Background()), ErrOffsetMismatch)
assert.Zero(t, uploader.upload.Offset())
fail.Store(false)
require.NoError(t, uploader.Upload(context.Background()))
assert.Equal(t, int64(len(contents)), uploader.upload.Offset())
assert.True(t, uploader.upload.Finished())
require.NoError(t, uploader.Upload(context.Background()))
}
+1 -1
View File
@@ -1783,7 +1783,7 @@ func (o *Object) updateSimple(ctx context.Context, body io.Reader, getBody func(
GetBody: getBody,
Body: body,
NoResponse: true,
ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud-snap/nextcloud-snap/issues/365
ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
ContentType: contentType,
Options: options,
ExtraHeaders: extraHeaders,
-2
View File
@@ -1277,8 +1277,6 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
if err != nil {
return nil, fmt.Errorf("failed to make multipart upload: %w", err)
}
// Stop the form writer reading in if the request fails before sending it all
defer func() { _ = formReader.Close() }()
contentLength := overhead + size
opts := rest.Opts{
-1
View File
@@ -66,7 +66,6 @@ docs = [
"internetarchive.md",
"internxt.md",
"jottacloud.md",
"kdrive.md",
"koofr.md",
"linkbox.md",
"mailru.md",
+19 -115
View File
@@ -19,7 +19,6 @@ import (
"sort"
"strconv"
"strings"
mutex "sync" // renamed as "sync" already in use
"testing"
"time"
"unicode/utf8"
@@ -62,12 +61,6 @@ const (
var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
// modtimeProbes holds the SetModTime error for each remote already probed by checkPreReqs
var (
modtimeProbesMu mutex.Mutex
modtimeProbes = map[string]error{}
)
/* Useful Command Shortcuts */
// go test ./cmd/bisync -remote local -race
// go test ./cmd/bisync -remote local -golden
@@ -95,7 +88,7 @@ var logReplacements = []string{
// ignore SFTP host key messages
`^NOTICE: .*?No host key validation is being performed.*$`, dropMe,
// ignore dropbox info messages
`^NOTICE: .*Too many requests or write operations.*$`, dropMe,
`^NOTICE: too_many_(requests|write_operations)/\.*: Too many requests or write operations.*$`, dropMe,
`^NOTICE: .*?: Forced to upload files to set modification times on this backend.$`, dropMe,
`^INFO : .*? Committing uploads - please wait...$`, dropMe,
`^INFO : .*? Committing uploads - done$`, dropMe,
@@ -109,9 +102,6 @@ var logReplacements = []string{
`^INFO : .*cache expired.*$`, dropMe,
// ignore "Implicitly create directory" messages (TestnStorage:)
`^INFO : .*Implicitly create directory.*$`, dropMe,
// ignore bucket creation messages from S3 providers which report success
// when creating a bucket which already exists (TestS3Wasabi:)
`^INFO : .*: Bucket ".*" created with ACL ".*"$`, dropMe,
// ignore differences in backend features
`^.*?"HashType1":.*?$`, dropMe,
`^.*?"HashType2":.*?$`, dropMe,
@@ -207,7 +197,6 @@ type bisyncTest struct {
testTitle string
testCase string
sessionName string
hooks map[string][]string // scenario steps to run once when the named hook fires
// test dirs
testDir string
dataDir string
@@ -241,7 +230,6 @@ type bisyncTest struct {
TestFn bisync.TestFunc
ignoreModtime bool // ignore modtimes when comparing final listings, for backends without support
ignoreBlankHash bool // ignore blank hashes for backends where we allow them to be blank
ignoreChecksum bool // a step of this test case used --ignore-checksum
}
var color = bisync.Color
@@ -447,7 +435,6 @@ func (b *bisyncTest) cleanupAll() {
func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase string) {
b.t = t
b.testCase = testCase
b.ignoreChecksum = false
var err error
b.fs1, b.parent1, b.path1, b.canonPath1 = b.makeTempRemote(ctx, b.argRemote1, "path1")
@@ -818,19 +805,6 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
case "test-func":
b.TestFn = testFunc
return
case "hook":
// hook <when> <step> queues a scenario step to run once, when the next bisync reaches <when>
b.checkArgs(args, 2, 0)
when := args[1]
if when != "before-check" && when != "during-sync" {
return fmt.Errorf("unknown hook %q (want before-check or during-sync)", when)
}
step := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(strings.TrimPrefix(line, args[0])), when))
if b.hooks == nil {
b.hooks = map[string][]string{}
}
b.hooks[when] = append(b.hooks[when], step)
return nil
case "concurrent-func":
b.TestFn = func() {
src := filepath.Join(b.dataDir, "file7.txt")
@@ -982,10 +956,6 @@ func splitLine(line string) (args []string) {
b := []byte(whitespaceReplacer.Replace(s))
b = regexChar.ReplaceAllFunc(b, func(b []byte) []byte {
c, _ := strconv.ParseUint(string(b[5:7]), 16, 8)
if c <= 0x1f || c == 0x7f {
// control characters are spelled with their glyph in rclone names
return []byte(encoder.Standard.Encode(string(rune(c))))
}
return []byte{byte(c)}
})
args = append(args, string(b))
@@ -995,8 +965,8 @@ func splitLine(line string) (args []string) {
var whitespaceReplacer = strings.NewReplacer(
"{spc}", " ",
"{tab}", "␉",
"{eol}", "␊",
"{tab}", "\t",
"{eol}", eol,
)
var regexChar = regexp.MustCompile(`\{chr:([0-9a-f]{2})\}`)
@@ -1069,7 +1039,7 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
b.ignoreModtime = true
}
// test if modtimes are writeable
probeSetModtime := func(f fs.Fs) error {
testSetModtime := func(f fs.Fs) {
ctx := accounting.WithStatsGroup(ctx, random.String(8)) // keep stats separate
in := bytes.NewBufferString("modtime_write_test")
objinfo := object.NewStaticObjectInfo("modtime_write_test", initDate, int64(len("modtime_write_test")), true, nil, nil)
@@ -1080,7 +1050,7 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
}
err = obj.SetModTime(ctx, initDate)
if err == fs.ErrorCantSetModTime {
return err
b.t.Skip("skipping test as at least one remote does not support setting modtime")
}
if err == fs.ErrorCantSetModTimeWithoutDelete { // transfers stats expected to differ on this backend
logReplacements = append(logReplacements, `^.*There was nothing to transfer.*$`, dropMe)
@@ -1090,52 +1060,34 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
if !f.Features().IsLocal {
time.Sleep(time.Second) // avoid GoogleCloudStorage Error 429 rateLimitExceeded
}
require.NoError(b.t, obj.Remove(ctx))
return err
}
// probing on every bisync step would sleep for minutes on non-local remotes
testSetModtime := func(f fs.Fs) {
modtimeProbesMu.Lock()
defer modtimeProbesMu.Unlock()
err, probed := modtimeProbes[f.Name()]
if !probed {
err = probeSetModtime(f)
modtimeProbes[f.Name()] = err
}
if err == fs.ErrorCantSetModTime {
b.t.Skip("skipping test as at least one remote does not support setting modtime")
}
err = obj.Remove(ctx)
require.NoError(b.t, err)
}
if b.testCase != "nomodtime" {
testSetModtime(b.fs1)
testSetModtime(b.fs2)
}
if b.testCase == "normalization" || b.testCase == "extended_filenames" {
if b.testCase == "normalization" || b.testCase == "extended_char_paths" || b.testCase == "extended_filenames" {
// test whether remote is capable of running test
const chars = "ě{chr:81}{chr:fe}{spc}áñhࢺ_測試Рускийěáñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö"
testfilename1 := splitLine(norm.NFD.String(norm.NFC.String(chars)))[0]
testfilename2 := splitLine(norm.NFC.String(norm.NFD.String(chars)))[0]
tempDir, err := cache.Get(ctx, b.tempDir)
require.NoError(b.t, err)
ctx := accounting.WithStatsGroup(ctx, random.String(8)) // keep stats separate
preTest := func(f fs.Fs, testfilename string) string {
// the path roots already hold the scenario's files, so probe in a fresh dir
dir := "pretest-" + strings.ToLower(random.String(8))
in := bytes.NewBufferString(testfilename)
objinfo := object.NewStaticObjectInfo(path.Join(dir, testfilename), initDate, int64(len(testfilename)), true, nil, nil)
objinfo := object.NewStaticObjectInfo(testfilename, initDate, int64(len(testfilename)), true, nil, nil)
obj, err := f.Put(ctx, in, objinfo)
if err != nil {
b.t.Skipf("Fs is incapable of running test, skipping: %s (expected: \n%s (%s) actual: \n%s (%v))\n (fs: %s) \n", b.testCase, testfilename, detectEncoding(testfilename), "upload failed", err, f)
}
entries, err := f.List(ctx, dir)
require.NoError(b.t, err)
require.Equal(b.t, 1, entries.Len())
got := path.Base(entries[0].Remote())
if got != testfilename {
diffStr, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: []string{testfilename}, B: []string{got}})
entries, err := f.List(ctx, "")
assert.NoError(b.t, err)
if entries.Len() == 1 && entries[0].Remote() != testfilename {
diffStr, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: []string{testfilename}, B: []string{entries[0].Remote()}})
// we can still deal with this as long as both remotes auto-convert the same way.
b.t.Logf("Warning: this remote seems to auto-convert special characters (testcase: %s) (expected: \n%s (%s) actual: \n%s (%s))\n (fs: %s) \n%v", b.testCase, testfilename, detectEncoding(testfilename), got, detectEncoding(got), f, diffStr)
b.t.Logf("Warning: this remote seems to auto-convert special characters (testcase: %s) (expected: \n%s (%s) actual: \n%s (%s))\n (fs: %s) \n%v", b.testCase, testfilename, detectEncoding(testfilename), entries[0].Remote(), detectEncoding(entries[0].Remote()), f, diffStr)
}
// test whether we can fix-case
ctxFixCase, ci := fs.AddConfig(ctx)
@@ -1152,12 +1104,7 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
if obj != nil {
require.NoError(b.t, upperObj.Remove(ctx))
}
for _, d := range []fs.Fs{f, tempDir} {
if err := d.Rmdir(ctx, dir); err != nil && !errors.Is(err, fs.ErrorDirNotFound) {
require.NoError(b.t, err)
}
}
return got
return entries[0].Remote()
}
got1 := preTest(b.fs1, testfilename1)
got1 += preTest(b.fs1, testfilename2)
@@ -1178,11 +1125,11 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
Workdir: b.workDir,
NoCleanup: true,
SaveQueues: true,
MaxDelete: bisync.DefaultMaxDelete,
CheckFilename: bisync.DefaultCheckFilename,
CheckSync: bisync.CheckSyncTrue,
TestFn: b.TestFn,
}
opt.TestFn = b.hookFn(ctx, "during-sync", b.TestFn)
opt.TestFnConflictCheck = b.hookFn(ctx, "before-check", nil)
ctx, opt = b.checkPreReqs(ctx, opt)
octx, ci := fs.AddConfig(ctx)
fs1, fs2 := b.fs1, b.fs2
@@ -1223,7 +1170,7 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
case "filters-file":
opt.FiltersFile = val
case "max-delete":
ci.MaxDelete, err = strconv.ParseInt(val, 10, 64)
opt.MaxDelete, err = strconv.Atoi(val)
require.NoError(b.t, err, "parsing max-delete=%q", val)
case "max-delete-renames-aware":
opt.MaxDeleteRenamesAware = true
@@ -1235,9 +1182,6 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
ci.SizeOnly = true
case "ignore-size":
ci.IgnoreSize = true
case "ignore-checksum":
ci.IgnoreChecksum = true
b.ignoreChecksum = true
case "checksum":
ci.CheckSum = true
opt.Compare.DownloadHash = true // allows us to test crypt and the like
@@ -1308,13 +1252,7 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
if err != nil {
b.logPrintf("Bisync error: %v", err)
}
for _, when := range []string{"before-check", "during-sync"} {
if len(b.hooks[when]) > 0 {
b.logPrintf("hook %s did not run, discarding %d steps", when, len(b.hooks[when]))
}
}
b.hooks = nil
if err == nil && !opt.DryRun && opt.TestFn == nil && opt.TestFnConflictCheck == nil && opt.FiltersFile == "" {
if err == nil && !opt.DryRun && opt.TestFn == nil && opt.FiltersFile == "" {
b.checkListingsMatchRemotes(ctx, fs1, fs2)
}
return nil
@@ -1359,30 +1297,6 @@ func (b *bisyncTest) checkListingsMatchRemotes(ctx context.Context, fs1, fs2 fs.
}
}
// hookFn returns a test hook that calls fn, if set, and then runs the
// scenario steps queued for the hook named when, the first time it is called.
func (b *bisyncTest) hookFn(ctx context.Context, when string, fn bisync.TestFunc) bisync.TestFunc {
if len(b.hooks[when]) == 0 {
return fn
}
return func() {
if fn != nil {
fn()
}
steps := b.hooks[when]
b.hooks[when] = nil
// log only through the captured bisync output, so hook steps appear once and in order
logFile, stepStr := b.logFile, b.stepStr
b.logFile, b.stepStr = nil, "hook "+when+":"
defer func() { b.logFile, b.stepStr = logFile, stepStr }()
for _, step := range steps {
if err := b.runTestStep(ctx, step); err != nil {
fs.Errorf(nil, "hook %s: %q failed: %v", when, step, err)
}
}
}
}
// saveTestListings creates a copy of test artifacts with given prefix
// including listings (.lst*), queues (.que) and filters (.flt, .flt.md5)
func (b *bisyncTest) saveTestListings(prefix string, keepSource bool) (err error) {
@@ -1808,16 +1722,6 @@ func (b *bisyncTest) mangleResult(dir, file string, golden bool) string {
if b.testCase == "dry_run" {
rep = append(rep, dryrunReplacements...)
}
if b.ignoreChecksum && b.fs1.Hashes().Overlap(b.fs2.Hashes()).Count() == 0 {
// with no hash in common, --ignore-checksum skips the check for identical files instead of running it
rep = append(rep,
`^.*Checking potential conflicts\.\.\.$`, dropMe,
`^.*: \d+ differences found$`, dropMe,
`^.*: \d+ matching files$`, dropMe,
`^.*Finished checking the potential conflicts.*$`, dropMe,
`^.*Not checking potential conflicts.*$`, dropMe,
)
}
repFrom := make([]*regexp.Regexp, len(rep)/2)
repTo := make([]string, len(rep)/2)
for i := 0; i < len(rep); i += 2 {
+7 -66
View File
@@ -5,7 +5,6 @@ import (
"context"
"fmt"
"strings"
"sync"
"github.com/rclone/rclone/backend/crypt"
"github.com/rclone/rclone/cmd/bisync/bilib"
@@ -80,7 +79,7 @@ func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool,
return true, false, err
}
if ht == hash.None {
return missingHashFn(ctx, dst, src)
return false, true, nil
}
if !same {
err = fmt.Errorf("%v differ", ht)
@@ -99,14 +98,14 @@ func (b *bisyncRun) CryptCheckFn(ctx context.Context, dst, src fs.Object) (diffe
return true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)
}
if underlyingHash == "" {
return missingHashFn(ctx, dst, src)
return false, true, nil
}
cryptHash, err := b.check.fcrypt.ComputeHash(ctx, cryptDst, src, b.check.hashType)
if err != nil {
return true, false, fmt.Errorf("error computing hash: %w", err)
}
if cryptHash == "" {
return missingHashFn(ctx, dst, src)
return false, true, nil
}
if cryptHash != underlyingHash {
err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", b.check.fdst.Name(), b.check.fdst.Root(), cryptHash, b.check.fsrc.Name(), b.check.fsrc.Root(), underlyingHash)
@@ -125,15 +124,6 @@ func (b *bisyncRun) ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object)
return b.CryptCheckFn(ctx, src, dst)
}
func missingHashFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
ci := fs.GetConfig(ctx)
if ci.SizeOnly || ci.IgnoreChecksum {
return false, true, nil
}
fs.Debugf(src, "hash is missing, so using --download")
return DownloadCheckFn(ctx, dst, src)
}
// DownloadCheckFn is a slightly modified version of Check with --download
func DownloadCheckFn(ctx context.Context, dst, src fs.Object) (equal bool, noHash bool, err error) {
equal, err = operations.CheckIdenticalDownload(ctx, src, dst)
@@ -144,37 +134,17 @@ func DownloadCheckFn(ctx context.Context, dst, src fs.Object) (equal bool, noHas
}
// check potential conflicts (to avoid renaming if already identical)
func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (matches bilib.Names, matches1, matches2 *fileList, err error) {
matches = bilib.Names{}
matches1, matches2 = newFileList(), newFileList()
func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (bilib.Names, error) {
matches := bilib.Names{}
if filterCheck.HaveFilesFrom() {
fs.Debugf(nil, "There are potential conflicts to check.")
if b.opt.TestFnConflictCheck != nil {
b.opt.TestFnConflictCheck()
}
// the sizes already match, and they are all a check could compare here
ci := fs.GetConfig(ctxCheck)
if ci.SizeOnly || (ci.IgnoreChecksum && fs1.Hashes().Overlap(fs2.Hashes()).Count() == 0) {
fs.Infof(nil, "Not checking potential conflicts, as only their sizes could be compared, and those already match.")
for file := range filterCheck.Files() {
matches.Add(file)
if b.march.ls1.has(file) {
b.march.ls1.getPut(file, matches1)
}
if b.march.ls2.has(file) {
b.march.ls2.getPut(file, matches2)
}
}
return matches, matches1, matches2, nil
}
opt, close, checkopterr := check.GetCheckOpt(fs1, fs2)
if checkopterr != nil {
b.critical = true
b.retryable = true
fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)
return matches, matches1, matches2, checkopterr
return matches, checkopterr
}
defer close()
@@ -182,22 +152,6 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter
opt = b.WhichCheck(ctxCheck, opt)
// record the objects found identical, so they can be compared with the listing snapshot
var mu sync.Mutex
checkFn := opt.Check
opt.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
differ, noHash, err = checkFn(ctx, dst, src)
if err == nil && !differ {
f1 := b.comparedFields(ctx, src, b.march.ls1.hash)
f2 := b.comparedFields(ctx, dst, b.march.ls2.hash)
mu.Lock()
matches1.put(src.Remote(), f1.size, f1.time, f1.hash, "", "-")
matches2.put(dst.Remote(), f2.size, f2.time, f2.hash, "", "-")
mu.Unlock()
}
return differ, noHash, err
}
fs.Infof(nil, "Checking potential conflicts...")
check := operations.CheckFn(ctxCheck, opt)
fs.Infof(nil, "Finished checking the potential conflicts. %s", check)
@@ -214,22 +168,9 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter
} else {
fs.Debugf(nil, "None of the conflicts were determined to be identical.")
}
}
return matches, matches1, matches2, nil
}
// comparedFields returns the fields of o that bisync compares, the same way
// the listing records them.
func (b *bisyncRun) comparedFields(ctx context.Context, o fs.Object, hashType hash.Type) (fi fileInfo) {
fi.size = o.Size()
if b.opt.Compare.Modtime {
fi.time = o.ModTime(ctx).In(TZ)
}
if b.opt.Compare.Checksum && hashType != hash.None {
fi.hash, _ = o.Hash(ctx, hashType)
fi.hash, _ = b.tryDownloadHash(ctx, o, fi.hash)
}
return fi
return matches, nil
}
// WhichEqual is similar to WhichCheck, but checks a single object.
-117
View File
@@ -1,117 +0,0 @@
package bisync
import (
"context"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fstest/mockfs"
"github.com/rclone/rclone/fstest/mockobject"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// blankHashObject supports MD5 through its Fs but returns a blank hash
type blankHashObject struct {
*mockobject.ContentMockObject
}
func (o blankHashObject) Hash(ctx context.Context, t hash.Type) (string, error) {
return "", nil
}
func newBlankHashObject(ctx context.Context, t *testing.T, fsName, content string) blankHashObject {
f, err := mockfs.NewFs(ctx, fsName, "", nil)
require.NoError(t, err)
f.(*mockfs.Fs).SetHashes(hash.NewHashSet(hash.MD5))
o := mockobject.New("file.txt").WithContent([]byte(content), mockobject.SeekModeNone)
o.SetFs(f)
return blankHashObject{o}
}
func TestCheckFnMissingHash(t *testing.T) {
for _, test := range []struct {
name string
content string
flag func(ci *fs.ConfigInfo)
wantDiffer bool
wantNoHash bool
}{
{name: "identical", content: "hello"},
{name: "same size, different content", content: "HELLO", wantDiffer: true},
{name: "size-only", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantNoHash: true},
{name: "ignore-checksum", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }, wantNoHash: true},
} {
t.Run(test.name, func(t *testing.T) {
ctx, ci := fs.AddConfig(context.Background())
if test.flag != nil {
test.flag(ci)
}
src := newBlankHashObject(ctx, t, "path1", "hello")
dst := newBlankHashObject(ctx, t, "path2", test.content)
differ, noHash, err := CheckFn(ctx, dst, src)
require.NoError(t, err)
assert.Equal(t, test.wantDiffer, differ, "differ")
assert.Equal(t, test.wantNoHash, noHash, "noHash")
})
}
}
func newEmptyFs(ctx context.Context, t *testing.T, fsName string, hashes hash.Set) fs.Fs {
f, err := mockfs.NewFs(ctx, fsName, "", nil)
require.NoError(t, err)
f.(*mockfs.Fs).SetHashes(hashes)
return f
}
func TestCheckConflictsSkipsCheck(t *testing.T) {
md5 := hash.NewHashSet(hash.MD5)
none := hash.NewHashSet()
for _, test := range []struct {
name string
hashes2 hash.Set
flag func(ci *fs.ConfigInfo)
wantSkip bool
}{
{name: "common hash", hashes2: md5},
{name: "no common hash", hashes2: none},
{name: "size-only with a common hash", hashes2: md5, flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantSkip: true},
{name: "size-only without a common hash", hashes2: none, flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantSkip: true},
{name: "ignore-checksum with a common hash", hashes2: md5, flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }},
{name: "ignore-checksum without a common hash", hashes2: none, flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }, wantSkip: true},
} {
t.Run(test.name, func(t *testing.T) {
ctx, ci := fs.AddConfig(context.Background())
if test.flag != nil {
test.flag(ci)
}
ctx, fi := filter.AddConfig(ctx)
require.NoError(t, fi.AddFile("file.txt"))
b := &bisyncRun{opt: &Options{}}
b.opt.Compare.Size = true
b.opt.Compare.Modtime = true
modtime := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
b.march.ls1, b.march.ls2 = newFileList(), newFileList()
b.march.ls1.put("file.txt", 5, modtime, "", "-", "-")
b.march.ls2.put("file.txt", 5, modtime, "", "-", "-")
fs1 := newEmptyFs(ctx, t, "path1", md5)
fs2 := newEmptyFs(ctx, t, "path2", test.hashes2)
matches, matches1, matches2, err := b.checkconflicts(ctx, fi, fs1, fs2)
require.NoError(t, err)
assert.Equal(t, test.wantSkip, matches.Has("file.txt"), "matched a file that is on neither path")
if test.wantSkip {
assert.True(t, b.sameCompared(b.march.ls1, matches1, "file.txt", fs1), "Path1 differs from the listing")
assert.True(t, b.sameCompared(b.march.ls2, matches2, "file.txt", fs2), "Path2 differs from the listing")
} else {
assert.False(t, matches1.has("file.txt"), "Path1 checked")
assert.False(t, matches2.has("file.txt"), "Path2 checked")
}
})
}
}
+7 -8
View File
@@ -38,7 +38,7 @@ type Options struct {
CheckSync CheckSyncMode
CreateEmptySrcDirs bool
RemoveEmptyDirs bool
MaxDelete int // percentage from 0 to 100, set by Bisync from --max-delete
MaxDelete int // percentage from 0 to 100
MaxDeleteRenamesAware bool
Force bool
FiltersFile string
@@ -46,14 +46,13 @@ type Options struct {
OrigBackupDir string
BackupDir1 string
BackupDir2 string
DryRun bool // set by Bisync from --dry-run
DryRun bool
NoCleanup bool
SaveQueues bool // save extra debugging files (test only flag)
IgnoreListingChecksum bool
Resilient bool
Recover bool
TestFn TestFunc // test-only option, for mocking errors
TestFnConflictCheck TestFunc // test-only option, called before checking potential conflicts
Compare CompareOpt
CompareFlag string
DebugName string
@@ -179,6 +178,7 @@ var commandDefinition = &cobra.Command{
ctx := context.Background()
opt := Opt
opt.applyContext(ctx)
if tzLocal {
TZ = time.Local
}
@@ -204,23 +204,22 @@ var commandDefinition = &cobra.Command{
},
}
func (opt *Options) applyContext(ctx context.Context) error {
func (opt *Options) applyContext(ctx context.Context) {
maxDelete := DefaultMaxDelete
ci := fs.GetConfig(ctx)
if ci.MaxDelete > 100 {
return fmt.Errorf("--max-delete is a percentage for bisync and must be from 0 to 100, not %d", ci.MaxDelete)
}
if ci.MaxDelete >= 0 {
maxDelete = int(ci.MaxDelete)
}
if maxDelete < 0 {
maxDelete = 0
}
if maxDelete > 100 {
maxDelete = 100
}
opt.MaxDelete = maxDelete
// reset MaxDelete for fs/operations, bisync handles this parameter specially
ci.MaxDelete = -1
opt.DryRun = ci.DryRun
return nil
}
func (opt *Options) setDryRun(ctx context.Context) context.Context {
+10 -23
View File
@@ -290,8 +290,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
handled := bilib.Names{}
renameSkipped := bilib.Names{}
deletedonboth := bilib.Names{}
skipped1 := newFileList()
skipped2 := newFileList()
skippedDirs1 := newFileList()
skippedDirs2 := newFileList()
b.renames = renames{}
ctxMove := b.opt.setDryRun(ctx)
@@ -361,7 +361,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
}
// if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop
matches, matches1, matches2, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2)
matches, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2)
for _, file := range ds1.sort() {
alias := b.aliases.Alias(file)
@@ -388,8 +388,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
// if files are identical, leave them alone instead of renaming
if (dirs1.has(file) || dirs1.has(alias)) && (dirs2.has(file) || dirs2.has(alias)) {
fs.Infof(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)
b.march.ls1.getPut(file, skipped1)
b.march.ls2.getPut(file, skipped2)
b.march.ls1.getPut(file, skippedDirs1)
b.march.ls2.getPut(file, skippedDirs2)
b.debugFn(file, func() {
b.debug(file, fmt.Sprintf("deltas dir: %s, ls1 has name?: %v, ls2 has name?: %v", file, b.march.ls1.has(b.DebugName), b.march.ls2.has(b.DebugName)))
})
@@ -415,14 +415,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
b.indent("Path1", p2, "Queue copy to Path2")
copy1to2.Add(b.march.ls1.getTryAlias(file, alias))
}
} else if b.sameCompared(b.march.ls1, matches1, b.march.ls1.getTryAlias(file, alias), b.fs1) &&
b.sameCompared(b.march.ls2, matches2, b.march.ls2.getTryAlias(file, alias), b.fs2) {
fs.Infof(nil, "Files are equal! Skipping: %s", file)
b.march.ls1.getPut(b.march.ls1.getTryAlias(file, alias), skipped1)
b.march.ls2.getPut(b.march.ls2.getTryAlias(file, alias), skipped2)
} else {
// the snapshot isn't the state we found equal, so leave the prior listing for the next run
fs.Infof(nil, "Files are equal but changed after the listing was made, so will check again next run: %s", file)
fs.Infof(nil, "Files are equal! Skipping: %s", file)
renameSkipped.Add(file)
renameSkipped.Add(alias)
}
} else {
fs.Debugf(nil, "Files are NOT equal: %s", file)
@@ -534,8 +530,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
queues.copy2to1 = copy2to1
queues.renameSkipped = renameSkipped
queues.deletedonboth = deletedonboth
queues.skipped1 = skipped1
queues.skipped2 = skipped2
queues.skippedDirs1 = skippedDirs1
queues.skippedDirs2 = skippedDirs2
return
}
@@ -636,12 +632,3 @@ func (b *bisyncRun) updateAliases(ctx context.Context, ds1, ds2 *deltaSet) {
addAliases(delMap1, fullMap2)
addAliases(delMap2, fullMap1)
}
// sameCompared reports whether file has the same values in the listing
// snapshot and in the checked list, in every field bisync compares.
func (b *bisyncRun) sameCompared(snapshot, checked *fileList, file string, f fs.Info) bool {
if !checked.has(file) {
return false
}
return len(b.fileInfoDiffs(file, file, snapshot, checked, f, f, snapshot.hash, snapshot.hash)) == 0
}
-1
View File
@@ -41,7 +41,6 @@ func RcHelp() string {
- path1 (required) - (string) a remote directory string e.g. ||drive:path1||
- path2 (required) - (string) a remote directory string e.g. ||drive:path2||
- dryRun - (bool) dry-run mode
- maxDelete - (int) Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: {MAXDELETE}%)
`+GenerateParams()+`
See [bisync command help](https://rclone.org/commands/rclone_bisync/)
and [full bisync description](https://rclone.org/bisync/)
+25 -28
View File
@@ -129,6 +129,12 @@ func (ls *fileList) put(file string, size int64, modtime time.Time, hash, id str
fi := ls.get(file)
if fi != nil {
fi.size = size
// if already have higher precision of same time, avoid overwriting it
if fi.time != modtime {
if modtime.Before(fi.time) && fi.time.Sub(modtime) < time.Second {
modtime = fi.time
}
}
fi.time = modtime
fi.hash = hash
fi.id = id
@@ -180,35 +186,29 @@ func (ls *fileList) getHash(file string) string {
}
func (b *bisyncRun) fileInfoEqual(file1, file2 string, ls1, ls2 *fileList) bool {
equal := true
if ls1.isDir(file1) && ls2.isDir(file2) {
return true
return equal
}
diffs := b.fileInfoDiffs(file1, file2, ls1, ls2, b.fs1, b.fs2, b.opt.Compare.HashType1, b.opt.Compare.HashType2)
for _, diff := range diffs {
b.indent("ERROR", file1, diff)
}
return len(diffs) == 0
}
// fileInfoDiffs describes each field bisync compares where file1 in ls1
// (on f1, with hash type ht1) differs from file2 in ls2 (on f2, with ht2).
func (b *bisyncRun) fileInfoDiffs(file1, file2 string, ls1, ls2 *fileList, f1, f2 fs.Info, ht1, ht2 hash.Type) (diffs []string) {
if b.opt.Compare.Size {
if sizeDiffers(ls1.getSize(file1), ls2.getSize(file2)) {
diffs = append(diffs, fmt.Sprintf("Size not equal in listing. Path1: %v, Path2: %v", ls1.getSize(file1), ls2.getSize(file2)))
b.indent("ERROR", file1, fmt.Sprintf("Size not equal in listing. Path1: %v, Path2: %v", ls1.getSize(file1), ls2.getSize(file2)))
equal = false
}
}
if b.opt.Compare.Modtime {
if timeDiffers(b.fctx, ls1.getTime(file1), ls2.getTime(file2), f1, f2) {
diffs = append(diffs, fmt.Sprintf("Modtime not equal in listing. Path1: %v, Path2: %v", ls1.getTime(file1), ls2.getTime(file2)))
if timeDiffers(b.fctx, ls1.getTime(file1), ls2.getTime(file2), b.fs1, b.fs2) {
b.indent("ERROR", file1, fmt.Sprintf("Modtime not equal in listing. Path1: %v, Path2: %v", ls1.getTime(file1), ls2.getTime(file2)))
equal = false
}
}
if b.opt.Compare.Checksum && !b.queueOpt.ignoreListingChecksum {
if b.hashDiffers(ls1.getHash(file1), ls2.getHash(file2), ht1, ht2, ls1.getSize(file1), ls2.getSize(file2)) {
diffs = append(diffs, fmt.Sprintf("Checksum not equal in listing. Path1: %v, Path2: %v", ls1.getHash(file1), ls2.getHash(file2)))
if b.hashDiffers(ls1.getHash(file1), ls2.getHash(file2), b.opt.Compare.HashType1, b.opt.Compare.HashType2, ls1.getSize(file1), ls2.getSize(file2)) {
b.indent("ERROR", file1, fmt.Sprintf("Checksum not equal in listing. Path1: %v, Path2: %v", ls1.getHash(file1), ls2.getHash(file2)))
equal = false
}
}
return diffs
return equal
}
// also returns false if not found
@@ -581,11 +581,6 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
// account for "deltaOthers" we handled separately
if queues.deletedonboth.NotEmpty() {
for file := range queues.deletedonboth {
// a conflict rename can reuse the name of a file deleted on
// both sides, in which case it was just copied and must stay
if queues.copy1to2.Has(file) || queues.copy2to1.Has(file) {
continue
}
srcList.remove(file)
dstList.remove(file)
}
@@ -632,7 +627,9 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
}
}
// recheck the ones we didn't rename due to --dry-run
// recheck the ones we skipped because they were equal
// we never got their info because they were never synced.
// TODO: add flag to skip this? (since it re-lists)
if queues.renameSkipped.NotEmpty() {
skippedList := queues.renameSkipped.ToList()
for _, file := range skippedList {
@@ -641,7 +638,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
}
}
}
// skipped dirs and equal files -- nothing to recheck, just add them from the snapshot
// skipped dirs -- nothing to recheck, just add them
// (they are not necessarily there already, if they are new)
path1List := srcList
path2List := dstList
@@ -649,11 +646,11 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
path1List = dstList
path2List = srcList
}
if !queues.skipped1.empty() {
queues.skipped1.getPutAll(path1List)
if !queues.skippedDirs1.empty() {
queues.skippedDirs1.getPutAll(path1List)
}
if !queues.skipped2.empty() {
queues.skipped2.getPutAll(path2List)
if !queues.skippedDirs2.empty() {
queues.skippedDirs2.getPutAll(path2List)
}
if filterRecheck.HaveFilesFrom() {
-114
View File
@@ -1,114 +0,0 @@
package bisync
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFileListPut(t *testing.T) {
old := time.Date(2026, 9, 29, 12, 0, 1, 555555555, time.UTC)
for _, test := range []struct {
name string
modtime time.Time
}{
{"slightly older", old.Add(-124 * time.Microsecond)},
{"slightly newer", old.Add(124 * time.Microsecond)},
{"same second, truncated", old.Truncate(time.Second)},
{"same second, rounded up", old.Round(time.Second)},
} {
t.Run(test.name, func(t *testing.T) {
ls := newFileList()
ls.put("file", 1, old, "", "-", "-")
ls.put("file", 2, test.modtime, "", "-", "-")
assert.True(t, test.modtime.Equal(ls.getTime("file")), "got %v, want %v", ls.getTime("file"), test.modtime)
assert.EqualValues(t, 2, ls.getSize("file"))
})
}
}
// Test a file whose new version is less than a second older than the old one
func TestBisyncCloseModtimes(t *testing.T) {
if *fstest.RemoteName != "" {
t.Skip("Skipping test on non local remote")
}
ctx := context.Background()
unchanged := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
newRun := func(t *testing.T) (path1, path2 string, run func(resync bool) error) {
// t.TempDir includes the test name, which makes the listing names too long
dir, err := os.MkdirTemp("", "bisync")
require.NoError(t, err)
t.Cleanup(func() { _ = os.RemoveAll(dir) })
path1, path2 = filepath.Join(dir, "path1"), filepath.Join(dir, "path2")
workdir := filepath.Join(dir, "workdir")
for _, path := range []string{path1, path2} {
require.NoError(t, os.Mkdir(path, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(path, "file1.txt"), []byte("unchanged"), 0o644))
require.NoError(t, os.Chtimes(filepath.Join(path, "file1.txt"), unchanged, unchanged))
}
run = func(resync bool) error {
f1, err := fs.NewFs(ctx, path1)
require.NoError(t, err)
f2, err := fs.NewFs(ctx, path2)
require.NoError(t, err)
return Bisync(ctx, f1, f2, &Options{Workdir: workdir, Resync: resync, CheckSync: CheckSyncTrue, MaxDelete: DefaultMaxDelete})
}
return path1, path2, run
}
writeFile := func(t *testing.T, path, content string, modtime time.Time) {
file := filepath.Join(path, "file0.txt")
require.NoError(t, os.WriteFile(file, []byte(content), 0o644))
require.NoError(t, os.Chtimes(file, modtime, modtime))
}
// listingTime returns the modtime recorded for file0.txt in the listing for path (1 or 2)
listingTime := func(t *testing.T, path1 string, path int) time.Time {
listings, err := filepath.Glob(filepath.Join(filepath.Dir(path1), "workdir", "*.path"+string(rune('0'+path))+".lst"))
require.NoError(t, err)
require.Len(t, listings, 1)
data, err := os.ReadFile(listings[0])
require.NoError(t, err)
for line := range strings.SplitSeq(string(data), "\n") {
if strings.HasSuffix(line, `"file0.txt"`) {
fields := strings.Fields(line)
modtime, err := time.Parse(timeFormat, fields[4])
require.NoError(t, err)
return modtime
}
}
t.Fatal("file0.txt not in listing")
return time.Time{}
}
// fileTime returns file0.txt's modtime as the filesystem stored it (Windows keeps 100ns)
fileTime := func(t *testing.T, path string) time.Time {
info, err := os.Stat(filepath.Join(path, "file0.txt"))
require.NoError(t, err)
return info.ModTime()
}
modtime := time.Date(2026, 9, 29, 12, 0, 1, 555555555, time.UTC)
t.Run("resync", func(t *testing.T) {
path1, path2, run := newRun(t)
writeFile(t, path1, "path1", modtime)
writeFile(t, path2, "path2, newer", modtime.Add(124*time.Microsecond))
require.NoError(t, run(true))
assert.True(t, fileTime(t, path2).Equal(listingTime(t, path1, 2)), "Path2 listing has %v, file has %v", listingTime(t, path1, 2), fileTime(t, path2))
})
t.Run("older edit", func(t *testing.T) {
path1, path2, run := newRun(t)
writeFile(t, path1, "original", modtime)
require.NoError(t, run(true))
writeFile(t, path1, "edited", modtime.Add(-200*time.Microsecond))
require.NoError(t, run(false))
assert.True(t, fileTime(t, path1).Equal(listingTime(t, path1, 1)), "Path1 listing has %v, file has %v", listingTime(t, path1, 1), fileTime(t, path1))
assert.True(t, fileTime(t, path2).Equal(listingTime(t, path1, 2)), "Path2 listing has %v, file has %v", listingTime(t, path1, 2), fileTime(t, path2))
})
}
+3 -8
View File
@@ -63,20 +63,15 @@ type bisyncRun struct {
type queues struct {
copy1to2 bilib.Names
copy2to1 bilib.Names
renameSkipped bilib.Names // not renamed due to --dry-run
skipped1 *fileList // dirs and equal files not renamed, from the Path1 snapshot
skipped2 *fileList // dirs and equal files not renamed, from the Path2 snapshot
renameSkipped bilib.Names // not renamed because it was equal
skippedDirs1 *fileList
skippedDirs2 *fileList
deletedonboth bilib.Names
}
// Bisync handles lock file, performs bisync run and checks exit status
func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
opt := *optArg // ensure that input is never changed
// applyContext changes the config, so give it a copy
ctx, _ = fs.AddConfig(ctx)
if err = opt.applyContext(ctx); err != nil {
return err
}
b := &bisyncRun{
fs1: fs1,
fs2: fs2,
+42 -38
View File
@@ -65,9 +65,9 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
opt := &Options{}
octx, ci := fs.AddConfig(ctx)
if dryRun, err := in.GetBool("dryRun"); err == nil && dryRun {
// dryRun can ask for a dry run, but not cancel one from the global config
ci.DryRun = true
if dryRun, err := in.GetBool("dryRun"); err == nil {
ci.DryRun = dryRun
opt.DryRun = dryRun
} else if rc.NotErrParamNotFound(err) {
return nil, err
}
@@ -76,58 +76,60 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
if maxDelete < 0 || maxDelete > 100 {
return nil, rc.NewErrParamInvalid(errors.New("maxDelete must be a percentage between 0 and 100"))
}
ci.MaxDelete = maxDelete
opt.MaxDelete = int(maxDelete)
} else if rc.NotErrParamNotFound(err) {
return nil, err
}
if opt.Resync, err = in.GetBool("resync"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("resync", "optional parameter is missing. using default value: %v", opt.Resync)
}
if opt.CheckAccess, err = in.GetBool("checkAccess"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("checkAccess", "optional parameter is missing. using default value: %v", opt.CheckAccess)
}
if opt.Force, err = in.GetBool("force"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("force", "optional parameter is missing. using default value: %v", opt.Force)
}
if opt.MaxDeleteRenamesAware, err = in.GetBool("maxDeleteRenamesAware"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("maxDeleteRenamesAware", "optional parameter is missing. using default value: %v", opt.MaxDeleteRenamesAware)
}
if opt.CreateEmptySrcDirs, err = in.GetBool("createEmptySrcDirs"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("createEmptySrcDirs", "optional parameter is missing. using default value: %v", opt.CreateEmptySrcDirs)
}
if opt.RemoveEmptyDirs, err = in.GetBool("removeEmptyDirs"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("removeEmptyDirs", "optional parameter is missing. using default value: %v", opt.RemoveEmptyDirs)
}
if opt.NoCleanup, err = in.GetBool("noCleanup"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("noCleanup", "optional parameter is missing. using default value: %v", opt.NoCleanup)
}
if opt.IgnoreListingChecksum, err = in.GetBool("ignoreListingChecksum"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("ignoreListingChecksum", "optional parameter is missing. using default value: %v", opt.IgnoreListingChecksum)
}
if opt.Resilient, err = in.GetBool("resilient"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("resilient", "optional parameter is missing. using default value: %v", opt.Resilient)
}
if opt.CheckFilename, err = in.GetString("checkFilename"); rc.NotErrParamNotFound(err) {
return nil, err
opt.CheckFilename = DefaultCheckFilename
fs.Debugf("checkFilename", "optional parameter is missing. using default value: %v", opt.CheckFilename)
}
if opt.FiltersFile, err = in.GetString("filtersFile"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("filtersFile", "optional parameter is missing. using default value: %v", opt.FiltersFile)
}
if opt.Workdir, err = in.GetString("workdir"); rc.NotErrParamNotFound(err) {
return nil, err
// "" sets correct default later
fs.Debugf("workdir", "optional parameter is missing. using default value: %v", opt.Workdir)
}
if opt.BackupDir1, err = in.GetString("backupDir1"); rc.IsErrParamNotFound(err) {
opt.BackupDir1, err = in.GetString("backupdir1") // the name before v1.74
if opt.BackupDir1, err = in.GetString("backupDir1"); rc.NotErrParamNotFound(err) {
// we accept an alternate capitalization here for backward compatibility.
if opt.BackupDir1, err = in.GetString("backupdir1"); rc.NotErrParamNotFound(err) {
fs.Debugf("backupDir1", "optional parameter is missing. using default value: %v", opt.BackupDir1)
}
}
if rc.NotErrParamNotFound(err) {
return nil, err
}
if opt.BackupDir2, err = in.GetString("backupDir2"); rc.IsErrParamNotFound(err) {
opt.BackupDir2, err = in.GetString("backupdir2") // the name before v1.74
}
if rc.NotErrParamNotFound(err) {
return nil, err
if opt.BackupDir2, err = in.GetString("backupDir2"); rc.NotErrParamNotFound(err) {
// we accept an alternate capitalization here for backward compatibility.
if opt.BackupDir2, err = in.GetString("backupdir2"); rc.NotErrParamNotFound(err) {
fs.Debugf("backupDir2", "optional parameter is missing. using default value: %v", opt.BackupDir2)
}
}
if err = setEnum(in, "checkSync", "true", opt.CheckSync.Set); err != nil {
return nil, err
@@ -142,25 +144,26 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
return nil, err
}
if opt.ConflictSuffixFlag, err = in.GetString("conflictSuffix"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("conflictSuffix", "optional parameter is missing. using default value: %v", opt.ConflictSuffixFlag)
}
if opt.Recover, err = in.GetBool("recover"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("recover", "optional parameter is missing. using default value: %v", opt.Recover)
}
if opt.CompareFlag, err = in.GetString("compare"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("compare", "optional parameter is missing. using default value: %v", opt.CompareFlag)
}
if opt.Compare.NoSlowHash, err = in.GetBool("noSlowHash"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("noSlowHash", "optional parameter is missing. using default value: %v", opt.Compare.NoSlowHash)
}
if opt.Compare.SlowHashSyncOnly, err = in.GetBool("slowHashSyncOnly"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("slowHashSyncOnly", "optional parameter is missing. using default value: %v", opt.Compare.SlowHashSyncOnly)
}
if opt.Compare.DownloadHash, err = in.GetBool("downloadHash"); rc.NotErrParamNotFound(err) {
return nil, err
fs.Debugf("downloadHash", "optional parameter is missing. using default value: %v", opt.Compare.DownloadHash)
}
if opt.MaxLock, err = in.GetFsDuration("maxLock"); rc.NotErrParamNotFound(err) {
return nil, err
opt.MaxLock = 0
fs.Debugf("maxLock", "optional parameter is missing. using default value: %v", opt.MaxLock)
}
fs1, err := rc.GetFsNamed(octx, in, "path1")
@@ -197,11 +200,12 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
func setEnum(in rc.Params, name string, defaultVal string, set func(s string) error) error {
v, err := in.GetString(name)
if rc.NotErrParamNotFound(err) {
if rc.NotErrParamNotFound(err) || v == "" {
v = defaultVal
fs.Debugf(name, "optional parameter is missing. using default value: %v", v)
}
if err := set(v); err != nil {
return err
}
if v == "" {
v = defaultVal
}
return set(v)
return nil
}
-4
View File
@@ -5,8 +5,6 @@ This takes the following parameters:
- path1 (required) - (string) a remote directory string e.g. `drive:path1`
- path2 (required) - (string) a remote directory string e.g. `drive:path2`
- dryRun - (bool) dry-run mode
- maxDelete - (int) Safety check on maximum percentage of deleted files
allowed. If exceeded, the bisync run will abort. (default: 50%)
- backupDir1 - (string) --backup-dir for Path1. Must be a non-overlapping path on
the same remote.
- backupDir2 - (string) --backup-dir for Path2. Must be a non-overlapping path on
@@ -36,8 +34,6 @@ unavailable. (warning: may be slow and use lots of data!)
using with --verbose
- ignoreListingChecksum - (bool) Do not use checksums for listings (add --ignore-
checksum to additionally skip post-copy checksum checks)
- maxDeleteRenamesAware - (bool) Exclude tracked renames from the --max-delete
safety check (requires --track-renames)
- maxLock - (Duration) Consider lock files older than this to be expired
(default: 0 (never expire)) (minimum: 2m)
- noCleanup - (bool) Retain working files (useful for troubleshooting and
-266
View File
@@ -1,266 +0,0 @@
package bisync_test
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/rc/jobs"
"github.com/rclone/rclone/fstest"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newRcBisync is newRcBisyncCtx with the rc jobs run in a plain context
func newRcBisync(t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {
return newRcBisyncCtx(context.Background(), t, n)
}
// newRcBisyncCtx creates Path1 with n files, resyncs it to Path2, and returns
// the paths and a function which runs sync/bisync on them as an rc job in
// ctx, whose config stands in for the flags the rcd was started with
func newRcBisyncCtx(ctx context.Context, t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {
if *fstest.RemoteName != "" {
t.Skip("Skipping test on non local remote")
}
dir := t.TempDir()
path1 = filepath.Join(dir, "path1")
path2 = filepath.Join(dir, "path2")
workdir := filepath.Join(dir, "workdir")
require.NoError(t, os.Mkdir(path1, 0o755))
require.NoError(t, os.Mkdir(path2, 0o755))
for i := range n {
require.NoError(t, os.WriteFile(filepath.Join(path1, fmt.Sprintf("file%d.txt", i)), []byte("content"), 0o644))
}
call := rc.Calls.Get("sync/bisync")
require.NotNil(t, call)
runIn := func(ctx context.Context, in rc.Params) error {
in["path1"] = path1
in["path2"] = path2
if _, ok := in["workdir"]; !ok {
in["workdir"] = workdir
}
_, _, err := jobs.NewJob(ctx, call.Fn, in)
return err
}
require.NoError(t, runIn(context.Background(), rc.Params{"resync": true}))
return path1, path2, func(in rc.Params) error { return runIn(ctx, in) }
}
func TestRcDryRun(t *testing.T) {
for _, test := range []struct {
name string
in rc.Params
dryRun bool
}{
{"dryRun", rc.Params{"dryRun": true}, true},
{"dry_run", rc.Params{"dry_run": true}, true},
{"_config", rc.Params{"_config": rc.Params{"DryRun": true}}, true},
{"dry_run and dryRun false", rc.Params{"dry_run": true, "dryRun": false}, true},
{"_config and dryRun false", rc.Params{"_config": rc.Params{"DryRun": true}, "dryRun": false}, true},
{"dryRun false", rc.Params{"dryRun": false}, false},
} {
t.Run(test.name, func(t *testing.T) {
path1, path2, run := newRcBisync(t, 1)
require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))
require.NoError(t, run(test.in))
if test.dryRun {
assert.NoFileExists(t, filepath.Join(path2, "new.txt"))
} else {
assert.FileExists(t, filepath.Join(path2, "new.txt"))
}
})
}
}
func TestRcMaxDelete(t *testing.T) {
for _, test := range []struct {
name string
in rc.Params
wantErr string
}{
{"default", rc.Params{}, ""},
{"maxDelete", rc.Params{"maxDelete": 5}, "too many deletes"},
{"max_delete allows", rc.Params{"max_delete": 20}, ""},
{"max_delete aborts", rc.Params{"max_delete": 5}, "too many deletes"},
{"_config MaxDelete aborts", rc.Params{"_config": rc.Params{"MaxDelete": 5}}, "too many deletes"},
{"max_delete over 100", rc.Params{"max_delete": 500}, "--max-delete"},
} {
t.Run(test.name, func(t *testing.T) {
path1, path2, run := newRcBisync(t, 10)
require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))
err := run(test.in)
if test.wantErr != "" {
assert.ErrorContains(t, err, test.wantErr)
assert.FileExists(t, filepath.Join(path2, "file0.txt"))
} else {
require.NoError(t, err)
assert.NoFileExists(t, filepath.Join(path2, "file0.txt"))
}
})
}
}
// Test a --max-delete the rcd was started with applies, as on the command line
func TestRcMaxDeleteFromRcd(t *testing.T) {
for _, test := range []struct {
maxDelete int64
wantErr string
}{
{5, "too many deletes"},
{500, "--max-delete"},
} {
t.Run(fmt.Sprint(test.maxDelete), func(t *testing.T) {
ctx, ci := fs.AddConfig(context.Background())
ci.MaxDelete = test.maxDelete
path1, path2, run := newRcBisyncCtx(ctx, t, 10)
require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))
assert.ErrorContains(t, run(rc.Params{}), test.wantErr)
assert.FileExists(t, filepath.Join(path2, "file0.txt"))
})
}
}
func TestRcInvalidParams(t *testing.T) {
for _, test := range []struct {
key string
value any
}{
{"force", "yes"},
{"maxDeleteRenamesAware", "yes"},
{"filtersFile", []string{"filters.txt"}},
{"checkSync", 1},
{"conflictResolve", "bogus"},
{"maxLock", "soon"},
} {
t.Run(test.key, func(t *testing.T) {
path1, path2, run := newRcBisync(t, 1)
require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))
err := run(rc.Params{test.key: test.value})
assert.Error(t, err)
assert.NoFileExists(t, filepath.Join(path2, "new.txt"))
})
}
}
func TestRcBackupDir(t *testing.T) {
for _, test := range []struct {
key string
changed int // the path whose change overwrites the file on the other path
}{
{"backupDir1", 2},
{"backupdir1", 2},
{"backupDir2", 1},
{"backupdir2", 1},
} {
t.Run(test.key, func(t *testing.T) {
path1, path2, run := newRcBisync(t, 2)
backupDir := filepath.Join(filepath.Dir(path1), "backup")
changedPath := map[int]string{1: path1, 2: path2}[test.changed]
require.NoError(t, os.WriteFile(filepath.Join(changedPath, "file0.txt"), []byte("changed"), 0o644))
require.NoError(t, run(rc.Params{test.key: backupDir}))
assert.FileExists(t, filepath.Join(backupDir, "file0.txt"))
})
}
}
func TestRcEnums(t *testing.T) {
read := func(t *testing.T, path string) string {
b, err := os.ReadFile(filepath.Join(path, "file0.txt"))
require.NoError(t, err)
return string(b)
}
// changeBoth makes file0.txt differ on both paths, with Path2 an hour newer
changeBoth := func(t *testing.T, path1, path2 string) {
now := time.Now()
for path, content := range map[string]string{path1: "path1", path2: "path2 wins"} {
file := filepath.Join(path, "file0.txt")
require.NoError(t, os.WriteFile(file, []byte(content), 0o644))
mtime := now
if path == path1 {
mtime = now.Add(-time.Hour)
}
require.NoError(t, os.Chtimes(file, mtime, mtime))
}
}
names := func(t *testing.T, path string) (names []string) {
entries, err := os.ReadDir(path)
require.NoError(t, err)
for _, e := range entries {
names = append(names, e.Name())
}
return names
}
t.Run("conflictResolve and conflictLoser", func(t *testing.T) {
path1, path2, run := newRcBisync(t, 2)
changeBoth(t, path1, path2)
require.NoError(t, run(rc.Params{"conflictResolve": "path2", "conflictLoser": "delete"}))
for _, path := range []string{path1, path2} {
assert.Equal(t, []string{"file0.txt", "file1.txt"}, names(t, path))
assert.Equal(t, "path2 wins", read(t, path))
}
})
t.Run("conflict defaults", func(t *testing.T) {
path1, path2, run := newRcBisync(t, 2)
changeBoth(t, path1, path2)
require.NoError(t, run(rc.Params{}))
assert.NotContains(t, names(t, path1), "file0.txt", "with no winner both versions are renamed")
})
for _, test := range []struct {
name string
in rc.Params
want string
}{
{"resyncMode default", rc.Params{"resync": true}, "path1"},
{"resyncMode path2", rc.Params{"resync": true, "resyncMode": "path2"}, "path2 wins"},
} {
t.Run(test.name, func(t *testing.T) {
path1, path2, run := newRcBisync(t, 2)
changeBoth(t, path1, path2)
require.NoError(t, run(test.in))
assert.Equal(t, test.want, read(t, path1))
assert.Equal(t, test.want, read(t, path2))
})
}
t.Run("checkSync only", func(t *testing.T) {
path1, path2, run := newRcBisync(t, 1)
require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))
require.NoError(t, run(rc.Params{"checkSync": "only"}))
assert.NoFileExists(t, filepath.Join(path2, "new.txt"))
})
}
// rcParamName returns the rc parameter name for a flag, as help.go does for rc.md
func rcParamName(flag string) string {
words := strings.Split(flag, "-")
for i := 1; i < len(words); i++ {
words[i] = strings.ToUpper(words[i][:1]) + words[i][1:]
}
return strings.Join(words, "")
}
// Test every bisync flag can be set through the rc
func TestRcFlagParity(t *testing.T) {
bisyncCmd, _, err := cmd.Root.Find([]string{"bisync"})
require.NoError(t, err)
bisyncCmd.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Hidden {
return
}
key := rcParamName(flag.Name)
t.Run(key, func(t *testing.T) {
_, _, run := newRcBisync(t, 1)
// no rc getter accepts a map, so this only fails if rcBisync reads the param
err := run(rc.Params{key: rc.Params{"wrong": "type"}})
assert.ErrorContains(t, err, key, "no rc parameter for --%s", flag.Name)
})
})
}
File diff suppressed because it is too large. Load diff
@@ -1 +0,0 @@
"file1.txt.conflict1"
Loaded 100 of 373 files, more files were not shown because too many files have changed in this diff. Show more