Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
687d264b68 | ||
|
|
4158f63d2f | ||
|
|
c47832ad34 | ||
|
|
60e96416df | ||
|
|
c841c3bba3 | ||
|
|
79fbc0842f | ||
|
|
22859b7e69 | ||
|
|
925fb4fb21 | ||
|
|
96f298bdec | ||
|
|
60fb55dc6a | ||
|
|
181ef190ce | ||
|
|
ad8cd41c49 | ||
|
|
5dae3adbf5 | ||
|
|
6507e13d5a | ||
|
|
e4913c9e91 | ||
|
|
28bf49d66f | ||
|
|
17b0c03338 | ||
|
|
a7ab39d3d1 | ||
|
|
7a594b09c6 | ||
|
|
f376c64784 | ||
|
|
afeb26ae74 | ||
|
|
c6af0b57c2 | ||
|
|
90595f34f2 | ||
|
|
b3cf0444a2 | ||
|
|
11ff6e49bb | ||
|
|
bcef98db3f | ||
|
|
f18aacf2c7 | ||
|
|
ab1f458013 | ||
|
|
7c1dfd99f3 | ||
|
|
57842c5ee4 | ||
|
|
739403963a | ||
|
|
1bec2133c5 | ||
|
|
64de81e6a0 | ||
|
|
0b187f3d7a | ||
|
|
0e5100203c | ||
|
|
ba3d34b9f8 | ||
|
|
77e6390f5b | ||
|
|
f190b34d95 | ||
|
|
49e7016d12 | ||
|
|
6240cbb694 | ||
|
|
66bc465d1a | ||
|
|
b576cbdf40 | ||
|
|
66dba8239f | ||
|
|
92ef010fd2 | ||
|
|
feb0664b64 | ||
|
|
61427e9729 | ||
|
|
3d1fd51f61 | ||
|
|
6c44400bf7 | ||
|
|
041b766428 | ||
|
|
3c505b1e99 | ||
|
|
6317faccc0 | ||
|
|
5a490a31c5 | ||
|
|
b8470e4cac | ||
|
|
e7ae39f42d | ||
|
|
43f107e049 | ||
|
|
748eaf9b28 | ||
|
|
11dd394670 | ||
|
|
21483522a6 | ||
|
|
f92262d48c | ||
|
|
ad87bf6c93 | ||
|
|
f1c06a047e | ||
|
|
d6fb148d9b | ||
|
|
930de88f69 | ||
|
|
fad0359e6b | ||
|
|
c14e507609 | ||
|
|
0720a194cf | ||
|
|
194d22ce71 | ||
|
|
7995d87cd5 | ||
|
|
2d261879dd | ||
|
|
2e0718167f | ||
|
|
9718ed3fc2 | ||
|
|
645e1ea5be | ||
|
|
40dab7ecdd | ||
|
|
43a2a92950 | ||
|
|
62f8f944e2 | ||
|
|
841ff2d418 | ||
|
|
33ab81ce6d | ||
|
|
632ff74375 | ||
|
|
39b926a2ce | ||
|
|
31f4c78c5e | ||
|
|
fcb317e226 | ||
|
|
82a4a63b92 | ||
|
|
d407018995 | ||
|
|
b5f0ea9b7b | ||
|
|
d166eaf635 | ||
|
|
62a1df7271 | ||
|
|
e171f996d6 | ||
|
|
2bf9e0d20e | ||
|
|
afb1daa8fa | ||
|
|
5775f0430b | ||
|
|
d0f00a22d5 | ||
|
|
ad7e0c64af | ||
|
|
674d7a8a70 | ||
|
|
1274301521 | ||
|
|
840755543f | ||
|
|
2472718b7c | ||
|
|
a4f1c1558e | ||
|
|
76d5c49ae6 | ||
|
|
9fec073ab7 | ||
|
|
a9c4548d92 | ||
|
|
d038b8b1c9 | ||
|
|
3ceea42329 | ||
|
|
6beb4a5bd2 | ||
|
|
7d70921bf5 | ||
|
|
76c6992260 | ||
|
|
f1297d6298 | ||
|
|
9a112f40c1 | ||
|
|
a3dcf7f0ad | ||
|
|
3084177049 | ||
|
|
e4b778b5e3 | ||
|
|
ee363b1826 | ||
|
|
e7f9c7bc10 | ||
|
|
10eb7b5b10 | ||
|
|
e0faeeb5d6 | ||
|
|
dcf15b5f9c | ||
|
|
c93cc1e1ae | ||
|
|
21c98ff5ff | ||
|
|
5fbc677ed4 | ||
|
|
00347ecf36 |
No files matched your search
+12
-11
@@ -32,12 +32,12 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.5'
|
||||
go: '~1.26.6'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +48,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.5'
|
||||
go: '~1.26.6'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.26.5'
|
||||
go: '~1.26.6'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +64,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.26.5'
|
||||
go: '~1.26.6'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.26.5'
|
||||
go: '~1.26.6'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,14 +81,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.5'
|
||||
go: '~1.26.6'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.25
|
||||
- job_name: go1.26
|
||||
os: ubuntu-latest
|
||||
go: '~1.25.12'
|
||||
go: '~1.26.6'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -179,6 +179,7 @@ jobs:
|
||||
unset HOMEBREW_NO_INSTALL_FROM_API
|
||||
brew untap --force homebrew/core
|
||||
brew untap --force homebrew/cask
|
||||
brew untap --force aws/tap
|
||||
brew update
|
||||
brew install --cask macfuse
|
||||
brew install git-annex git-annex-remote-rclone
|
||||
@@ -283,7 +284,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.26.5'
|
||||
go-version: '~1.26.6'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
@@ -398,7 +399,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.26.5'
|
||||
go-version: '~1.26.6'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
@@ -46,6 +46,11 @@ linters:
|
||||
- -ST1022
|
||||
# Disable quickfix checks
|
||||
- -QF*
|
||||
# Disabled because its dataflow analysis (new in staticcheck
|
||||
# 0.8.0) makes linting large packages more than 10x slower,
|
||||
# which takes the CI lint job past its timeout, and it
|
||||
# produces false positives (eg on operations.Delete).
|
||||
- -SA4023
|
||||
gocritic:
|
||||
# With gocritic there are different settings, but since enabled-checks
|
||||
# and disabled-checks cannot both be set, for full customization the
|
||||
|
||||
@@ -630,10 +630,37 @@ remote or an fs.
|
||||
- **Remember** we have >50 backends to maintain so keeping them as similar as
|
||||
possible to each other is a high priority!
|
||||
|
||||
### Managing memory
|
||||
|
||||
Use rclone's memory pool for upload buffers rather than allocating them
|
||||
yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
|
||||
rclone's memory usage smaller.
|
||||
|
||||
- If the provider supports multipart or parallel chunk uploads, implement
|
||||
[fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
|
||||
and drive it with
|
||||
[multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
|
||||
This gives you pooled chunk buffers, concurrency, retries and
|
||||
`--multi-thread-streams` support for free. See s3/b2 backends for examples.
|
||||
- If you must buffer data yourself take the buffer from the global pool with
|
||||
[multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
|
||||
and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
|
||||
be rewound for a retry.
|
||||
- When using a pooled buffer `Close()` it on all paths so its pages go back to
|
||||
the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
|
||||
attempt, set the request's `ContentLength` explicitly, and if the buffer
|
||||
reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
|
||||
in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
|
||||
so the transport can't close it and free its pages before the retry.
|
||||
|
||||
### Unit tests
|
||||
|
||||
- Create a config entry called `TestRemote` for the unit tests to use
|
||||
- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
|
||||
- If your backend does chunked uploads, set `ChunkedUpload` in the
|
||||
`fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
|
||||
on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
|
||||
`FsPutRetry` tests can run the chunked path with small chunks.
|
||||
- Make sure all tests pass with `go test -v`
|
||||
|
||||
### Integration tests
|
||||
|
||||
Generated
+2233
-1480
File diff suppressed because it is too large.
Load diff
Generated
+1489
-813
File diff suppressed because it is too large.
Load diff
@@ -93,11 +93,15 @@ test: rclone test_all
|
||||
@echo "Written logs in test_all.log"
|
||||
|
||||
# Quick test
|
||||
#
|
||||
# The timeout is raised above the go test default of 10m as the
|
||||
# cmd/gitannex end to end tests can take longer than that on slow CI
|
||||
# runners.
|
||||
quicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
|
||||
|
||||
racequicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
|
||||
|
||||
compiletest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
|
||||
|
||||
@@ -185,8 +185,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
foundArchive := subArchive(remote)
|
||||
if foundArchive != nil {
|
||||
fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
|
||||
// Archive path
|
||||
foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
|
||||
// Archive path, in canonical form so that it compares equal
|
||||
// to the cleaned entry names inside the archive
|
||||
foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
|
||||
if foundArchive.root == "." {
|
||||
foundArchive.root = ""
|
||||
}
|
||||
// Path to the archive
|
||||
archiveRemote := remote[:len(foundArchive.remote)]
|
||||
// Remote is archive leaf name
|
||||
@@ -491,24 +495,50 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entries, err = subFs.List(ctx, dir)
|
||||
subEntries, err := subFs.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, entry := range entries {
|
||||
entries = subEntries[:0]
|
||||
skipped := 0
|
||||
for _, entry := range subEntries {
|
||||
remote := entry.Remote()
|
||||
// Only pass on direct children of dir - anything else
|
||||
// could escape the archive's namespace
|
||||
if !isDirectChild(dir, remote) {
|
||||
fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
// Can only unarchive files
|
||||
if o, ok := entry.(fs.Object); ok {
|
||||
remote := o.Remote()
|
||||
archive := f.findArchive(remote)
|
||||
if archive != nil {
|
||||
// Overwrite entry with directory
|
||||
entries[i] = fs.NewDir(remote, o.ModTime(ctx))
|
||||
entry = fs.NewDir(remote, o.ModTime(ctx))
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// isDirectChild reports whether remote names an entry directly in dir
|
||||
// ("" being the root), with no ".." or other components in between.
|
||||
func isDirectChild(dir, remote string) bool {
|
||||
if remote == "" || remote == "." || remote == ".." || strings.HasSuffix(remote, "/") {
|
||||
return false
|
||||
}
|
||||
parent := path.Dir(remote)
|
||||
if parent == "." || parent == "/" {
|
||||
parent = ""
|
||||
}
|
||||
return parent == dir && path.Clean(remote) == remote
|
||||
}
|
||||
|
||||
// NewObject creates a new remote archive file object
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
@@ -526,6 +556,11 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Don't trust the archiver to have returned the object asked for
|
||||
if o.Remote() != remote {
|
||||
fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
@@ -277,3 +279,103 @@ func TestArchiveSquashfsIssue9004(t *testing.T) {
|
||||
assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveUncleanRoot checks that a path into an archive which isn't
|
||||
// in canonical form (with "./" or doubled slashes) still finds its
|
||||
// directory.
|
||||
func TestArchiveUncleanRoot(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create("sub/dir/a.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
zipPath := filepath.Join(t.TempDir(), "test.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
|
||||
for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
|
||||
t.Run(root, func(t *testing.T) {
|
||||
f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
|
||||
require.NoError(t, err)
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "a.txt", entries[0].Remote())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// escapingObject is an object whose remote is not where it was asked for.
|
||||
type escapingObject struct {
|
||||
fs.Object
|
||||
remote string
|
||||
}
|
||||
|
||||
func (o *escapingObject) Remote() string { return o.remote }
|
||||
func (o *escapingObject) String() string { return o.remote }
|
||||
|
||||
// escapingFs stands in for a badly behaved archiver which exposes entry
|
||||
// names outside the directory being listed.
|
||||
type escapingFs struct {
|
||||
fs.Fs
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (f *escapingFs) Name() string { return "escaping" }
|
||||
func (f *escapingFs) Root() string { return "" }
|
||||
func (f *escapingFs) String() string { return "escaping" }
|
||||
func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
|
||||
|
||||
func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return fs.DirEntries{
|
||||
&escapingObject{remote: path.Join(dir, "good.txt")},
|
||||
fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
&escapingObject{remote: path.Join(dir, "../escape.txt")},
|
||||
&escapingObject{remote: "../../escape.txt"},
|
||||
&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
|
||||
fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
return &escapingObject{remote: "../escape.txt"}, nil
|
||||
}
|
||||
|
||||
// TestArchiveEscapingArchiver checks that the archive backend does not
|
||||
// pass on entries from an archiver which escape the directory being
|
||||
// listed, whatever the archiver does.
|
||||
func TestArchiveEscapingArchiver(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
return &escapingFs{prefix: prefix}, nil
|
||||
},
|
||||
Extension: ".escaping",
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
|
||||
f, err := cache.Get(ctx, ":archive:"+dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Archives are discovered when their parent directory is listed
|
||||
_, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "test.escaping")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
|
||||
|
||||
_, err = f.NewObject(ctx, "test.escaping/file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
@@ -8,6 +8,11 @@ import (
|
||||
)
|
||||
|
||||
// Archiver describes an archive package
|
||||
//
|
||||
// Entry names inside an archive are attacker controlled. An archiver
|
||||
// must not expose an entry whose remote escapes the archive's own
|
||||
// namespace (for example one with a ".." component), typically by
|
||||
// validating names with lib/sanitize.
|
||||
type Archiver struct {
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
@@ -237,8 +238,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
skipped := 0
|
||||
for _, item := range items {
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
// Skip entry names that aren't safe
|
||||
if err := sanitize.Leaf(item.Name()); err != nil {
|
||||
fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
var entry fs.DirEntry
|
||||
if item.IsDir() {
|
||||
remote := f.fromNative(nativeDir, item.Name())
|
||||
@@ -260,6 +268,12 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
// Log once per listing rather than per entry so a crafted image
|
||||
// can't flood the log
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
fscache "github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A backslash is an ordinary character in a file name on the systems
|
||||
// squashfs images are made on, so an entry containing one must be
|
||||
// listed and openable like any other file.
|
||||
func TestBackslashInName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localFs, err := fscache.Get(ctx, "testdata")
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, "backslash.sqfs", "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, `back\slash.txt`, entries[0].Remote())
|
||||
|
||||
o, err := f.NewObject(ctx, `back\slash.txt`)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(6), o.Size())
|
||||
}
|
||||
Binary file not shown.
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
@@ -135,15 +136,20 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
return singleObject, fmt.Errorf("failed to read zip file: %w", err)
|
||||
}
|
||||
dt := dirtree.New()
|
||||
skipped := 0
|
||||
for _, file := range zr.File {
|
||||
remote := strings.Trim(path.Clean(file.Name), "/")
|
||||
if remote == "." {
|
||||
remote = ""
|
||||
// Skip entries whose name escapes the archive's own namespace
|
||||
remote, err := sanitize.Path(file.Name)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root
|
||||
if !strings.HasPrefix(remote, f.root) {
|
||||
// Ignore all files outside the root, requiring a path
|
||||
// boundary so that root "foo" does not also match a
|
||||
// sibling entry such as "foobar"
|
||||
if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
|
||||
continue
|
||||
}
|
||||
if remote == f.root {
|
||||
@@ -157,6 +163,14 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
dt.AddDir(dir)
|
||||
} else {
|
||||
if remote == "" {
|
||||
// A file at the root itself can only be the
|
||||
// archive member f.root points at - with no root
|
||||
// it is a crafted name for the archive's own
|
||||
// directory, which can't be a file
|
||||
if f.root == "" {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
remote = path.Base(f.root)
|
||||
singleObject = true
|
||||
dt = dirtree.New()
|
||||
@@ -173,6 +187,9 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
package zip
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// writeZip builds a zip file with the given entry names (a name ending
|
||||
// in "/" is a directory) in dir and returns its leaf name.
|
||||
func writeZip(t *testing.T, dir, name string, names ...string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, n := range names {
|
||||
w, err := zw.Create(n)
|
||||
require.NoError(t, err)
|
||||
if !strings.HasSuffix(n, "/") {
|
||||
_, err = w.Write([]byte("data for " + n))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// allRemotes returns every Object remote in the mounted archive's dirtree.
|
||||
func allRemotes(t *testing.T, f fs.Fs) []string {
|
||||
zf, ok := f.(*Fs)
|
||||
require.True(t, ok)
|
||||
var remotes []string
|
||||
for _, entries := range zf.dt {
|
||||
for _, entry := range entries {
|
||||
if o, ok := entry.(*Object); ok {
|
||||
remotes = append(remotes, o.Remote())
|
||||
}
|
||||
}
|
||||
}
|
||||
// dt iteration order is nondeterministic, so sort for stable comparison.
|
||||
sort.Strings(remotes)
|
||||
return remotes
|
||||
}
|
||||
|
||||
// A malicious zip whose entry names escape the archive's own namespace
|
||||
// must not be exposed by the zip backend (Zip Slip).
|
||||
func TestReadZipSlip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "evil.zip",
|
||||
"good.txt",
|
||||
"../../etc/cron.d/evil",
|
||||
"../escape.txt",
|
||||
"sub/../../above.txt",
|
||||
`..\..\evil.exe`,
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
|
||||
// The one benign entry must survive.
|
||||
assert.Contains(t, remotes, "good.txt")
|
||||
|
||||
// No escaping entry may be exposed.
|
||||
for _, remote := range remotes {
|
||||
assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
|
||||
"escaping remote exposed: %q", remote)
|
||||
}
|
||||
assert.Equal(t, []string{"good.txt"}, remotes)
|
||||
}
|
||||
|
||||
// Mounting with a non-empty root must only expose entries within that
|
||||
// root directory, not sibling directories that merely share a name
|
||||
// prefix (root "foo" must not match "foobar").
|
||||
func TestReadZipRootBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip",
|
||||
"foo/a.txt",
|
||||
"foobar/b.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "foo")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
assert.Equal(t, []string{"a.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A file entry whose name refers to the archive's own root (".", "/",
|
||||
// "./" or "") must be skipped, not turn the whole archive into a single
|
||||
// file which hides every other entry.
|
||||
func TestReadZipRootNamedEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "dot.zip",
|
||||
".",
|
||||
"/",
|
||||
"./",
|
||||
"",
|
||||
"good.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
assert.Equal(t, []string{"good.txt"}, remotes)
|
||||
}
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -52,6 +51,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/transferaccounter"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -70,6 +70,7 @@ const (
|
||||
defaultChunkSize = 4 * fs.Mebi
|
||||
defaultAccessTier = blob.AccessTier("") // FIXME AccessTierNone
|
||||
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
|
||||
sasCopyStartSkew = 15 * time.Minute // how far back SAS should start to allow for clock skew
|
||||
)
|
||||
|
||||
// setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
|
||||
@@ -1827,7 +1828,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
|
||||
}
|
||||
|
||||
// Validity window
|
||||
start := time.Now().UTC()
|
||||
start := time.Now().UTC().Add(-sasCopyStartSkew)
|
||||
expiry := start.Add(2 * sasCopyValidity)
|
||||
startStr := start.Format(time.RFC3339)
|
||||
expiryStr := expiry.Format(time.RFC3339)
|
||||
@@ -1876,7 +1877,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
|
||||
// Build the SAS values
|
||||
perms := sas.BlobPermissions{Read: true}
|
||||
container, containerPath := o.split()
|
||||
start := time.Now().UTC()
|
||||
start := time.Now().UTC().Add(-sasCopyStartSkew)
|
||||
expiry := start.Add(sasCopyValidity)
|
||||
vals := sas.BlobSignatureValues{
|
||||
StartTime: start,
|
||||
@@ -2346,6 +2347,16 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
|
||||
} else {
|
||||
size = *info.ContentLength
|
||||
}
|
||||
// On a range request Content-Length is the length of the range, not the object, so take the
|
||||
// object's size from the total in Content-Range instead.
|
||||
if info.ContentRange != nil {
|
||||
contentRange, err := rest.ParseContentRange(*info.ContentRange)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
|
||||
} else if contentRange.Size >= 0 {
|
||||
size = contentRange.Size
|
||||
}
|
||||
}
|
||||
if isDirectoryMarker(size, metadata, o.remote) {
|
||||
return fs.ErrorNotAFile
|
||||
}
|
||||
@@ -2379,22 +2390,6 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
|
||||
// o.accessTier = blob.AccessTier(*info.AccessTier)
|
||||
// }
|
||||
o.setMetadata(metadata)
|
||||
|
||||
// If it was a Range request, the size is wrong, so correct it
|
||||
if info.ContentRange != nil {
|
||||
contentRange := *info.ContentRange
|
||||
slash := strings.IndexRune(contentRange, '/')
|
||||
if slash >= 0 {
|
||||
i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
|
||||
if err == nil {
|
||||
o.size = i
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find length in %q", contentRange)
|
||||
}
|
||||
}
|
||||
o.contentEncoding = info.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
|
||||
@@ -54,6 +54,58 @@ func TestBlockIDCreator(t *testing.T) {
|
||||
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
|
||||
}
|
||||
|
||||
func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
|
||||
pInt64 := func(i int64) *int64 { return &i }
|
||||
pString := func(s string) *string { return &s }
|
||||
newTestObject := func() *Object {
|
||||
return &Object{
|
||||
fs: &Fs{},
|
||||
remote: "test.bin",
|
||||
size: -2, // sentinel to check it gets set
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("WholeBlob", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(12345)
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(12345), o.size)
|
||||
})
|
||||
|
||||
t.Run("RangeRequest", func(t *testing.T) {
|
||||
// On a ranged download Content-Length is the chunk length so the
|
||||
// size must come from the total in Content-Range
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("bytes 67108864-134217727/169721004032")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(169721004032), o.size)
|
||||
})
|
||||
|
||||
t.Run("BadContentRange", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("potato")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(67108864), o.size)
|
||||
})
|
||||
|
||||
t.Run("UnknownLength", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(-1), o.size)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCopySASTimingConstants(t *testing.T) {
|
||||
require.Greater(t, sasCopyStartSkew, time.Duration(0))
|
||||
require.Greater(t, sasCopyValidity, sasCopyStartSkew)
|
||||
}
|
||||
|
||||
func (f *Fs) testFeatures(t *testing.T) {
|
||||
// Check first feature flags are set on this remote
|
||||
enabled := f.Features().SetTier
|
||||
|
||||
@@ -696,11 +696,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
// Measure the size if it is unknown
|
||||
if sizeUnknown {
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
}
|
||||
// Count the bytes read so short reads from the source can be detected
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
|
||||
// Check we have a source MD5 hash...
|
||||
if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
|
||||
@@ -735,6 +733,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if sizeUnknown {
|
||||
// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
|
||||
size = int64(counter.BytesRead())
|
||||
} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
|
||||
// The file was created at the declared size so a short source
|
||||
// leaves the remainder padded with zeroes
|
||||
if isNewlyCreated {
|
||||
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
|
||||
fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
|
||||
}
|
||||
} else if bytesRead < size {
|
||||
// Truncate the file to the bytes actually received so it
|
||||
// is not left padded to the declared size with zeroes
|
||||
if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
|
||||
fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if hashUnknown {
|
||||
md5Hash = hasher.Sum(nil)
|
||||
|
||||
+10
-3
@@ -45,6 +45,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/youmark/pkcs8"
|
||||
)
|
||||
@@ -1691,7 +1692,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// upload does a single non-multipart upload
|
||||
//
|
||||
// This is recommended for less than 50 MiB of content
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
upload := api.UploadFile{
|
||||
Name: o.fs.opt.Enc.FromStandardName(leaf),
|
||||
ContentModifiedAt: api.Time(modTime),
|
||||
@@ -1703,9 +1704,10 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
|
||||
var resp *http.Response
|
||||
var result api.FolderItems
|
||||
counter := readers.NewCountingReader(in)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Body: in,
|
||||
Body: counter,
|
||||
MultipartMetadataName: "attributes",
|
||||
MultipartContentName: "contents",
|
||||
MultipartFileName: upload.Name,
|
||||
@@ -1725,6 +1727,11 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload.
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
if result.TotalCount != 1 || len(result.Entries) != 1 {
|
||||
return fmt.Errorf("failed to upload %v - not sure why", o)
|
||||
}
|
||||
@@ -1754,7 +1761,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Upload with simple or multipart
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
|
||||
err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
} else {
|
||||
err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
}
|
||||
|
||||
@@ -698,6 +698,16 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared -
|
||||
// compressed data is stored under a name containing the
|
||||
// original size so a mismatch makes the object unreadable.
|
||||
if size := src.Size(); size >= 0 && meta.Size != size {
|
||||
if removeErr := dataObject.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
|
||||
|
||||
// meta data upload may fail. in this case we try to remove the original object
|
||||
|
||||
+35
-8
@@ -11,6 +11,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -486,7 +487,12 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
}
|
||||
|
||||
// encryptFileName encrypts a file path
|
||||
func (c *Cipher) encryptFileName(in string) string {
|
||||
//
|
||||
// If stripVersion is set, a version string (as used by --b2-versions)
|
||||
// on the last segment is kept in plain text. Set it for file names
|
||||
// only: directory names must encrypt identically whether passed alone
|
||||
// or as the parent of a file name.
|
||||
func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
// Skip directory name encryption if the user chose to
|
||||
@@ -499,7 +505,7 @@ func (c *Cipher) encryptFileName(in string) string {
|
||||
// of the file name gets encrypted/obfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -529,19 +535,24 @@ func (c *Cipher) EncryptFileName(in string) string {
|
||||
if c.mode == NameEncryptionOff {
|
||||
return in + c.encryptedSuffix
|
||||
}
|
||||
return c.encryptFileName(in)
|
||||
return c.encryptFileName(in, true)
|
||||
}
|
||||
|
||||
// EncryptDirName encrypts a directory path
|
||||
//
|
||||
// Unlike EncryptFileName, a version string on the last segment is
|
||||
// encrypted along with the rest of the name.
|
||||
func (c *Cipher) EncryptDirName(in string) string {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in
|
||||
}
|
||||
return c.encryptFileName(in)
|
||||
return c.encryptFileName(in, false)
|
||||
}
|
||||
|
||||
// decryptFileName decrypts a file path
|
||||
func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
//
|
||||
// stripVersion is as for encryptFileName.
|
||||
func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
var err error
|
||||
@@ -555,7 +566,7 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
// of the file name gets decrypted/deobfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -601,15 +612,31 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
|
||||
// Leave the version string on, if it was there
|
||||
return decrypted, nil
|
||||
}
|
||||
return c.decryptFileName(in)
|
||||
return c.decryptFileName(in, true)
|
||||
}
|
||||
|
||||
// DecryptDirName decrypts a directory path
|
||||
//
|
||||
// Unlike DecryptFileName, a version string on the last segment is
|
||||
// expected to be part of the encrypted name. Directory names created
|
||||
// by rclone before v1.76 had a version-like suffix left in plain
|
||||
// text; these are still decrypted so that they appear in listings,
|
||||
// but they can't be opened or removed until renamed on the
|
||||
// underlying remote.
|
||||
func (c *Cipher) DecryptDirName(in string) (string, error) {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in, nil
|
||||
}
|
||||
return c.decryptFileName(in)
|
||||
out, err := c.decryptFileName(in, false)
|
||||
if err != nil && version.Match(path.Base(in)) {
|
||||
var legacyErr error
|
||||
out, legacyErr = c.decryptFileName(in, true)
|
||||
if legacyErr == nil {
|
||||
fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// NameEncryptionMode returns the encryption mode in use for names
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -682,6 +683,43 @@ func TestNonStandardDecryptDirName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test directories whose name looks like it has a version string -
|
||||
// these encrypt verbatim, so that EncryptDirName agrees with the same
|
||||
// directory encrypted as the parent of a file name, whereas file names
|
||||
// keep the version string in plain text
|
||||
func TestVersionedDirName(t *testing.T) {
|
||||
const dir = "dir-v2001-02-03-040506-123"
|
||||
enc, err := NewNameEncoding("base32")
|
||||
require.NoError(t, err)
|
||||
for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
|
||||
c, err := newCipher(mode, "", "", true, enc)
|
||||
require.NoError(t, err)
|
||||
what := fmt.Sprintf("Testing mode=%v", mode)
|
||||
// Check EncryptDirName matches the parent of an encrypted file name
|
||||
encryptedDir := c.EncryptDirName(dir)
|
||||
encryptedFile := c.EncryptFileName(dir + "/file.txt")
|
||||
assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
|
||||
// Check the encrypted directory name round trips OK
|
||||
decryptedDir, err := c.DecryptDirName(encryptedDir)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// Check a file with the same name keeps its version string in plain text
|
||||
encryptedFile = c.EncryptFileName(dir)
|
||||
assert.NotEqual(t, encryptedDir, encryptedFile, what)
|
||||
assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
|
||||
decryptedFile, err := c.DecryptFileName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedFile, what)
|
||||
// Check a directory with a legacy encrypted name (the file form) still decrypts
|
||||
decryptedDir, err = c.DecryptDirName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// but a name which is bad in both forms is still an error
|
||||
_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
|
||||
assert.Error(t, err, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSize(t *testing.T) {
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
|
||||
for _, test := range []struct {
|
||||
|
||||
@@ -1179,6 +1179,11 @@ func (o *ObjectInfo) Size() int64 {
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
|
||||
// If the data is unchanged then the hash of the source is the
|
||||
// hash of the object which will be uploaded
|
||||
if o.f.opt.NoDataEncryption {
|
||||
return o.ObjectInfo.Hash(ctx, hash)
|
||||
}
|
||||
var srcObj fs.Object
|
||||
var ok bool
|
||||
// Get the underlying object if there is one
|
||||
|
||||
@@ -84,6 +84,10 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
|
||||
|
||||
// Test ObjectInfo.Hash
|
||||
wantHash := md5.Sum(outBuf.Bytes())
|
||||
if f.opt.NoDataEncryption {
|
||||
// If the data isn't encrypted, the hash should be that of the plaintext
|
||||
wantHash = md5.Sum([]byte(contents))
|
||||
}
|
||||
gotHash, err := src.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
|
||||
|
||||
@@ -2766,7 +2766,12 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
}
|
||||
trashedFiles := false
|
||||
if check {
|
||||
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
|
||||
// trashedFiles is only consulted to decide between trashing and hard deleting
|
||||
// below, so when the directory is going to be trashed anyway there is no reason
|
||||
// to enumerate trashed children: let the server filter them out instead of
|
||||
// paging through them. Hard deletes still need to see them (#1040).
|
||||
includeAll := !f.opt.UseTrash || f.opt.TrashedOnly
|
||||
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, includeAll, func(item *drive.File) bool {
|
||||
if !item.Trashed {
|
||||
fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
|
||||
return true
|
||||
|
||||
@@ -19,7 +19,7 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
|
||||
Entries: items,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
complete, err = f.srv.UploadSessionFinishBatchV2(arg)
|
||||
complete, err = f.srv.UploadSessionFinishBatchV2Context(ctx, arg)
|
||||
if retry, err := shouldRetryExclude(ctx, err); !retry {
|
||||
return retry, err
|
||||
}
|
||||
|
||||
+121
-72
@@ -382,20 +382,20 @@ type Options struct {
|
||||
|
||||
// Fs represents a remote dropbox server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
srv files.Client // the connection to the dropbox server
|
||||
svc files.Client // the connection to the dropbox server (unauthorized)
|
||||
sharing sharing.Client // as above, but for generating sharing links
|
||||
users users.Client // as above, but for accessing user information
|
||||
team team.Client // for the Teams API
|
||||
slashRoot string // root with "/" prefix, lowercase
|
||||
slashRootSlash string // root with "/" prefix and postfix, lowercase
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
ns string // The namespace we are using or "" for none
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
srv files.ContextClient // the connection to the dropbox server
|
||||
svc files.ContextClient // the connection to the dropbox server (unauthorized)
|
||||
sharing sharing.ContextClient // as above, but for generating sharing links
|
||||
users users.ContextClient // as above, but for accessing user information
|
||||
team team.ContextClient // for the Teams API
|
||||
slashRoot string // root with "/" prefix
|
||||
slashRootSlash string // root with "/" prefix and postfix
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
ns string // The namespace we are using or "" for none
|
||||
batcher *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
|
||||
exportExts []exportExtension
|
||||
}
|
||||
@@ -577,7 +577,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
|
||||
// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
|
||||
f.team = team.New(cfg)
|
||||
f.team = team.NewContext(cfg)
|
||||
|
||||
if opt.Impersonate != "" {
|
||||
user := team.UserSelectorArg{
|
||||
@@ -588,7 +588,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
members := []*team.UserSelectorArg{&user}
|
||||
args := team.NewMembersGetInfoArgs(members)
|
||||
|
||||
memberIDs, err := f.team.MembersGetInfo(args)
|
||||
memberIDs, err := f.team.MembersGetInfoContext(ctx, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
|
||||
}
|
||||
@@ -603,10 +603,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
cfg.AsAdminID = opt.ImpersonateAdmin
|
||||
}
|
||||
|
||||
f.srv = files.New(cfg)
|
||||
f.svc = files.New(ucfg)
|
||||
f.sharing = sharing.New(cfg)
|
||||
f.users = users.New(cfg)
|
||||
f.srv = files.NewContext(cfg)
|
||||
f.svc = files.NewContext(ucfg)
|
||||
f.sharing = sharing.NewContext(cfg)
|
||||
f.users = users.NewContext(cfg)
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: true,
|
||||
ReadMimeType: false,
|
||||
@@ -673,7 +673,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// If root starts with / then use the actual root
|
||||
var acc *users.FullAccount
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
acc, err = f.users.GetCurrentAccount()
|
||||
acc, err = f.users.GetCurrentAccountContext(ctx)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -693,7 +693,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
// See if the root is actually an object
|
||||
if f.root != "" {
|
||||
_, err = f.getFileMetadata(ctx, f.slashRoot)
|
||||
_, _, err = f.getFileMetadata(ctx, f.slashRoot)
|
||||
if err == nil {
|
||||
newRoot := path.Dir(f.root)
|
||||
if newRoot == "." {
|
||||
@@ -728,15 +728,16 @@ func (f *Fs) setRoot(root string) {
|
||||
}
|
||||
|
||||
type getMetadataResult struct {
|
||||
entry files.IsMetadata
|
||||
notFound bool
|
||||
err error
|
||||
entry files.IsMetadata
|
||||
remoteIsExportPath bool
|
||||
notFound bool
|
||||
err error
|
||||
}
|
||||
|
||||
// getMetadata gets the metadata for a file or directory
|
||||
func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
|
||||
res.err = f.pacer.Call(func() (bool, error) {
|
||||
res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
|
||||
res.entry, res.err = f.srv.GetMetadataContext(ctx, &files.GetMetadataArg{
|
||||
Path: f.opt.Enc.FromStandardPath(objPath),
|
||||
})
|
||||
return shouldRetry(ctx, res.err)
|
||||
@@ -779,6 +780,7 @@ func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportE
|
||||
ch <- getMetadataResult{notFound: true}
|
||||
return
|
||||
}
|
||||
res.remoteIsExportPath = true
|
||||
}
|
||||
|
||||
// Return our real result or error
|
||||
@@ -824,7 +826,7 @@ func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-ch
|
||||
}
|
||||
|
||||
// getFileMetadata gets the metadata for a file
|
||||
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
|
||||
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, bool, error) {
|
||||
var res getMetadataResult
|
||||
|
||||
// Try all possible metadatas
|
||||
@@ -833,7 +835,7 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
|
||||
res = <-ch
|
||||
|
||||
if res.err != nil {
|
||||
return nil, res.err
|
||||
return nil, false, res.err
|
||||
}
|
||||
if !res.notFound {
|
||||
break
|
||||
@@ -841,17 +843,17 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
|
||||
}
|
||||
|
||||
if res.notFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
return nil, false, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
fileInfo, ok := res.entry.(*files.FileMetadata)
|
||||
if !ok {
|
||||
if _, ok = res.entry.(*files.FolderMetadata); ok {
|
||||
return nil, fs.ErrorIsDir
|
||||
return nil, false, fs.ErrorIsDir
|
||||
}
|
||||
return nil, fs.ErrorNotAFile
|
||||
return nil, false, fs.ErrorNotAFile
|
||||
}
|
||||
return fileInfo, nil
|
||||
return fileInfo, res.remoteIsExportPath, nil
|
||||
}
|
||||
|
||||
// getDirMetadata gets the metadata for a directory
|
||||
@@ -910,7 +912,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Limit: 100,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFolders(&arg)
|
||||
res, err = f.sharing.ListFoldersContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -922,7 +924,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFoldersContinue(&arg)
|
||||
res, err = f.sharing.ListFoldersContinueContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -971,7 +973,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
|
||||
SharedFolderId: id,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.sharing.MountFolder(&arg)
|
||||
_, err := f.sharing.MountFolderContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -988,7 +990,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Limit: 100,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFiles(&arg)
|
||||
res, err = f.sharing.ListReceivedFilesContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1000,7 +1002,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFilesContinue(&arg)
|
||||
res, err = f.sharing.ListReceivedFilesContinueContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1008,7 +1010,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
}
|
||||
}
|
||||
for _, entry := range res.Entries {
|
||||
entryPath := entry.Name
|
||||
entryPath := f.opt.Enc.ToStandardName(entry.Name)
|
||||
o := &Object{
|
||||
fs: f,
|
||||
url: entry.PreviewUrl,
|
||||
@@ -1107,7 +1109,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
arg.Path = "" // Specify root folder as empty string
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolder(arg)
|
||||
res, err = f.srv.ListFolderContext(ctx, arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1125,7 +1127,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContinue(&arg)
|
||||
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1162,7 +1164,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
var sfMeta *sharing.SharedFolderMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var apiErr error
|
||||
sfMeta, apiErr = f.sharing.GetFolderMetadata(sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
|
||||
sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
|
||||
return shouldRetry(ctx, apiErr)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1249,7 +1251,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return cErr
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.CreateFolderV2(&arg2)
|
||||
_, err = f.srv.CreateFolderV2Context(ctx, &arg2)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -1277,7 +1279,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
}
|
||||
var res *files.ListFolderResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolder(arg)
|
||||
res, err = f.srv.ListFolderContext(ctx, arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1301,7 +1303,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
|
||||
// remove it
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
|
||||
_, err = f.srv.DeleteV2Context(ctx, &files.DeleteArg{Path: encRoot})
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -1360,7 +1362,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
}
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.CopyV2(&arg)
|
||||
result, err = f.srv.CopyV2Context(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1421,7 +1423,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var err error
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.MoveV2(&arg)
|
||||
result, err = f.srv.MoveV2Context(ctx, &arg)
|
||||
switch e := err.(type) {
|
||||
case files.MoveV2APIError:
|
||||
// There seems to be a bit of eventual consistency here which causes this to
|
||||
@@ -1475,7 +1477,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
|
||||
var linkRes sharing.IsSharedLinkMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -1484,7 +1486,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
fs.Debugf(absPath, "can't create link with expiry, trying without")
|
||||
createArg.Settings.Expires = nil
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
@@ -1498,7 +1500,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
}
|
||||
var listRes *sharing.ListSharedLinksResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
listRes, err = f.sharing.ListSharedLinks(&listArg)
|
||||
listRes, err = f.sharing.ListSharedLinksContext(ctx, &listArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1559,7 +1561,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
},
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.MoveV2(&arg)
|
||||
_, err = f.srv.MoveV2Context(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1573,7 +1575,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
var q *users.SpaceUsage
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
q, err = f.users.GetSpaceUsage()
|
||||
q, err = f.users.GetSpaceUsageContext(ctx)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1660,7 +1662,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
|
||||
arg.Path = ""
|
||||
}
|
||||
|
||||
startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
|
||||
startCursor, err = f.srv.ListFolderGetLatestCursorContext(ctx, arg)
|
||||
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -1670,6 +1672,34 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
|
||||
return startCursor.Cursor, nil
|
||||
}
|
||||
|
||||
// trimPrefixFold returns s with the leading prefix removed, matching
|
||||
// case insensitively rune by rune so folded runes of differing UTF-8
|
||||
// length still match.
|
||||
//
|
||||
// If s is exactly the root that prefix names (prefix without its
|
||||
// trailing "/") it returns "". If prefix doesn't match it returns s
|
||||
// unchanged.
|
||||
func trimPrefixFold(s, prefix string) string {
|
||||
if strings.HasSuffix(prefix, "/") && strings.EqualFold(s, strings.TrimSuffix(prefix, "/")) {
|
||||
return ""
|
||||
}
|
||||
|
||||
offset := 0
|
||||
for prefix != "" {
|
||||
if offset >= len(s) {
|
||||
return s
|
||||
}
|
||||
sRune, sSize := utf8.DecodeRuneInString(s[offset:])
|
||||
prefixRune, prefixSize := utf8.DecodeRuneInString(prefix)
|
||||
if !strings.EqualFold(string(sRune), string(prefixRune)) {
|
||||
return s
|
||||
}
|
||||
offset += sSize
|
||||
prefix = prefix[prefixSize:]
|
||||
}
|
||||
return s[offset:]
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
|
||||
cursor := startCursor
|
||||
var res *files.ListFolderLongpollResult
|
||||
@@ -1693,7 +1723,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
Timeout: timeout,
|
||||
}
|
||||
|
||||
res, err = f.svc.ListFolderLongpoll(&args)
|
||||
res, err = f.svc.ListFolderLongpollContext(ctx, &args)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1716,7 +1746,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
Cursor: cursor,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
changeList, err = f.srv.ListFolderContinue(&arg)
|
||||
changeList, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1729,17 +1759,18 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
switch info := entry.(type) {
|
||||
case *files.FolderMetadata:
|
||||
entryType = fs.EntryDirectory
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
entryPath = info.PathDisplay
|
||||
case *files.FileMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
entryPath = info.PathDisplay
|
||||
case *files.DeletedMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
entryPath = info.PathDisplay
|
||||
default:
|
||||
fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
|
||||
continue
|
||||
}
|
||||
entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
|
||||
|
||||
if entryPath != "" {
|
||||
notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
|
||||
@@ -1851,7 +1882,7 @@ func (o *Object) Size() int64 {
|
||||
return o.bytes
|
||||
}
|
||||
|
||||
func (o *Object) setMetadataForExport(info *files.FileMetadata) {
|
||||
func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPath bool) {
|
||||
o.bytes = -1
|
||||
o.hash = ""
|
||||
|
||||
@@ -1872,8 +1903,10 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata) {
|
||||
o.exportType = exportExportable
|
||||
// get rid of any paper extension, if present
|
||||
o.remote = strings.TrimSuffix(o.remote, paperExtension)
|
||||
// add the export extension
|
||||
o.remote += "." + string(exportExt)
|
||||
if !remoteIsExportPath {
|
||||
// add the export extension
|
||||
o.remote += "." + string(exportExt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1881,19 +1914,23 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata) {
|
||||
//
|
||||
// This isn't a complete set of metadata and has an inaccurate date
|
||||
func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
|
||||
return o.setMetadataFromEntryWithExportPath(info, false)
|
||||
}
|
||||
|
||||
func (o *Object) setMetadataFromEntryWithExportPath(info *files.FileMetadata, remoteIsExportPath bool) error {
|
||||
o.id = info.Id
|
||||
o.bytes = int64(info.Size)
|
||||
o.modTime = time.Time(info.ClientModified)
|
||||
o.hash = info.ContentHash
|
||||
|
||||
if !info.IsDownloadable {
|
||||
o.setMetadataForExport(info)
|
||||
o.setMetadataForExport(info, remoteIsExportPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reads the entry for a file from dropbox
|
||||
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
|
||||
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, bool, error) {
|
||||
return o.fs.getFileMetadata(ctx, o.remotePath())
|
||||
}
|
||||
|
||||
@@ -1903,11 +1940,11 @@ func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
|
||||
if !o.modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
entry, err := o.readEntry(ctx)
|
||||
entry, remoteIsExportPath, err := o.readEntry(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return o.setMetadataFromEntry(entry)
|
||||
return o.setMetadataFromEntryWithExportPath(entry, remoteIsExportPath)
|
||||
}
|
||||
|
||||
// Returns the remote path for the object
|
||||
@@ -1961,7 +1998,7 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
|
||||
arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
|
||||
var exportResult *files.ExportResult
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
exportResult, in, err = o.fs.srv.Export(&arg)
|
||||
exportResult, in, err = o.fs.srv.ExportContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1983,7 +2020,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
Url: o.url,
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
|
||||
_, in, err = o.fs.sharing.GetSharedLinkFileContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2003,7 +2040,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
ExtraHeaders: headers,
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.srv.Download(&arg)
|
||||
_, in, err = o.fs.srv.DownloadContext(ctx, &arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -2027,7 +2064,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
// start upload
|
||||
var res *files.UploadSessionStartResult
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, nil)
|
||||
res, err = o.fs.srv.UploadSessionStartContext(ctx, &files.UploadSessionStartArg{}, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2073,7 +2110,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, chunk)
|
||||
// after session is started, we retry everything
|
||||
if err != nil {
|
||||
// Check for incorrect offset error and retry with new offset
|
||||
@@ -2098,11 +2135,23 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
}
|
||||
}
|
||||
// Don't waste the low level retries if the context has
|
||||
// been cancelled
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
return err != nil, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if size >= 0 {
|
||||
// Check for sources which truncate early
|
||||
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
|
||||
if in.BytesRead() < expected {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
}
|
||||
if appendArg.Close {
|
||||
break
|
||||
}
|
||||
@@ -2132,7 +2181,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
entry, err = o.fs.srv.UploadSessionFinish(args, nil)
|
||||
entry, err = o.fs.srv.UploadSessionFinishContext(ctx, args, nil)
|
||||
if retry, err := shouldRetryExclude(ctx, err); !retry {
|
||||
return retry, err
|
||||
}
|
||||
@@ -2200,7 +2249,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
|
||||
} else {
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
|
||||
entry, err = o.fs.srv.UploadContext(ctx, &files.UploadArg{CommitInfo: *commitInfo}, in)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
@@ -2225,7 +2274,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
return errNotSupportedInSharedMode
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
|
||||
_, err = o.fs.srv.DeleteV2Context(ctx, &files.DeleteArg{
|
||||
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
|
||||
})
|
||||
return shouldRetry(ctx, err)
|
||||
|
||||
@@ -2,17 +2,155 @@ package dropbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type paperMetadataClient struct {
|
||||
files.ContextClient
|
||||
info *files.FileMetadata
|
||||
}
|
||||
|
||||
func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
|
||||
if arg.Path == "document" {
|
||||
return c.info, nil
|
||||
}
|
||||
return nil, files.GetMetadataAPIError{
|
||||
APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
|
||||
EndpointError: &files.GetMetadataError{
|
||||
Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
|
||||
Path: &files.LookupError{
|
||||
Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetMetadataCancellation(t *testing.T) {
|
||||
requestStarted := make(chan struct{})
|
||||
releaseRequest := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(requestStarted)
|
||||
<-releaseRequest
|
||||
}))
|
||||
defer server.Close()
|
||||
defer close(releaseRequest)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
f := &Fs{
|
||||
srv: files.NewContext(dropbox.Config{
|
||||
Client: server.Client(),
|
||||
URLGenerator: func(hostType string, namespace string, route string) string {
|
||||
return server.URL
|
||||
},
|
||||
}),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
|
||||
}
|
||||
|
||||
result := make(chan getMetadataResult, 1)
|
||||
go func() {
|
||||
result <- f.getMetadata(ctx, "/file")
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-requestStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dropbox request did not start")
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case res := <-result:
|
||||
require.ErrorIs(t, res.err, context.Canceled)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dropbox request did not observe cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
type changeNotifyClient struct {
|
||||
files.ContextClient
|
||||
entries []files.IsMetadata
|
||||
}
|
||||
|
||||
func (c changeNotifyClient) ListFolderLongpollContext(context.Context, *files.ListFolderLongpollArg) (*files.ListFolderLongpollResult, error) {
|
||||
return files.NewListFolderLongpollResult(true), nil
|
||||
}
|
||||
|
||||
func (c changeNotifyClient) ListFolderContinueContext(context.Context, *files.ListFolderContinueArg) (*files.ListFolderResult, error) {
|
||||
return files.NewListFolderResult(c.entries, "next", false), nil
|
||||
}
|
||||
|
||||
func TestTrimPrefixFold(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path string
|
||||
prefix string
|
||||
want string
|
||||
}{
|
||||
{name: "exact casing", path: "/Docs/Sub", prefix: "/Docs/", want: "Sub"},
|
||||
{name: "different casing", path: "/docs/Sub", prefix: "/Docs/", want: "Sub"},
|
||||
{name: "different UTF-8 lengths", path: "/K/Sub", prefix: "/K/", want: "Sub"},
|
||||
{name: "exact root", path: "/docs", prefix: "/Docs/", want: ""},
|
||||
{name: "sibling boundary", path: "/Docs2/File", prefix: "/Docs/", want: "/Docs2/File"},
|
||||
{name: "root remote", path: "/File", prefix: "/", want: "File"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, trimPrefixFold(test.path, test.prefix))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeNotifyTrimsRootCaseInsensitively(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := changeNotifyClient{entries: []files.IsMetadata{
|
||||
&files.FolderMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub"}},
|
||||
&files.FileMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub/File.TXT"}},
|
||||
&files.DeletedMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Gone.md"}},
|
||||
}}
|
||||
f := &Fs{
|
||||
ci: fs.GetConfig(ctx),
|
||||
srv: client,
|
||||
svc: client,
|
||||
slashRootSlash: "/Docs/",
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
|
||||
}
|
||||
|
||||
type notification struct {
|
||||
path string
|
||||
entryType fs.EntryType
|
||||
}
|
||||
var notifications []notification
|
||||
cursor, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
|
||||
notifications = append(notifications, notification{path: path, entryType: entryType})
|
||||
}, "start")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "next", cursor)
|
||||
assert.Equal(t, []notification{
|
||||
{path: "Sub", entryType: fs.EntryDirectory},
|
||||
{path: "Sub/File.TXT", entryType: fs.EntryObject},
|
||||
{path: "Gone.md", entryType: fs.EntryObject},
|
||||
}, notifications)
|
||||
}
|
||||
|
||||
func TestInternalCheckPathLength(t *testing.T) {
|
||||
rep := func(n int, r rune) (out string) {
|
||||
rs := make([]rune, n)
|
||||
@@ -50,6 +188,155 @@ func TestInternalCheckPathLength(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaperExportRemote(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
info := &files.FileMetadata{
|
||||
ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
|
||||
}
|
||||
f := &Fs{
|
||||
exportExts: []exportExtension{"md"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
|
||||
srv: paperMetadataClient{info: info},
|
||||
}
|
||||
|
||||
direct, err := f.NewObject(ctx, "document.md")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md", direct.Remote())
|
||||
|
||||
listed, err := f.newObjectWithInfo(ctx, "document.md", info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md.md", listed.Remote())
|
||||
|
||||
legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md", legacy.Remote())
|
||||
}
|
||||
|
||||
// uploadSessionClient is a mock files.ContextClient which records the
|
||||
// chunked upload calls made to it
|
||||
type uploadSessionClient struct {
|
||||
files.ContextClient
|
||||
appends int // number of UploadSessionAppendV2Context calls
|
||||
maxAppends int // fail the append after this many calls to stop runaway loops
|
||||
bytesWritten int64 // bytes received by UploadSessionAppendV2Context
|
||||
finishCalled bool // set if UploadSessionFinishContext was called
|
||||
appended func() // if set, called after each successful append
|
||||
}
|
||||
|
||||
var errTooManyAppends = errors.New("too many appends - upload looping?")
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
|
||||
return &files.UploadSessionStartResult{SessionId: "session"}, nil
|
||||
}
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
|
||||
// the real client fails the request if the context is cancelled
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
c.appends++
|
||||
if c.appends > c.maxAppends {
|
||||
return errTooManyAppends
|
||||
}
|
||||
n, err := io.Copy(io.Discard, content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.bytesWritten += n
|
||||
if c.appended != nil {
|
||||
c.appended()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
|
||||
c.finishCalled = true
|
||||
return &files.FileMetadata{}, nil
|
||||
}
|
||||
|
||||
// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
|
||||
func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
|
||||
srv: srv,
|
||||
}
|
||||
f.opt.ChunkSize = chunkSize
|
||||
batcherOptions := defaultBatcherOptions
|
||||
batcherOptions.Mode = "off"
|
||||
var err error
|
||||
f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
|
||||
// endlessReader supplies bytes forever
|
||||
type endlessReader struct{}
|
||||
|
||||
func (endlessReader) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = 'x'
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestUploadChunkedEarlyEOF(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("MultiChunk", func(t *testing.T) {
|
||||
// The declared size spans 4 chunks but the source ends after 1.5
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
||||
})
|
||||
|
||||
t.Run("SingleChunk", func(t *testing.T) {
|
||||
// The declared size fits in one chunk but the source ends early
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 500)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
||||
})
|
||||
|
||||
t.Run("Complete", func(t *testing.T) {
|
||||
// A source which supplies exactly the declared size uploads OK
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.True(t, client.finishCalled)
|
||||
assert.Equal(t, int64(250), client.bytesWritten)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUploadChunkedCancel(t *testing.T) {
|
||||
// Cancelling the context must stop the upload even though every
|
||||
// append is succeeding
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
client.appended = func() {
|
||||
if client.appends == 2 {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, endlessReader{}, files.NewCommitInfo("/test.bin"), -1)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.False(t, client.finishCalled)
|
||||
}
|
||||
|
||||
func (f *Fs) importPaperForTest(t *testing.T) {
|
||||
content := `# test doc
|
||||
|
||||
@@ -64,7 +351,7 @@ Lorem ipsum __dolor__ sit amet
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
reader := strings.NewReader(content)
|
||||
_, err = f.srv.PaperCreate(&arg, reader)
|
||||
_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
|
||||
return shouldRetry(context.Background(), err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -100,3 +387,91 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// newSharingTestFs builds a minimal *Fs whose sharing client talks to a
|
||||
// local mock server instead of the real Dropbox API.
|
||||
func newSharingTestFs(t *testing.T, handler http.HandlerFunc) *Fs {
|
||||
t.Helper()
|
||||
ts := httptest.NewServer(handler)
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
cfg := dropbox.Config{
|
||||
Client: &http.Client{Transport: http.DefaultTransport},
|
||||
URLGenerator: func(hostType, namespace, route string) string {
|
||||
return fmt.Sprintf("%s/2/%s/%s", ts.URL, namespace, route)
|
||||
},
|
||||
}
|
||||
|
||||
return &Fs{
|
||||
opt: Options{
|
||||
Enc: encoder.Base |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeDel |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8,
|
||||
},
|
||||
sharing: sharing.NewContext(cfg),
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond))),
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseDBXTime(t *testing.T, s string) *dropbox.DBXTime {
|
||||
t.Helper()
|
||||
tm, err := time.Parse(time.RFC3339, s)
|
||||
require.NoError(t, err)
|
||||
return (*dropbox.DBXTime)(&tm)
|
||||
}
|
||||
|
||||
func receivedFilesHandler(t *testing.T, rawName string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.URL.Path, "list_received_files") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
resp := sharing.ListFilesResult{
|
||||
Entries: []*sharing.SharedFileMetadata{{
|
||||
Id: "id:123",
|
||||
Name: rawName,
|
||||
PreviewUrl: "https://dropbox.com/preview/123",
|
||||
Policy: &sharing.FolderPolicy{},
|
||||
TimeInvited: mustParseDBXTime(t, "2024-01-01T00:00:00Z"),
|
||||
}},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListReceivedFilesDecodesName confirms received-file names are decoded
|
||||
// via Enc.ToStandardName before being returned, matching listSharedFolders'
|
||||
// handling of shared-folder names.
|
||||
func TestListReceivedFilesDecodesName(t *testing.T) {
|
||||
const rawName = "report.txt␠"
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
|
||||
|
||||
wantName := f.opt.Enc.ToStandardName(rawName)
|
||||
require.NotEqual(t, rawName, wantName)
|
||||
|
||||
var gotNames []string
|
||||
err := f.listReceivedFiles(context.Background(), func(entry fs.DirEntry) error {
|
||||
gotNames = append(gotNames, entry.Remote())
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, gotNames, 1)
|
||||
assert.Equal(t, wantName, gotNames[0])
|
||||
}
|
||||
|
||||
// TestFindSharedFileResolvesDecodedName confirms findSharedFile can look up
|
||||
// a received file by its standard, decoded rclone-visible name.
|
||||
func TestFindSharedFileResolvesDecodedName(t *testing.T) {
|
||||
const rawName = "report.txt␠"
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
|
||||
|
||||
encodedName := f.opt.Enc.ToStandardName(rawName)
|
||||
require.NotEqual(t, rawName, encodedName)
|
||||
|
||||
o, err := f.findSharedFile(context.Background(), encodedName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, encodedName, o.remote)
|
||||
}
|
||||
@@ -25,7 +25,9 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
}
|
||||
|
||||
if dir != "" {
|
||||
_ = f.Mkdir(ctx, dir)
|
||||
if _, err := f.createFolder(ctx, dir); err != nil {
|
||||
return fmt.Errorf("failed to create multipart folder: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
folder := strings.Trim(dir, "/")
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -141,6 +142,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
// Update updates the object with new data
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
size := src.Size()
|
||||
counter := readers.NewCountingReader(in)
|
||||
in = counter
|
||||
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err := o.fs.uploadFile(ctx, in, o.remote)
|
||||
@@ -155,6 +158,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
if removeErr := o.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
o.size = size
|
||||
o.modTime = src.ModTime(ctx)
|
||||
return nil
|
||||
|
||||
+51
-1
@@ -57,7 +57,13 @@ The input format is comma separated list of key,value pairs. Standard
|
||||
|
||||
For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
|
||||
|
||||
You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.`,
|
||||
You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
|
||||
|
||||
The headers are only sent to the host in the configured URL. If the
|
||||
server redirects to another host (including a subdomain or a different
|
||||
port) the headers are not sent to it, or to any further hop in that
|
||||
redirect chain. When headers are set, a redirect from https to http is
|
||||
refused as it would send them in cleartext.`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -283,6 +289,11 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
|
||||
}
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
// Without configured headers keep the default policy which
|
||||
// public mirrors that redirect from https to http rely on.
|
||||
if len(opt.Headers) > 0 {
|
||||
client.CheckRedirect = checkRedirect(opt)
|
||||
}
|
||||
|
||||
endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
|
||||
fs.Debugf(nil, "Root: %s", endpoint)
|
||||
@@ -510,6 +521,45 @@ func addHeaders(req *http.Request, opt *Options) {
|
||||
}
|
||||
}
|
||||
|
||||
// checkRedirect returns an http.Client.CheckRedirect function which
|
||||
// follows redirects but refuses an HTTPS to HTTP downgrade with
|
||||
// rest.ErrHTTPSDowngrade and strips the configured headers when the
|
||||
// redirect chain has left the originally requested host at any point.
|
||||
func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
if err := rest.RefuseHTTPSDowngradeRedirectFn(req, via); err != nil {
|
||||
if errors.Is(err, rest.ErrHTTPSDowngrade) {
|
||||
err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if redirectLeavesHost(req, via) {
|
||||
for i := 0; i < len(opt.Headers); i += 2 {
|
||||
req.Header.Del(opt.Headers[i])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// redirectLeavesHost reports whether any hop in the redirect chain
|
||||
// via plus the pending request req is to a different host from the
|
||||
// original request via[0].
|
||||
//
|
||||
// net/http copies the headers afresh from the original request for
|
||||
// every hop, so once the chain has visited another host the headers
|
||||
// must be stripped from every subsequent hop, even one back to the
|
||||
// original host, as the other host chose the URL.
|
||||
func redirectLeavesHost(req *http.Request, via []*http.Request) bool {
|
||||
origin := via[0].URL
|
||||
for _, hop := range via {
|
||||
if !rest.SameHost(hop.URL, origin) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return !rest.SameHost(req.URL, origin)
|
||||
}
|
||||
|
||||
// Adds the configured headers to the request if any
|
||||
func (f *Fs) addHeaders(req *http.Request) {
|
||||
addHeaders(req, &f.opt)
|
||||
|
||||
@@ -581,3 +581,182 @@ func TestFsNoSlashRoots(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readRedirected makes an http remote with the configured headers
|
||||
// pointing at url and reads file.txt from it, checking the content
|
||||
func readRedirected(t *testing.T, url string) {
|
||||
configfile.Install()
|
||||
m := configmap.Simple{
|
||||
"type": "http",
|
||||
"url": url + "/",
|
||||
"headers": strings.Join(headers, ","),
|
||||
}
|
||||
f, err := NewFs(context.Background(), remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
o, err := f.NewObject(context.Background(), "file.txt")
|
||||
require.NoError(t, err)
|
||||
fd, err := o.Open(context.Background())
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(fd)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fd.Close())
|
||||
assert.Equal(t, "hello", string(data))
|
||||
}
|
||||
|
||||
func TestRedirectKeepsHeadersOnSameHost(t *testing.T) {
|
||||
var got http.Header
|
||||
var ts *httptest.Server
|
||||
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/moved/") {
|
||||
http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
readRedirected(t, ts.URL)
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Equal(t, headers[i+1], got.Get(headers[i]))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
|
||||
// A plaintext server to be redirected to
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
// An HTTPS server which redirects to plaintext HTTP
|
||||
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
// open makes a remote pointing at ts with or without headers
|
||||
// and opens file.txt on it
|
||||
open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
|
||||
configfile.Install()
|
||||
m := configmap.Simple{
|
||||
"type": "http",
|
||||
"url": ts.URL + "/",
|
||||
"no_head": "true",
|
||||
}
|
||||
if withHeaders {
|
||||
m.Set("headers", strings.Join(headers, ","))
|
||||
}
|
||||
ctx, ci := fs.AddConfig(context.Background())
|
||||
ci.InsecureSkipVerify = true
|
||||
f, err := NewFs(ctx, remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
// no_head means NewObject doesn't make a request so use Open to make one
|
||||
o, err := f.NewObject(ctx, "file.txt")
|
||||
require.NoError(t, err)
|
||||
return o.Open(ctx)
|
||||
}
|
||||
|
||||
t.Run("WithHeaders", func(t *testing.T) {
|
||||
// Headers could leak so the downgrade must be refused
|
||||
_, err := open(t, true)
|
||||
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
|
||||
})
|
||||
|
||||
t.Run("WithoutHeaders", func(t *testing.T) {
|
||||
// Nothing to leak so the downgrade is followed
|
||||
fd, err := open(t, false)
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(fd)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fd.Close())
|
||||
assert.Equal(t, "hello", string(data))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRedirectKeepsHeadersOnSameHostDifferentCase(t *testing.T) {
|
||||
// The redirect spells the host differently but it is the same
|
||||
// host so the headers must be kept
|
||||
var got http.Header
|
||||
var ts *httptest.Server
|
||||
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/moved/") {
|
||||
http.Redirect(w, r, "http://LOCALHOST:"+port(t, ts)+"/moved"+r.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
readRedirected(t, "http://localhost:"+port(t, ts))
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Equal(t, headers[i+1], got.Get(headers[i]))
|
||||
}
|
||||
}
|
||||
|
||||
// port returns the port ts is listening on
|
||||
func port(t *testing.T, ts *httptest.Server) string {
|
||||
u, err := url.Parse(ts.URL)
|
||||
require.NoError(t, err)
|
||||
return u.Port()
|
||||
}
|
||||
|
||||
func TestRedirectStripsHeadersAfterHostChange(t *testing.T) {
|
||||
// The configured server redirects to a second server which
|
||||
// redirects back to the configured server. The headers must not
|
||||
// be sent on the way back as the other host chose the path.
|
||||
var got http.Header
|
||||
var ts *httptest.Server
|
||||
bouncer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
|
||||
}))
|
||||
defer bouncer.Close()
|
||||
|
||||
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/moved/") {
|
||||
http.Redirect(w, r, bouncer.URL+r.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
readRedirected(t, ts.URL)
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Empty(t, got.Values(headers[i]), "header %q sent after cross-host redirect", headers[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
|
||||
// Serve the final response from a second server. httptest servers
|
||||
// listen on 127.0.0.1:port so the two servers have different hosts
|
||||
// for the purposes of the redirect check.
|
||||
var got http.Header
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
|
||||
}))
|
||||
defer redirector.Close()
|
||||
|
||||
readRedirected(t, redirector.URL)
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
@@ -1771,11 +1772,18 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
}
|
||||
|
||||
// Read and write file content
|
||||
_, err = io.Copy(fileWriter, in)
|
||||
counter := readers.NewCountingReader(in)
|
||||
_, err = io.Copy(fileWriter, counter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy file content: %w", err)
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload.
|
||||
if int64(counter.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
|
||||
@@ -498,7 +498,7 @@ func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*Dri
|
||||
}
|
||||
|
||||
// CreateUpload creates an url for an upload.
|
||||
func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string) (*UploadResponse, *http.Response, error) {
|
||||
func (d *DriveService) CreateUpload(ctx context.Context, size int64, name, zone string) (*UploadResponse, *http.Response, error) {
|
||||
// first we need to request an upload url
|
||||
values := map[string]any{
|
||||
"filename": name,
|
||||
@@ -513,7 +513,7 @@ func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/ws/" + defaultZone + "/upload/web",
|
||||
Path: "/ws/" + zone + "/upload/web",
|
||||
ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
|
||||
RootURL: d.docsEndpoint,
|
||||
Body: body,
|
||||
@@ -552,14 +552,14 @@ func (d *DriveService) Upload(ctx context.Context, in io.Reader, size int64, nam
|
||||
// ctx: the context.Context object for the request.
|
||||
// r: a pointer to the UpdateFileInfo struct containing the information for the file update.
|
||||
// Returns a pointer to the DriveItem struct representing the updated file, the http.Response object, and an error if any.
|
||||
func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*DriveItem, *http.Response, error) {
|
||||
func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo, zone string) (*DriveItem, *http.Response, error) {
|
||||
body, err := IntoReader(r)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/ws/" + defaultZone + "/update/documents",
|
||||
Path: "/ws/" + zone + "/update/documents",
|
||||
ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
|
||||
RootURL: d.docsEndpoint,
|
||||
Body: body,
|
||||
@@ -572,7 +572,7 @@ func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*Driv
|
||||
|
||||
doc := responseInfo.Results[0].Document
|
||||
item := DriveItem{
|
||||
Drivewsid: "FILE::com.apple.CloudDocs::" + doc.DocumentID,
|
||||
Drivewsid: ConstructDriveID(doc.DocumentID, zone, "FILE"),
|
||||
Docwsid: doc.DocumentID,
|
||||
Itemid: doc.ItemID,
|
||||
Etag: doc.Etag,
|
||||
@@ -892,6 +892,18 @@ func DeconstructDriveID(id string) (docType, zone, docid string) {
|
||||
return split[0], split[1], split[2]
|
||||
}
|
||||
|
||||
// ZoneFromDriveID returns the zone a drive ID belongs to, or the default zone
|
||||
// for an ID that carries none. Items in an app container -- Obsidian, Pages,
|
||||
// Shortcuts -- live in that app's zone rather than com.apple.CloudDocs, and a
|
||||
// write addressed to the wrong zone cannot resolve its parent.
|
||||
func ZoneFromDriveID(id string) string {
|
||||
_, zone, _ := DeconstructDriveID(id)
|
||||
if zone == "" {
|
||||
return defaultZone
|
||||
}
|
||||
return zone
|
||||
}
|
||||
|
||||
// ConstructDriveID constructs a drive ID from the given components.
|
||||
func ConstructDriveID(id string, zone string, t string) string {
|
||||
return strings.Join([]string{t, zone, id}, "::")
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestZoneFromDriveID(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
id string
|
||||
want string
|
||||
}{
|
||||
{"app container", "FOLDER::iCloud.md.obsidian::documents#o2v", "iCloud.md.obsidian"},
|
||||
{"another app container", "FOLDER::com.apple.Pages::documents#7qt", "com.apple.Pages"},
|
||||
{"ordinary folder", "FOLDER::com.apple.CloudDocs::B847FE2D", "com.apple.CloudDocs"},
|
||||
{"file in a container", "FILE::iCloud.md.obsidian::abc123", "iCloud.md.obsidian"},
|
||||
{"no zone", "root", defaultZone},
|
||||
{"empty", "", defaultZone},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, ZoneFromDriveID(test.id))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestZoneFromDriveIDRoundTrip(t *testing.T) {
|
||||
id := ConstructDriveID("abc123", "iCloud.md.obsidian", "FILE")
|
||||
assert.Equal(t, "FILE::iCloud.md.obsidian::abc123", id)
|
||||
assert.Equal(t, "iCloud.md.obsidian", ZoneFromDriveID(id))
|
||||
}
|
||||
@@ -343,7 +343,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
|
||||
var item *api.DriveItem
|
||||
if err = f.pacer.Call(func() (bool, error) {
|
||||
item, resp, err = f.service.UpdateFile(ctx, &r)
|
||||
item, resp, err = f.service.UpdateFile(ctx, &r, api.ZoneFromDriveID(pathID))
|
||||
return retryResultUnknown(ctx, resp, err)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
@@ -949,7 +949,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Create document
|
||||
var uploadInfo *api.UploadResponse
|
||||
if err = o.fs.pacer.Call(func() (bool, error) {
|
||||
uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name)
|
||||
uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name, api.ZoneFromDriveID(dirID))
|
||||
return ignoreResultUnknown(ctx, resp, err)
|
||||
}); err != nil {
|
||||
return err
|
||||
@@ -988,7 +988,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Update metadata
|
||||
var item *api.DriveItem
|
||||
if err = o.fs.pacer.Call(func() (bool, error) {
|
||||
item, resp, err = o.fs.service.UpdateFile(ctx, &r)
|
||||
item, resp, err = o.fs.service.UpdateFile(ctx, &r, api.ZoneFromDriveID(dirID))
|
||||
return ignoreResultUnknown(ctx, resp, err)
|
||||
}); err != nil {
|
||||
return err
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -854,11 +855,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
|
||||
// make a PUT request at (IAS3)/encoded(:item/:path)
|
||||
counter := readers.NewCountingReader(in)
|
||||
var resp *http.Response
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/" + url.PathEscape(path.Join(bucket, bucketPath)),
|
||||
Body: in,
|
||||
Body: counter,
|
||||
ContentLength: &size,
|
||||
ExtraHeaders: headers,
|
||||
}
|
||||
@@ -868,6 +870,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload
|
||||
if err == nil && size >= 0 && int64(counter.BytesRead()) != size {
|
||||
err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
// we can't update/find metadata here as IA will "ingest" uploaded file(s)
|
||||
// upon uploads. (you can find its progress at https://archive.org/history/ItemNameHere )
|
||||
// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)
|
||||
|
||||
@@ -28,6 +28,7 @@ type userInfo struct {
|
||||
Bucket string
|
||||
BridgeUser string
|
||||
UserID string
|
||||
NewToken string
|
||||
}
|
||||
|
||||
type userInfoConfig struct {
|
||||
@@ -61,6 +62,7 @@ func getUserInfo(ctx context.Context, cfg *userInfoConfig) (*userInfo, error) {
|
||||
Bucket: resp.User.Bucket,
|
||||
BridgeUser: resp.User.BridgeUser,
|
||||
UserID: resp.User.UserID,
|
||||
NewToken: resp.NewToken,
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "User info: rootFolderId=%s, bucket=%s",
|
||||
|
||||
@@ -367,6 +367,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.bridgeUser = userInfo.BridgeUser
|
||||
f.userID = userInfo.UserID
|
||||
|
||||
// The refresh endpoint rotates the token on every successful call.
|
||||
// Persist the rotated token so routine use keeps the stored token
|
||||
// current; otherwise it keeps its original expiry and accounts that
|
||||
// cannot re-login non-interactively (2FA) eventually strand.
|
||||
if userInfo.NewToken != "" {
|
||||
if rotated, rotErr := jwtToOAuth2Token(userInfo.NewToken); rotErr != nil {
|
||||
fs.Debugf(f, "Not adopting rotated token from user info: %v", rotErr)
|
||||
} else {
|
||||
// Use the rotated token for this session even if saving it
|
||||
// fails; persistence is best-effort.
|
||||
f.cfg.Token = userInfo.NewToken
|
||||
if putErr := oauthutil.PutToken(name, m, rotated, false); putErr != nil {
|
||||
fs.Debugf(f, "Failed to save rotated token from user info: %v", putErr)
|
||||
} else {
|
||||
fs.Debugf(f, "Persisted rotated token from user info, expiry: %v", rotated.Expiry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
+50
-1
@@ -688,6 +688,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}()
|
||||
|
||||
for {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
var fis []os.FileInfo
|
||||
if useReadDir {
|
||||
// Windows and Plan9 read the directory entries with the stat information in which
|
||||
@@ -1315,7 +1318,7 @@ func (o *Object) setTimes(atime, mtime time.Time) (err error) {
|
||||
if o.translatedLink {
|
||||
err = lChtimes(o.path, atime, mtime)
|
||||
} else {
|
||||
err = os.Chtimes(o.path, atime, mtime)
|
||||
err = o.fs.chtimes(o.path, atime, mtime)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -1420,6 +1423,13 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Clamp offset into range to avoid panic
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset > int64(len(linkdst)) {
|
||||
offset = int64(len(linkdst))
|
||||
}
|
||||
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
|
||||
}
|
||||
|
||||
@@ -1565,6 +1575,45 @@ func (f *Fs) symlink(target, localPath string) (err error) {
|
||||
return root.Symlink(target, rel)
|
||||
}
|
||||
|
||||
// chmod changes the mode of localPath.
|
||||
func (f *Fs) chmod(localPath string, mode os.FileMode) (err error) {
|
||||
if !f.opt.TranslateSymlinks {
|
||||
return os.Chmod(localPath, mode)
|
||||
}
|
||||
root, rel, err := f.osRoot(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fs.CheckClose(root, &err)
|
||||
return root.Chmod(rel, mode)
|
||||
}
|
||||
|
||||
// chown changes the ownership of localPath.
|
||||
func (f *Fs) chown(localPath string, uid, gid int) (err error) {
|
||||
if !f.opt.TranslateSymlinks {
|
||||
return os.Chown(localPath, uid, gid)
|
||||
}
|
||||
root, rel, err := f.osRoot(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fs.CheckClose(root, &err)
|
||||
return root.Chown(rel, uid, gid)
|
||||
}
|
||||
|
||||
// chtimes changes the atime and mtime of localPath.
|
||||
func (f *Fs) chtimes(localPath string, atime, mtime time.Time) (err error) {
|
||||
if !f.opt.TranslateSymlinks {
|
||||
return os.Chtimes(localPath, atime, mtime)
|
||||
}
|
||||
root, rel, err := f.osRoot(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fs.CheckClose(root, &err)
|
||||
return root.Chtimes(rel, atime, mtime)
|
||||
}
|
||||
|
||||
// mkdirAll makes all the directories needed to store the object
|
||||
func (o *Object) mkdirAll() error {
|
||||
return o.fs.mkdirAll(filepath.Dir(o.path))
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -198,6 +199,32 @@ func TestSymlink(t *testing.T) {
|
||||
require.NoError(t, in.Close())
|
||||
}
|
||||
|
||||
// TestSymlinkRangeBeyondEnd checks range requests on a translated
|
||||
// symlink's target string don't panic.
|
||||
func TestSymlinkRangeBeyondEnd(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
const target = "file.txt"
|
||||
require.NoError(t, putLink(ctx, f, "symlink.txt", target))
|
||||
|
||||
o, err := f.NewObject(ctx, "symlink.txt"+fs.LinkSuffix)
|
||||
require.NoError(t, err)
|
||||
|
||||
// An offset just past the end and a wildly large offset must both read
|
||||
// empty rather than panicking.
|
||||
for _, start := range []int64{int64(len(target)), int64(len(target)) + 1, math.MaxInt64} {
|
||||
in, err := o.Open(ctx, &fs.RangeOption{Start: start, End: -1})
|
||||
require.NoError(t, err)
|
||||
contents, err := io.ReadAll(in)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, string(contents))
|
||||
require.NoError(t, in.Close())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSymlinkError(t *testing.T) {
|
||||
m := configmap.Simple{
|
||||
"links": "true",
|
||||
@@ -341,6 +368,133 @@ func TestSymlinkInTreeWriteThroughWorks(t *testing.T) {
|
||||
require.Equal(t, "world", string(got))
|
||||
}
|
||||
|
||||
// TestDirMetadataThroughPlantedSymlinkBlocked checks metadata +
|
||||
// symlinks can't write outside the root.
|
||||
func TestDirMetadataThroughPlantedSymlinkBlocked(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlinks and unix modes not applicable on Windows")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
// A directory outside the destination whose metadata the attacker targets.
|
||||
evil := t.TempDir()
|
||||
evilDir := filepath.Join(evil, "secret.d")
|
||||
require.NoError(t, os.Mkdir(evilDir, 0700))
|
||||
evilMtime := fstest.Time("2016-06-07T08:09:10Z")
|
||||
require.NoError(t, os.Chtimes(evilDir, evilMtime, evilMtime))
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
|
||||
require.NoError(t, putLink(ctx, f, "pwn", evilDir))
|
||||
|
||||
// The source now presents "pwn" as a directory with attacker-chosen mode
|
||||
// and mtime. Applying it must not reach through the planted symlink.
|
||||
metadata := fs.Metadata{
|
||||
"mode": "0777",
|
||||
"mtime": "2001-02-03T04:05:06Z",
|
||||
}
|
||||
_, err := f.MkdirMetadata(ctx, "pwn", metadata)
|
||||
require.Error(t, err, "applying metadata through a planted symlink should be refused")
|
||||
|
||||
// The outside directory's mode and mtime must be unchanged.
|
||||
fi, err := os.Stat(evilDir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, os.FileMode(0700), fi.Mode().Perm(), "chmod escaped through planted symlink to %q", evilDir)
|
||||
require.True(t, fi.ModTime().Equal(evilMtime), "chtimes escaped through planted symlink to %q", evilDir)
|
||||
}
|
||||
|
||||
// TestDirSetModTimeThroughPlantedSymlinkBlocked checks we can't
|
||||
// chtimes outside the root with --links
|
||||
func TestDirSetModTimeThroughPlantedSymlinkBlocked(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("symlinks not applicable on Windows")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
evil := t.TempDir()
|
||||
evilDir := filepath.Join(evil, "secret.d")
|
||||
require.NoError(t, os.Mkdir(evilDir, 0700))
|
||||
evilMtime := fstest.Time("2016-06-07T08:09:10Z")
|
||||
require.NoError(t, os.Chtimes(evilDir, evilMtime, evilMtime))
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
require.NoError(t, putLink(ctx, f, "pwn", evilDir))
|
||||
|
||||
err := f.DirSetModTime(ctx, "pwn", fstest.Time("2001-02-03T04:05:06Z"))
|
||||
require.Error(t, err, "setting dir modtime through a planted symlink should be refused")
|
||||
|
||||
fi, err := os.Stat(evilDir)
|
||||
require.NoError(t, err)
|
||||
require.True(t, fi.ModTime().Equal(evilMtime), "chtimes escaped through planted symlink to %q", evilDir)
|
||||
}
|
||||
|
||||
// TestDirMetadataInTreeWorks checks the root confinement doesn't
|
||||
// break legitimate dir metadata.
|
||||
func TestDirMetadataInTreeWorks(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("unix modes not applicable on Windows")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
metadata := fs.Metadata{
|
||||
"mode": "0705",
|
||||
"mtime": "2001-02-03T04:05:06Z",
|
||||
}
|
||||
_, err := f.MkdirMetadata(ctx, "sub", metadata)
|
||||
require.NoError(t, err)
|
||||
|
||||
fi, err := os.Stat(filepath.Join(f.root, "sub"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, os.FileMode(0705), fi.Mode().Perm())
|
||||
require.True(t, fi.ModTime().Equal(fstest.Time("2001-02-03T04:05:06Z")))
|
||||
}
|
||||
|
||||
// TestDirBTimeThroughPlantedSymlinkBlocked checks a btime write
|
||||
// through a symlink can't escape the root.
|
||||
func TestDirBTimeThroughPlantedSymlinkBlocked(t *testing.T) {
|
||||
if !haveSetBTime {
|
||||
t.Skip("birth time is not settable on this OS")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
evil := t.TempDir()
|
||||
evilDir := filepath.Join(evil, "secret.d")
|
||||
require.NoError(t, os.Mkdir(evilDir, 0700))
|
||||
|
||||
// Read the outside dir's btime through a local Fs rooted at evil.
|
||||
evilFsRaw, err := NewFs(ctx, "local", evil, configmap.Simple{})
|
||||
require.NoError(t, err)
|
||||
evilFs := evilFsRaw.(*Fs)
|
||||
readBTime := func() string {
|
||||
o, err := evilFs.newObject("secret.d")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, o.lstat())
|
||||
m, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
return m["btime"]
|
||||
}
|
||||
before := readBTime()
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
require.NoError(t, putLink(ctx, f, "pwn", evilDir))
|
||||
|
||||
_, _ = f.MkdirMetadata(ctx, "pwn", fs.Metadata{"btime": "2001-02-03T04:05:06Z"})
|
||||
|
||||
require.Equal(t, before, readBTime(), "btime escaped through planted symlink to %q", evilDir)
|
||||
}
|
||||
|
||||
// TestEncodingEscapeBlocked checks that a name from a malicious source can't
|
||||
// be decoded into path syntax which writes outside the destination.
|
||||
func TestEncodingEscapeBlocked(t *testing.T) {
|
||||
|
||||
@@ -105,7 +105,10 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
|
||||
}
|
||||
if haveSetBTime {
|
||||
if btimeOK {
|
||||
if o.translatedLink {
|
||||
// When translating symlinks, never follow the path. A planted symlink must
|
||||
// not redirect the birth-time write out of the root. The NOFOLLOW open is a
|
||||
// no-op on a real file or directory
|
||||
if o.translatedLink || o.fs.opt.TranslateSymlinks {
|
||||
err = lsetBTime(o.path, btime)
|
||||
} else {
|
||||
err = setBTime(o.path, btime)
|
||||
@@ -128,7 +131,7 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
|
||||
if o.translatedLink {
|
||||
err = os.Lchown(o.path, uid, gid)
|
||||
} else {
|
||||
err = os.Chown(o.path, uid, gid)
|
||||
err = o.fs.chown(o.path, uid, gid)
|
||||
}
|
||||
if err != nil {
|
||||
outErr = fmt.Errorf("failed to change ownership: %w", err)
|
||||
@@ -155,7 +158,7 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
|
||||
err = nil
|
||||
}
|
||||
} else {
|
||||
err = os.Chmod(o.path, fileMode)
|
||||
err = o.fs.chmod(o.path, fileMode)
|
||||
}
|
||||
if err != nil {
|
||||
outErr = fmt.Errorf("failed to change permissions: %w", err)
|
||||
|
||||
@@ -549,31 +549,34 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
|
||||
// We don't have the final ID yet?
|
||||
// query Microsoft Graph
|
||||
if opt.finalDriveID == "" {
|
||||
_, err := srv.CallJSON(ctx, &opt.opts, nil, &drives)
|
||||
if err != nil {
|
||||
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
|
||||
_, drivesErr := srv.CallJSON(ctx, &opt.opts, nil, &drives)
|
||||
if drivesErr != nil {
|
||||
fs.Debugf(nil, "Failed to query /me/drives: %v - trying /me/drive", drivesErr)
|
||||
}
|
||||
|
||||
// Also call /me/drive as sometimes /me/drives doesn't return it #4068
|
||||
if opt.opts.Path == "/me/drives" {
|
||||
opt.opts.Path = "/me/drive"
|
||||
meDriveOpts := opt.opts
|
||||
meDriveOpts.Path = "/me/drive"
|
||||
meDrive := api.DriveResource{}
|
||||
_, err := srv.CallJSON(ctx, &opt.opts, nil, &meDrive)
|
||||
if err != nil {
|
||||
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
|
||||
}
|
||||
found := false
|
||||
for _, drive := range drives.Drives {
|
||||
if drive.DriveID == meDrive.DriveID {
|
||||
found = true
|
||||
break
|
||||
_, meDriveErr := srv.CallJSON(ctx, &meDriveOpts, nil, &meDrive)
|
||||
if meDriveErr == nil {
|
||||
found := false
|
||||
for _, drive := range drives.Drives {
|
||||
if drive.DriveID == meDrive.DriveID {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
// add the me drive if not found already
|
||||
if !found {
|
||||
fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
|
||||
drives.Drives = append(drives.Drives, meDrive)
|
||||
}
|
||||
} else if drivesErr != nil {
|
||||
return fs.ConfigError("driveid", fmt.Sprintf("Failed to query available drives: /me/drives: %v; /me/drive: %v\nEnter the drive ID manually instead", drivesErr, meDriveErr))
|
||||
}
|
||||
// add the me drive if not found already
|
||||
if !found {
|
||||
fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
|
||||
drives.Drives = append(drives.Drives, meDrive)
|
||||
}
|
||||
} else if drivesErr != nil {
|
||||
return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", drivesErr))
|
||||
}
|
||||
} else {
|
||||
drives.Drives = append(drives.Drives, api.DriveResource{
|
||||
@@ -2678,9 +2681,8 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
|
||||
}
|
||||
return true, fmt.Errorf("retry this chunk skipping %d bytes: %w", skip, err)
|
||||
} else if err != nil && resp != nil && resp.StatusCode == http.StatusNotFound {
|
||||
fs.Debugf(o, "Received 404 error: assuming eventual consistency problem with session - retrying chunk: %v", err)
|
||||
time.Sleep(5 * time.Second) // a little delay to help things along
|
||||
return true, err
|
||||
fs.Debugf(o, "Received 404 error: upload session not found - not retrying: %v", err)
|
||||
return false, fserrors.NoLowLevelRetryError(err)
|
||||
}
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
|
||||
@@ -26,7 +26,7 @@ type BackendConfig struct {
|
||||
Remote string `yaml:"remote"`
|
||||
Features []string `yaml:"features"`
|
||||
Hashes []string `yaml:"hashes"`
|
||||
Precision int `yaml:"precision"`
|
||||
Precision int64 `yaml:"precision"`
|
||||
}
|
||||
|
||||
// GetBackendConfig from docs/data/backends
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package overview
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/docs/data/backends"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -53,3 +55,17 @@ func TestGetBackendConfig(t *testing.T) {
|
||||
assert.Equal(t, expectedMemoryConfig, conf, "parsed memory.yaml should match")
|
||||
|
||||
}
|
||||
|
||||
// Check every embedded backend YAML file parses on all architectures
|
||||
func TestGetBackendConfigAll(t *testing.T) {
|
||||
entries, err := backends.BackendFS.ReadDir(".")
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, entries)
|
||||
|
||||
for _, entry := range entries {
|
||||
name := strings.TrimSuffix(entry.Name(), ".yaml")
|
||||
conf, err := GetBackendConfig(name)
|
||||
assert.NoError(t, err, "failed to load %s", entry.Name())
|
||||
assert.NotNil(t, conf)
|
||||
}
|
||||
}
|
||||
@@ -128,6 +128,13 @@ func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared before
|
||||
// finalising, otherwise a truncated file would be stored as a good
|
||||
// upload. Returning an error here aborts the upload.
|
||||
if size >= 0 && off != size {
|
||||
return fmt.Errorf("multipart upload: expected %d bytes in input, but got %d: %w", size, off, io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
err = w.Close(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multipart upload: failed to finalise: %w", err)
|
||||
|
||||
@@ -58,6 +58,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
@@ -1447,10 +1448,11 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, s
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create upload client: %w", err)
|
||||
}
|
||||
counter := readers.NewCountingReader(in)
|
||||
req := &s3.PutObjectInput{
|
||||
Bucket: &p.Bucket,
|
||||
Key: &p.Key,
|
||||
Body: io.NopCloser(in),
|
||||
Body: io.NopCloser(counter),
|
||||
}
|
||||
// Apply upload options
|
||||
for _, option := range options {
|
||||
@@ -1478,7 +1480,14 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, s
|
||||
_, err = client.PutObject(ctx, req, s3opts...)
|
||||
return f.shouldRetry(ctx, nil, err)
|
||||
})
|
||||
return
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check for truncation of input
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, gcid string, size int64, options ...fs.OpenOption) (info *api.File, err error) {
|
||||
|
||||
@@ -1114,7 +1114,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
modTime := src.ModTime(ctx)
|
||||
var linkID string
|
||||
var fileSystemAttrs *proton.RevisionXAttrCommon
|
||||
if err = o.fs.pacer.Call(func() (bool, error) {
|
||||
if err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
linkID, fileSystemAttrs, err = o.fs.protonDrive.UploadFileByReader(ctx, folderLinkID, leaf, modTime, in, 0)
|
||||
return shouldRetry(ctx, err)
|
||||
}); err != nil {
|
||||
|
||||
@@ -1126,12 +1126,14 @@ func (o *Object) dynamicUpload(ctx context.Context, size int64, modTime time.Tim
|
||||
|
||||
_, err := io.CopyN(rw, in, localChunk)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("read chunk with offset %d size %d: %w", offset, localChunk, err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
||||
err = o.upload(ctx, uploadSession.UploadKey, rw, size, offset, localChunk, options...)
|
||||
_ = rw.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload chunk with offset %d size %d: %w", offset, localChunk, err)
|
||||
}
|
||||
@@ -1218,7 +1220,7 @@ func (o *Object) uploadSession(ctx context.Context, parentID, name string) (uplo
|
||||
return o.fs.uploadLink(ctx, parentID, encName)
|
||||
}
|
||||
|
||||
func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.Reader, fullSize int64, offset int64, chunkSize int64, options ...fs.OpenOption) (err error) {
|
||||
func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.ReadSeeker, fullSize int64, offset int64, chunkSize int64, options ...fs.OpenOption) (err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
RootURL: fmt.Sprintf(uploadURL, o.fs.opt.Host) + uploadKey,
|
||||
@@ -1231,6 +1233,11 @@ func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.Reader,
|
||||
var fileID string
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// Rewind the chunk so a retry re-sends it in full
|
||||
_, err := chunk.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &fileID)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
name: Mega
|
||||
description: MEGA S4 Object Storage
|
||||
endpoint:
|
||||
s3.eu-amsterdam.megas4.com: Mega S4 Amsterdam
|
||||
s3.eu-luxembourg.megas4.com: Mega S4 Luxembourg
|
||||
s3.eu-paris.megas4.com: Mega S4 Paris
|
||||
s3.eu-barcelona.megas4.com: Mega S4 Barcelona
|
||||
s3.ca-montreal.megas4.com: Mega S4 Montreal
|
||||
s3.ca-vancouver.megas4.com: Mega S4 Vancouver
|
||||
s3.ap-tokyo.megas4.com: Mega S4 Tokyo
|
||||
s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam, legacy)
|
||||
s3.eu-central-2.s4.mega.io: Mega S4 eu-central-2 (Bettembourg, legacy)
|
||||
s3.ca-central-1.s4.mega.io: Mega S4 ca-central-1 (Montreal, legacy)
|
||||
s3.ca-west-1.s4.mega.io: Mega S4 ca-west-1 (Vancouver, legacy)
|
||||
s3.eu-luxembourg-1.megas4.com: Mega S4 Luxembourg 1
|
||||
s3.eu-luxembourg-2.megas4.com: Mega S4 Luxembourg 2
|
||||
s3.eu-amsterdam-1.megas4.com: Mega S4 Amsterdam 1
|
||||
s3.eu-amsterdam-2.megas4.com: Mega S4 Amsterdam 2
|
||||
s3.eu-paris-1.megas4.com: Mega S4 Paris 1
|
||||
s3.eu-paris-2.megas4.com: Mega S4 Paris 2
|
||||
s3.eu-barcelona-1.megas4.com: Mega S4 Barcelona 1
|
||||
s3.eu-barcelona-2.megas4.com: Mega S4 Barcelona 2
|
||||
s3.ca-montreal-1.megas4.com: Mega S4 Montreal 1
|
||||
s3.ca-montreal-2.megas4.com: Mega S4 Montreal 2
|
||||
s3.ca-vancouver-1.megas4.com: Mega S4 Vancouver 1
|
||||
s3.ca-vancouver-2.megas4.com: Mega S4 Vancouver 2
|
||||
s3.ap-tokyo-1.megas4.com: Mega S4 Tokyo 1
|
||||
s3.ap-tokyo-2.megas4.com: Mega S4 Tokyo 2
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
list_version: 2
|
||||
|
||||
@@ -3278,6 +3278,16 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// With NoHeadObject no metadata was read for the new object, so carry
|
||||
// the size and MD5 over from the source as a server-side copy produces
|
||||
// an object with identical content.
|
||||
if f.opt.NoHeadObject {
|
||||
if dstObject, ok := dstObj.(*Object); ok {
|
||||
dstObject.bytes = srcObj.bytes
|
||||
dstObject.md5 = srcObj.md5
|
||||
}
|
||||
}
|
||||
|
||||
// Set Object Lock via separate API calls if requested
|
||||
if f.opt.ObjectLockSetAfterUpload {
|
||||
if dstObject, ok := dstObj.(*Object); ok {
|
||||
@@ -4657,6 +4667,16 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
|
||||
// retry all chunks once have done the first few
|
||||
return true, err
|
||||
}
|
||||
if uout == nil || uout.ETag == nil {
|
||||
// A successful UploadPart without an ETag header is unusable: the
|
||||
// part ETag is required by CompleteMultipartUpload. Proxies and
|
||||
// load balancers have been observed emitting empty 200 responses
|
||||
// under load - see #9822. Treat it as a retryable error so the
|
||||
// pacer retries this chunk, instead of dereferencing a nil ETag
|
||||
// in the debug log below or completing the upload with a broken
|
||||
// part list.
|
||||
return true, fmt.Errorf("UploadPart response for chunk %d has no ETag", chunkNumber+1)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -144,6 +144,33 @@ func (f *Fs) InternalTestNoHead(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestNoHeadObjectCopy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
contents := random.String(1000)
|
||||
item := fstest.NewItem("test-no-head-object-copy-src", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
src := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
assert.NoError(t, src.Remove(ctx))
|
||||
}()
|
||||
// Set NoHeadObject for this test so the copied object's metadata is not read back
|
||||
f.opt.NoHeadObject = true
|
||||
defer func() {
|
||||
f.opt.NoHeadObject = false
|
||||
}()
|
||||
dst, err := f.Copy(ctx, src, "test-no-head-object-copy-dst")
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
assert.NoError(t, dst.Remove(ctx))
|
||||
}()
|
||||
assert.Equal(t, src.Size(), dst.Size())
|
||||
srcHash, err := src.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
dstHash, err := dst.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, srcHash, dstHash)
|
||||
assert.NotEqual(t, "", dstHash)
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestHasChildren(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
contents := random.String(100)
|
||||
@@ -810,6 +837,7 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Metadata", f.InternalTestMetadata)
|
||||
t.Run("NoHead", f.InternalTestNoHead)
|
||||
t.Run("NoHeadObjectCopy", f.InternalTestNoHeadObjectCopy)
|
||||
t.Run("HasChildren", f.InternalTestHasChildren)
|
||||
t.Run("Versions", f.InternalTestVersions)
|
||||
t.Run("ObjectLock", f.InternalTestObjectLock)
|
||||
|
||||
+11
-1
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -176,11 +177,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// Update the object with the contents of the io.Reader
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
size := src.Size()
|
||||
counter := readers.NewCountingReader(in)
|
||||
var resp *http.Response
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: path.Join("/renter/uploadstream/", o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
|
||||
Body: in,
|
||||
Body: counter,
|
||||
ContentLength: &size,
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
@@ -191,6 +193,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return o.fs.shouldRetry(resp, err)
|
||||
})
|
||||
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload.
|
||||
// The partially uploaded file is cleaned up by Put which copes
|
||||
// with it appearing asynchronously after the failed upload.
|
||||
if err == nil && size >= 0 && int64(counter.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = o.readMetaData(ctx)
|
||||
}
|
||||
|
||||
@@ -836,6 +836,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return fmt.Errorf("Update Close failed: %w", err)
|
||||
}
|
||||
|
||||
// Return the connection so the SetModTime below can reuse it rather than
|
||||
// dialling a second one - the file is closed so remove() no longer needs
|
||||
// it. This nils cn out, making the deferred putConnection a no-op.
|
||||
o.fs.putConnection(&cn, nil)
|
||||
|
||||
// Set the modified time and also o.statResult
|
||||
err = o.SetModTime(ctx, src.ModTime(ctx))
|
||||
if err != nil {
|
||||
|
||||
@@ -1,7 +1,97 @@
|
||||
// Unit tests for internal SMB functions
|
||||
package smb
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDialClosesConnectionOnSetupError(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, listener.Close()) }()
|
||||
|
||||
type acceptResult struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
accepted := make(chan acceptResult, 1)
|
||||
go func() {
|
||||
conn, err := listener.Accept()
|
||||
accepted <- acceptResult{conn: conn, err: err}
|
||||
}()
|
||||
|
||||
f := &Fs{opt: Options{Pass: "invalid"}}
|
||||
_, err = f.dial(context.Background(), "tcp", listener.Addr().String())
|
||||
require.Error(t, err)
|
||||
|
||||
var result acceptResult
|
||||
select {
|
||||
case result = <-accepted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("timed out waiting for server to accept connection")
|
||||
}
|
||||
require.NoError(t, result.err)
|
||||
defer func() { require.NoError(t, result.conn.Close()) }()
|
||||
require.NoError(t, result.conn.SetReadDeadline(time.Now().Add(time.Second)))
|
||||
|
||||
buffer := make([]byte, 1)
|
||||
n, err := result.conn.Read(buffer)
|
||||
require.Zero(t, n)
|
||||
require.ErrorIs(t, err, io.EOF)
|
||||
}
|
||||
|
||||
// TestUploadConnectionReuse checks an upload leaves only one connection in the
|
||||
// pool, ie the connection it used is available again for the SetModTime which
|
||||
// follows it rather than a second one being dialled.
|
||||
//
|
||||
// This needs a real SMB server so it is skipped if one isn't configured.
|
||||
func TestUploadConnectionReuse(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fstest.Initialise()
|
||||
remoteName := *fstest.RemoteName
|
||||
if remoteName == "" {
|
||||
remoteName = "TestSMB:rclone"
|
||||
}
|
||||
remote, err := fs.NewFs(ctx, remoteName)
|
||||
if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
|
||||
t.Skipf("skipping as %q is not configured", remoteName)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
f, ok := remote.(*Fs)
|
||||
if !ok {
|
||||
t.Skipf("skipping as %q is not an SMB remote", remoteName)
|
||||
}
|
||||
|
||||
defer func() { require.NoError(t, f.Shutdown(ctx)) }()
|
||||
|
||||
// Empty the pool so the connections counted below are only the upload's
|
||||
require.NoError(t, f.drainPool(ctx))
|
||||
|
||||
const contents = "connection reuse test"
|
||||
remotePath := fmt.Sprintf("rclone-test-connection-reuse-%d.txt", time.Now().UnixNano())
|
||||
src := object.NewStaticObjectInfo(remotePath, time.Now(), int64(len(contents)), true, nil, nil)
|
||||
o, err := f.Put(ctx, strings.NewReader(contents), src)
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, o.Remove(ctx)) }()
|
||||
|
||||
f.poolMu.Lock()
|
||||
pooled := len(f.pool)
|
||||
f.poolMu.Unlock()
|
||||
assert.Equal(t, 1, pooled, "upload should leave exactly one connection in the pool")
|
||||
}
|
||||
|
||||
// TestIsPathDir tests the isPathDir function logic
|
||||
func TestIsPathDir(t *testing.T) {
|
||||
|
||||
@@ -233,6 +233,7 @@ type Fs struct {
|
||||
ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips
|
||||
chunksUploadURL string // upload URL for nextcloud chunked
|
||||
canChunk bool // set if nextcloud and nextcloud_chunk_size is set
|
||||
canRecalcHash bool // set if the server can recalculate checksums with PATCH (nextcloud)
|
||||
authSingleflight *singleflight.Group
|
||||
}
|
||||
|
||||
@@ -664,6 +665,7 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
|
||||
f.propsetMtime = true
|
||||
f.hasOCSHA1 = true
|
||||
f.canChunk = true
|
||||
f.canRecalcHash = true
|
||||
|
||||
if f.opt.ChunkSize == 0 {
|
||||
fs.Logf(nil, "Chunked uploads are disabled because nextcloud_chunk_size is set to 0")
|
||||
@@ -1471,11 +1473,16 @@ var owncloudPropsetWithChecksum = `<?xml version="1.0" encoding="utf-8" ?>
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
if o.fs.propsetMtime {
|
||||
// Setting the modification time discards the stored
|
||||
// checksums so they are set again in the same request.
|
||||
// Servers which can't do that recalculate them afterwards.
|
||||
checksums := ""
|
||||
if o.fs.hasOCSHA1 && o.sha1 != "" {
|
||||
checksums = "SHA1:" + o.sha1
|
||||
} else if o.fs.hasOCMD5 && o.md5 != "" {
|
||||
checksums = "MD5:" + o.md5
|
||||
if !o.fs.canRecalcHash {
|
||||
if o.fs.hasOCSHA1 && o.sha1 != "" {
|
||||
checksums = "SHA1:" + o.sha1
|
||||
} else if o.fs.hasOCMD5 && o.md5 != "" {
|
||||
checksums = "MD5:" + o.md5
|
||||
}
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
@@ -1510,6 +1517,12 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
// in-memory modtime identical to the one a fresh
|
||||
// listing returns
|
||||
o.modTime = modTime.Truncate(time.Second)
|
||||
if o.fs.canRecalcHash && (o.sha1 != "" || o.md5 != "") {
|
||||
err = o.recalculateHash(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't restore checksum after setting modified time: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// got an error, but it's possible it actually worked, so double-check
|
||||
@@ -1596,7 +1609,54 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
// read metadata from remote
|
||||
o.hasMetaData = false
|
||||
return o.readMetaData(ctx)
|
||||
err = o.readMetaData(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The server only stores a checksum supplied with the upload,
|
||||
// so ask the server to calculate one if it is missing.
|
||||
if o.fs.canRecalcHash && o.sha1 == "" && o.md5 == "" {
|
||||
err = o.recalculateHash(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't calculate checksum after upload: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recalculateHash asks the server to calculate and store the checksum
|
||||
// of the object's contents, updating the cached hash from the result.
|
||||
//
|
||||
// This uses the nextcloud PATCH extension with the X-Recalculate-Hash
|
||||
// header which returns the checksum in the OC-Checksum header.
|
||||
func (o *Object) recalculateHash(ctx context.Context) error {
|
||||
hashName := ""
|
||||
if o.fs.hasOCSHA1 {
|
||||
hashName = "sha1"
|
||||
} else if o.fs.hasOCMD5 {
|
||||
hashName = "md5"
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: o.filePath(),
|
||||
NoResponse: true,
|
||||
ExtraHeaders: map[string]string{"X-Recalculate-Hash": hashName},
|
||||
}
|
||||
var resp *http.Response
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashes := (&api.Prop{Checksums: []string{resp.Header.Get("OC-Checksum")}}).Hashes()
|
||||
o.sha1 = hashes[hash.SHA1]
|
||||
o.md5 = hashes[hash.MD5]
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string]string {
|
||||
|
||||
@@ -1189,6 +1189,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return err
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload.
|
||||
if size := src.Size(); size >= 0 && int64(in1.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in1.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
// Set the modTime of the uploaded file and re-read the metadata
|
||||
// so the object has the md5sum the server computed for the upload.
|
||||
//
|
||||
|
||||
@@ -4,13 +4,29 @@ A demo proxy for rclone serve sftp/webdav/ftp, etc.
|
||||
|
||||
This takes the incoming user/pass and converts it into an sftp backend
|
||||
running on localhost.
|
||||
|
||||
Logins from outside ALLOWED_NETWORKS are refused.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import ipaddress
|
||||
|
||||
ALLOWED_NETWORKS = ["127.0.0.0/8", "::1/128"]
|
||||
|
||||
def allowed(ip):
|
||||
"""Return True if ip is in one of ALLOWED_NETWORKS."""
|
||||
if ip is None:
|
||||
return False
|
||||
address = ipaddress.ip_address(ip)
|
||||
return any(address in ipaddress.ip_network(network) for network in ALLOWED_NETWORKS)
|
||||
|
||||
def main():
|
||||
i = json.load(sys.stdin)
|
||||
# Exiting non zero refuses the login - rclone logs whatever we
|
||||
# write on stderr, so say why.
|
||||
if not allowed(i.get("client_ip")):
|
||||
sys.exit("client_ip %s not allowed" % i.get("client_ip"))
|
||||
o = {
|
||||
"type": "sftp", # type of backend
|
||||
"_root": "", # root of the fs
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -201,28 +202,20 @@ func ArchiveExtract(ctx context.Context, dst fs.Fs, dstDir string, src fs.Fs, sr
|
||||
// destPath maps an archive entry name onto its destination remote within
|
||||
// dstDir, returning an error if the name is unsafe.
|
||||
//
|
||||
// Archive entry names are attacker controlled. A leading "./" is stripped:
|
||||
// tar archives created with relative paths (e.g. "tar -czf archive.tar.gz .")
|
||||
// use "./" prefixed entries and, without stripping, rclone would encode the
|
||||
// "." as a full-width dot character creating a spurious directory.
|
||||
//
|
||||
// Any entry with a ".." path component is then rejected to prevent a path
|
||||
// traversal ("Zip Slip") attack: path.Join collapses "..", so an entry such
|
||||
// as "../escaped.txt" joined onto "dir" would resolve to "escaped.txt" and be
|
||||
// written outside the selected destination directory. Both "/" and "\" are
|
||||
// treated as separators when looking for ".." segments: archive names should
|
||||
// use "/", but a crafted archive may use "\", which the local backend treats
|
||||
// as a path separator on Windows.
|
||||
// Archive entry names are attacker controlled so they are sanitized with
|
||||
// sanitize.Path before being joined onto dstDir. This cleans the name
|
||||
// (stripping the "./" prefix which tar archives created with relative
|
||||
// paths, e.g. "tar -czf archive.tar.gz .", put on their entries) and
|
||||
// rejects any name with a ".." path component which would otherwise
|
||||
// escape the destination directory (a path traversal, or "Zip Slip",
|
||||
// attack).
|
||||
//
|
||||
// The returned remote is empty for the archive root entry ("./"), which the
|
||||
// caller should skip.
|
||||
func destPath(nameInArchive, dstDir string) (string, error) {
|
||||
remote := strings.TrimPrefix(nameInArchive, "./")
|
||||
isSeparator := func(r rune) bool { return r == '/' || r == '\\' }
|
||||
for _, segment := range strings.FieldsFunc(remote, isSeparator) {
|
||||
if segment == ".." {
|
||||
return "", fmt.Errorf("refusing to extract archive entry %q with a %q path component", nameInArchive, "..")
|
||||
}
|
||||
remote, err := sanitize.Path(nameInArchive)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("refusing to extract archive entry: %w", err)
|
||||
}
|
||||
if remote == "" {
|
||||
return "", nil
|
||||
|
||||
@@ -18,25 +18,10 @@ func TestDestPath(t *testing.T) {
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "strip leading dot-slash from file",
|
||||
input: "./file.txt",
|
||||
expected: "file.txt",
|
||||
},
|
||||
{
|
||||
name: "strip leading dot-slash from nested path",
|
||||
name: "sanitized name returned",
|
||||
input: "./subdir/file.txt",
|
||||
expected: "subdir/file.txt",
|
||||
},
|
||||
{
|
||||
name: "no prefix unchanged",
|
||||
input: "file.txt",
|
||||
expected: "file.txt",
|
||||
},
|
||||
{
|
||||
name: "nested path unchanged",
|
||||
input: "dir/file.txt",
|
||||
expected: "dir/file.txt",
|
||||
},
|
||||
{
|
||||
name: "joined onto destination directory",
|
||||
input: "file.txt",
|
||||
@@ -48,44 +33,22 @@ func TestDestPath(t *testing.T) {
|
||||
input: "./",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "only single leading dot-slash stripped",
|
||||
input: "././file.txt",
|
||||
expected: "./file.txt",
|
||||
},
|
||||
{
|
||||
name: "leading dot-dot rejected",
|
||||
input: "../etc/passwd",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "leading dot-dot rejected with destination",
|
||||
input: "../escaped.txt",
|
||||
dstDir: "safe/prefix",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "interior dot-dot rejected",
|
||||
name: "interior dot-dot rejected with destination",
|
||||
input: "dir/../../escaped.txt",
|
||||
dstDir: "safe/prefix",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "trailing dot-dot rejected",
|
||||
input: "dir/..",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "backslash dot-dot rejected",
|
||||
input: `..\escaped.txt`,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "nested backslash dot-dot rejected",
|
||||
input: `dir\..\..\escaped.txt`,
|
||||
dstDir: "safe/prefix",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
|
||||
@@ -675,7 +675,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
|
||||
b.debugFn(tr.Name, func() {
|
||||
prettyprint(tr, tr.Name, fs.LogLevelInfo)
|
||||
})
|
||||
if tr.Error == nil && tr.Bytes > 0 || tr.Size <= 0 {
|
||||
if tr.Error == nil && (tr.Bytes > 0 || tr.Size <= 0) {
|
||||
prettyprint(tr, "keeping: "+tr.Name, fs.LogLevelDebug)
|
||||
toKeep = append(toKeep, tr.Name)
|
||||
}
|
||||
|
||||
+3
-1
@@ -247,7 +247,9 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
|
||||
|
||||
b.SyncCI = fs.GetConfig(ctxCopy) // allows us to request graceful shutdown
|
||||
accounting.Stats(ctxCopy).SetMaxCompletedTransfers(-1) // we need a complete list in the event of graceful shutdown
|
||||
ctxCopy, b.CancelSync = context.WithCancel(ctxCopy)
|
||||
ctxCopy, cancel := context.WithCancel(ctxCopy)
|
||||
defer cancel()
|
||||
b.CancelSync = cancel
|
||||
b.testFn()
|
||||
err := sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
|
||||
prettyprint(b.queueOpt.logger, "b.queueOpt.logger", fs.LogLevelDebug)
|
||||
|
||||
@@ -280,6 +280,11 @@ for all `mount` and `serve` commands on macOS. For details, see [vfs-case-sensit
|
||||
|
||||
#### NFS mount
|
||||
|
||||
For macOS (and other platforms where this path is supported), prefer the dedicated
|
||||
[rclone nfsmount](/commands/rclone_nfsmount/) command. It starts the NFS server and
|
||||
performs the mount for you. `rclone mount` itself still uses FUSE (macFUSE/FUSE-T)
|
||||
and does not switch to NFS via a flag.
|
||||
|
||||
This method spins up an NFS server using [serve nfs](/commands/rclone_serve_nfs/)
|
||||
command and mounts it to the specified mountpoint. If you run this in background
|
||||
mode using |--daemon|, you will need to send SIGTERM signal to the rclone process
|
||||
|
||||
@@ -244,6 +244,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
}
|
||||
listener, err := net.Listen(network, s.httpListenAddr)
|
||||
if err != nil {
|
||||
s.vfs.Shutdown()
|
||||
return nil, err
|
||||
}
|
||||
s.HTTPConn = listener
|
||||
@@ -402,6 +403,7 @@ func (s *server) Wait() {
|
||||
// Shutdown the DLNA server
|
||||
func (s *server) Shutdown() error {
|
||||
err := s.HTTPConn.Close()
|
||||
s.vfs.Shutdown()
|
||||
close(s.waitChan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to shutdown DLNA server: %w", err)
|
||||
|
||||
@@ -204,6 +204,101 @@ func TestDockerPluginLogic(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestDockerPluginVolumeNameConfined checks that a crafted volume name
|
||||
// containing ".." components cannot escape the base directory to create a
|
||||
// mountpoint at an arbitrary host path.
|
||||
func TestDockerPluginVolumeNameConfined(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
oldCacheDir := config.GetCacheDir()
|
||||
testDir, testFs := initialise(ctx, t)
|
||||
err := config.SetCacheDir(testDir)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = config.SetCacheDir(oldCacheDir)
|
||||
if !t.Failed() {
|
||||
fstest.Purge(testFs)
|
||||
_ = os.RemoveAll(testDir)
|
||||
}
|
||||
}()
|
||||
|
||||
drv, err := docker.NewDriver(ctx, testDir, nil, nil, true, true)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, drv)
|
||||
|
||||
// A volume name with ".." components resolves outside testDir, to a
|
||||
// sibling path that does not exist so the test never touches a real
|
||||
// system directory.
|
||||
escape := filepath.Join("..", "docker-escape-target")
|
||||
outside := filepath.Join(filepath.Dir(testDir), "docker-escape-target")
|
||||
require.NoDirExists(t, outside)
|
||||
volReq := &docker.CreateRequest{
|
||||
Name: escape,
|
||||
Options: docker.VolOpts{"remote": testDir},
|
||||
}
|
||||
err = drv.Create(volReq)
|
||||
assertErrorContains(t, err, "resolves outside the base directory")
|
||||
|
||||
// The escaping directory must not have been created.
|
||||
_, statErr := os.Stat(outside)
|
||||
assert.True(t, os.IsNotExist(statErr), "escaping mountpoint %q should not exist", outside)
|
||||
|
||||
// The volume must not have been registered.
|
||||
_, err = drv.Get(&docker.GetRequest{Name: escape})
|
||||
assert.Error(t, err)
|
||||
|
||||
// A name that resolves to the base directory itself (empty or ".")
|
||||
// must also be rejected, since mounting there would shadow every
|
||||
// other volume.
|
||||
for _, name := range []string{"", ".", filepath.Join("a", "..")} {
|
||||
volReq.Name = name
|
||||
err = drv.Create(volReq)
|
||||
assertErrorContains(t, err, "resolves outside the base directory",
|
||||
"name %q should be rejected", name)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDockerPluginRestoreStateConfined checks that a persisted state file
|
||||
// with a mountpoint escaping the base directory is not trusted verbatim on
|
||||
// restore: the mountpoint is re-derived from the base directory and name.
|
||||
func TestDockerPluginRestoreStateConfined(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
oldCacheDir := config.GetCacheDir()
|
||||
testDir, testFs := initialise(ctx, t)
|
||||
err := config.SetCacheDir(testDir)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = config.SetCacheDir(oldCacheDir)
|
||||
if !t.Failed() {
|
||||
fstest.Purge(testFs)
|
||||
_ = os.RemoveAll(testDir)
|
||||
}
|
||||
}()
|
||||
|
||||
// Persist a state file as an older, vulnerable rclone might have: a
|
||||
// benign name but a mountpoint that escapes the base directory.
|
||||
escaped := filepath.Join(filepath.Dir(testDir), "docker-escape-restore")
|
||||
require.NoDirExists(t, escaped)
|
||||
state := fmt.Sprintf(`[{"name":"vol1","mountpoint":%q,"created":%q,"fs":%q,"options":{"remote":%q},"mounts":[]}]`,
|
||||
escaped, time.Now().Format(time.RFC3339), testDir, testDir)
|
||||
statePath := filepath.Join(testDir, "docker-plugin.state")
|
||||
require.NoError(t, os.WriteFile(statePath, []byte(state), 0600))
|
||||
|
||||
// Restore the state into a new dummy driver.
|
||||
drv, err := docker.NewDriver(ctx, testDir, nil, nil, true, false)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, drv)
|
||||
|
||||
// The restored volume must point back inside the base directory, and
|
||||
// the escaping directory must not have been created.
|
||||
getRes, err := drv.Get(&docker.GetRequest{Name: "vol1"})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, getRes)
|
||||
assert.Equal(t, filepath.Join(testDir, "vol1"), getRes.Volume.Mountpoint)
|
||||
assert.NotEqual(t, escaped, getRes.Volume.Mountpoint)
|
||||
_, statErr := os.Stat(escaped)
|
||||
assert.True(t, os.IsNotExist(statErr), "escaping mountpoint %q should not exist", escaped)
|
||||
}
|
||||
|
||||
const (
|
||||
httpTimeout = 2 * time.Second
|
||||
tempDelay = 10 * time.Millisecond
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd/mountlib"
|
||||
@@ -54,8 +55,26 @@ type VolInfo struct {
|
||||
Status map[string]any `json:",omitempty"`
|
||||
}
|
||||
|
||||
// volumeMountPath returns the mountpoint for a volume called name below
|
||||
// root, together with whether that path is a strict descendant of root.
|
||||
//
|
||||
// filepath.Join cleans its result, collapsing any ".." components in the
|
||||
// volume name, so a crafted name could otherwise resolve to an arbitrary
|
||||
// host path outside root, or to root itself (for an empty or "." name)
|
||||
// where a mount would shadow every other volume. Callers must reject names
|
||||
// for which confined is false rather than creating a directory and mounting
|
||||
// there.
|
||||
func volumeMountPath(root, name string) (path string, confined bool) {
|
||||
path = filepath.Join(root, name)
|
||||
root = filepath.Clean(root)
|
||||
return path, strings.HasPrefix(path, root+string(filepath.Separator))
|
||||
}
|
||||
|
||||
func newVolume(ctx context.Context, name string, volOpt VolOpts, drv *Driver) (*Volume, error) {
|
||||
path := filepath.Join(drv.root, name)
|
||||
path, confined := volumeMountPath(drv.root, name)
|
||||
if !confined {
|
||||
return nil, fmt.Errorf("invalid volume name %q: resolves outside the base directory", name)
|
||||
}
|
||||
mnt := &mountlib.MountPoint{
|
||||
MountPoint: path,
|
||||
}
|
||||
@@ -104,8 +123,17 @@ func (vol *Volume) prepareState() {
|
||||
// retrieved with getPendingMounts for deferred mounting.
|
||||
func (vol *Volume) restoreState(ctx context.Context, drv *Driver) error {
|
||||
vol.drv = drv
|
||||
// Re-derive the mountpoint from the base directory and name rather
|
||||
// than trusting the persisted path, which an older rclone or a
|
||||
// tampered state file could have left pointing outside the base
|
||||
// directory.
|
||||
path, confined := volumeMountPath(drv.root, vol.Name)
|
||||
if !confined {
|
||||
return fmt.Errorf("invalid volume name %q: resolves outside the base directory", vol.Name)
|
||||
}
|
||||
vol.MountPoint = path
|
||||
vol.mnt = &mountlib.MountPoint{
|
||||
MountPoint: vol.MountPoint,
|
||||
MountPoint: path,
|
||||
}
|
||||
// Save pending mounts before applyOptions clears them
|
||||
vol.pendingMounts = vol.Mounts
|
||||
|
||||
+47
-35
@@ -16,7 +16,6 @@ import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
@@ -166,17 +165,23 @@ You can set a single username and password with the --user and --pass flags.
|
||||
|
||||
// driver contains everything to run the driver for the FTP server
|
||||
type driver struct {
|
||||
f fs.Fs
|
||||
srv *ftp.Server
|
||||
ctx context.Context // for global config
|
||||
opt Options
|
||||
globalVFS *vfs.VFS // the VFS if not using auth proxy
|
||||
proxy *proxy.Proxy // may be nil if not in use
|
||||
useTLS bool
|
||||
userPassMu sync.Mutex // to protect userPass
|
||||
userPass map[string]string // cache of username => password when using vfs proxy
|
||||
f fs.Fs
|
||||
srv *ftp.Server
|
||||
ctx context.Context // for global config
|
||||
opt Options
|
||||
provider *proxy.Provider
|
||||
useTLS bool
|
||||
}
|
||||
|
||||
// sessionObscuredPassKey is the key under which the obscured password is
|
||||
// stored in the per-session ftp.Session.Data map when using the auth proxy.
|
||||
//
|
||||
// The credential must be bound to the FTP session, not to the username: two
|
||||
// sessions can share a username but resolve to different proxy backends, so a
|
||||
// username-keyed store would let a later login rebind an earlier session's
|
||||
// operations to the later session's backend.
|
||||
const sessionObscuredPassKey = "rclone-obscured-pass"
|
||||
|
||||
func init() {
|
||||
fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "ftp", Opt: &Opt, Options: OptionsInfo})
|
||||
}
|
||||
@@ -195,16 +200,17 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
}
|
||||
|
||||
d := &driver{
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
}
|
||||
if proxy.Opt.AuthProxy != "" {
|
||||
d.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
|
||||
d.userPass = make(map[string]string, 16)
|
||||
} else {
|
||||
d.globalVFS = vfs.New(ctx, f, vfsOpt)
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
provider: proxy.NewProvider(ctx, f, vfsOpt, proxyOpt),
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
d.provider.Shutdown()
|
||||
}
|
||||
}()
|
||||
|
||||
d.useTLS = d.opt.TLSKey != ""
|
||||
|
||||
// Check PassivePorts format since the server library doesn't!
|
||||
@@ -250,7 +256,9 @@ func (d *driver) Serve() error {
|
||||
//lint:ignore U1000 unused when not building linux
|
||||
func (d *driver) Shutdown() error {
|
||||
fs.Logf(d.f, "Stopping FTP on %s", d.srv.Hostname+":"+strconv.Itoa(d.srv.Port))
|
||||
return d.srv.Shutdown()
|
||||
err := d.srv.Shutdown()
|
||||
d.provider.Shutdown()
|
||||
return err
|
||||
}
|
||||
|
||||
// Return the first address of the server
|
||||
@@ -316,23 +324,24 @@ func (l *Logger) PrintResponse(sessionID string, code int, message string) {
|
||||
|
||||
// CheckPasswd handle auth based on configuration
|
||||
func (d *driver) CheckPasswd(sctx *ftp.Context, user, pass string) (ok bool, err error) {
|
||||
if d.proxy != nil {
|
||||
_, _, err = d.proxy.Call(user, pass, false)
|
||||
if d.provider.IsProxy() {
|
||||
_, _, err = d.provider.Proxy().Call(user, pass, false, sctx.Sess.RemoteAddr().String())
|
||||
if err != nil {
|
||||
fs.Infof(nil, "proxy login failed: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
// Cache obscured password for later lookup.
|
||||
// Cache the obscured password on the session for later lookup.
|
||||
//
|
||||
// We don't cache the VFS directly in the driver as we want them
|
||||
// to be expired and the auth proxy does that for us.
|
||||
// We don't cache the VFS directly as we want it to be expired and
|
||||
// the auth proxy does that for us. We bind the credential to this
|
||||
// FTP session rather than to the username so a later login with the
|
||||
// same username but a different credential can't rebind this
|
||||
// session's operations to a different backend.
|
||||
oPass, err := obscure.Obscure(pass)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
d.userPassMu.Lock()
|
||||
d.userPass[user] = oPass
|
||||
d.userPassMu.Unlock()
|
||||
sctx.Sess.Data[sessionObscuredPassKey] = oPass
|
||||
} else {
|
||||
userOK := subtle.ConstantTimeCompare([]byte(d.opt.User), []byte(user))
|
||||
// No password configured means any password is accepted
|
||||
@@ -349,16 +358,19 @@ func (d *driver) CheckPasswd(sctx *ftp.Context, user, pass string) (ok bool, err
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Get the VFS for this connection
|
||||
// getVFS returns the VFS for this connection.
|
||||
//
|
||||
// In proxy mode, getVFS calls proxy.Call on each FTP command which refreshes
|
||||
// the proxy cache timer (like http/webdav). Therefore, connection-level pinning
|
||||
// is not used; only individual transfers exceeding the cache expiry window
|
||||
// could be affected.
|
||||
func (d *driver) getVFS(sctx *ftp.Context) (VFS *vfs.VFS, err error) {
|
||||
if d.proxy == nil {
|
||||
if !d.provider.IsProxy() {
|
||||
// If no proxy always use the same VFS
|
||||
return d.globalVFS, nil
|
||||
return d.provider.VFS(), nil
|
||||
}
|
||||
user := sctx.Sess.LoginUser()
|
||||
d.userPassMu.Lock()
|
||||
oPass, ok := d.userPass[user]
|
||||
d.userPassMu.Unlock()
|
||||
oPass, ok := sctx.Sess.Data[sessionObscuredPassKey].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("proxy user not logged in")
|
||||
}
|
||||
@@ -366,7 +378,7 @@ func (d *driver) getVFS(sctx *ftp.Context) (VFS *vfs.VFS, err error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
VFS, _, err = d.proxy.Call(user, pass, false)
|
||||
VFS, _, err = d.provider.Proxy().Call(user, pass, false, sctx.Sess.RemoteAddr().String())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("proxy login failed: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
// Test that auth-proxy credentials are bound to the FTP session and not
|
||||
// shared between sessions that happen to use the same username.
|
||||
|
||||
//go:build !windows && !darwin && !plan9
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ftpclient "github.com/jlaffaye/ftp"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/cmd/serve/proxy"
|
||||
"github.com/rclone/rclone/lib/israce"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestProxyCrossSession checks that two FTP sessions authenticating with the
|
||||
// same username but different auth-proxy credentials stay bound to their own
|
||||
// backends. A later login must not rebind an earlier, still-open session to
|
||||
// the later session's backend.
|
||||
func TestProxyCrossSession(t *testing.T) {
|
||||
if israce.Enabled {
|
||||
t.Skip("Skipping under race detector as underlying library is racy")
|
||||
}
|
||||
|
||||
// Two roots reached with the same username but different passwords.
|
||||
attackerRoot := t.TempDir()
|
||||
victimRoot := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(attackerRoot, "attacker.txt"), []byte("attacker-only\n"), 0600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(victimRoot, "victim.txt"), []byte("victim-secret\n"), 0600))
|
||||
t.Setenv("RCLONE_TEST_ATTACKER_ROOT", attackerRoot)
|
||||
t.Setenv("RCLONE_TEST_VICTIM_ROOT", victimRoot)
|
||||
|
||||
const addr = "127.0.0.1:52121"
|
||||
|
||||
opt := Opt
|
||||
opt.ListenAddr = addr
|
||||
opt.PassivePorts = testPASSIVEPORTRANGE
|
||||
|
||||
// The auth-proxy branch is selected from the global proxy.Opt.
|
||||
oldAuthProxy := proxy.Opt.AuthProxy
|
||||
proxy.Opt.AuthProxy = "go run proxy_crosssession_code.go"
|
||||
defer func() { proxy.Opt.AuthProxy = oldAuthProxy }()
|
||||
proxyOpt := proxy.Opt
|
||||
|
||||
w, err := newServer(context.Background(), nil, &opt, &vfscommon.Opt, &proxyOpt)
|
||||
require.NoError(t, err)
|
||||
|
||||
quit := make(chan struct{})
|
||||
go func() {
|
||||
assert.NoError(t, w.Serve())
|
||||
close(quit)
|
||||
}()
|
||||
defer func() {
|
||||
require.NoError(t, w.Shutdown())
|
||||
<-quit
|
||||
}()
|
||||
|
||||
dial := func(pass string) *ftpclient.ServerConn {
|
||||
var c *ftpclient.ServerConn
|
||||
var err error
|
||||
for range 100 {
|
||||
c, err = ftpclient.Dial(addr, ftpclient.DialWithTimeout(5*time.Second))
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, c.Login("shared", pass))
|
||||
return c
|
||||
}
|
||||
|
||||
// The attacker logs in and establishes its authority over its own root.
|
||||
attacker := dial("attacker-token")
|
||||
defer func() { _ = attacker.Quit() }()
|
||||
|
||||
names, err := attacker.NameList("/")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, names, "attacker.txt")
|
||||
assert.NotContains(t, names, "victim.txt")
|
||||
|
||||
// A second principal logs in with the same username but a different token.
|
||||
victim := dial("victim-token")
|
||||
defer func() { _ = victim.Quit() }()
|
||||
|
||||
vnames, err := victim.NameList("/")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, vnames, "victim.txt")
|
||||
|
||||
// The still-open attacker session must remain bound to the attacker
|
||||
// backend rather than being rebound to the victim's.
|
||||
names, err = attacker.NameList("/")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, names, "attacker.txt")
|
||||
assert.NotContains(t, names, "victim.txt", "attacker session was rebound to the victim backend")
|
||||
|
||||
// And the attacker must not be able to read the victim's file.
|
||||
_, err = attacker.Retr("victim.txt")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/israce"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -100,6 +101,27 @@ func TestCheckPasswd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewServerPerServerAuthProxy checks that a per-server proxyOpt.AuthProxy
|
||||
// enables proxy mode even when the process-global proxy.Opt.AuthProxy is empty,
|
||||
// which is the normal case when the server is configured via serve/start.
|
||||
func TestNewServerPerServerAuthProxy(t *testing.T) {
|
||||
// Ensure the global is empty so we only test the per-server option.
|
||||
assert.Equal(t, "", proxy.Opt.AuthProxy)
|
||||
|
||||
opt := Opt
|
||||
opt.ListenAddr = testHOST + ":" + testPORT
|
||||
opt.PassivePorts = testPASSIVEPORTRANGE
|
||||
|
||||
proxyOpt := proxy.Opt
|
||||
proxyOpt.AuthProxy = "/path/to/auth/proxy"
|
||||
|
||||
d, err := newServer(context.Background(), nil, &opt, &vfscommon.Opt, &proxyOpt)
|
||||
require.NoError(t, err)
|
||||
defer d.provider.Shutdown()
|
||||
assert.True(t, d.provider.IsProxy(), "expected auth proxy to be enabled by per-server option")
|
||||
assert.Nil(t, d.provider.VFS(), "expected no fixed VFS when auth proxy is in use")
|
||||
}
|
||||
|
||||
func TestRc(t *testing.T) {
|
||||
if israce.Enabled {
|
||||
t.Skip("Skipping under race detector as underlying library is racy")
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
//go:build ignore
|
||||
|
||||
// A test auth proxy that maps the supplied password to a backend root.
|
||||
//
|
||||
// Both roots require the same FTP username ("shared") but different
|
||||
// passwords, so it exercises two credentials that share a username but
|
||||
// resolve to different backends. The roots are passed in the environment.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
var in map[string]string
|
||||
if err := json.NewDecoder(os.Stdin).Decode(&in); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
roots := map[string]string{
|
||||
"attacker-token": os.Getenv("RCLONE_TEST_ATTACKER_ROOT"),
|
||||
"victim-token": os.Getenv("RCLONE_TEST_VICTIM_ROOT"),
|
||||
}
|
||||
root, ok := roots[in["pass"]]
|
||||
if in["user"] != "shared" || !ok {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
out := map[string]string{
|
||||
"type": "local",
|
||||
"_root": root,
|
||||
}
|
||||
if err := json.NewEncoder(os.Stdout).Encode(&out); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
+21
-30
@@ -4,7 +4,6 @@ package http
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
@@ -150,33 +149,21 @@ control the stats printing.
|
||||
|
||||
// HTTP contains everything to run the server
|
||||
type HTTP struct {
|
||||
f fs.Fs
|
||||
_vfs *vfs.VFS // don't use directly, use getVFS
|
||||
server *libhttp.Server
|
||||
opt Options
|
||||
proxy *proxy.Proxy
|
||||
ctx context.Context // for global config
|
||||
f fs.Fs
|
||||
provider *proxy.Provider
|
||||
server *libhttp.Server
|
||||
opt Options
|
||||
ctx context.Context // for global config
|
||||
}
|
||||
|
||||
// Gets the VFS in use for this request
|
||||
func (s *HTTP) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
|
||||
if s._vfs != nil {
|
||||
return s._vfs, nil
|
||||
}
|
||||
value := libhttp.CtxGetAuth(ctx)
|
||||
if value == nil {
|
||||
return nil, errors.New("no VFS found in context")
|
||||
}
|
||||
VFS, ok := value.(*vfs.VFS)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("context value is not VFS: %#v", value)
|
||||
}
|
||||
return VFS, nil
|
||||
return s.provider.Get(ctx)
|
||||
}
|
||||
|
||||
// auth does proxy authorization
|
||||
func (s *HTTP) auth(user, pass string) (value any, err error) {
|
||||
VFS, _, err := s.proxy.Call(user, pass, false)
|
||||
func (s *HTTP) auth(r *http.Request, user, pass string) (value any, err error) {
|
||||
VFS, _, err := s.provider.Proxy().Call(user, pass, false, r.RemoteAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -185,17 +172,19 @@ func (s *HTTP) auth(user, pass string) (value any, err error) {
|
||||
|
||||
func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Options, proxyOpt *proxy.Options) (s *HTTP, err error) {
|
||||
s = &HTTP{
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
provider: proxy.NewProvider(ctx, f, vfsOpt, proxyOpt),
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
s.provider.Shutdown()
|
||||
}
|
||||
}()
|
||||
|
||||
if proxyOpt.AuthProxy != "" {
|
||||
s.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
|
||||
// override auth
|
||||
if s.provider.IsProxy() {
|
||||
s.opt.Auth.CustomAuthFn = s.auth
|
||||
} else {
|
||||
s._vfs = vfs.New(ctx, f, vfsOpt)
|
||||
}
|
||||
|
||||
s.server, err = libhttp.NewServer(ctx,
|
||||
@@ -235,7 +224,9 @@ func (s *HTTP) Addr() net.Addr {
|
||||
|
||||
// Shutdown the server
|
||||
func (s *HTTP) Shutdown() error {
|
||||
return s.server.Shutdown()
|
||||
err := s.server.Shutdown()
|
||||
s.provider.Shutdown()
|
||||
return err
|
||||
}
|
||||
|
||||
// serveFavicon serves the remote's favicon.ico if it exists, otherwise
|
||||
|
||||
@@ -186,16 +186,7 @@ func (f *FS) Readlink(link string) (result string, err error) {
|
||||
func (f *FS) Chmod(name string, mode os.FileMode) (err error) {
|
||||
name = f.fullPath(name)
|
||||
defer log.Trace(name, "mode=%v", mode)("err=%v", &err)
|
||||
file, err := f.vfs.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
fs.Logf(f, "Error while closing file: %e", err)
|
||||
}
|
||||
}()
|
||||
err = file.Chmod(mode)
|
||||
err = f.vfs.Chmod(name, mode)
|
||||
// Mask Chmod not implemented
|
||||
if err == vfs.ENOSYS {
|
||||
err = nil
|
||||
@@ -213,16 +204,12 @@ func (f *FS) Lchown(name string, uid, gid int) (err error) {
|
||||
func (f *FS) Chown(name string, uid, gid int) (err error) {
|
||||
name = f.fullPath(name)
|
||||
defer log.Trace(name, "uid=%d, gid=%d", uid, gid)("err=%v", &err)
|
||||
file, err := f.vfs.Open(name)
|
||||
if err != nil {
|
||||
return err
|
||||
err = f.vfs.Chown(name, uid, gid)
|
||||
// Mask Chown not implemented
|
||||
if err == vfs.ENOSYS {
|
||||
err = nil
|
||||
}
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
fs.Logf(f, "Error while closing file: %e", err)
|
||||
}
|
||||
}()
|
||||
return file.Chown(uid, gid)
|
||||
return err
|
||||
}
|
||||
|
||||
// Chtimes changes the access time and modified time
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
//go:build unix
|
||||
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Chmod/Chown arrive as plain SETATTR calls on the link path after a
|
||||
// SYMLINK RPC, so they must not follow symlinks - a freshly created
|
||||
// symlink usually dangles and following it would fail with ENOENT,
|
||||
// which the NFS layer surfaces as an IO error. See #9627.
|
||||
func TestChmodDanglingSymlink(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
f, err := fs.NewFs(ctx, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
opt := vfscommon.Opt
|
||||
opt.Links = true
|
||||
opt.CacheMode = vfscommon.CacheModeWrites
|
||||
v := vfs.New(ctx, f, &opt)
|
||||
defer v.Shutdown()
|
||||
bfs := &FS{vfs: v}
|
||||
|
||||
// Create a symlink pointing at a target which doesn't exist yet
|
||||
require.NoError(t, bfs.Symlink("does-not-exist", "link"))
|
||||
|
||||
// SETATTR after SYMLINK must not fail
|
||||
assert.NoError(t, bfs.Chmod("link", 0777))
|
||||
assert.NoError(t, bfs.Chown("link", 1000, 1000))
|
||||
assert.NoError(t, bfs.Lchown("link", 1000, 1000))
|
||||
|
||||
// A genuinely missing node must still report ENOENT
|
||||
assert.ErrorIs(t, bfs.Chmod("missing", 0777), vfs.ENOENT)
|
||||
assert.ErrorIs(t, bfs.Chown("missing", 1000, 1000), vfs.ENOENT)
|
||||
}
|
||||
+21
-2
@@ -82,6 +82,17 @@ func AddFlags(flagSet *pflag.FlagSet) {
|
||||
flags.AddFlagsFromOptions(flagSet, "", OptionsInfo)
|
||||
}
|
||||
|
||||
type nfsHandler struct {
|
||||
*Server
|
||||
vfs *vfs.VFS
|
||||
}
|
||||
|
||||
func (h *nfsHandler) Shutdown() error {
|
||||
err := h.Server.Shutdown()
|
||||
h.vfs.Shutdown()
|
||||
return err
|
||||
}
|
||||
|
||||
func init() {
|
||||
vfsflags.AddFlags(Command.Flags())
|
||||
AddFlags(Command.Flags())
|
||||
@@ -98,10 +109,16 @@ func init() {
|
||||
var opt = Opt // set default opts
|
||||
err = rc.ParseOptions(in, "opt", &opt)
|
||||
if err != nil {
|
||||
VFS.Shutdown()
|
||||
return nil, err
|
||||
}
|
||||
// Create server
|
||||
return NewServer(ctx, VFS, &opt)
|
||||
s, err := NewServer(ctx, VFS, &opt)
|
||||
if err != nil {
|
||||
VFS.Shutdown()
|
||||
return nil, err
|
||||
}
|
||||
return &nfsHandler{Server: s, vfs: VFS}, nil
|
||||
})
|
||||
}
|
||||
|
||||
@@ -111,7 +128,9 @@ func Run(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
f = cmd.NewFsSrc(args)
|
||||
cmd.Run(false, true, command, func() error {
|
||||
s, err := NewServer(context.Background(), vfs.New(context.Background(), f, &vfscommon.Opt), &Opt)
|
||||
VFS := vfs.New(context.Background(), f, &vfscommon.Opt)
|
||||
defer VFS.Shutdown()
|
||||
s, err := NewServer(context.Background(), VFS, &Opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+335
-41
@@ -12,8 +12,11 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -21,6 +24,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
libcache "github.com/rclone/rclone/lib/cache"
|
||||
libhttp "github.com/rclone/rclone/lib/http"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
@@ -52,9 +56,10 @@ This config generated must have this extra parameter
|
||||
|
||||
- |_root| - root to use for the backend
|
||||
|
||||
And it may have this parameter
|
||||
And it may have these parameters
|
||||
|
||||
- |_obscure| - comma separated strings for parameters to obscure
|
||||
- |_secret_access_key| - the secret for S3 access key auth (see below)
|
||||
|
||||
If password authentication was used by the client, input to the proxy
|
||||
process (on STDIN) would look similar to this:
|
||||
@@ -62,7 +67,8 @@ process (on STDIN) would look similar to this:
|
||||
|||json
|
||||
{
|
||||
"user": "me",
|
||||
"pass": "mypassword"
|
||||
"pass": "mypassword",
|
||||
"client_ip": "192.168.1.1"
|
||||
}
|
||||
|||
|
||||
|
||||
@@ -72,10 +78,44 @@ proxy process (on STDIN) would look similar to this:
|
||||
|||json
|
||||
{
|
||||
"user": "me",
|
||||
"public_key": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDuwESFdAe14hVS6omeyX7edc...JQdf"
|
||||
"public_key": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDuwESFdAe14hVS6omeyX7edc...JQdf",
|
||||
"client_ip": "192.168.1.1"
|
||||
}
|
||||
|||
|
||||
|
||||
If the client authenticated with an S3 access key (|rclone serve s3|),
|
||||
the client never sends its secret, only a signature made with it, so
|
||||
the input contains just the access key ID as the |user| with no |pass|
|
||||
or |public_key|:
|
||||
|
||||
|||json
|
||||
{
|
||||
"user": "AKIAIOSFODNN7EXAMPLE",
|
||||
"client_ip": "192.168.1.1"
|
||||
}
|
||||
|||
|
||||
|
||||
In this case the program must look up the secret access key for that
|
||||
access key ID and return it in the |_secret_access_key| field of the
|
||||
output. Rclone then uses that secret to verify the signature on the
|
||||
request, refusing the request if it does not match. This means the
|
||||
proxy program is the source of truth for both the credentials and the
|
||||
backend they map to. If the program does not return
|
||||
|_secret_access_key| or returns it empty the request is refused.
|
||||
|
||||
The program's answer for an access key ID is cached (see below) but
|
||||
is checked with the program again after 5 minutes even if the access
|
||||
key ID is in constant use, so revoking an access key ID in the
|
||||
program takes effect within 5 minutes. A rotated secret takes effect
|
||||
on the first request signed with it.
|
||||
|
||||
The |client_ip| key holds the IP address the client connected from,
|
||||
without a port number. It can be used to restrict logins to certain
|
||||
networks, or to log authentication attempts centrally. It is omitted if
|
||||
the client has no IP address, for example when connecting over a unix
|
||||
socket. Note that if rclone is behind a reverse proxy this will be the
|
||||
address of the reverse proxy and not the original client.
|
||||
|
||||
And as an example return this on STDOUT
|
||||
|
||||
|||json
|
||||
@@ -101,11 +141,12 @@ to make proxy to many different sftp backends, you could make the
|
||||
in the output and the user to |user|. For security you'd probably want
|
||||
to restrict the |host| to a limited list.
|
||||
|
||||
An internal cache of backends is keyed on the |user| and a hash of the
|
||||
|pass| or |public_key|. This means that if a user's password or
|
||||
public-key changes, or the proxy returns different config parameters
|
||||
(eg a rotated |api_key|), a fresh backend will be created on the next
|
||||
request rather than the cached one being reused.
|
||||
An internal cache of backends is keyed on the |user|, a hash of the
|
||||
|pass| or |public_key|, and the |client_ip|. This means that if a
|
||||
user's password or public-key changes, the client connects from a new IP
|
||||
address, or the proxy returns different config parameters (eg a rotated
|
||||
|api_key|), a fresh backend will be created on the next request rather
|
||||
than the cached one being reused.
|
||||
|
||||
This can be used to build general purpose proxies to any kind of
|
||||
backend that rclone supports.
|
||||
@@ -133,30 +174,65 @@ func init() {
|
||||
|
||||
// Proxy represents a proxy to turn auth requests into a VFS
|
||||
type Proxy struct {
|
||||
cmdLine []string // broken down command line
|
||||
vfsCache *libcache.Cache
|
||||
ctx context.Context // for global config
|
||||
Opt Options
|
||||
vfsOpt vfscommon.Options
|
||||
cmdLine []string // broken down command line
|
||||
vfsCache *libcache.Cache
|
||||
ctx context.Context // for global config
|
||||
Opt Options
|
||||
vfsOpt vfscommon.Options
|
||||
accessKeyMu sync.Mutex // serialises replacing a cached access key entry
|
||||
}
|
||||
|
||||
// cacheEntry is what is stored in the vfsCache
|
||||
type cacheEntry struct {
|
||||
vfs *vfs.VFS // stored VFS
|
||||
pwHash [sha256.Size]byte // sha256 hash of the password/publicKey
|
||||
vfs *vfs.VFS // stored VFS
|
||||
pwHash [sha256.Size]byte // sha256 hash of the password/publicKey
|
||||
secret string // secret access key returned by the proxy for access key auth
|
||||
refreshed *atomic.Int64 // unix nanoseconds when the proxy last confirmed this entry
|
||||
}
|
||||
|
||||
// accessKeyRefreshInterval is the minimum time between consulting the
|
||||
// proxy again for a cached access key ID whose secret failed to
|
||||
// verify a request, so a stream of bad signatures for a known access
|
||||
// key ID can't make the proxy run for every request.
|
||||
//
|
||||
// A variable so tests can adjust it.
|
||||
var accessKeyRefreshInterval = 10 * time.Second
|
||||
|
||||
// accessKeyRevalidateInterval is how long a cached access key ID is
|
||||
// trusted before the proxy is asked about it again even though its
|
||||
// signatures are still verifying, so a revoked access key ID stops
|
||||
// working within this time rather than for as long as it stays in
|
||||
// use.
|
||||
//
|
||||
// A variable so tests can adjust it.
|
||||
var accessKeyRevalidateInterval = 5 * time.Minute
|
||||
|
||||
// authKind is the kind of credential the client presented
|
||||
type authKind int
|
||||
|
||||
const (
|
||||
authPassword authKind = iota // auth is a password
|
||||
authPublicKey // auth is a public key
|
||||
authAccessKey // user is an S3 access key ID and auth is empty
|
||||
)
|
||||
|
||||
// New creates a new proxy with the Options passed in
|
||||
//
|
||||
// Any VFS are created with the vfsOpt passed in.
|
||||
func New(ctx context.Context, opt *Options, vfsOpt *vfscommon.Options) *Proxy {
|
||||
return &Proxy{
|
||||
p := &Proxy{
|
||||
ctx: ctx,
|
||||
Opt: *opt,
|
||||
cmdLine: strings.Fields(opt.AuthProxy),
|
||||
vfsCache: libcache.New(),
|
||||
vfsOpt: *vfsOpt,
|
||||
}
|
||||
p.vfsCache.SetFinalizer(func(value any) {
|
||||
if entry, ok := value.(cacheEntry); ok && entry.vfs != nil {
|
||||
entry.vfs.Shutdown()
|
||||
}
|
||||
})
|
||||
return p
|
||||
}
|
||||
|
||||
// run the proxy command returning a config map
|
||||
@@ -212,29 +288,42 @@ var cacheKeyHMACKey = func() []byte {
|
||||
return key
|
||||
}()
|
||||
|
||||
// generateCacheKey creates a composite cache key from user and auth credentials
|
||||
func generateCacheKey(user, auth string) string {
|
||||
// ipFromAddr returns the bare IP from a "host:port" address, or "" if it has none.
|
||||
func ipFromAddr(addr string) string {
|
||||
ap, err := netip.ParseAddrPort(addr)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return ap.Addr().Unmap().String()
|
||||
}
|
||||
|
||||
// generateCacheKey creates a composite cache key from the user, the auth
|
||||
// credentials and the client's IP address.
|
||||
func generateCacheKey(user, auth, clientIP string) string {
|
||||
mac := hmac.New(sha256.New, cacheKeyHMACKey)
|
||||
mac.Write([]byte(auth))
|
||||
// Separate the two so ("ab", "c") can't collide with ("a", "bc")
|
||||
mac.Write([]byte{0})
|
||||
mac.Write([]byte(clientIP))
|
||||
return user + "-" + hex.EncodeToString(mac.Sum(nil)[:8])
|
||||
}
|
||||
|
||||
// call runs the auth proxy and returns a cacheEntry and an error
|
||||
func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error) {
|
||||
var config configmap.Simple
|
||||
func (p *Proxy) call(user, auth string, kind authKind, clientIP string) (value any, err error) {
|
||||
// Contact the proxy
|
||||
if isPublicKey {
|
||||
config, err = p.run(map[string]string{
|
||||
"user": user,
|
||||
"public_key": auth,
|
||||
})
|
||||
} else {
|
||||
config, err = p.run(map[string]string{
|
||||
"user": user,
|
||||
"pass": auth,
|
||||
})
|
||||
in := map[string]string{
|
||||
"user": user,
|
||||
}
|
||||
|
||||
switch kind {
|
||||
case authPublicKey:
|
||||
in["public_key"] = auth
|
||||
case authPassword:
|
||||
in["pass"] = auth
|
||||
}
|
||||
if clientIP != "" {
|
||||
in["client_ip"] = clientIP
|
||||
}
|
||||
config, err := p.run(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -248,6 +337,19 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)
|
||||
if !ok {
|
||||
return nil, errors.New("proxy: _root not set in result")
|
||||
}
|
||||
var secret string
|
||||
if kind == authAccessKey {
|
||||
secret, ok = config.Get("_secret_access_key")
|
||||
if !ok {
|
||||
return nil, errors.New("proxy: _secret_access_key not set in result")
|
||||
}
|
||||
// An empty secret would let anyone sign for this access key ID
|
||||
if secret == "" {
|
||||
return nil, errors.New("proxy: _secret_access_key is empty in result")
|
||||
}
|
||||
// Keep the secret out of the backend config
|
||||
delete(config, "_secret_access_key")
|
||||
}
|
||||
|
||||
// Find the backend
|
||||
fsInfo, err := fs.Find(fsName)
|
||||
@@ -255,10 +357,32 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)
|
||||
return nil, fmt.Errorf("proxy: couldn't find backend for %q: %w", fsName, err)
|
||||
}
|
||||
|
||||
// Make the cache key include the auth so that changes to the
|
||||
// auth (eg the proxy returning new config) create a fresh
|
||||
// backend rather than reusing the cached one.
|
||||
cacheKey := generateCacheKey(user, auth)
|
||||
// Make the cache key include the auth and the client IP so that
|
||||
// changes to either (eg the proxy returning new config) create a
|
||||
// fresh backend rather than reusing the cached one.
|
||||
cacheKey := generateCacheKey(user, auth, clientIP)
|
||||
|
||||
// For access key auth the secret isn't part of the cache key as
|
||||
// it isn't known until the proxy has been run, so retire any
|
||||
// cached entry whose secret the proxy has since changed.
|
||||
//
|
||||
// The stale entry is moved to a key including its old secret
|
||||
// rather than deleted so that its VFS isn't shut down under
|
||||
// requests still using it - it expires with the rest of the cache
|
||||
// once it has been unused for long enough.
|
||||
//
|
||||
// The check, rename and the creation below are done under a lock
|
||||
// so that two concurrent refreshes during a rotation can't have
|
||||
// the second retire the backend the first has just created.
|
||||
if kind == authAccessKey {
|
||||
p.accessKeyMu.Lock()
|
||||
defer p.accessKeyMu.Unlock()
|
||||
if old, ok := p.vfsCache.GetMaybe(cacheKey); ok {
|
||||
if entry, ok := old.(cacheEntry); ok && entry.secret != secret {
|
||||
p.vfsCache.Rename(cacheKey, generateCacheKey(user, entry.secret, clientIP))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// base name of config on user name and auth hash. This may appear in logs
|
||||
name := "proxy-" + cacheKey
|
||||
@@ -285,29 +409,47 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)
|
||||
// need to in memory. An attacker would find it easier to go
|
||||
// after the unencrypted password in memory most likely.
|
||||
entry := cacheEntry{
|
||||
vfs: vfs.New(p.ctx, f, &p.vfsOpt),
|
||||
pwHash: sha256.Sum256([]byte(auth)),
|
||||
vfs: vfs.New(p.ctx, f, &p.vfsOpt),
|
||||
pwHash: sha256.Sum256([]byte(auth)),
|
||||
secret: secret,
|
||||
refreshed: new(atomic.Int64),
|
||||
}
|
||||
return entry, true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("proxy: failed to create backend: %w", err)
|
||||
}
|
||||
// The proxy has just confirmed this entry whether it was created
|
||||
// or reused
|
||||
if entry, ok := value.(cacheEntry); ok {
|
||||
entry.refreshed.Store(time.Now().UnixNano())
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// Call runs the auth proxy with the username and password/public key provided
|
||||
// returning a *vfs.VFS and the key used in the VFS cache.
|
||||
func (p *Proxy) Call(user, auth string, isPublicKey bool) (VFS *vfs.VFS, vfsKey string, err error) {
|
||||
// Cache key includes the auth so credential changes don't hit a stale entry
|
||||
cacheKey := generateCacheKey(user, auth)
|
||||
//
|
||||
// remoteAddr is the address of the client as returned by net.Addr.String().
|
||||
// It may be empty if the client has no IP address, for example when
|
||||
// connecting over a unix socket.
|
||||
func (p *Proxy) Call(user, auth string, isPublicKey bool, remoteAddr string) (VFS *vfs.VFS, vfsKey string, err error) {
|
||||
clientIP := ipFromAddr(remoteAddr)
|
||||
|
||||
// Cache key includes the auth and the client IP so credential or
|
||||
// address changes don't hit a stale entry
|
||||
cacheKey := generateCacheKey(user, auth, clientIP)
|
||||
|
||||
// Look in the cache first with the credential-aware key
|
||||
value, ok := p.vfsCache.GetMaybe(cacheKey)
|
||||
|
||||
// If not found then call the proxy for a fresh answer
|
||||
if !ok {
|
||||
value, err = p.call(user, auth, isPublicKey)
|
||||
kind := authPassword
|
||||
if isPublicKey {
|
||||
kind = authPublicKey
|
||||
}
|
||||
value, err = p.call(user, auth, kind, clientIP)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
@@ -335,6 +477,53 @@ func (p *Proxy) Call(user, auth string, isPublicKey bool) (VFS *vfs.VFS, vfsKey
|
||||
return entry.vfs, cacheKey, nil
|
||||
}
|
||||
|
||||
// CallAccessKey runs the auth proxy for an S3 access key ID returning
|
||||
// a *vfs.VFS and the secret access key the proxy supplied for it.
|
||||
//
|
||||
// The caller must verify the request's signature against the returned
|
||||
// secret - the proxy only maps the access key ID to a backend and
|
||||
// secret, it cannot authenticate the client itself.
|
||||
//
|
||||
// If refresh is true the proxy is consulted rather than using a cached
|
||||
// answer, unless it was consulted for this access key ID less than
|
||||
// accessKeyRefreshInterval ago. A refresh never shuts down a cached
|
||||
// backend, even if the proxy returns a different secret, so this is
|
||||
// safe to do when a signature fails to verify in case the secret has
|
||||
// been rotated.
|
||||
//
|
||||
// A cached answer older than accessKeyRevalidateInterval is always
|
||||
// checked with the proxy so a revoked access key ID is refused even
|
||||
// if it is in constant use.
|
||||
//
|
||||
// remoteAddr is the address of the client as returned by net.Addr.String().
|
||||
func (p *Proxy) CallAccessKey(accessKeyID, remoteAddr string, refresh bool) (VFS *vfs.VFS, secret string, err error) {
|
||||
clientIP := ipFromAddr(remoteAddr)
|
||||
cacheKey := generateCacheKey(accessKeyID, "", clientIP)
|
||||
value, ok := p.vfsCache.GetMaybe(cacheKey)
|
||||
if ok {
|
||||
if entry, isEntry := value.(cacheEntry); isEntry {
|
||||
age := time.Since(time.Unix(0, entry.refreshed.Load()))
|
||||
switch {
|
||||
case age >= accessKeyRevalidateInterval:
|
||||
refresh = true
|
||||
case age < accessKeyRefreshInterval:
|
||||
refresh = false
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok || refresh {
|
||||
value, err = p.call(accessKeyID, "", authAccessKey, clientIP)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
entry, ok := value.(cacheEntry)
|
||||
if !ok {
|
||||
return nil, "", fmt.Errorf("proxy: value is not cache entry: %#v", value)
|
||||
}
|
||||
return entry.vfs, entry.secret, nil
|
||||
}
|
||||
|
||||
// Get VFS from the cache using key - returns nil if not found
|
||||
func (p *Proxy) Get(key string) *vfs.VFS {
|
||||
value, ok := p.vfsCache.GetMaybe(key)
|
||||
@@ -344,3 +533,108 @@ func (p *Proxy) Get(key string) *vfs.VFS {
|
||||
entry := value.(cacheEntry)
|
||||
return entry.vfs
|
||||
}
|
||||
|
||||
// Shutdown shuts down all cached VFS instances
|
||||
func (p *Proxy) Shutdown() {
|
||||
if p != nil && p.vfsCache != nil {
|
||||
p.vfsCache.Clear()
|
||||
}
|
||||
}
|
||||
|
||||
// Provider hands out VFS instances, either a fixed one or per-user via an auth proxy.
|
||||
type Provider struct {
|
||||
vfs *vfs.VFS // set if not using an auth proxy
|
||||
proxy *Proxy // set if using an auth proxy
|
||||
}
|
||||
|
||||
// NewProvider creates a Provider. If proxyOpt.AuthProxy is set it creates an
|
||||
// auth proxy; otherwise it creates a fixed VFS from f and vfsOpt.
|
||||
func NewProvider(ctx context.Context, f fs.Fs, vfsOpt *vfscommon.Options, proxyOpt *Options) *Provider {
|
||||
p := &Provider{}
|
||||
if proxyOpt != nil && proxyOpt.AuthProxy != "" {
|
||||
p.proxy = New(ctx, proxyOpt, vfsOpt)
|
||||
} else {
|
||||
p.vfs = vfs.New(ctx, f, vfsOpt)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Get returns the VFS for the current request context.
|
||||
// For fixed-VFS providers it returns the single VFS.
|
||||
// For proxy providers it reads the VFS from the request's auth context.
|
||||
func (p *Provider) Get(ctx context.Context) (*vfs.VFS, error) {
|
||||
if p.vfs != nil {
|
||||
return p.vfs, nil
|
||||
}
|
||||
value := libhttp.CtxGetAuth(ctx)
|
||||
if value == nil {
|
||||
return nil, errors.New("no VFS found in context")
|
||||
}
|
||||
VFS, ok := value.(*vfs.VFS)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("context value is not VFS: %#v", value)
|
||||
}
|
||||
return VFS, nil
|
||||
}
|
||||
|
||||
// VFS returns the fixed VFS, or nil if using an auth proxy.
|
||||
func (p *Provider) VFS() *vfs.VFS {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
return p.vfs
|
||||
}
|
||||
|
||||
// Proxy returns the Proxy instance, or nil if not using an auth proxy.
|
||||
func (p *Provider) Proxy() *Proxy {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
return p.proxy
|
||||
}
|
||||
|
||||
// IsProxy returns true if using an auth proxy.
|
||||
func (p *Provider) IsProxy() bool {
|
||||
return p != nil && p.proxy != nil
|
||||
}
|
||||
|
||||
// Shutdown tears down the provider: shuts down the fixed VFS or flushes all proxy cache entries.
|
||||
func (p *Provider) Shutdown() {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
if p.vfs != nil {
|
||||
p.vfs.Shutdown()
|
||||
}
|
||||
if p.proxy != nil {
|
||||
p.proxy.Shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
// Pin pins the cache entry for key so it won't be evicted by expire
|
||||
func (p *Proxy) Pin(key string) {
|
||||
if p != nil && p.vfsCache != nil {
|
||||
p.vfsCache.Pin(key)
|
||||
}
|
||||
}
|
||||
|
||||
// Unpin unpins the cache entry for key
|
||||
func (p *Proxy) Unpin(key string) {
|
||||
if p != nil && p.vfsCache != nil {
|
||||
p.vfsCache.Unpin(key)
|
||||
}
|
||||
}
|
||||
|
||||
// Pin pins the cache entry for key if using an auth proxy
|
||||
func (p *Provider) Pin(key string) {
|
||||
if p != nil && p.proxy != nil {
|
||||
p.proxy.Pin(key)
|
||||
}
|
||||
}
|
||||
|
||||
// Unpin unpins the cache entry for key if using an auth proxy
|
||||
func (p *Provider) Unpin(key string) {
|
||||
if p != nil && p.proxy != nil {
|
||||
p.proxy.Unpin(key)
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,25 @@ func main() {
|
||||
if out["_root"] == "" {
|
||||
out["_root"] = ""
|
||||
}
|
||||
// S3 access key auth has neither pass nor public_key and needs
|
||||
// the secret returned, unless the user asks for it to be omitted
|
||||
// or empty. The secret's suffix can be changed to simulate a
|
||||
// rotation and an access key ID can be revoked.
|
||||
_, havePass := in["pass"]
|
||||
_, havePublicKey := in["public_key"]
|
||||
switch {
|
||||
case havePass || havePublicKey || in["user"] == "nosecret":
|
||||
case in["user"] == os.Getenv("RCLONE_TEST_PROXY_REVOKED"):
|
||||
log.Fatalf("access key ID %q revoked", in["user"])
|
||||
case in["user"] == "emptysecret":
|
||||
out["_secret_access_key"] = ""
|
||||
default:
|
||||
suffix := os.Getenv("RCLONE_TEST_PROXY_SECRET_SUFFIX")
|
||||
if suffix == "" {
|
||||
suffix = "-secret"
|
||||
}
|
||||
out["_secret_access_key"] = in["user"] + suffix
|
||||
}
|
||||
json.NewEncoder(os.Stdout).Encode(&out)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
||||
+221
-13
@@ -7,12 +7,15 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -42,6 +45,25 @@ func TestRun(t *testing.T) {
|
||||
}, config)
|
||||
})
|
||||
|
||||
t.Run("ClientIP", func(t *testing.T) {
|
||||
config, err := p.run(map[string]string{
|
||||
"type": "ftp",
|
||||
"user": "me",
|
||||
"pass": "pass",
|
||||
"host": "127.0.0.1",
|
||||
"client_ip": "192.0.2.1",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, configmap.Simple{
|
||||
"type": "ftp",
|
||||
"user": "me-test",
|
||||
"pass": "pass",
|
||||
"host": "127.0.0.1",
|
||||
"client_ip": "192.0.2.1",
|
||||
"_root": "",
|
||||
}, config)
|
||||
})
|
||||
|
||||
t.Run("Error", func(t *testing.T) {
|
||||
config, err := p.run(map[string]string{
|
||||
"error": "potato",
|
||||
@@ -74,6 +96,24 @@ func TestRun(t *testing.T) {
|
||||
|
||||
const testUser = "testUser"
|
||||
const testPass = "testPass"
|
||||
const testIP = "192.0.2.1"
|
||||
const testAddr = testIP + ":1024"
|
||||
const otherAddr = "198.51.100.1:1024"
|
||||
|
||||
t.Run("CacheKey", func(t *testing.T) {
|
||||
// The source port differs on every connection so it must not
|
||||
// affect the cache key, otherwise the proxy would be run for
|
||||
// every connection rather than once per client.
|
||||
assert.Equal(t,
|
||||
generateCacheKey(testUser, testPass, ipFromAddr(testIP+":1024")),
|
||||
generateCacheKey(testUser, testPass, ipFromAddr(testIP+":2048")))
|
||||
|
||||
// A different client IP must produce a different key so the
|
||||
// proxy is consulted again
|
||||
assert.NotEqual(t,
|
||||
generateCacheKey(testUser, testPass, ipFromAddr(testAddr)),
|
||||
generateCacheKey(testUser, testPass, ipFromAddr(otherAddr)))
|
||||
})
|
||||
|
||||
t.Run("call w/Password", func(t *testing.T) {
|
||||
// check cache empty
|
||||
@@ -81,7 +121,7 @@ func TestRun(t *testing.T) {
|
||||
defer p.vfsCache.Clear()
|
||||
|
||||
passwordBytes := []byte(testPass)
|
||||
value, err := p.call(testUser, testPass, false)
|
||||
value, err := p.call(testUser, testPass, authPassword, testIP)
|
||||
require.NoError(t, err)
|
||||
entry, ok := value.(cacheEntry)
|
||||
require.True(t, ok)
|
||||
@@ -91,7 +131,7 @@ func TestRun(t *testing.T) {
|
||||
require.NotNil(t, entry.vfs)
|
||||
f := entry.vfs.Fs()
|
||||
require.NotNil(t, f)
|
||||
cacheKey := generateCacheKey(testUser, testPass)
|
||||
cacheKey := generateCacheKey(testUser, testPass, testIP)
|
||||
assert.Equal(t, "proxy-"+cacheKey, f.Name())
|
||||
assert.True(t, strings.HasPrefix(f.String(), "Local file system"))
|
||||
|
||||
@@ -107,8 +147,8 @@ func TestRun(t *testing.T) {
|
||||
assert.Equal(t, 0, p.vfsCache.Entries())
|
||||
defer p.vfsCache.Clear()
|
||||
|
||||
cacheKey := generateCacheKey(testUser, testPass)
|
||||
vfs, vfsKey, err := p.Call(testUser, testPass, false)
|
||||
cacheKey := generateCacheKey(testUser, testPass, testIP)
|
||||
vfs, vfsKey, err := p.Call(testUser, testPass, false, testAddr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs)
|
||||
assert.Equal(t, "proxy-"+cacheKey, vfs.Fs().Name())
|
||||
@@ -129,7 +169,7 @@ func TestRun(t *testing.T) {
|
||||
})
|
||||
|
||||
// now try again from the cache
|
||||
vfs, vfsKey, err = p.Call(testUser, testPass, false)
|
||||
vfs, vfsKey, err = p.Call(testUser, testPass, false, testAddr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs)
|
||||
assert.Equal(t, "proxy-"+cacheKey, vfs.Fs().Name())
|
||||
@@ -141,7 +181,7 @@ func TestRun(t *testing.T) {
|
||||
// A different password produces a different cache key, so it
|
||||
// creates a fresh cache entry rather than hitting the existing
|
||||
// one. Authentication itself is the proxy script's job.
|
||||
vfs2, vfsKey2, err := p.Call(testUser, testPass+"different", false)
|
||||
vfs2, vfsKey2, err := p.Call(testUser, testPass+"different", false, testAddr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs2)
|
||||
assert.NotEqual(t, cacheKey, vfsKey2)
|
||||
@@ -152,18 +192,40 @@ func TestRun(t *testing.T) {
|
||||
t.Error("fs/cache returned the stale backend after auth change")
|
||||
}
|
||||
|
||||
// A different client IP also produces a different cache key, so
|
||||
// the proxy is consulted again rather than the cached backend
|
||||
// being reused - the proxy may be filtering on the IP.
|
||||
vfs3, vfsKey3, err := p.Call(testUser, testPass, false, otherAddr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs3)
|
||||
assert.NotEqual(t, cacheKey, vfsKey3)
|
||||
assert.Equal(t, 3, p.vfsCache.Entries())
|
||||
|
||||
// If a cached entry's pwHash somehow doesn't match the supplied
|
||||
// auth (eg a hash collision on the cache key), Call must reject
|
||||
// it. Simulate by corrupting the cached pwHash.
|
||||
entry := cacheEntry{vfs: vfs, pwHash: sha256.Sum256([]byte("tampered"))}
|
||||
p.vfsCache.Put(cacheKey, entry)
|
||||
vfs, vfsKey, err = p.Call(testUser, testPass, false)
|
||||
vfs, vfsKey, err = p.Call(testUser, testPass, false, testAddr)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "incorrect password")
|
||||
require.Nil(t, vfs)
|
||||
require.Equal(t, "", vfsKey)
|
||||
})
|
||||
|
||||
t.Run("Call w/o Address", func(t *testing.T) {
|
||||
// A client with no address, eg on a unix socket, must still
|
||||
// authenticate
|
||||
assert.Equal(t, 0, p.vfsCache.Entries())
|
||||
defer p.vfsCache.Clear()
|
||||
|
||||
vfs, vfsKey, err := p.Call(testUser, testPass, false, "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs)
|
||||
assert.Equal(t, generateCacheKey(testUser, testPass, ""), vfsKey)
|
||||
assert.Equal(t, 1, p.vfsCache.Entries())
|
||||
})
|
||||
|
||||
privateKey, privateKeyErr := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if privateKeyErr != nil {
|
||||
fs.Fatal(nil, "error generating test private key "+privateKeyErr.Error())
|
||||
@@ -180,7 +242,7 @@ func TestRun(t *testing.T) {
|
||||
assert.Equal(t, 0, p.vfsCache.Entries())
|
||||
defer p.vfsCache.Clear()
|
||||
|
||||
value, err := p.call(testUser, publicKeyString, true)
|
||||
value, err := p.call(testUser, publicKeyString, authPublicKey, testIP)
|
||||
require.NoError(t, err)
|
||||
entry, ok := value.(cacheEntry)
|
||||
require.True(t, ok)
|
||||
@@ -190,7 +252,7 @@ func TestRun(t *testing.T) {
|
||||
require.NotNil(t, entry.vfs)
|
||||
f := entry.vfs.Fs()
|
||||
require.NotNil(t, f)
|
||||
cacheKey := generateCacheKey(testUser, publicKeyString)
|
||||
cacheKey := generateCacheKey(testUser, publicKeyString, testIP)
|
||||
assert.Equal(t, "proxy-"+cacheKey, f.Name())
|
||||
assert.True(t, strings.HasPrefix(f.String(), "Local file system"))
|
||||
|
||||
@@ -206,11 +268,12 @@ func TestRun(t *testing.T) {
|
||||
assert.Equal(t, 0, p.vfsCache.Entries())
|
||||
defer p.vfsCache.Clear()
|
||||
|
||||
cacheKey := generateCacheKey(testUser, publicKeyString)
|
||||
cacheKey := generateCacheKey(testUser, publicKeyString, testIP)
|
||||
vfs, vfsKey, err := p.Call(
|
||||
testUser,
|
||||
publicKeyString,
|
||||
true,
|
||||
testAddr,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs)
|
||||
@@ -232,7 +295,7 @@ func TestRun(t *testing.T) {
|
||||
})
|
||||
|
||||
// now try again from the cache
|
||||
vfs, vfsKey, err = p.Call(testUser, publicKeyString, true)
|
||||
vfs, vfsKey, err = p.Call(testUser, publicKeyString, true, testAddr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs)
|
||||
assert.Equal(t, "proxy-"+cacheKey, vfs.Fs().Name())
|
||||
@@ -244,7 +307,7 @@ func TestRun(t *testing.T) {
|
||||
// A different public key produces a different cache key, so it
|
||||
// creates a fresh cache entry rather than hitting the existing
|
||||
// one. Authentication itself is the proxy script's job.
|
||||
vfs2, vfsKey2, err := p.Call(testUser, publicKeyString+"different", true)
|
||||
vfs2, vfsKey2, err := p.Call(testUser, publicKeyString+"different", true, testAddr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, vfs2)
|
||||
assert.NotEqual(t, cacheKey, vfsKey2)
|
||||
@@ -260,10 +323,155 @@ func TestRun(t *testing.T) {
|
||||
// it. Simulate by corrupting the cached pwHash.
|
||||
entry := cacheEntry{vfs: vfs, pwHash: sha256.Sum256([]byte("tampered"))}
|
||||
p.vfsCache.Put(cacheKey, entry)
|
||||
vfs, vfsKey, err = p.Call(testUser, publicKeyString, true)
|
||||
vfs, vfsKey, err = p.Call(testUser, publicKeyString, true, testAddr)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "incorrect public key")
|
||||
require.Nil(t, vfs)
|
||||
require.Equal(t, "", vfsKey)
|
||||
})
|
||||
}
|
||||
|
||||
// TestCallAccessKeyConcurrentRefresh checks that concurrent refreshes
|
||||
// which all see a rotated secret end up sharing one backend and one
|
||||
// cache entry rather than the later ones retiring the entry an
|
||||
// earlier one created and returned.
|
||||
func TestCallAccessKeyConcurrentRefresh(t *testing.T) {
|
||||
opt := Opt
|
||||
opt.AuthProxy = "go run proxy_code.go"
|
||||
p := New(context.Background(), &opt, &vfscommon.Opt)
|
||||
defer p.Shutdown()
|
||||
const remoteAddr = "192.0.2.1:1234"
|
||||
|
||||
oldInterval := accessKeyRefreshInterval
|
||||
accessKeyRefreshInterval = 0
|
||||
defer func() { accessKeyRefreshInterval = oldInterval }()
|
||||
|
||||
VFS, _, err := p.CallAccessKey("CONCURRENT", remoteAddr, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Rotate the secret then refresh from many goroutines at once
|
||||
t.Setenv("RCLONE_TEST_PROXY_SECRET_SUFFIX", "-rotated")
|
||||
const n = 8
|
||||
results := make([]*vfs.VFS, n)
|
||||
var wg sync.WaitGroup
|
||||
for i := range n {
|
||||
wg.Go(func() {
|
||||
newVFS, secret, err := p.CallAccessKey("CONCURRENT", remoteAddr, true)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "CONCURRENT-rotated", secret)
|
||||
results[i] = newVFS
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Every caller must have got the same backend and only one
|
||||
// entry was retired
|
||||
for i := range n {
|
||||
assert.Same(t, VFS, results[i], "goroutine %d got a different backend", i)
|
||||
}
|
||||
assert.Equal(t, 2, p.vfsCache.Entries())
|
||||
}
|
||||
|
||||
func TestIPFromAddr(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"192.0.2.1:1024", "192.0.2.1"},
|
||||
{"[2001:db8::1]:1024", "2001:db8::1"},
|
||||
{"[::ffff:192.0.2.1]:1024", "192.0.2.1"},
|
||||
{"[fe80::1%eth0]:1024", "fe80::1%eth0"},
|
||||
{"/tmp/rclone.sock", ""},
|
||||
{"/tmp/foo:bar.sock", ""},
|
||||
{`C:\Users\me\rclone.sock`, ""},
|
||||
{"@", ""},
|
||||
{"<nil>", ""},
|
||||
{"", ""},
|
||||
} {
|
||||
assert.Equal(t, test.want, ipFromAddr(test.in), test.in)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCallAccessKey(t *testing.T) {
|
||||
opt := Opt
|
||||
opt.AuthProxy = "go run proxy_code.go"
|
||||
p := New(context.Background(), &opt, &vfscommon.Opt)
|
||||
defer p.Shutdown()
|
||||
const remoteAddr = "192.0.2.1:1234"
|
||||
|
||||
// Disable refresh rate limiting for this test
|
||||
oldInterval := accessKeyRefreshInterval
|
||||
accessKeyRefreshInterval = 0
|
||||
defer func() { accessKeyRefreshInterval = oldInterval }()
|
||||
|
||||
VFS, secret, err := p.CallAccessKey("AKID", remoteAddr, false)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, VFS)
|
||||
assert.Equal(t, "AKID-secret", secret)
|
||||
|
||||
// Check the cached entry is returned on the next call
|
||||
VFS2, secret2, err := p.CallAccessKey("AKID", remoteAddr, false)
|
||||
require.NoError(t, err)
|
||||
assert.Same(t, VFS, VFS2)
|
||||
assert.Equal(t, secret, secret2)
|
||||
|
||||
// Check a different access key ID gets a different backend
|
||||
VFS3, secret3, err := p.CallAccessKey("OTHER", remoteAddr, false)
|
||||
require.NoError(t, err)
|
||||
assert.NotSame(t, VFS, VFS3)
|
||||
assert.Equal(t, "OTHER-secret", secret3)
|
||||
|
||||
// Check a refresh with an unchanged secret keeps the cached backend
|
||||
VFS4, secret4, err := p.CallAccessKey("AKID", remoteAddr, true)
|
||||
require.NoError(t, err)
|
||||
assert.Same(t, VFS, VFS4)
|
||||
assert.Equal(t, secret, secret4)
|
||||
|
||||
// Check a refresh with a changed secret returns the new secret.
|
||||
// The VFS is the same object as vfs.New shares a live VFS for the
|
||||
// same backend and options, which keeps requests in flight under
|
||||
// the old secret working.
|
||||
t.Setenv("RCLONE_TEST_PROXY_SECRET_SUFFIX", "-rotated")
|
||||
entries := p.vfsCache.Entries()
|
||||
VFS5, secret5, err := p.CallAccessKey("AKID", remoteAddr, true)
|
||||
require.NoError(t, err)
|
||||
assert.Same(t, VFS, VFS5)
|
||||
assert.Equal(t, "AKID-rotated", secret5)
|
||||
// The old entry is retired rather than dropped
|
||||
assert.Equal(t, entries+1, p.vfsCache.Entries())
|
||||
|
||||
// Check a proxy which doesn't return the secret is an error
|
||||
_, _, err = p.CallAccessKey("nosecret", remoteAddr, false)
|
||||
require.ErrorContains(t, err, "_secret_access_key not set")
|
||||
|
||||
// Check a proxy which returns an empty secret is an error
|
||||
_, _, err = p.CallAccessKey("emptysecret", remoteAddr, false)
|
||||
require.ErrorContains(t, err, "_secret_access_key is empty")
|
||||
|
||||
// Check refreshes are rate limited: with a long interval a
|
||||
// refresh returns the cached secret without running the proxy
|
||||
accessKeyRefreshInterval = time.Hour
|
||||
t.Setenv("RCLONE_TEST_PROXY_SECRET_SUFFIX", "-rotated-again")
|
||||
VFS6, secret6, err := p.CallAccessKey("AKID", remoteAddr, true)
|
||||
require.NoError(t, err)
|
||||
assert.Same(t, VFS5, VFS6)
|
||||
assert.Equal(t, "AKID-rotated", secret6)
|
||||
|
||||
// And with no interval the proxy is run and the rotation seen
|
||||
accessKeyRefreshInterval = 0
|
||||
_, secret7, err := p.CallAccessKey("AKID", remoteAddr, true)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "AKID-rotated-again", secret7)
|
||||
|
||||
// Check a cached entry is revalidated with the proxy once it is
|
||||
// old enough even without a refresh being asked for
|
||||
oldRevalidate := accessKeyRevalidateInterval
|
||||
defer func() { accessKeyRevalidateInterval = oldRevalidate }()
|
||||
t.Setenv("RCLONE_TEST_PROXY_REVOKED", "AKID")
|
||||
_, secret8, err := p.CallAccessKey("AKID", remoteAddr, false)
|
||||
require.NoError(t, err, "cached entry should still be trusted")
|
||||
assert.Equal(t, "AKID-rotated-again", secret8)
|
||||
accessKeyRevalidateInterval = 0
|
||||
_, _, err = p.CallAccessKey("AKID", remoteAddr, false)
|
||||
require.ErrorContains(t, err, "revoked")
|
||||
}
|
||||
+75
-11
@@ -12,16 +12,27 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/ncw/swift/v2"
|
||||
"github.com/rclone/gofakes3"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
|
||||
var (
|
||||
emptyPrefix = &gofakes3.Prefix{}
|
||||
)
|
||||
|
||||
// tempObjectPrefix is reserved for serve s3's temporary objects: an object
|
||||
// whose leaf name starts with it is hidden from S3 listings.
|
||||
const tempObjectPrefix = ".rclone_temp_"
|
||||
|
||||
// putObjectPrefix is prepended to the leaf name of the temporary object a
|
||||
// PutObject upload is written to before it is renamed into place.
|
||||
const putObjectPrefix = tempObjectPrefix + "put_"
|
||||
|
||||
// s3Backend implements the gofacess3.Backend interface to make an S3
|
||||
// backend for gofakes3. It also implements gofakes3.MultipartBackend so that
|
||||
// multipart uploads stream straight through to the underlying Fs via
|
||||
@@ -37,13 +48,17 @@ type s3Backend struct {
|
||||
// warnInMemoryOnce logs a single NOTICE the first time a multipart
|
||||
// upload falls back to being buffered in memory.
|
||||
warnInMemoryOnce sync.Once
|
||||
|
||||
reaperQuit chan struct{} // closed to stop the abandoned upload reaper
|
||||
reaperStop sync.Once
|
||||
}
|
||||
|
||||
// newBackend creates a new SimpleBucketBackend.
|
||||
func newBackend(s *Server) gofakes3.Backend {
|
||||
func newBackend(s *Server) *s3Backend {
|
||||
return &s3Backend{
|
||||
s: s,
|
||||
meta: new(sync.Map),
|
||||
s: s,
|
||||
meta: new(sync.Map),
|
||||
reaperQuit: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -311,6 +326,9 @@ func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]
|
||||
}
|
||||
|
||||
// PutObject creates or overwrites the object with the given name.
|
||||
//
|
||||
// A failed or interrupted upload should never disturb what is stored
|
||||
// at the key.
|
||||
func (b *s3Backend) PutObject(
|
||||
ctx context.Context,
|
||||
bucketName, objectName string,
|
||||
@@ -343,24 +361,63 @@ func (b *s3Backend) PutObject(
|
||||
}
|
||||
}
|
||||
|
||||
f, err := _vfs.Create(fp)
|
||||
// Upload via a temporary object renamed into place if needed. The handle
|
||||
// is opened read-write, which the VFS routes through the cache - where a
|
||||
// failed upload commits what it has on Close and can't be abandoned -
|
||||
// from --vfs-cache-mode minimal up, not just writes.
|
||||
fsys := _vfs.Fs()
|
||||
tmpFp := fp
|
||||
if (fsys.Features().PartialUploads || _vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal) && operations.CanServerSideMove(fsys) {
|
||||
tmpFp = path.Join(objectDir, putObjectPrefix+uuid.New().String())
|
||||
}
|
||||
|
||||
// cleanup discards a failed upload, removing the temporary object
|
||||
// (never the object at fp) and any stale VFS state for fp.
|
||||
cleanup := func() {
|
||||
if tmpFp != fp {
|
||||
b.forgetPath(_vfs, tmpFp)
|
||||
_ = _vfs.Remove(tmpFp)
|
||||
} else {
|
||||
b.forgetPath(_vfs, fp)
|
||||
}
|
||||
}
|
||||
|
||||
f, err := _vfs.Create(tmpFp)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(f, input); err != nil {
|
||||
// remove file when i/o error occurred (FsPutErr)
|
||||
_ = f.Close()
|
||||
_ = _vfs.Remove(fp)
|
||||
n, err := io.Copy(f, input)
|
||||
if err == nil && size >= 0 && n != size {
|
||||
// The body ended cleanly but short of its declared size
|
||||
err = gofakes3.ErrIncompleteBody
|
||||
}
|
||||
if err != nil {
|
||||
// The upload from the client failed part way through - abort the
|
||||
// write so a streaming upload fails rather than committing a
|
||||
// truncated object.
|
||||
if aborter, ok := f.(interface{ CloseWithError(error) error }); ok {
|
||||
_ = aborter.CloseWithError(err)
|
||||
} else {
|
||||
_ = f.Close()
|
||||
}
|
||||
cleanup()
|
||||
return result, err
|
||||
}
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
// remove file when close error occurred (FsPutErr)
|
||||
_ = _vfs.Remove(fp)
|
||||
cleanup()
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Rename the temporary object into place
|
||||
if tmpFp != fp {
|
||||
if err := _vfs.Rename(tmpFp, fp); err != nil {
|
||||
cleanup()
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
|
||||
_, err = _vfs.Stat(fp)
|
||||
if err != nil {
|
||||
return result, err
|
||||
@@ -375,8 +432,11 @@ func (b *s3Backend) PutObject(
|
||||
return result, _vfs.Chtimes(fp, ti, ti)
|
||||
}
|
||||
// ignore error since the file is successfully created
|
||||
}
|
||||
|
||||
if val, ok := meta["mtime"]; ok {
|
||||
if val, ok := meta["mtime"]; ok {
|
||||
ti, err := swift.FloatStringToTime(val)
|
||||
if err == nil {
|
||||
b.storeModtime(fp, meta, val)
|
||||
return result, _vfs.Chtimes(fp, ti, ti)
|
||||
}
|
||||
@@ -541,6 +601,10 @@ func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket
|
||||
meta["mtime"] = swift.TimeToFloatString(cStat.ModTime())
|
||||
}
|
||||
|
||||
// PutObject rejects a body shorter than its declared size, so a copy
|
||||
// whose source is being overwritten as it is read - its stated size no
|
||||
// longer matching its readable bytes - fails with IncompleteBody rather
|
||||
// than storing a mixture.
|
||||
_, err = b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -37,7 +37,7 @@ func newTestBackend(t *testing.T) (*s3Backend, string) {
|
||||
w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
|
||||
require.NoError(t, err)
|
||||
|
||||
return newBackend(w).(*s3Backend), root
|
||||
return newBackend(w), root
|
||||
}
|
||||
|
||||
const rootSecret = "ROOT_LEVEL_SECRET_MARKER"
|
||||
|
||||
@@ -8,6 +8,11 @@ import (
|
||||
"github.com/rclone/rclone/vfs"
|
||||
)
|
||||
|
||||
// legacyMultipartUploadPrefix marked the temporary objects of in-progress
|
||||
// multipart uploads before the tempObjectPrefix namespace was reserved
|
||||
// (rclone v1.75); leftovers from an older server are still hidden.
|
||||
const legacyMultipartUploadPrefix = ".rclone_multipart_upload_"
|
||||
|
||||
func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {
|
||||
fp, err := bucketDirPath(bucketName, fdPath)
|
||||
if err != nil {
|
||||
@@ -23,8 +28,8 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, a
|
||||
for _, entry := range dirEntries {
|
||||
object := entry.Name()
|
||||
|
||||
// Hide the in-progress multipart uploads
|
||||
if strings.HasPrefix(object, multipartUploadPrefix) {
|
||||
// Hide the temporary objects of in-progress uploads
|
||||
if strings.HasPrefix(object, tempObjectPrefix) || strings.HasPrefix(object, legacyMultipartUploadPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
+251
-157
@@ -1,13 +1,17 @@
|
||||
// Multipart upload support for serve s3.
|
||||
//
|
||||
// Multipart uploads received by serve s3 are streamed, in part-number order,
|
||||
// into a single PutStream upload to the underlying Fs, so the whole file is
|
||||
// never buffered in memory. This implements the gofakes3.MultipartBackend
|
||||
// Multipart uploads received by serve s3 are written, in part-number order,
|
||||
// through the VFS, exactly like a plain PutObject: to a temporary object
|
||||
// which is renamed into place on completion, so the object at the key only
|
||||
// ever changes atomically on success. With the default --vfs-cache-mode off
|
||||
// the parts stream through the VFS into a single upload to the remote; with
|
||||
// --vfs-cache-mode writes or above they are buffered in the VFS cache and
|
||||
// uploaded by its write-back. This implements the gofakes3.MultipartBackend
|
||||
// interface on s3Backend.
|
||||
//
|
||||
// When streaming is disabled (--disable-multipart-streaming) or the Fs has no
|
||||
// PutStream, ErrMultipartUploadNotSupported is returned so that gofakes3 falls
|
||||
// back to buffering the parts in memory.
|
||||
// When streaming is disabled (--disable-multipart-streaming),
|
||||
// ErrMultipartUploadNotSupported is returned so that gofakes3 falls back to
|
||||
// buffering the parts in memory.
|
||||
|
||||
package s3
|
||||
|
||||
@@ -29,40 +33,44 @@ import (
|
||||
"github.com/ncw/swift/v2"
|
||||
"github.com/rclone/gofakes3"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
|
||||
// multipartUploadPrefix is prepended to the leaf name of the temporary object
|
||||
// a streamed multipart upload is written to before it is moved into place.
|
||||
const multipartUploadPrefix = ".rclone_multipart_upload_"
|
||||
const multipartUploadPrefix = tempObjectPrefix + "multipart_"
|
||||
|
||||
// multipartUpload tracks one in-flight S3 multipart upload that is being
|
||||
// streamed, in part order, into a single PutStream upload to the underlying Fs.
|
||||
// multipartUpload tracks one in-flight S3 multipart upload. The parts are
|
||||
// written, in part-number order, into fh - a VFS file handle which either
|
||||
// streams straight through to the remote (the default) or is backed by the
|
||||
// VFS cache (when the VFS is caching writes).
|
||||
type multipartUpload struct {
|
||||
bucket, key string
|
||||
fp string // final object path
|
||||
streamFp string // path the parts are streamed to (fp when the backend uploads atomically)
|
||||
streamFp string // path the parts are written to (fp when the remote has no server-side move or copy)
|
||||
meta map[string]string
|
||||
|
||||
pipeW *io.PipeWriter // parts are streamed here, in part-number order
|
||||
fh io.WriteCloser // sink the in-order parts are written to
|
||||
vfs *vfs.VFS // the VFS fh was created on, used for all later operations
|
||||
|
||||
mu sync.Mutex
|
||||
cond *sync.Cond // signalled when buffered shrinks, nextPart advances or the upload closes
|
||||
partMD5s map[int][]byte // raw MD5 sums per part (for the final S3 multipart ETag)
|
||||
partSizes map[int]int64 // observed part sizes
|
||||
closed bool
|
||||
aborted bool // closed by abort, so nothing was committed
|
||||
active int // requests in flight for this upload, protecting it from the reaper
|
||||
lastUsed time.Time // when the last request for this upload finished
|
||||
|
||||
putCancel context.CancelFunc // cancels the background PutStream
|
||||
putDone chan struct{} // closed when the background PutStream returns
|
||||
putErr error // PutStream result (read only after putDone is closed)
|
||||
nextPart int // next part number to stream (1-based)
|
||||
streamBuf map[int]*pool.RW // parts received ahead of nextPart, awaiting their turn
|
||||
pumping bool // a goroutine is currently writing to the pipe
|
||||
buffered int64 // bytes of parts admitted but not yet streamed or released
|
||||
bufferLimit int64 // max buffered before parts ahead of nextPart must wait (<= 0 for no limit)
|
||||
nextPart int // next part number to stream (1-based)
|
||||
streamBuf map[int]*pool.RW // parts received ahead of nextPart, awaiting their turn
|
||||
pumping bool // a goroutine is currently writing to the sink
|
||||
buffered int64 // bytes of parts admitted but not yet streamed or released
|
||||
bufferLimit int64 // max buffered before parts ahead of nextPart must wait (<= 0 for no limit)
|
||||
}
|
||||
|
||||
// newMultipartUpload allocates an upload struct.
|
||||
@@ -78,11 +86,27 @@ func newMultipartUpload(bucket, key, fp, streamFp string, meta map[string]string
|
||||
nextPart: 1,
|
||||
streamBuf: map[int]*pool.RW{},
|
||||
bufferLimit: bufferLimit,
|
||||
lastUsed: time.Now(),
|
||||
}
|
||||
up.cond = sync.NewCond(&up.mu)
|
||||
return up
|
||||
}
|
||||
|
||||
// startActivity marks the upload as having a request in flight.
|
||||
func (up *multipartUpload) startActivity() {
|
||||
up.mu.Lock()
|
||||
up.active++
|
||||
up.mu.Unlock()
|
||||
}
|
||||
|
||||
// endActivity marks the request done and restarts the upload's idle time.
|
||||
func (up *multipartUpload) endActivity() {
|
||||
up.mu.Lock()
|
||||
up.active--
|
||||
up.lastUsed = time.Now()
|
||||
up.mu.Unlock()
|
||||
}
|
||||
|
||||
// loadUpload looks up an in-flight upload by ID.
|
||||
func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, error) {
|
||||
v, ok := b.multipartUploads.Load(uploadID)
|
||||
@@ -92,22 +116,26 @@ func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, er
|
||||
return v.(*multipartUpload), nil
|
||||
}
|
||||
|
||||
// CreateMultipartUpload begins a new multipart upload that streams the parts,
|
||||
// in part-number order, into a single PutStream upload to the underlying Fs.
|
||||
// CreateMultipartUpload begins a new multipart upload.
|
||||
//
|
||||
// Backends that upload atomically (PartialUploads=false) are streamed
|
||||
// straight to the final object. An aborted or failed upload never
|
||||
// makes a partial object visible or disturbs a pre-existing one.
|
||||
// Backends where a partial upload is visible (PartialUploads=true)
|
||||
// are instead streamed to a temporary object that is moved into
|
||||
// place, server-side, on completion, giving the same atomic
|
||||
// behaviour.
|
||||
// The parts are written, in part-number order, through the VFS to a temporary
|
||||
// object which is renamed into place on completion. With the default
|
||||
// --vfs-cache-mode off the write streams through to the remote as the parts
|
||||
// arrive; with --vfs-cache-mode writes or above it lands in the VFS cache and
|
||||
// is uploaded by the write-back. Either way an aborted or failed upload never
|
||||
// makes a partial object visible at the final path or disturbs a pre-existing
|
||||
// one.
|
||||
//
|
||||
// If streaming is disabled (--disable-multipart-streaming), the Fs has no
|
||||
// PutStream, or a non-atomic Fs can't move/copy objects server-side,
|
||||
// ErrMultipartUploadNotSupported is returned so that gofakes3 falls back to
|
||||
// buffering the whole upload in memory; a one-off NOTICE warns about the
|
||||
// memory use.
|
||||
// On a remote with no server-side move or copy the parts are written straight
|
||||
// to the final object instead, trading some atomicity for never buffering in
|
||||
// memory: the in-flight upload is visible at the key, and a failed or aborted
|
||||
// upload can leave partial data there when the remote doesn't upload atomically
|
||||
// or the VFS is caching writes (a write to the cache can't be abandoned).
|
||||
//
|
||||
// With --disable-multipart-streaming, ErrMultipartUploadNotSupported is
|
||||
// returned so that gofakes3 falls back to buffering the whole upload in
|
||||
// memory; a one-off NOTICE warns about the memory use. A caching VFS needs
|
||||
// no streaming support, so it ignores the flag.
|
||||
func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objectName string, meta map[string]string) (gofakes3.UploadID, error) {
|
||||
_vfs, err := b.s.getVFS(ctx)
|
||||
if err != nil {
|
||||
@@ -117,11 +145,9 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
|
||||
return "", gofakes3.BucketNotFound(bucketName)
|
||||
}
|
||||
|
||||
f := _vfs.Fs()
|
||||
features := f.Features()
|
||||
if reason := b.noStreamingReason(f); reason != "" {
|
||||
if b.s.opt.DisableMultipartStreaming && _vfs.Opt.CacheMode < vfscommon.CacheModeMinimal {
|
||||
b.warnInMemoryOnce.Do(func() {
|
||||
fs.Logf(nil, "serve s3: buffering multipart uploads in memory because %s - this may use a lot of memory", reason)
|
||||
fs.Logf(nil, "serve s3: buffering multipart uploads in memory because --disable-multipart-streaming is set - this may use a lot of memory")
|
||||
})
|
||||
return "", gofakes3.ErrMultipartUploadNotSupported
|
||||
}
|
||||
@@ -139,53 +165,36 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
|
||||
|
||||
uploadID := gofakes3.UploadID(uuid.New().String())
|
||||
streamFp := fp
|
||||
// If partial uploads visible, stream to temporary object
|
||||
if features.PartialUploads {
|
||||
// Write to a temporary object moved into place on completion if the
|
||||
// remote supports server-side move (if not write directly to the final
|
||||
// object). Unlike a plain PutObject all remotes use a temporary object.
|
||||
// S3 semantics say the key must not show it until it completes. This is
|
||||
// at the cost of a server-side copy and delete where the remote has no
|
||||
// server side move.
|
||||
if operations.CanServerSideMove(_vfs.Fs()) {
|
||||
streamFp = path.Join(objectDir, multipartUploadPrefix+string(uploadID))
|
||||
}
|
||||
|
||||
up := newMultipartUpload(bucketName, objectName, fp, streamFp, meta, int64(b.s.opt.MultipartStreamingBufferLimit))
|
||||
|
||||
src := object.NewStaticObjectInfo(streamFp, time.Now(), -1, true, nil, f)
|
||||
pr, pw := io.Pipe()
|
||||
// Use a context that outlives this request (it's cancelled on abort) but
|
||||
// keeps its values.
|
||||
putCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
|
||||
up.pipeW = pw
|
||||
up.putCancel = cancel
|
||||
up.putDone = make(chan struct{})
|
||||
go func() {
|
||||
_, err := features.PutStream(putCtx, pr, src)
|
||||
up.putErr = err
|
||||
_ = pr.CloseWithError(err)
|
||||
close(up.putDone)
|
||||
}()
|
||||
fh, err := _vfs.Create(streamFp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
up.fh = fh
|
||||
up.vfs = _vfs
|
||||
|
||||
b.multipartUploads.Store(uploadID, up)
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// noStreamingReason returns a non-empty reason why streamed multipart uploads
|
||||
// can't be used for f, or "" if they can.
|
||||
func (b *s3Backend) noStreamingReason(f fs.Fs) string {
|
||||
switch {
|
||||
case b.s.opt.DisableMultipartStreaming:
|
||||
return "--disable-multipart-streaming is set"
|
||||
case f.Features().PutStream == nil:
|
||||
return "this backend doesn't support streaming uploads"
|
||||
case f.Features().PartialUploads && !operations.CanServerSideMove(f):
|
||||
return "this backend can't upload atomically and has no server-side move or copy"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// UploadPart writes a single part from the S3 client into the streaming upload.
|
||||
func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName string, uploadID gofakes3.UploadID, partNumber int, contentLength int64, body io.Reader) (string, error) {
|
||||
up, err := b.loadUpload(uploadID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
up.startActivity()
|
||||
defer up.endActivity()
|
||||
|
||||
// Wait until there is room to buffer this part, bounding the memory a
|
||||
// client which uploads faster than the backend drains can consume.
|
||||
@@ -194,8 +203,9 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
|
||||
}
|
||||
|
||||
// Buffer the part in a pool-backed RW so we can MD5 it (for the ETag) and
|
||||
// stream it once it is this part's turn.
|
||||
rw := multipart.NewRW().Reserve(contentLength)
|
||||
// stream it once it is this part's turn. The RW grows a page at a time as
|
||||
// the body is read.
|
||||
rw := multipart.NewRW()
|
||||
hasher := md5.New()
|
||||
n, err := io.Copy(rw, io.TeeReader(body, hasher))
|
||||
if err != nil {
|
||||
@@ -222,17 +232,24 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
|
||||
// client sends parts faster than the backend drains them.
|
||||
//
|
||||
// The next part the stream needs (and any retry of an earlier one) is always
|
||||
// admitted so the pipe can keep draining; so is a single part bigger than the
|
||||
// admitted so the sink can keep draining; so is a single part bigger than the
|
||||
// limit when the buffer is empty, to guarantee progress. Reserved bytes are
|
||||
// returned with release, or by the pump as the part is streamed.
|
||||
//
|
||||
// size is the client-declared part length and is not trusted: a negative value
|
||||
// is rejected, and the admission test is written so a huge value can't overflow
|
||||
// the running total and wrongly admit further parts past the limit.
|
||||
func (up *multipartUpload) waitForTurn(partNumber int, size int64) error {
|
||||
if size < 0 {
|
||||
return gofakes3.ErrInvalidArgument
|
||||
}
|
||||
up.mu.Lock()
|
||||
defer up.mu.Unlock()
|
||||
for {
|
||||
if up.closed {
|
||||
return gofakes3.ErrNoSuchUpload
|
||||
}
|
||||
if up.bufferLimit <= 0 || partNumber <= up.nextPart || up.buffered == 0 || up.buffered+size <= up.bufferLimit {
|
||||
if up.bufferLimit <= 0 || partNumber <= up.nextPart || up.buffered == 0 || size <= up.bufferLimit-up.buffered {
|
||||
up.buffered += size
|
||||
return nil
|
||||
}
|
||||
@@ -249,11 +266,11 @@ func (up *multipartUpload) release(size int64) {
|
||||
up.mu.Unlock()
|
||||
}
|
||||
|
||||
// streamPart records a part and streams the parts into the pipe in order.
|
||||
// streamPart records a part and streams the parts into the sink in order.
|
||||
//
|
||||
// Parts must be uploaded in ascending, contiguous part-number order. A part
|
||||
// that arrives ahead of the next expected one is buffered until its turn; the
|
||||
// parts are then pumped into the pipe in order. Whichever goroutine finds the
|
||||
// parts are then pumped into the sink in order. Whichever goroutine finds the
|
||||
// next part available does the pumping, so concurrent (but in-order) clients
|
||||
// are tolerated, with the buffering bounded by waitForTurn.
|
||||
//
|
||||
@@ -265,6 +282,15 @@ func (up *multipartUpload) release(size int64) {
|
||||
// one is rejected.
|
||||
func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte, rw *pool.RW) error {
|
||||
up.mu.Lock()
|
||||
if up.closed {
|
||||
// The upload was aborted or completed while the part body was
|
||||
// being received.
|
||||
up.buffered -= size
|
||||
up.cond.Broadcast()
|
||||
up.mu.Unlock()
|
||||
_ = rw.Close()
|
||||
return gofakes3.ErrNoSuchUpload
|
||||
}
|
||||
if oldMD5, exists := up.partMD5s[partNumber]; exists {
|
||||
if old, buffered := up.streamBuf[partNumber]; buffered {
|
||||
_ = old.Close()
|
||||
@@ -289,13 +315,21 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
|
||||
up.streamBuf[partNumber] = rw
|
||||
|
||||
if up.pumping {
|
||||
// Another goroutine owns the pipe and will pump this part in turn.
|
||||
// Another goroutine owns the sink and will pump this part in turn.
|
||||
up.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
up.pumping = true
|
||||
|
||||
for {
|
||||
if up.closed {
|
||||
// Aborted while pumping: the sink is closed and any
|
||||
// parts still buffered were released by the abort, so
|
||||
// report the upload gone rather than success.
|
||||
up.pumping = false
|
||||
up.mu.Unlock()
|
||||
return gofakes3.ErrNoSuchUpload
|
||||
}
|
||||
prw, ok := up.streamBuf[up.nextPart]
|
||||
if !ok {
|
||||
up.pumping = false
|
||||
@@ -306,14 +340,21 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
|
||||
psize := prw.Size()
|
||||
up.mu.Unlock()
|
||||
|
||||
err := pipePart(up.pipeW, prw)
|
||||
err := pipePart(up.fh, prw)
|
||||
_ = prw.Close()
|
||||
if err != nil {
|
||||
up.mu.Lock()
|
||||
up.pumping = false
|
||||
up.buffered -= psize
|
||||
closed := up.closed
|
||||
up.cond.Broadcast()
|
||||
up.mu.Unlock()
|
||||
if closed {
|
||||
// The write failed because the upload was aborted while
|
||||
// this part was being pumped: report the upload gone
|
||||
// rather than the sink's ECLOSED as an internal error.
|
||||
return gofakes3.ErrNoSuchUpload
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -324,7 +365,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
|
||||
}
|
||||
}
|
||||
|
||||
// pipePart writes the whole of rw into w (the pipe).
|
||||
// pipePart writes the whole of rw into w (the sink).
|
||||
func pipePart(w io.Writer, rw *pool.RW) error {
|
||||
if _, err := rw.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
@@ -333,110 +374,144 @@ func pipePart(w io.Writer, rw *pool.RW) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload finalises a streamed multipart upload. It closes the
|
||||
// pipe (so PutStream finishes), registers the new file with the VFS, computes
|
||||
// the S3-style multipart ETag, and stores the user metadata so HeadObject and
|
||||
// CompleteMultipartUpload finalises a multipart upload. It closes the sink,
|
||||
// committing the upload, renames the temporary object into place, computes the
|
||||
// S3-style multipart ETag, and stores the user metadata so HeadObject and
|
||||
// GetObject see the same fields the in-memory PutObject path produces.
|
||||
func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, objectName string, uploadID gofakes3.UploadID, input *gofakes3.CompleteMultipartUploadRequest) (gofakes3.VersionID, string, error) {
|
||||
up, err := b.loadUpload(uploadID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer b.multipartUploads.Delete(uploadID)
|
||||
up.startActivity()
|
||||
defer up.endActivity()
|
||||
|
||||
if err := up.validate(input); err != nil {
|
||||
_ = up.abort(ctx)
|
||||
b.forgetPath(ctx, up.streamFp)
|
||||
b.multipartUploads.Delete(uploadID)
|
||||
_ = up.abort()
|
||||
b.discardUpload(up)
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// All parts must have been streamed: contiguous part numbers from 1 with
|
||||
// nothing left buffered. A leftover means the client used non-contiguous
|
||||
// part numbers, which the in-order stream can't place.
|
||||
up.mu.Lock()
|
||||
streamed := up.nextPart - 1
|
||||
total := len(up.partSizes)
|
||||
leftover := len(up.streamBuf)
|
||||
up.mu.Unlock()
|
||||
if leftover != 0 || streamed != total {
|
||||
_ = up.abort(ctx)
|
||||
b.forgetPath(ctx, up.streamFp)
|
||||
return "", "", gofakes3.ErrInvalidPart
|
||||
}
|
||||
|
||||
if err := up.close(ctx); err != nil {
|
||||
b.forgetPath(ctx, up.streamFp)
|
||||
// close commits the upload, failing with the upload left open if the
|
||||
// streamed parts don't form the complete object; abort then tears it
|
||||
// down. (After a successful or failed commit the abort is a no-op.)
|
||||
if err := up.close(); err != nil {
|
||||
b.multipartUploads.Delete(uploadID)
|
||||
_ = up.abort()
|
||||
b.discardUpload(up)
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
_vfs, err := b.s.getVFS(ctx)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
// If the parts were streamed to a temporary object move it into place
|
||||
// Rename the temporary object into place: on a caching VFS the
|
||||
// write-back then uploads it under the final name, otherwise it is
|
||||
// moved server-side on the remote. On failure the upload record is
|
||||
// kept, because gofakes3 keeps its own record when the backend errors
|
||||
// so that the client can retry the CompleteMultipartUpload: the
|
||||
// committed close is idempotent, so the retry just renames again.
|
||||
if up.streamFp != up.fp {
|
||||
if err := b.moveIntoPlace(ctx, _vfs.Fs(), up.streamFp, up.fp); err != nil {
|
||||
b.forgetPath(ctx, up.streamFp)
|
||||
b.forgetPath(ctx, up.fp)
|
||||
if err := up.vfs.Rename(up.streamFp, up.fp); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
b.forgetPath(ctx, up.streamFp)
|
||||
}
|
||||
b.forgetPath(ctx, up.fp)
|
||||
b.multipartUploads.Delete(uploadID)
|
||||
|
||||
b.meta.Store(up.fp, up.meta)
|
||||
if val, ok := up.meta["X-Amz-Meta-Mtime"]; ok {
|
||||
if ti, err := swift.FloatStringToTime(val); err == nil {
|
||||
b.storeModtime(up.fp, up.meta, val)
|
||||
_ = _vfs.Chtimes(up.fp, ti, ti)
|
||||
_ = up.vfs.Chtimes(up.fp, ti, ti)
|
||||
}
|
||||
} else if val, ok := up.meta["mtime"]; ok {
|
||||
if ti, err := swift.FloatStringToTime(val); err == nil {
|
||||
b.storeModtime(up.fp, up.meta, val)
|
||||
_ = _vfs.Chtimes(up.fp, ti, ti)
|
||||
_ = up.vfs.Chtimes(up.fp, ti, ti)
|
||||
}
|
||||
}
|
||||
|
||||
return "", up.multipartETag(input), nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload tears down an in-progress upload, asking the background
|
||||
// PutStream to discard any data already sent.
|
||||
// AbortMultipartUpload tears down an in-progress upload, discarding any data
|
||||
// already received.
|
||||
func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, objectName string, uploadID gofakes3.UploadID) error {
|
||||
up, err := b.loadUpload(uploadID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.multipartUploads.Delete(uploadID)
|
||||
err = up.abort(ctx)
|
||||
// An atomic backend leaves the final object untouched by the aborted
|
||||
// upload; a non-atomic one only ever wrote the temporary object. Either way
|
||||
// invalidating streamFp is enough.
|
||||
b.forgetPath(ctx, up.streamFp)
|
||||
return err
|
||||
if err := up.abort(); err != nil {
|
||||
fs.Errorf(up.fp, "aborting multipart upload: %v", err)
|
||||
}
|
||||
b.discardUpload(up)
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveIntoPlace moves the temporary object srcFp to its final path dstFp on f,
|
||||
// server-side, overwriting any object already there.
|
||||
func (b *s3Backend) moveIntoPlace(ctx context.Context, f fs.Fs, srcFp, dstFp string) error {
|
||||
srcObj, err := f.NewObject(ctx, srcFp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find uploaded object: %w", err)
|
||||
// startReaper starts a goroutine which aborts incomplete multipart
|
||||
// uploads once they have been idle for expiry. Stopped by stopReaper.
|
||||
func (b *s3Backend) startReaper(expiry time.Duration) {
|
||||
interval := min(expiry/2, time.Minute)
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-b.reaperQuit:
|
||||
return
|
||||
case <-ticker.C:
|
||||
b.reapExpiredUploads(time.Now(), expiry)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// stopReaper stops the abandoned upload reaper, if running.
|
||||
func (b *s3Backend) stopReaper() {
|
||||
b.reaperStop.Do(func() {
|
||||
close(b.reaperQuit)
|
||||
})
|
||||
}
|
||||
|
||||
// reapExpiredUploads aborts and cleans up multipart uploads which have had
|
||||
// no request activity for longer than expiry.
|
||||
func (b *s3Backend) reapExpiredUploads(now time.Time, expiry time.Duration) {
|
||||
b.multipartUploads.Range(func(key, value any) bool {
|
||||
uploadID := key.(gofakes3.UploadID)
|
||||
up := value.(*multipartUpload)
|
||||
up.mu.Lock()
|
||||
expired := up.active == 0 && now.Sub(up.lastUsed) >= expiry
|
||||
up.mu.Unlock()
|
||||
if !expired {
|
||||
return true
|
||||
}
|
||||
fs.Logf(up.fp, "aborting multipart upload %s idle for more than %v", uploadID, expiry)
|
||||
b.multipartUploads.Delete(uploadID)
|
||||
if err := up.abort(); err != nil {
|
||||
fs.Errorf(up.fp, "aborting abandoned multipart upload: %v", err)
|
||||
}
|
||||
b.discardUpload(up)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// discardUpload cleans up after a failed or aborted upload, removing the
|
||||
// temporary object and any stale VFS state for it. It never removes the
|
||||
// object at the final path: when the parts were written straight to the
|
||||
// final object (a remote with no server-side move or copy) it may hold a
|
||||
// pre-existing object the abandoned streaming write never disturbed, so
|
||||
// only the VFS's view of the path is refreshed. (A caching VFS on such a
|
||||
// remote commits the partial data instead - see abort.)
|
||||
func (b *s3Backend) discardUpload(up *multipartUpload) {
|
||||
b.forgetPath(up.vfs, up.streamFp)
|
||||
if up.streamFp == up.fp {
|
||||
return
|
||||
}
|
||||
if _, err := operations.Move(ctx, f, nil, dstFp, srcObj); err != nil {
|
||||
return fmt.Errorf("failed to move uploaded object into place: %w", err)
|
||||
}
|
||||
return nil
|
||||
_ = up.vfs.Remove(up.streamFp)
|
||||
}
|
||||
|
||||
// forgetPath invalidates the parent directory's cached VFS listing so that
|
||||
// subsequent VFS Stat / List calls re-read fp from the underlying Fs.
|
||||
func (b *s3Backend) forgetPath(ctx context.Context, fp string) {
|
||||
_vfs, err := b.s.getVFS(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
func (b *s3Backend) forgetPath(_vfs *vfs.VFS, fp string) {
|
||||
if root, err := _vfs.Root(); err == nil {
|
||||
root.ForgetPath(fp, fs.EntryObject)
|
||||
}
|
||||
@@ -469,39 +544,59 @@ func (up *multipartUpload) validate(input *gofakes3.CompleteMultipartUploadReque
|
||||
return nil
|
||||
}
|
||||
|
||||
// close finalises the upload by signalling EOF to the background PutStream and
|
||||
// waiting for it to finish.
|
||||
func (up *multipartUpload) close(ctx context.Context) error {
|
||||
// close finalises the upload, committing it: closing the sink completes the
|
||||
// streaming upload to the remote, or the write to the VFS cache whose
|
||||
// write-back then uploads it.
|
||||
//
|
||||
// The upload is only committed if the streamed parts form the complete
|
||||
// object: contiguous part numbers from 1 with nothing left buffered (a
|
||||
// leftover means the client used non-contiguous part numbers, which the
|
||||
// in-order stream can't place). Otherwise ErrInvalidPart is returned and
|
||||
// the upload is left open. The check and the commit share one critical
|
||||
// section so no late part can slip into the stream between them.
|
||||
//
|
||||
// Returns ErrNoSuchUpload if the upload was aborted by a concurrent
|
||||
// AbortMultipartUpload, in which case nothing has been committed - the
|
||||
// upload must not be reported as complete.
|
||||
func (up *multipartUpload) close() error {
|
||||
up.mu.Lock()
|
||||
if up.closed {
|
||||
aborted := up.aborted
|
||||
up.mu.Unlock()
|
||||
if aborted {
|
||||
return gofakes3.ErrNoSuchUpload
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(up.streamBuf) != 0 || up.nextPart-1 != len(up.partSizes) {
|
||||
up.mu.Unlock()
|
||||
return gofakes3.ErrInvalidPart
|
||||
}
|
||||
up.closed = true
|
||||
up.cond.Broadcast()
|
||||
up.mu.Unlock()
|
||||
|
||||
err := up.pipeW.Close()
|
||||
<-up.putDone
|
||||
up.putCancel()
|
||||
if up.putErr != nil {
|
||||
return up.putErr
|
||||
}
|
||||
return err
|
||||
return up.fh.Close()
|
||||
}
|
||||
|
||||
// errMultipartAborted makes the background PutStream fail when an upload is
|
||||
// aborted, so it tears down its partial object instead of completing.
|
||||
// errMultipartAborted is the reason an aborted upload's write is abandoned
|
||||
// with, so the streaming upload fails instead of committing what it has.
|
||||
var errMultipartAborted = errors.New("serve s3: multipart upload aborted")
|
||||
|
||||
// abort cancels the background PutStream and releases any buffered parts.
|
||||
func (up *multipartUpload) abort(ctx context.Context) error {
|
||||
// abort tears down the upload and releases any buffered parts. The write is
|
||||
// abandoned so nothing is committed; a sink which can't abandon (a caching
|
||||
// one) is closed normally, committing what it has to the cache - the caller
|
||||
// then removes its temporary file, except on a remote with no server-side
|
||||
// move or copy, where the parts went straight to the final path and the
|
||||
// partial data is left to be written back.
|
||||
func (up *multipartUpload) abort() error {
|
||||
up.mu.Lock()
|
||||
if up.closed {
|
||||
up.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
up.closed = true
|
||||
up.aborted = true
|
||||
streamBuf := up.streamBuf
|
||||
up.streamBuf = nil
|
||||
up.cond.Broadcast()
|
||||
@@ -511,12 +606,11 @@ func (up *multipartUpload) abort(ctx context.Context) error {
|
||||
_ = rw.Close()
|
||||
}
|
||||
|
||||
// Fail the background PutStream (so it discards its partial object) and
|
||||
// wait for it to return.
|
||||
up.putCancel()
|
||||
_ = up.pipeW.CloseWithError(errMultipartAborted)
|
||||
<-up.putDone
|
||||
return nil
|
||||
if aborter, ok := up.fh.(interface{ CloseWithError(error) error }); ok {
|
||||
_ = aborter.CloseWithError(errMultipartAborted)
|
||||
return nil
|
||||
}
|
||||
return up.fh.Close()
|
||||
}
|
||||
|
||||
// multipartETag computes the S3 multipart ETag for the assembled object:
|
||||
|
||||
@@ -5,8 +5,10 @@ package s3
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/url"
|
||||
"path"
|
||||
"sync"
|
||||
@@ -16,10 +18,15 @@ import (
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/rclone/gofakes3"
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/cmd/serve/proxy"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
@@ -40,9 +47,8 @@ func newMultipartTestServer(t *testing.T, disableStreaming bool) (*minio.Core, f
|
||||
|
||||
// newMultipartTestServerBacking is like newMultipartTestServer but backed by
|
||||
// the named remote (a fresh local temp directory if empty). ":memory:" gives
|
||||
// an atomic (PartialUploads=false) backing, so the streamed-straight-to-the-
|
||||
// destination path is exercised as well as the temporary-object path that
|
||||
// local (PartialUploads=true) uses.
|
||||
// an atomic (PartialUploads=false) backing, so both flavours of remote go
|
||||
// through the temporary-object-plus-rename path.
|
||||
func newMultipartTestServerBacking(t *testing.T, backing string, disableStreaming bool) (*minio.Core, fs.Fs, string) {
|
||||
return newMultipartTestServerOpt(t, backing, disableStreaming, nil)
|
||||
}
|
||||
@@ -50,8 +56,22 @@ func newMultipartTestServerBacking(t *testing.T, backing string, disableStreamin
|
||||
// newMultipartTestServerOpt is like newMultipartTestServerBacking but also
|
||||
// applies tweak (if non-nil) to the server Options before starting it.
|
||||
func newMultipartTestServerOpt(t *testing.T, backing string, disableStreaming bool, tweak func(*Options)) (*minio.Core, fs.Fs, string) {
|
||||
return newMultipartTestServerVFS(t, backing, disableStreaming, tweak, nil)
|
||||
}
|
||||
|
||||
// newMultipartTestServerVFS is like newMultipartTestServerOpt but also
|
||||
// overrides the VFS options (nil for the defaults) and disables the named
|
||||
// features on the backing remote. A backing with features disabled must have
|
||||
// a unique config string (e.g. a distinct description=) so an active VFS
|
||||
// wrapping a fully-featured instance of the same remote isn't reused.
|
||||
func newMultipartTestServerVFS(t *testing.T, backing string, disableStreaming bool, tweak func(*Options), vfsOpt *vfscommon.Options, disableFeatures ...string) (*minio.Core, fs.Fs, string) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
if len(disableFeatures) > 0 {
|
||||
var ci *fs.ConfigInfo
|
||||
ctx, ci = fs.AddConfig(ctx)
|
||||
ci.DisableFeatures = disableFeatures
|
||||
}
|
||||
if backing == "" {
|
||||
backing = t.TempDir()
|
||||
}
|
||||
@@ -61,10 +81,13 @@ func newMultipartTestServerOpt(t *testing.T, backing string, disableStreaming bo
|
||||
// process-wide store, so a fixed name would leak objects between tests.
|
||||
bucket := fmt.Sprintf("test-%d", testBackingCounter.Add(1))
|
||||
require.NoError(t, f.Mkdir(ctx, bucket))
|
||||
if vfsOpt == nil {
|
||||
vfsOpt = &vfscommon.Opt
|
||||
}
|
||||
// The VFS is cached per remote (fs.ConfigString), so a shared ":memory:"
|
||||
// server reuses a VFS whose cached root listing predates the bucket just
|
||||
// created; forget it so the new bucket is visible.
|
||||
if root, err := vfs.New(ctx, f, &vfscommon.Opt).Root(); err == nil {
|
||||
if root, err := vfs.New(ctx, f, vfsOpt).Root(); err == nil {
|
||||
root.ForgetAll()
|
||||
}
|
||||
|
||||
@@ -77,7 +100,7 @@ func newMultipartTestServerOpt(t *testing.T, backing string, disableStreaming bo
|
||||
if tweak != nil {
|
||||
tweak(&opt)
|
||||
}
|
||||
w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
|
||||
w, err := newServer(ctx, f, &opt, vfsOpt, &proxy.Opt)
|
||||
require.NoError(t, err)
|
||||
go func() { _ = w.Serve() }()
|
||||
t.Cleanup(func() { _ = w.Shutdown() })
|
||||
@@ -448,6 +471,491 @@ func TestMultipartBufferLimit(t *testing.T) {
|
||||
assert.Equal(t, want, readObject(t, f, bucket, object))
|
||||
}
|
||||
|
||||
// stubSink is a multipartUpload sink which records how it was closed.
|
||||
type stubSink struct {
|
||||
closed bool
|
||||
abortErr error // the reason passed to CloseWithError
|
||||
}
|
||||
|
||||
func (s *stubSink) Write(p []byte) (int, error) { return len(p), nil }
|
||||
|
||||
func (s *stubSink) Close() error {
|
||||
s.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubSink) CloseWithError(err error) error {
|
||||
s.closed = true
|
||||
s.abortErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestMultipartAbortDuringUploadPart aborts the upload between a part's
|
||||
// waitForTurn and its streamPart - as happens when the abort arrives while
|
||||
// the part body is still being received from the client - and checks that
|
||||
// streamPart fails cleanly instead of panicking on the torn-down upload.
|
||||
func TestMultipartAbortDuringUploadPart(t *testing.T) {
|
||||
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)
|
||||
sink := &stubSink{}
|
||||
up.fh = sink
|
||||
|
||||
// An UploadPart in progress: the part is admitted, then the abort lands
|
||||
// while its body is still being received.
|
||||
contents := []byte("hello")
|
||||
require.NoError(t, up.waitForTurn(1, int64(len(contents))))
|
||||
require.NoError(t, up.abort())
|
||||
|
||||
// The abort must abandon the write rather than committing it.
|
||||
assert.True(t, sink.closed)
|
||||
assert.Equal(t, errMultipartAborted, sink.abortErr)
|
||||
|
||||
// The UploadPart resumes: it buffers the part and calls streamPart.
|
||||
rw := multipart.NewRW()
|
||||
_, err := rw.Write(contents)
|
||||
require.NoError(t, err)
|
||||
md5Sum := md5.Sum(contents)
|
||||
err = up.streamPart(1, int64(len(contents)), md5Sum[:], rw)
|
||||
require.ErrorIs(t, err, gofakes3.ErrNoSuchUpload)
|
||||
|
||||
// The reorder buffer reservation must have been returned
|
||||
up.mu.Lock()
|
||||
assert.Equal(t, int64(0), up.buffered)
|
||||
up.mu.Unlock()
|
||||
}
|
||||
|
||||
// TestMultipartCloseAfterAbort checks that a close racing an abort reports
|
||||
// the abort: gofakes3 can dispatch CompleteMultipartUpload and
|
||||
// AbortMultipartUpload for the same uploadID concurrently, and a Complete
|
||||
// that loses the race must not report success for data that was never
|
||||
// committed.
|
||||
func TestMultipartCloseAfterAbort(t *testing.T) {
|
||||
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)
|
||||
up.fh = &stubSink{}
|
||||
require.NoError(t, up.abort())
|
||||
require.ErrorIs(t, up.close(), gofakes3.ErrNoSuchUpload)
|
||||
|
||||
// close stays idempotent after a successful close.
|
||||
up = newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)
|
||||
up.fh = &stubSink{}
|
||||
require.NoError(t, up.close())
|
||||
require.NoError(t, up.close())
|
||||
}
|
||||
|
||||
// TestMultipartCloseIncomplete checks that close refuses to commit while a
|
||||
// part is still buffered awaiting an earlier one, leaving the upload open
|
||||
// for more parts or an abort.
|
||||
func TestMultipartCloseIncomplete(t *testing.T) {
|
||||
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)
|
||||
sink := &stubSink{}
|
||||
up.fh = sink
|
||||
|
||||
// Part 2 arrives ahead of part 1 so it is buffered, not streamed.
|
||||
contents := []byte("hello")
|
||||
rw := multipart.NewRW()
|
||||
_, err := rw.Write(contents)
|
||||
require.NoError(t, err)
|
||||
md5Sum := md5.Sum(contents)
|
||||
require.NoError(t, up.waitForTurn(2, int64(len(contents))))
|
||||
require.NoError(t, up.streamPart(2, int64(len(contents)), md5Sum[:], rw))
|
||||
|
||||
require.ErrorIs(t, up.close(), gofakes3.ErrInvalidPart)
|
||||
assert.False(t, sink.closed)
|
||||
|
||||
// The upload is still open so abort tears it down.
|
||||
require.NoError(t, up.abort())
|
||||
assert.True(t, sink.closed)
|
||||
}
|
||||
|
||||
// failingSink is a sink whose Close fails and which cannot abandon writes,
|
||||
// like a caching VFS handle whose synchronous write-back fails.
|
||||
type failingSink struct{}
|
||||
|
||||
func (failingSink) Write(p []byte) (int, error) { return len(p), nil }
|
||||
func (failingSink) Close() error { return errBoom }
|
||||
|
||||
// TestMultipartAbortAlwaysSucceeds checks that AbortMultipartUpload reports
|
||||
// success even when closing the sink fails: the upload is torn down either
|
||||
// way, and an error reply would leave gofakes3's record of the upload alive
|
||||
// with ours consumed, so every retried abort would 404 on a ghost upload.
|
||||
func TestMultipartAbortAlwaysSucceeds(t *testing.T) {
|
||||
b, _, bucket := newPutTestBackend(t, "", nil)
|
||||
ctx := context.Background()
|
||||
_vfs, err := b.s.getVFS(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
up := newMultipartUpload(bucket, "key", bucket+"/key", bucket+"/"+multipartUploadPrefix+"x", nil, 0)
|
||||
up.fh = failingSink{}
|
||||
up.vfs = _vfs
|
||||
const uploadID = gofakes3.UploadID("failing-close")
|
||||
b.multipartUploads.Store(uploadID, up)
|
||||
|
||||
require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "key", uploadID))
|
||||
|
||||
// The upload record is consumed - a second abort is NoSuchUpload.
|
||||
require.ErrorIs(t, b.AbortMultipartUpload(ctx, bucket, "key", uploadID), gofakes3.ErrNoSuchUpload)
|
||||
}
|
||||
|
||||
// TestMultipartCompleteRenameFailureKeepsUpload checks that a Complete
|
||||
// failing after the commit (here: the rename of the temporary object) keeps
|
||||
// the upload record, so the retried CompleteMultipartUpload which gofakes3
|
||||
// allows on a backend error finds the upload instead of a NoSuchUpload.
|
||||
func TestMultipartCompleteRenameFailureKeepsUpload(t *testing.T) {
|
||||
b, _, bucket := newPutTestBackend(t, "", nil)
|
||||
ctx := context.Background()
|
||||
_vfs, err := b.s.getVFS(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// A committed upload whose temporary object is missing: the rename
|
||||
// fails after the commit succeeded.
|
||||
up := newMultipartUpload(bucket, "key", bucket+"/key", bucket+"/"+multipartUploadPrefix+"missing", nil, 0)
|
||||
up.fh = &stubSink{}
|
||||
up.vfs = _vfs
|
||||
const uploadID = gofakes3.UploadID("rename-fails")
|
||||
b.multipartUploads.Store(uploadID, up)
|
||||
|
||||
_, _, err = b.CompleteMultipartUpload(ctx, bucket, "key", uploadID, &gofakes3.CompleteMultipartUploadRequest{})
|
||||
require.Error(t, err)
|
||||
_, err = b.loadUpload(uploadID)
|
||||
require.NoError(t, err, "the upload record must survive a retryable Complete failure")
|
||||
}
|
||||
|
||||
// TestMultipartReaper checks that an incomplete multipart upload abandoned
|
||||
// by its client is aborted and cleaned up after --multipart-expiry, and that
|
||||
// late operations on it fail with NoSuchUpload.
|
||||
func TestMultipartReaper(t *testing.T) {
|
||||
core, f, bucket := newMultipartTestServerOpt(t, "", false, func(opt *Options) {
|
||||
opt.MultipartExpiry = fs.Duration(100 * time.Millisecond)
|
||||
})
|
||||
ctx := context.Background()
|
||||
const object = "abandoned.bin"
|
||||
|
||||
uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
data := []byte(random.String(50 * 1024))
|
||||
_, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for well over the expiry and the reaper interval, then the
|
||||
// upload must be gone.
|
||||
time.Sleep(time.Second)
|
||||
_, err = core.PutObjectPart(ctx, bucket, object, uploadID, 2, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "NoSuchUpload")
|
||||
err = core.AbortMultipartUpload(ctx, bucket, object, uploadID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "NoSuchUpload")
|
||||
|
||||
// Nothing is left at the key or as a temporary object.
|
||||
_, err = f.NewObject(ctx, path.Join(bucket, object))
|
||||
require.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
requireOnly(t, f, bucket)
|
||||
}
|
||||
|
||||
// TestMultipartReapExpiredUploads checks the reaper's rules directly: an
|
||||
// idle upload past the expiry is aborted and cleaned up, one with a request
|
||||
// in flight is left alone however stale its idle time, and a fresh one is
|
||||
// kept.
|
||||
func TestMultipartReapExpiredUploads(t *testing.T) {
|
||||
b, _, bucket := newPutTestBackend(t, "", nil)
|
||||
ctx := context.Background()
|
||||
_vfs, err := b.s.getVFS(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
newUp := func(id string) *multipartUpload {
|
||||
up := newMultipartUpload(bucket, id, bucket+"/"+id, bucket+"/"+multipartUploadPrefix+id, nil, 0)
|
||||
up.fh = &stubSink{}
|
||||
up.vfs = _vfs
|
||||
b.multipartUploads.Store(gofakes3.UploadID(id), up)
|
||||
return up
|
||||
}
|
||||
|
||||
const expiry = time.Hour
|
||||
now := time.Now()
|
||||
|
||||
newUp("fresh")
|
||||
idle := newUp("idle")
|
||||
busy := newUp("busy")
|
||||
busy.startActivity()
|
||||
for _, up := range []*multipartUpload{idle, busy} {
|
||||
up.mu.Lock()
|
||||
up.lastUsed = now.Add(-2 * expiry)
|
||||
up.mu.Unlock()
|
||||
}
|
||||
|
||||
b.reapExpiredUploads(now, expiry)
|
||||
|
||||
_, err = b.loadUpload("fresh")
|
||||
assert.NoError(t, err, "a fresh upload must not be reaped")
|
||||
_, err = b.loadUpload("busy")
|
||||
assert.NoError(t, err, "an upload with a request in flight must not be reaped")
|
||||
_, err = b.loadUpload("idle")
|
||||
assert.ErrorIs(t, err, gofakes3.ErrNoSuchUpload, "an idle upload past the expiry must be reaped")
|
||||
|
||||
// The reaped upload was aborted, not committed.
|
||||
sink := idle.fh.(*stubSink)
|
||||
assert.True(t, sink.closed)
|
||||
assert.Equal(t, errMultipartAborted, sink.abortErr)
|
||||
}
|
||||
|
||||
// cacheWritesVFSOpt returns VFS options with --vfs-cache-mode writes and the
|
||||
// given write-back delay.
|
||||
func cacheWritesVFSOpt(writeBack time.Duration) *vfscommon.Options {
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.CacheMode = vfscommon.CacheModeWrites
|
||||
vfsOpt.WriteBack = fs.Duration(writeBack)
|
||||
return &vfsOpt
|
||||
}
|
||||
|
||||
// waitForContent waits for bucket/object on the backing Fs to hold want
|
||||
// (e.g. after the VFS write-back delay).
|
||||
func waitForContent(t *testing.T, f fs.Fs, bucket, object string, want []byte) {
|
||||
ctx := context.Background()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if o, err := f.NewObject(ctx, path.Join(bucket, object)); err == nil {
|
||||
if rc, err := o.Open(ctx); err == nil {
|
||||
got, err := io.ReadAll(rc)
|
||||
_ = rc.Close()
|
||||
if err == nil && bytes.Equal(got, want) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("object %s/%s never reached the expected content on the backing remote", bucket, object)
|
||||
}
|
||||
|
||||
// TestMultipartCacheModeWrites checks that with --vfs-cache-mode writes a
|
||||
// multipart upload goes through the VFS cache and is written back to the
|
||||
// backing remote, with no temporary object left behind. Also run with
|
||||
// --disable-multipart-streaming, which only affects the streaming path - the
|
||||
// cache needs no PutStream, so backends without one take this path instead
|
||||
// of buffering in memory.
|
||||
func TestMultipartCacheModeWrites(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
disableStreaming bool
|
||||
}{
|
||||
{"Streaming", false},
|
||||
{"NoStreaming", true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
core, f, bucket := newMultipartTestServerVFS(t, "", tc.disableStreaming, nil, cacheWritesVFSOpt(100*time.Millisecond))
|
||||
const object = "cached.bin"
|
||||
want, err := multipartUploadParts(t, core, bucket, object, []int{120 * 1024, 100 * 1024, 53 * 1024})
|
||||
require.NoError(t, err)
|
||||
waitForContent(t, f, bucket, object, want)
|
||||
requireOnly(t, f, bucket, object)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMultipartCacheModeMinimal checks that --vfs-cache-mode minimal takes
|
||||
// the cached path just like writes - the parts are written through a
|
||||
// read-write handle, which the VFS caches from minimal up - including with
|
||||
// --disable-multipart-streaming set, which only affects the streaming path.
|
||||
func TestMultipartCacheModeMinimal(t *testing.T) {
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.CacheMode = vfscommon.CacheModeMinimal
|
||||
vfsOpt.WriteBack = fs.Duration(100 * time.Millisecond)
|
||||
core, f, bucket := newMultipartTestServerVFS(t, "", true, nil, &vfsOpt)
|
||||
const object = "cached-minimal.bin"
|
||||
|
||||
want, err := multipartUploadParts(t, core, bucket, object, []int{120 * 1024, 100 * 1024, 53 * 1024})
|
||||
require.NoError(t, err)
|
||||
waitForContent(t, f, bucket, object, want)
|
||||
requireOnly(t, f, bucket, object)
|
||||
}
|
||||
|
||||
// TestMultipartCacheModeWritesAbort checks that with --vfs-cache-mode writes
|
||||
// an aborted upload is discarded from the cache: nothing reaches the backing
|
||||
// remote and an existing object at the key survives.
|
||||
func TestMultipartCacheModeWritesAbort(t *testing.T) {
|
||||
core, f, bucket := newMultipartTestServerVFS(t, "", false, nil, cacheWritesVFSOpt(100*time.Millisecond))
|
||||
ctx := context.Background()
|
||||
const object = "cached-abort.bin"
|
||||
|
||||
existing := []byte(random.String(100))
|
||||
_, err := core.PutObject(ctx, bucket, object, bytes.NewReader(existing), int64(len(existing)), "", "", minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
waitForContent(t, f, bucket, object, existing)
|
||||
|
||||
uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
data := []byte(random.String(50 * 1024))
|
||||
_, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, core.AbortMultipartUpload(ctx, bucket, object, uploadID))
|
||||
|
||||
// Wait out several write-back intervals: the aborted upload must not be
|
||||
// written back, neither over the object nor as a temporary object.
|
||||
time.Sleep(time.Second)
|
||||
assert.Equal(t, existing, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
}
|
||||
|
||||
// TestMultipartCacheModeWritesSupersedesPut checks that a multipart upload
|
||||
// completed while an earlier PUT to the same key is still in the write-back
|
||||
// window ends up with the multipart data: both writes go through the same
|
||||
// cache item, so the earlier PUT's write-back cannot land on top of the
|
||||
// newer multipart object.
|
||||
func TestMultipartCacheModeWritesSupersedesPut(t *testing.T) {
|
||||
core, f, bucket := newMultipartTestServerVFS(t, "", false, nil, cacheWritesVFSOpt(500*time.Millisecond))
|
||||
ctx := context.Background()
|
||||
const object = "supersede.bin"
|
||||
|
||||
// PUT an object; it sits in the cache awaiting write-back.
|
||||
old := []byte(random.String(100))
|
||||
_, err := core.PutObject(ctx, bucket, object, bytes.NewReader(old), int64(len(old)), "", "", minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Immediately replace it with a multipart upload to the same key.
|
||||
want, err := multipartUploadParts(t, core, bucket, object, []int{60 * 1024, 40 * 1024})
|
||||
require.NoError(t, err)
|
||||
|
||||
// After all write-backs settle the multipart data must have won.
|
||||
waitForContent(t, f, bucket, object, want)
|
||||
time.Sleep(time.Second)
|
||||
assert.Equal(t, want, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
}
|
||||
|
||||
// TestMultipartNoPutStream checks that a multipart upload to a remote
|
||||
// without streaming upload support works with the default cache mode: the
|
||||
// parts are spooled to a temporary file on local disk and uploaded with a
|
||||
// known size, rather than being buffered in memory.
|
||||
func TestMultipartNoPutStream(t *testing.T) {
|
||||
core, f, bucket := newMultipartTestServerVFS(t, "", false, nil, nil, "PutStream")
|
||||
require.Nil(t, f.Features().PutStream)
|
||||
const object = "no-putstream.bin"
|
||||
|
||||
want, err := multipartUploadParts(t, core, bucket, object, []int{120 * 1024, 100 * 1024, 53 * 1024})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
}
|
||||
|
||||
// TestMultipartNoServerSideMove checks multipart uploads to an atomic remote
|
||||
// with no server-side move or copy, where the parts stream straight to the
|
||||
// final object: an upload round-trips, and an aborted upload leaves an
|
||||
// existing object at the key untouched.
|
||||
func TestMultipartNoServerSideMove(t *testing.T) {
|
||||
// The distinct description= gives this remote its own config string so
|
||||
// it doesn't share a VFS with the fully-featured ":memory:" servers.
|
||||
core, f, bucket := newMultipartTestServerVFS(t, ":memory,description=no-server-side-move:", false, nil, nil, "Copy")
|
||||
require.False(t, operations.CanServerSideMove(f))
|
||||
ctx := context.Background()
|
||||
const object = "direct.bin"
|
||||
|
||||
existing := []byte(random.String(100))
|
||||
_, err := core.PutObject(ctx, bucket, object, bytes.NewReader(existing), int64(len(existing)), "", "", minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// An aborted upload must leave the existing object untouched.
|
||||
uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
data := []byte(random.String(50 * 1024))
|
||||
_, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, core.AbortMultipartUpload(ctx, bucket, object, uploadID))
|
||||
assert.Equal(t, existing, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
|
||||
// A completed upload replaces it.
|
||||
want, err := multipartUploadParts(t, core, bucket, object, []int{60 * 1024, 40 * 1024})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
}
|
||||
|
||||
// TestMultipartNoServerSideMovePartialUploads checks that a multipart upload
|
||||
// to a remote where partial uploads are visible and which has no server-side
|
||||
// move or copy round-trips: the parts are written straight to the final
|
||||
// object rather than being buffered in memory.
|
||||
func TestMultipartNoServerSideMovePartialUploads(t *testing.T) {
|
||||
core, f, bucket := newMultipartTestServerVFS(t, "", false, nil, nil, "Move", "Copy")
|
||||
require.False(t, operations.CanServerSideMove(f))
|
||||
require.True(t, f.Features().PartialUploads)
|
||||
const object = "direct-partial.bin"
|
||||
|
||||
want, err := multipartUploadParts(t, core, bucket, object, []int{60 * 1024, 40 * 1024})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
}
|
||||
|
||||
// TestMultipartCacheModeWritesNoServerSideMove checks that with
|
||||
// --vfs-cache-mode writes a remote with no server-side move or copy is still
|
||||
// written through the cache rather than falling back to buffering the upload
|
||||
// in memory, and pins the documented trade-offs of that path: the parts go
|
||||
// into the cache under the final key, so the in-flight upload is visible
|
||||
// there, and an aborted upload cannot be abandoned once in the cache, so its
|
||||
// partial data is written back as if it were a completed object.
|
||||
func TestMultipartCacheModeWritesNoServerSideMove(t *testing.T) {
|
||||
// The distinct description= gives this remote its own config string so
|
||||
// it doesn't share a VFS with the fully-featured ":memory:" servers.
|
||||
core, f, bucket := newMultipartTestServerVFS(t, ":memory,description=no-move-cache:", false, nil, cacheWritesVFSOpt(100*time.Millisecond), "Copy")
|
||||
require.False(t, operations.CanServerSideMove(f))
|
||||
ctx := context.Background()
|
||||
|
||||
// A round trip goes through the cache under the final key.
|
||||
const object = "cached-direct.bin"
|
||||
want, err := multipartUploadParts(t, core, bucket, object, []int{120 * 1024, 100 * 1024, 53 * 1024})
|
||||
require.NoError(t, err)
|
||||
waitForContent(t, f, bucket, object, want)
|
||||
|
||||
// The parts are written to the cache, not buffered in memory, so the
|
||||
// in-flight upload is visible at the key.
|
||||
const object2 = "cached-direct-inflight.bin"
|
||||
uploadID, err := core.NewMultipartUpload(ctx, bucket, object2, minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
data := []byte(random.String(50 * 1024))
|
||||
_, err = core.PutObjectPart(ctx, bucket, object2, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})
|
||||
require.NoError(t, err)
|
||||
_, err = core.StatObject(ctx, bucket, object2, minio.StatObjectOptions{})
|
||||
assert.NoError(t, err, "in-flight upload should be visible at the key")
|
||||
|
||||
// An aborted upload's partial data is committed to the cache and
|
||||
// written back.
|
||||
require.NoError(t, core.AbortMultipartUpload(ctx, bucket, object2, uploadID))
|
||||
waitForContent(t, f, bucket, object2, data)
|
||||
requireOnly(t, f, bucket, object, object2)
|
||||
}
|
||||
|
||||
// TestTempObjectsHiddenFromListings checks that the reserved .rclone_temp_
|
||||
// prefix, and the multipart prefix used before it was reserved, are hidden
|
||||
// from S3 listings while remaining visible to rclone itself for cleanup.
|
||||
func TestTempObjectsHiddenFromListings(t *testing.T) {
|
||||
core, f, bucket := newMultipartTestServer(t, false)
|
||||
ctx := context.Background()
|
||||
|
||||
names := []string{
|
||||
"visible.bin",
|
||||
tempObjectPrefix + "anything",
|
||||
multipartUploadPrefix + "leftover",
|
||||
putObjectPrefix + "leftover",
|
||||
legacyMultipartUploadPrefix + "leftover",
|
||||
}
|
||||
for _, name := range names {
|
||||
data := []byte("x")
|
||||
src := object.NewStaticObjectInfo(path.Join(bucket, name), time.Now(), int64(len(data)), true, nil, f)
|
||||
_, err := f.Put(ctx, bytes.NewReader(data), src)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
// All the objects really exist on the backing remote...
|
||||
requireOnly(t, f, bucket, names...)
|
||||
|
||||
// ...but only the visible one appears in an S3 listing.
|
||||
result, err := core.ListObjects(bucket, "", "", "", 1000)
|
||||
require.NoError(t, err)
|
||||
var keys []string
|
||||
for _, o := range result.Contents {
|
||||
keys = append(keys, o.Key)
|
||||
}
|
||||
assert.Equal(t, []string{"visible.bin"}, keys)
|
||||
}
|
||||
|
||||
// TestMultipartOverwrite checks that a completed multipart upload atomically
|
||||
// replaces an existing object of the same name.
|
||||
func TestMultipartOverwrite(t *testing.T) {
|
||||
@@ -469,3 +977,84 @@ func TestMultipartOverwrite(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// poolProbeReader records the pool's in-use buffer count the first time it is
|
||||
// read - after UploadPart has created its buffer but before any body bytes have
|
||||
// been delivered - then reports a short body by returning io.EOF.
|
||||
type poolProbeReader struct {
|
||||
baseline int
|
||||
recorded int
|
||||
read bool
|
||||
}
|
||||
|
||||
func (r *poolProbeReader) Read(p []byte) (int, error) {
|
||||
if !r.read {
|
||||
r.read = true
|
||||
r.recorded = pool.Global().InUse() - r.baseline
|
||||
}
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// TestUploadPartNoReserveBeforeBody checks that UploadPart does not preallocate
|
||||
// pool memory proportional to the client-declared Content-Length before any
|
||||
// body bytes have been received. A part declaring a large size but sending no
|
||||
// body must not reserve pages up front, so an unverified header cannot exhaust
|
||||
// process memory.
|
||||
func TestUploadPartNoReserveBeforeBody(t *testing.T) {
|
||||
b, _, bucket := newPutTestBackend(t, "", nil)
|
||||
ctx := context.Background()
|
||||
|
||||
uploadID, err := b.CreateMultipartUpload(ctx, bucket, "object", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
const declared = int64(64 << 20) // 64 MiB declared by the client
|
||||
wantPages := int(declared / int64(pool.BufferSize))
|
||||
|
||||
reader := &poolProbeReader{baseline: pool.Global().InUse()}
|
||||
_, err = b.UploadPart(ctx, bucket, "object", uploadID, 1, declared, reader)
|
||||
// No body bytes arrive, so the part is rejected as incomplete.
|
||||
require.ErrorIs(t, err, gofakes3.ErrIncompleteBody)
|
||||
|
||||
// The fixed path allocates nothing before the first read, so recorded is 0;
|
||||
// the vulnerable path preallocated wantPages (64). The pool is process-wide,
|
||||
// so recorded could pick up a few unrelated in-use buffers, but never the
|
||||
// 64-page reservation the bug produced - the margin distinguishes them.
|
||||
require.True(t, reader.read, "the body must have been read")
|
||||
require.Less(t, reader.recorded, wantPages,
|
||||
"UploadPart preallocated pool pages from the declared Content-Length before any body bytes arrived")
|
||||
}
|
||||
|
||||
// TestWaitForTurnRejectsBogusSize checks that the reorder-buffer admission
|
||||
// rejects a negative client-declared part length and that a huge declared
|
||||
// length cannot overflow the running total so as to admit a further part past
|
||||
// the buffer limit.
|
||||
func TestWaitForTurnRejectsBogusSize(t *testing.T) {
|
||||
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 1<<20)
|
||||
|
||||
// A part length can never be negative.
|
||||
require.ErrorIs(t, up.waitForTurn(1, -1), gofakes3.ErrInvalidArgument)
|
||||
|
||||
// A huge out-of-order part is admitted once because the buffer is empty,
|
||||
// driving buffered near the top of the int64 range.
|
||||
require.NoError(t, up.waitForTurn(2, math.MaxInt64))
|
||||
|
||||
// A further out-of-order part must wait, not be wrongly admitted by an
|
||||
// overflow of buffered+size.
|
||||
admitted := make(chan struct{})
|
||||
go func() {
|
||||
_ = up.waitForTurn(3, math.MaxInt64)
|
||||
close(admitted)
|
||||
}()
|
||||
select {
|
||||
case <-admitted:
|
||||
t.Fatal("out-of-order part admitted past the buffer limit via overflow")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
// Wake the blocked goroutine so it doesn't leak.
|
||||
up.mu.Lock()
|
||||
up.closed = true
|
||||
up.cond.Broadcast()
|
||||
up.mu.Unlock()
|
||||
<-admitted
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
// PutObject error handling tests for serve s3.
|
||||
//
|
||||
// A PUT which fails part way through its body must never disturb the object
|
||||
// already stored at the key.
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ncw/swift/v2"
|
||||
"github.com/rclone/gofakes3"
|
||||
"github.com/rclone/rclone/cmd/serve/proxy"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// errorReader yields data then fails with err.
|
||||
type errorReader struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *errorReader) Read(p []byte) (int, error) {
|
||||
if len(r.data) == 0 {
|
||||
return 0, r.err
|
||||
}
|
||||
n := copy(p, r.data)
|
||||
r.data = r.data[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// newPutTestBackend returns an s3Backend for direct PutObject tests, backed by
|
||||
// the named remote, plus the backing Fs and a bucket created on it. vfsOpt
|
||||
// overrides the VFS options (nil for the defaults).
|
||||
func newPutTestBackend(t *testing.T, backing string, vfsOpt *vfscommon.Options) (*s3Backend, fs.Fs, string) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
if backing == "" {
|
||||
backing = t.TempDir()
|
||||
}
|
||||
f, err := fs.NewFs(ctx, backing)
|
||||
require.NoError(t, err)
|
||||
bucket := fmt.Sprintf("test-%d", testBackingCounter.Add(1))
|
||||
require.NoError(t, f.Mkdir(ctx, bucket))
|
||||
if vfsOpt == nil {
|
||||
vfsOpt = &vfscommon.Opt
|
||||
}
|
||||
// The VFS is cached per remote, so a shared ":memory:" backing reuses a
|
||||
// VFS whose cached root listing predates the bucket just created; forget
|
||||
// it so the new bucket is visible.
|
||||
if root, err := vfs.New(ctx, f, vfsOpt).Root(); err == nil {
|
||||
root.ForgetAll()
|
||||
}
|
||||
opt := Opt
|
||||
opt.HTTP.ListenAddr = []string{endpoint}
|
||||
w, err := newServer(ctx, f, &opt, vfsOpt, &proxy.Opt)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = w.Shutdown() })
|
||||
return newBackend(w), f, bucket
|
||||
}
|
||||
|
||||
var errBoom = errors.New("boom")
|
||||
|
||||
// failureModes enumerates the ways a PUT body can fail to arrive in full:
|
||||
// an arbitrary read error, the unexpected EOF a dropped connection gives
|
||||
// (which must not be mistaken for a clean end of stream), and a body which
|
||||
// ends cleanly short of its declared size.
|
||||
var failureModes = []struct {
|
||||
name string
|
||||
readerErr error
|
||||
wantErr error
|
||||
}{
|
||||
{"ReadError", errBoom, errBoom},
|
||||
{"UnexpectedEOF", io.ErrUnexpectedEOF, io.ErrUnexpectedEOF},
|
||||
{"ShortBody", io.EOF, gofakes3.ErrIncompleteBody},
|
||||
}
|
||||
|
||||
// failPut makes a PutObject call to bucket/object whose body fails with
|
||||
// readerErr part way through and asserts wantErr is passed back.
|
||||
func failPut(t *testing.T, b *s3Backend, bucket, object string, readerErr, wantErr error) {
|
||||
_, err := b.PutObject(context.Background(), bucket, object, map[string]string{},
|
||||
&errorReader{data: []byte(random.String(50)), err: readerErr}, 1000)
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
}
|
||||
|
||||
// TestPutObjectFailurePreservesExisting checks that a failed PUT leaves the
|
||||
// object stored at the key untouched - neither removed nor overwritten with
|
||||
// truncated data - with no temporary object left behind.
|
||||
func TestPutObjectFailurePreservesExisting(t *testing.T) {
|
||||
for _, tc := range testRemotes {
|
||||
for _, fm := range failureModes {
|
||||
t.Run(tc.name+"/"+fm.name, func(t *testing.T) {
|
||||
b, f, bucket := newPutTestBackend(t, tc.backing, nil)
|
||||
ctx := context.Background()
|
||||
const object = "existing.txt"
|
||||
|
||||
existing := []byte(random.String(100))
|
||||
_, err := b.PutObject(ctx, bucket, object, map[string]string{}, bytes.NewReader(existing), int64(len(existing)))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, existing, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
|
||||
failPut(t, b, bucket, object, fm.readerErr, fm.wantErr)
|
||||
|
||||
assert.Equal(t, existing, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
|
||||
// The object must still be served correctly - in particular
|
||||
// HeadObject must not report the failed upload's size.
|
||||
head, err := b.HeadObject(ctx, bucket, object)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(existing)), head.Size)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPutObjectFailureNewKey checks that a failed PUT to a key with no
|
||||
// existing object leaves nothing behind - no partial object at the key and
|
||||
// no temporary object.
|
||||
func TestPutObjectFailureNewKey(t *testing.T) {
|
||||
for _, tc := range testRemotes {
|
||||
for _, fm := range failureModes {
|
||||
t.Run(tc.name+"/"+fm.name, func(t *testing.T) {
|
||||
b, f, bucket := newPutTestBackend(t, tc.backing, nil)
|
||||
ctx := context.Background()
|
||||
const object = "new.txt"
|
||||
|
||||
failPut(t, b, bucket, object, fm.readerErr, fm.wantErr)
|
||||
|
||||
_, err := f.NewObject(ctx, path.Join(bucket, object))
|
||||
require.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
requireOnly(t, f, bucket)
|
||||
|
||||
// The failed upload must not be served either
|
||||
_, err = b.HeadObject(ctx, bucket, object)
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPutObjectMtime checks that the object's modtime is set from the
|
||||
// "X-Amz-Meta-Mtime" or "mtime" metadata supplied with the PUT.
|
||||
func TestPutObjectMtime(t *testing.T) {
|
||||
want := fstest.Time("2011-12-25T12:59:59.123456789Z")
|
||||
for _, metaKey := range []string{"X-Amz-Meta-Mtime", "mtime"} {
|
||||
t.Run(metaKey, func(t *testing.T) {
|
||||
b, f, bucket := newPutTestBackend(t, "", nil)
|
||||
ctx := context.Background()
|
||||
const object = "mtime.txt"
|
||||
|
||||
contents := []byte(random.String(50))
|
||||
meta := map[string]string{metaKey: swift.TimeToFloatString(want)}
|
||||
_, err := b.PutObject(ctx, bucket, object, meta, bytes.NewReader(contents), int64(len(contents)))
|
||||
require.NoError(t, err)
|
||||
|
||||
o, err := f.NewObject(ctx, path.Join(bucket, object))
|
||||
require.NoError(t, err)
|
||||
fstest.AssertTimeEqualWithPrecision(t, object, want, o.ModTime(ctx), f.Precision())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// waitForObject waits for bucket/object to appear on the backing Fs (e.g.
|
||||
// after the VFS write-back delay).
|
||||
func waitForObject(t *testing.T, f fs.Fs, bucket, object string) {
|
||||
ctx := context.Background()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := f.NewObject(ctx, path.Join(bucket, object)); err == nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("object %s/%s never appeared on the backing remote", bucket, object)
|
||||
}
|
||||
|
||||
// TestPutObjectFailureCached checks the failed PUT semantics with the cache
|
||||
// modes that write through the VFS cache - minimal and above, since the PUT
|
||||
// opens its handle read-write: the truncated file the failed upload leaves
|
||||
// in the cache must be discarded, not written back over the object at the
|
||||
// key.
|
||||
func TestPutObjectFailureCached(t *testing.T) {
|
||||
for _, cm := range []vfscommon.CacheMode{vfscommon.CacheModeMinimal, vfscommon.CacheModeWrites} {
|
||||
for _, tc := range testRemotes {
|
||||
t.Run(cm.String()+"/"+tc.name, func(t *testing.T) {
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.CacheMode = cm
|
||||
vfsOpt.WriteBack = fs.Duration(100 * time.Millisecond)
|
||||
b, f, bucket := newPutTestBackend(t, tc.backing, &vfsOpt)
|
||||
const object = "cached.txt"
|
||||
|
||||
existing := []byte(random.String(100))
|
||||
_, err := b.PutObject(context.Background(), bucket, object, map[string]string{}, bytes.NewReader(existing), int64(len(existing)))
|
||||
require.NoError(t, err)
|
||||
waitForObject(t, f, bucket, object)
|
||||
assert.Equal(t, existing, readObject(t, f, bucket, object))
|
||||
|
||||
failPut(t, b, bucket, object, errBoom, errBoom)
|
||||
|
||||
// Wait out several write-back intervals to catch the truncated
|
||||
// data being written back before checking the object is
|
||||
// untouched.
|
||||
time.Sleep(time.Second)
|
||||
assert.Equal(t, existing, readObject(t, f, bucket, object))
|
||||
requireOnly(t, f, bucket, object)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-3
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/serve"
|
||||
@@ -39,11 +40,15 @@ var OptionsInfo = fs.Options{{
|
||||
}, {
|
||||
Name: "disable_multipart_streaming",
|
||||
Default: false,
|
||||
Help: "Buffer multipart uploads in memory instead of streaming them to the backend (see the Multipart uploads docs section)",
|
||||
Help: "Buffer multipart uploads in memory instead of streaming them to the backend",
|
||||
}, {
|
||||
Name: "multipart_streaming_buffer_limit",
|
||||
Default: fs.SizeSuffix(256 * 1024 * 1024),
|
||||
Help: "Maximum memory buffered per streamed multipart upload for parts arriving out of order, 0 for unlimited (see the Multipart uploads docs section)",
|
||||
Help: "Maximum memory buffered per streamed multipart upload for parts arriving out of order, 0 for unlimited",
|
||||
}, {
|
||||
Name: "multipart_expiry",
|
||||
Default: fs.Duration(24 * time.Hour),
|
||||
Help: "Abort incomplete multipart uploads idle for longer than this, 0 to keep forever",
|
||||
}}.
|
||||
Add(httplib.ConfigInfo).
|
||||
Add(httplib.AuthConfigInfo)
|
||||
@@ -57,6 +62,7 @@ type Options struct {
|
||||
NoCleanup bool `config:"no_cleanup"`
|
||||
DisableMultipartStreaming bool `config:"disable_multipart_streaming"`
|
||||
MultipartStreamingBufferLimit fs.SizeSuffix `config:"multipart_streaming_buffer_limit"`
|
||||
MultipartExpiry fs.Duration `config:"multipart_expiry"`
|
||||
Auth httplib.AuthConfig
|
||||
HTTP httplib.Config
|
||||
}
|
||||
@@ -114,7 +120,7 @@ var Command = &cobra.Command{
|
||||
},
|
||||
Use: "s3 remote:path",
|
||||
Short: `Serve remote:path over s3.`,
|
||||
Long: help() + strings.TrimSpace(httplib.AuthHelp(flagPrefix)+httplib.Help(flagPrefix)+vfs.Help()),
|
||||
Long: help() + strings.TrimSpace(httplib.AuthHelp(flagPrefix)+httplib.Help(flagPrefix)+vfs.Help()+proxy.Help),
|
||||
RunE: func(command *cobra.Command, args []string) error {
|
||||
var f fs.Fs
|
||||
if proxy.Opt.AuthProxy == "" {
|
||||
|
||||
+157
-4
@@ -8,6 +8,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -15,8 +16,11 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/rclone/gofakes3/signature"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/s3" // for TestS3Minio backing remote
|
||||
"github.com/rclone/rclone/cmd/serve/proxy"
|
||||
@@ -39,11 +43,19 @@ const (
|
||||
)
|
||||
|
||||
// Configure and serve the server
|
||||
//
|
||||
// If f is nil the server is expected to be using an auth proxy, in
|
||||
// which case the credentials are handed to the test proxy via the
|
||||
// environment rather than --auth-key.
|
||||
func serveS3(t *testing.T, f fs.Fs) (testURL string, keyid string, keysec string, w *Server) {
|
||||
keyid = random.String(16)
|
||||
keysec = random.String(16)
|
||||
opt := Opt // copy default options
|
||||
opt.AuthKey = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
|
||||
if f == nil {
|
||||
t.Setenv("RCLONE_TEST_PROXY_AUTH_KEY", fmt.Sprintf("%s,%s", keyid, keysec))
|
||||
} else {
|
||||
opt.AuthKey = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
|
||||
}
|
||||
opt.HTTP.ListenAddr = []string{endpoint}
|
||||
w, _ = newServer(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)
|
||||
go func() {
|
||||
@@ -283,9 +295,6 @@ func TestListBucketsAuthProxy(t *testing.T) {
|
||||
{
|
||||
description: "list buckets",
|
||||
bucket: "mybucket",
|
||||
// request with random keyid
|
||||
// instead of what was set in 'authPair'
|
||||
keyID: random.String(16),
|
||||
files: []FileStuct{
|
||||
{
|
||||
path: "",
|
||||
@@ -297,6 +306,12 @@ func TestListBucketsAuthProxy(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "list buckets: unknown s3 key",
|
||||
bucket: "mybucket",
|
||||
keyID: random.String(16),
|
||||
shouldFail: true,
|
||||
},
|
||||
{
|
||||
description: "list buckets: wrong s3 secret",
|
||||
bucket: "mybucket",
|
||||
@@ -308,6 +323,144 @@ func TestListBucketsAuthProxy(t *testing.T) {
|
||||
testListBuckets(t, cases, true)
|
||||
}
|
||||
|
||||
// TestNewServerPerServerAuthProxy checks that a per-server proxyOpt.AuthProxy
|
||||
// enables proxy mode even when the process-global proxy.Opt.AuthProxy is empty,
|
||||
// which is the normal case when the server is configured via serve/start.
|
||||
func TestNewServerPerServerAuthProxy(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
|
||||
// Ensure the global is empty so we only test the per-server option.
|
||||
assert.Equal(t, "", proxy.Opt.AuthProxy)
|
||||
|
||||
f, err := fs.NewFs(context.Background(), "testdata")
|
||||
require.NoError(t, err)
|
||||
|
||||
opt := Opt
|
||||
opt.AuthKey = []string{"access-key,secret-key"}
|
||||
opt.HTTP.ListenAddr = []string{endpoint}
|
||||
|
||||
proxyOpt := proxy.Opt
|
||||
proxyOpt.AuthProxy = "/path/to/auth/proxy"
|
||||
|
||||
w, err := newServer(context.Background(), f, &opt, &vfscommon.Opt, &proxyOpt)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
assert.NoError(t, w.Shutdown())
|
||||
}()
|
||||
assert.True(t, w.provider.IsProxy(), "expected auth proxy to be enabled by per-server option")
|
||||
assert.Nil(t, w.provider.VFS(), "expected no fixed VFS when auth proxy is in use")
|
||||
}
|
||||
|
||||
// TestAuthProxyEmptySecret checks that a request for an arbitrary
|
||||
// access key ID signed with an empty secret is refused when using an
|
||||
// auth proxy without --auth-key.
|
||||
func TestAuthProxyEmptySecret(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
|
||||
prog, err := filepath.Abs("../servetest/proxy_code.go")
|
||||
require.NoError(t, err)
|
||||
files, err := filepath.Abs("testdata")
|
||||
require.NoError(t, err)
|
||||
proxy.Opt.AuthProxy = "go run " + prog + " " + files
|
||||
defer func() {
|
||||
proxy.Opt.AuthProxy = ""
|
||||
}()
|
||||
|
||||
endpoint, keyid, _, s := serveS3(t, nil)
|
||||
defer func() {
|
||||
assert.NoError(t, s.server.Shutdown())
|
||||
}()
|
||||
|
||||
for _, accessKeyID := range []string{keyid, random.String(16)} {
|
||||
req, err := http.NewRequest("GET", endpoint+"/", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
|
||||
signer := v4.NewSigner()
|
||||
err = signer.SignHTTP(context.Background(), aws.Credentials{AccessKeyID: accessKeyID, SecretAccessKey: ""}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
assert.Equal(t, http.StatusForbidden, resp.StatusCode, string(body))
|
||||
assert.NotContains(t, string(body), "ListAllMyBucketsResult")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuthProxyKeysNotRegistered checks that secrets supplied by the
|
||||
// auth proxy are never registered in gofakes3's process wide key
|
||||
// store where any other serve s3 instance in the process would honour
|
||||
// them.
|
||||
func TestAuthProxyKeysNotRegistered(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
|
||||
prog, err := filepath.Abs("../servetest/proxy_code.go")
|
||||
require.NoError(t, err)
|
||||
files, err := filepath.Abs("testdata")
|
||||
require.NoError(t, err)
|
||||
proxy.Opt.AuthProxy = "go run " + prog + " " + files
|
||||
defer func() {
|
||||
proxy.Opt.AuthProxy = ""
|
||||
}()
|
||||
|
||||
endpoint, keyid, keysec, s := serveS3(t, nil)
|
||||
defer func() {
|
||||
assert.NoError(t, s.server.Shutdown())
|
||||
}()
|
||||
|
||||
sign := func() *http.Request {
|
||||
req, err := http.NewRequest("GET", endpoint+"/", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
|
||||
signer := v4.NewSigner()
|
||||
err = signer.SignHTTP(context.Background(), aws.Credentials{AccessKeyID: keyid, SecretAccessKey: keysec}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
|
||||
require.NoError(t, err)
|
||||
return req
|
||||
}
|
||||
|
||||
// A correctly signed request is accepted by the server
|
||||
resp, err := http.DefaultClient.Do(sign())
|
||||
require.NoError(t, err)
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode, string(body))
|
||||
|
||||
// But the key it used must not be in the global key store
|
||||
assert.NotEqual(t, signature.ErrNone, signature.V4SignVerify(sign()), "proxy secret was registered in the gofakes3 key store")
|
||||
}
|
||||
|
||||
// TestAuthKeyPerServer checks that two servers in the same process
|
||||
// with different --auth-key pairs only accept their own credentials.
|
||||
func TestAuthKeyPerServer(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
f, err := fs.NewFs(context.Background(), "testdata")
|
||||
require.NoError(t, err)
|
||||
|
||||
urlA, keyA, secA, sA := serveS3(t, f)
|
||||
defer func() { assert.NoError(t, sA.server.Shutdown()) }()
|
||||
urlB, keyB, secB, sB := serveS3(t, f)
|
||||
defer func() { assert.NoError(t, sB.server.Shutdown()) }()
|
||||
|
||||
signedGet := func(url, accessKeyID, secret string) int {
|
||||
req, err := http.NewRequest("GET", url+"/", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
|
||||
err = v4.NewSigner().SignHTTP(context.Background(), aws.Credentials{AccessKeyID: accessKeyID, SecretAccessKey: secret}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
|
||||
require.NoError(t, err)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
_, _ = io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
assert.Equal(t, http.StatusOK, signedGet(urlA, keyA, secA), "A with A's key")
|
||||
assert.Equal(t, http.StatusOK, signedGet(urlB, keyB, secB), "B with B's key")
|
||||
assert.Equal(t, http.StatusForbidden, signedGet(urlA, keyB, secB), "A with B's key")
|
||||
assert.Equal(t, http.StatusForbidden, signedGet(urlB, keyA, secA), "B with A's key")
|
||||
}
|
||||
|
||||
func TestRc(t *testing.T) {
|
||||
servetest.TestRc(t, rc.Params{
|
||||
"type": "s3",
|
||||
|
||||
+190
-41
@@ -13,6 +13,12 @@ docs](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html)).
|
||||
`--auth-key` is not provided then `serve s3` will allow anonymous
|
||||
access.
|
||||
|
||||
Alternatively `--auth-proxy` can be used to look up the secret for each
|
||||
access key ID and choose the backend it maps to (see [Auth
|
||||
Proxy](#auth-proxy) below). When an auth proxy is in use `--auth-key`
|
||||
is ignored and every request must be signed with the secret the proxy
|
||||
returns for its access key ID.
|
||||
|
||||
Like all rclone flags `--auth-key` can be set via environment
|
||||
variables, in this case `RCLONE_AUTH_KEY`. Since this flag can be
|
||||
repeated, the input to `RCLONE_AUTH_KEY` is CSV encoded. Because the
|
||||
@@ -88,24 +94,73 @@ access_key_id = ACCESS_KEY_ID
|
||||
secret_access_key = SECRET_ACCESS_KEY
|
||||
```
|
||||
|
||||
### Object uploads (PUT)
|
||||
|
||||
A `PutObject` upload only ever changes the object at its key atomically, on
|
||||
success, a failed or interrupted PUT neither removes nor overwrites the
|
||||
object already stored at the key, and never leaves a partial object visible
|
||||
at it.
|
||||
|
||||
Remotes that upload atomically (e.g. object stores such as `s3`) are streamed
|
||||
straight to the destination. On remotes where a partial upload would
|
||||
otherwise be visible (e.g. `local`), and whenever `--vfs-cache-mode` is
|
||||
`writes` or above, the upload is written to a temporary object that is
|
||||
renamed into place on success; these remotes need to support a server-side
|
||||
move or copy for this (nearly all do - without move or copy the upload is
|
||||
written directly and a failed PUT may leave a partial object at the key). If
|
||||
`serve s3` is killed part-way through an upload the temporary object (named
|
||||
with a leading `.rclone_temp_put_`) may be left behind; it is hidden from
|
||||
S3 listings but must be removed manually.
|
||||
|
||||
### Multipart uploads
|
||||
|
||||
By default `serve s3` **streams** each multipart upload, in part-number
|
||||
order, into a single `PutStream` upload to the underlying remote, so the
|
||||
whole file is never buffered in memory - memory use stays bounded by the
|
||||
parts in flight. The remote then performs its own internal upload (for
|
||||
example its own multipart upload, still with bounded memory). This works
|
||||
for any remote that supports `PutStream`, which is nearly all of them,
|
||||
including through `crypt`.
|
||||
|
||||
The upload is atomic so the destination object only ever changes on a
|
||||
Multipart uploads are written, in part-number order, to a temporary
|
||||
object which is renamed into place, server-side, on completion, so the
|
||||
upload is atomic. The object at the key only ever changes on a
|
||||
successful completion. A failed or aborted upload never affects any
|
||||
object already stored under that name. Remotes that upload atomically
|
||||
already (object stores such as `s3`) are streamed straight to the
|
||||
destination. On remotes where a partial upload would otherwise be visible
|
||||
(such as `local`), the parts are streamed to a temporary object that is
|
||||
moved into place, server-side, on completion; these remotes therefore
|
||||
also need to support a server-side move or copy.
|
||||
object already stored under that name and a partly-uploaded object
|
||||
never becomes visible under it.
|
||||
|
||||
With the default `--vfs-cache-mode off` `serve s3` **streams** each
|
||||
multipart upload, in part-number order, into a single streaming upload
|
||||
to the underlying remote, so the whole file is never buffered in
|
||||
memory. Memory use stays bounded by the parts in flight. The remote
|
||||
then performs its own internal upload (for example its own multipart
|
||||
upload, still with bounded memory). Remotes that don't support
|
||||
streaming uploads (those that must know the file size before the
|
||||
upload starts, such as `onedrive`, `pcloud`, `jottacloud`, `mailru`,
|
||||
`opendrive`, `putio`, `protondrive` and `zoho`) have the parts spooled
|
||||
to a temporary file on **local disk** instead, and uploaded with the
|
||||
size then known on completion, so they need local disk space for the
|
||||
largest objects in flight rather than memory.
|
||||
|
||||
With `--vfs-cache-mode writes` (or `full`) the parts are written to a
|
||||
temporary file in the VFS cache and uploaded by the VFS write-back -
|
||||
see [Multipart uploads and the VFS
|
||||
cache](#multipart-uploads-and-the-vfs-cache) below.
|
||||
|
||||
The rename into place needs the remote to support a server-side move
|
||||
or copy, which nearly all do. It is a cheap rename on most remotes,
|
||||
but on object stores without a real rename (such as `s3` itself) the
|
||||
move is performed as a server-side copy and delete of the whole
|
||||
object, which can take time and API calls for large objects.
|
||||
Concurrent multipart uploads of the same key (which S3 permits) are
|
||||
safe. Each writes its own temporary object and the last to complete
|
||||
wins.
|
||||
|
||||
On the few remotes that support neither server side move nor copy, the
|
||||
parts are written straight to the destination object instead and never
|
||||
buffered in memory. This is at some cost in atomicity - the incomplete
|
||||
object is visible under its final name while the upload is in flight,
|
||||
as it also is for a plain object PUT on such remotes, and concurrent
|
||||
multipart uploads of the same key write to the same object and can
|
||||
interleave. A failed or aborted upload still leaves any pre-existing
|
||||
object untouched provided the remote uploads atomically and the VFS
|
||||
cache is off; on a remote where partial uploads are visible it may
|
||||
leave partial data at the key (like a plain PUT there), and with
|
||||
`--vfs-cache-mode writes` (or `full`) a write to the cache cannot be
|
||||
abandoned, so an aborted upload's partial data is written back to the
|
||||
remote as if it had completed.
|
||||
|
||||
**Features**
|
||||
|
||||
@@ -120,10 +175,14 @@ also need to support a server-side move or copy.
|
||||
as one continuous stream.
|
||||
- The destination object only ever changes atomically, on completion: an
|
||||
aborted or failed upload leaves any pre-existing object of the same
|
||||
name untouched, and a partly-uploaded object never becomes visible.
|
||||
- Backend-agnostic - it only needs the remote to support `PutStream`
|
||||
(plus a server-side move or copy on remotes that don't upload
|
||||
atomically).
|
||||
name untouched, and a partly-uploaded object never becomes visible
|
||||
(except on the few remotes with no server-side move or copy, as
|
||||
above).
|
||||
- Multipart uploads go through the VFS like any other upload, so they
|
||||
show in rclone's transfer stats and obey `--bwlimit`.
|
||||
- Backend-agnostic - it only needs the remote to support a server-side
|
||||
move or copy for the rename into place, which nearly all do; a remote
|
||||
without streaming upload support spools to local disk as above.
|
||||
|
||||
**Limitations**
|
||||
|
||||
@@ -149,31 +208,121 @@ also need to support a server-side move or copy.
|
||||
upload and the client must start it again. (The remote's own upload
|
||||
still retries its internal chunks.)
|
||||
- Parts are serialised into one stream, so ingest from the client is
|
||||
effectively single-threaded, although the remote's own upload still
|
||||
runs concurrently.
|
||||
- On remotes that don't upload atomically (such as `local`), the
|
||||
completed object is moved into place with a server-side operation.
|
||||
This is a cheap rename on most such remotes. On these remotes, if
|
||||
`serve s3` is killed part-way through an upload the temporary object
|
||||
(named with a leading `.rclone_multipart_upload_`) may be left behind;
|
||||
it is hidden from S3 listings but must be removed manually.
|
||||
effectively single-threaded. When streaming, the remote's own upload
|
||||
runs concurrently with the parts arriving; with the local disk spool
|
||||
or the VFS cache the upload to the remote only starts on completion.
|
||||
- If `serve s3` is killed part-way through an upload the temporary
|
||||
object (named with a leading `.rclone_temp_multipart_`) may be left
|
||||
behind; it is hidden from S3 listings but must be removed manually.
|
||||
|
||||
#### Multipart uploads and the VFS cache
|
||||
|
||||
With `--vfs-cache-mode writes` (or `full`) multipart uploads do not
|
||||
stream to the remote at all. The parts are written, in part-number
|
||||
order, to a temporary file in the VFS cache. On completion the file is
|
||||
renamed into place and uploaded by the VFS write-back, exactly like a
|
||||
plain object PUT. This needs no streaming upload support from the
|
||||
remote. The rename normally happens in the cache before the upload has
|
||||
started, but the VFS requires the remote to support a server-side move
|
||||
or copy to rename files at all (and uses one if the temporary file has
|
||||
already been written back, e.g. with `--vfs-write-back 0`). On remotes
|
||||
without either, the parts are written to the cache directly under the
|
||||
final key instead: the upload still never touches memory, but it loses
|
||||
its atomicity - the in-flight upload is visible at the key, and an
|
||||
aborted upload cannot be abandoned once in the cache, so its partial
|
||||
data is written back to the remote as if it were a completed object.
|
||||
|
||||
Remotes that benefit from `--vfs-cache-mode writes`:
|
||||
|
||||
- **Remotes over slow or unreliable links.** A failure in a streamed
|
||||
upload aborts the whole multipart upload and the client must start
|
||||
again from the first part; a failed write-back upload is retried by
|
||||
the VFS (see `--vfs-cache-max-age` and friends) without the client
|
||||
being involved. Ingest from the client also runs at local disk speed
|
||||
rather than being throttled to the remote's pace.
|
||||
- **Workloads that read back or overwrite what they just wrote.** The
|
||||
completed object stays in the cache, so subsequent `GET`/`HEAD`
|
||||
requests are served locally, and plain PUTs and multipart uploads to
|
||||
the same key go through the same cache entry so the last write wins
|
||||
regardless of upload style.
|
||||
|
||||
The trade-offs of the VFS cache:
|
||||
|
||||
- The whole object lands on local disk, so the cache (`--cache-dir`)
|
||||
needs space for the largest objects in flight; `--vfs-cache-max-size`
|
||||
cannot evict files which are still being uploaded.
|
||||
- The `200 OK` for `CompleteMultipartUpload` means the data is safely
|
||||
in the **local cache**, not yet on the remote - the same durability
|
||||
the cache gives plain PUTs. If an acknowledgement must mean the data
|
||||
has reached the remote (for example WAL archiving), use the default
|
||||
`--vfs-cache-mode off`.
|
||||
- The upload to the remote only starts on completion, rather than
|
||||
overlapping with the parts arriving, so the data reaches the remote
|
||||
later than with streaming.
|
||||
- If `serve s3` is killed part-way through an upload, the temporary
|
||||
file survives in the cache and the VFS cache recovery uploads it to
|
||||
the remote on restart as a temporary object (named with a leading
|
||||
`.rclone_temp_multipart_`); as with the streaming path, it is
|
||||
hidden from S3 listings but must be removed manually.
|
||||
|
||||
#### Cleaning up temporary objects
|
||||
|
||||
If `serve s3` is killed part-way through an upload it can leave a
|
||||
temporary object behind, named with a leading `.rclone_temp_`. This
|
||||
whole prefix is reserved: any object whose name (the last
|
||||
`/`-separated segment of its key) starts with `.rclone_temp_` is
|
||||
hidden from S3 listings, so don't give real objects such names - an
|
||||
existing object with such a name disappears from listings (though it
|
||||
stays accessible directly by its key: only listings hide reserved
|
||||
names, `GET`, `HEAD` and `DELETE` of the exact key still work). A
|
||||
temporary object never holds acknowledged data - uploads whose
|
||||
temporary object survived were never confirmed to the client - so old
|
||||
ones are safe to delete:
|
||||
|
||||
rclone delete --min-age 24h --include ".rclone_temp_*" remote:path
|
||||
|
||||
The `--min-age` protects uploads which are still in progress: make sure
|
||||
it is longer than your longest upload, especially if several `serve s3`
|
||||
instances share the same remote.
|
||||
|
||||
rclone v1.75 named its temporary multipart objects
|
||||
`.rclone_multipart_upload_*`; leftovers from an older server are also
|
||||
hidden from listings and can be cleaned up the same way.
|
||||
|
||||
#### Abandoned uploads
|
||||
|
||||
A client which starts a multipart upload and vanishes without either
|
||||
completing or aborting it would otherwise hold on to its resources
|
||||
forever.
|
||||
|
||||
An incomplete multipart upload which has had no activity for
|
||||
`--multipart-expiry` (default `24h`) is therefore aborted and cleaned
|
||||
up, exactly as if the client had called `AbortMultipartUpload`, and a
|
||||
`NOTICE` is logged.
|
||||
|
||||
An upload with a part still being received is never expired, however
|
||||
slowly the part is arriving, and each completed part restarts the
|
||||
clock, so the expiry only needs to outlast the client's pauses
|
||||
*between* parts, not the whole upload.
|
||||
|
||||
Late operations on an expired upload fail with `NoSuchUpload`, as they
|
||||
do on real S3 when a lifecycle rule has aborted the upload. Set
|
||||
`--multipart-expiry 0` to keep incomplete uploads forever.
|
||||
|
||||
#### Disabling streaming
|
||||
|
||||
If you pass `--disable-multipart-streaming`, or the remote doesn't
|
||||
support `PutStream` (or doesn't upload atomically and can't move or copy
|
||||
server-side), multipart uploads are instead **buffered in memory**
|
||||
by the underlying S3 library: every part is held in memory and the whole
|
||||
object is written out in one go when the upload completes (the previous
|
||||
behaviour). This removes the in-order/contiguous-part restriction above,
|
||||
so parts can be uploaded in any order, but **memory use grows with the
|
||||
size of the upload**, so it is only suitable for small objects. A one-off
|
||||
`NOTICE` is logged the first time this happens.
|
||||
|
||||
Alternatively, if the client is an rclone `s3` remote (like the
|
||||
`[serves3]` example above), you can set `use_multipart_uploads = false`
|
||||
on it so it uploads each object as a single stream and skips multipart
|
||||
uploads altogether.
|
||||
If you pass `--disable-multipart-streaming`, multipart uploads are
|
||||
instead **buffered in memory** by the underlying S3 library: every
|
||||
part is held in memory and the whole object is written out in one go
|
||||
when the upload completes. This removes the in-order/contiguous-part
|
||||
restriction above, so parts can be uploaded in any order, but **memory
|
||||
use grows with the size of the upload**, so it is only suitable for
|
||||
small objects. A one-off `NOTICE` is logged the first time this
|
||||
happens. This flag is the only thing that makes multipart uploads
|
||||
buffer in memory - it is never done because of missing remote
|
||||
capabilities. Consider `--vfs-cache-mode writes` instead, which
|
||||
buffers the upload in the VFS cache on disk and takes precedence over
|
||||
`--disable-multipart-streaming`.
|
||||
|
||||
### Bugs
|
||||
|
||||
|
||||
+102
-71
@@ -3,14 +3,12 @@ package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/rclone/gofakes3"
|
||||
@@ -34,12 +32,11 @@ type Server struct {
|
||||
server *httplib.Server
|
||||
opt Options
|
||||
f fs.Fs
|
||||
_vfs *vfs.VFS // don't use directly, use getVFS
|
||||
provider *proxy.Provider
|
||||
faker *gofakes3.GoFakeS3
|
||||
backend *s3Backend
|
||||
handler http.Handler
|
||||
proxy *proxy.Proxy
|
||||
ctx context.Context // for global config
|
||||
s3Secret string
|
||||
etagHashType hash.Type
|
||||
}
|
||||
|
||||
@@ -49,8 +46,17 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
provider: proxy.NewProvider(ctx, f, vfsOpt, proxyOpt),
|
||||
etagHashType: hash.None,
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if w.backend != nil {
|
||||
w.backend.stopReaper()
|
||||
}
|
||||
w.provider.Shutdown()
|
||||
}
|
||||
}()
|
||||
|
||||
if w.opt.EtagHash == "auto" {
|
||||
w.etagHashType = f.Hashes().GetOne()
|
||||
@@ -64,20 +70,31 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
fs.Debugf(f, "Using hash %v for ETag", w.etagHashType)
|
||||
}
|
||||
|
||||
if len(opt.AuthKey) == 0 {
|
||||
if len(opt.AuthKey) == 0 && !w.provider.IsProxy() {
|
||||
fs.Logf("serve s3", "No auth provided so allowing anonymous access")
|
||||
} else {
|
||||
w.s3Secret = getAuthSecret(opt.AuthKey)
|
||||
}
|
||||
|
||||
authList, err := authlistResolver(opt.AuthKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing auth list failed: %q", err)
|
||||
}
|
||||
if w.provider.IsProxy() {
|
||||
// The proxy middleware authenticates every request itself so
|
||||
// gofakes3's own auth must be left empty.
|
||||
if len(authList) > 0 {
|
||||
fs.Logf("serve s3", "--auth-key is ignored when --auth-proxy is set - the proxy must supply the secret for each access key ID")
|
||||
}
|
||||
authList = nil
|
||||
}
|
||||
|
||||
w.backend = newBackend(w)
|
||||
if w.opt.MultipartExpiry > 0 {
|
||||
w.backend.startReaper(time.Duration(w.opt.MultipartExpiry))
|
||||
}
|
||||
|
||||
var newLogger logger
|
||||
w.faker = gofakes3.New(
|
||||
newBackend(w),
|
||||
w.backend,
|
||||
gofakes3.WithHostBucket(!opt.ForcePathStyle),
|
||||
gofakes3.WithLogger(newLogger),
|
||||
gofakes3.WithRequestID(rand.Uint64()),
|
||||
@@ -88,17 +105,10 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
|
||||
w.handler = w.faker.Server()
|
||||
|
||||
if proxy.Opt.AuthProxy != "" {
|
||||
w.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
|
||||
// proxy auth middleware
|
||||
if w.provider.IsProxy() {
|
||||
w.handler = proxyAuthMiddleware(w.handler, w)
|
||||
w.handler = authPairMiddleware(w.handler, w)
|
||||
} else {
|
||||
w._vfs = vfs.New(ctx, f, vfsOpt)
|
||||
|
||||
if len(opt.AuthKey) > 0 {
|
||||
w.faker.AddAuthKeys(authList)
|
||||
}
|
||||
} else if len(opt.AuthKey) > 0 {
|
||||
w.faker.AddAuthKeys(authList)
|
||||
}
|
||||
|
||||
w.server, err = httplib.NewServer(ctx,
|
||||
@@ -116,8 +126,8 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
}
|
||||
|
||||
func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
|
||||
if w._vfs != nil {
|
||||
return w._vfs, nil
|
||||
if w.provider.VFS() != nil {
|
||||
return w.provider.VFS(), nil
|
||||
}
|
||||
|
||||
value := ctx.Value(ctxKeyID)
|
||||
@@ -132,13 +142,41 @@ func (w *Server) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
|
||||
return VFS, nil
|
||||
}
|
||||
|
||||
// auth does proxy authorization
|
||||
func (w *Server) auth(accessKeyID string) (value any, err error) {
|
||||
VFS, _, err := w.proxy.Call(stringToMd5Hash(accessKeyID), accessKeyID, false)
|
||||
// auth authenticates the request via the auth proxy.
|
||||
//
|
||||
// The proxy maps the access key ID to a VFS and a secret access key
|
||||
// and the request's signature is verified against that secret. If it
|
||||
// fails against a cached secret the proxy is consulted again in case
|
||||
// the secret has been rotated.
|
||||
//
|
||||
// The secret is only ever used here and is never registered with
|
||||
// gofakes3, whose key store is shared by every instance in the
|
||||
// process.
|
||||
func (w *Server) auth(r *http.Request, accessKeyID string) (VFS *vfs.VFS, err error) {
|
||||
p := w.provider.Proxy()
|
||||
VFS, secret, err := p.CallAccessKey(accessKeyID, r.RemoteAddr, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return VFS, err
|
||||
errCode := signature.V4SignVerifyWithSecret(r, secret)
|
||||
if errCode == signature.ErrNone {
|
||||
return VFS, nil
|
||||
}
|
||||
// Only a signature mismatch can be cured by a rotated secret, so
|
||||
// only then is the proxy worth consulting again - other failures
|
||||
// (expired request, bad date, missing headers) must not make
|
||||
// every bad request run the proxy.
|
||||
if signature.GetAPIError(errCode).Code == "SignatureDoesNotMatch" {
|
||||
VFS, secret, err = p.CallAccessKey(accessKeyID, r.RemoteAddr, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
errCode = signature.V4SignVerifyWithSecret(r, secret)
|
||||
if errCode == signature.ErrNone {
|
||||
return VFS, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("signature verification failed: %s", signature.GetAPIError(errCode).Code)
|
||||
}
|
||||
|
||||
// Bind register the handler to http.Router
|
||||
@@ -161,62 +199,55 @@ func (w *Server) Addr() net.Addr {
|
||||
|
||||
// Shutdown the server
|
||||
func (w *Server) Shutdown() error {
|
||||
return w.server.Shutdown()
|
||||
}
|
||||
|
||||
func authPairMiddleware(next http.Handler, ws *Server) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
accessKey, _ := parseAccessKeyID(r)
|
||||
// set the auth pair
|
||||
authPair := map[string]string{
|
||||
accessKey: ws.s3Secret,
|
||||
}
|
||||
ws.faker.AddAuthKeys(authPair)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
w.backend.stopReaper()
|
||||
err := w.server.Shutdown()
|
||||
w.provider.Shutdown()
|
||||
return err
|
||||
}
|
||||
|
||||
// proxyAuthMiddleware authenticates each request via the auth proxy,
|
||||
// storing the VFS it returns in the request context, and refuses
|
||||
// requests the proxy does not accept.
|
||||
func proxyAuthMiddleware(next http.Handler, ws *Server) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
accessKey, _ := parseAccessKeyID(r)
|
||||
value, err := ws.auth(accessKey)
|
||||
accessKey, errCode := parseAccessKeyID(r)
|
||||
if errCode != signature.ErrNone {
|
||||
fs.Infof(r.URL.Path, "%s: Auth failed: no access key ID in request", r.RemoteAddr)
|
||||
accessDenied(w)
|
||||
return
|
||||
}
|
||||
VFS, err := ws.auth(r, accessKey)
|
||||
if err != nil {
|
||||
fs.Infof(r.URL.Path, "%s: Auth failed: %v", r.RemoteAddr, err)
|
||||
accessDenied(w)
|
||||
return
|
||||
}
|
||||
if value != nil {
|
||||
r = r.WithContext(context.WithValue(r.Context(), ctxKeyID, value))
|
||||
}
|
||||
|
||||
r = r.WithContext(context.WithValue(r.Context(), ctxKeyID, VFS))
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func parseAccessKeyID(r *http.Request) (accessKey string, error signature.ErrorCode) {
|
||||
v4Auth := r.Header.Get("Authorization")
|
||||
req, err := signature.ParseSignV4(v4Auth)
|
||||
if err != signature.ErrNone {
|
||||
return "", err
|
||||
}
|
||||
// accessDenied writes an S3 AccessDenied error response
|
||||
func accessDenied(w http.ResponseWriter) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte(`<?xml version="1.0" encoding="UTF-8"?><Error><Code>AccessDenied</Code><Message>Access Denied</Message></Error>`))
|
||||
}
|
||||
|
||||
// parseAccessKeyID returns the access key ID from the request's
|
||||
// Authorization header or presigned URL query parameters
|
||||
func parseAccessKeyID(r *http.Request) (accessKey string, errCode signature.ErrorCode) {
|
||||
v4Auth := r.Header.Get("Authorization")
|
||||
if v4Auth == "" {
|
||||
// Presigned URLs carry the credential in the query string
|
||||
q := r.URL.Query()
|
||||
if q.Get("X-Amz-Signature") != "" {
|
||||
v4Auth = fmt.Sprintf("%s Credential=%s, SignedHeaders=%s, Signature=%s", q.Get("X-Amz-Algorithm"), q.Get("X-Amz-Credential"), q.Get("X-Amz-SignedHeaders"), q.Get("X-Amz-Signature"))
|
||||
}
|
||||
}
|
||||
req, errCode := signature.ParseSignV4(v4Auth)
|
||||
if errCode != signature.ErrNone {
|
||||
return "", errCode
|
||||
}
|
||||
return req.Credential.GetAccessKey(), signature.ErrNone
|
||||
}
|
||||
|
||||
func stringToMd5Hash(s string) string {
|
||||
hasher := md5.New()
|
||||
hasher.Write([]byte(s))
|
||||
return hex.EncodeToString(hasher.Sum(nil))
|
||||
}
|
||||
|
||||
func getAuthSecret(authPair []string) string {
|
||||
if len(authPair) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
splited := strings.Split(authPair[0], ",")
|
||||
if len(splited) != 2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
secret := strings.TrimSpace(splited[1])
|
||||
return secret
|
||||
}
|
||||
@@ -1,12 +1,18 @@
|
||||
//go:build ignore
|
||||
|
||||
// A simple auth proxy for testing purposes
|
||||
//
|
||||
// For S3 access key auth (no "pass" or "public_key" in the input) the
|
||||
// access key ID and secret are read from the environment variable
|
||||
// RCLONE_TEST_PROXY_AUTH_KEY as "accessKeyID,secretAccessKey" and any
|
||||
// other access key ID is refused.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -28,6 +34,19 @@ func main() {
|
||||
"_root": root,
|
||||
"_obscure": "pass",
|
||||
}
|
||||
|
||||
_, havePass := in["pass"]
|
||||
_, havePublicKey := in["public_key"]
|
||||
if !havePass && !havePublicKey {
|
||||
accessKeyID, secret, ok := strings.Cut(os.Getenv("RCLONE_TEST_PROXY_AUTH_KEY"), ",")
|
||||
if !ok {
|
||||
log.Fatal("RCLONE_TEST_PROXY_AUTH_KEY not set")
|
||||
}
|
||||
if in["user"] != accessKeyID {
|
||||
log.Fatalf("unknown access key ID %q", in["user"])
|
||||
}
|
||||
out["_secret_access_key"] = secret
|
||||
}
|
||||
json.NewEncoder(os.Stdout).Encode(&out)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
||||
+27
-22
@@ -37,28 +37,24 @@ import (
|
||||
type server struct {
|
||||
f fs.Fs
|
||||
opt Options
|
||||
vfs *vfs.VFS
|
||||
provider *proxy.Provider
|
||||
ctx context.Context // for global config
|
||||
config *ssh.ServerConfig
|
||||
listener net.Listener
|
||||
stopped chan struct{} // for waiting on the listener to stop
|
||||
proxy *proxy.Proxy
|
||||
}
|
||||
|
||||
func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Options, proxyOpt *proxy.Options) (*server, error) {
|
||||
s := &server{
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
stopped: make(chan struct{}),
|
||||
}
|
||||
if proxy.Opt.AuthProxy != "" {
|
||||
s.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
|
||||
} else {
|
||||
s.vfs = vfs.New(ctx, f, vfsOpt)
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
provider: proxy.NewProvider(ctx, f, vfsOpt, proxyOpt),
|
||||
stopped: make(chan struct{}),
|
||||
}
|
||||
err := s.configure()
|
||||
if err != nil {
|
||||
s.provider.Shutdown()
|
||||
return nil, fmt.Errorf("sftp configuration failed: %w", err)
|
||||
}
|
||||
return s, nil
|
||||
@@ -66,8 +62,8 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
|
||||
// getVFS gets the vfs from s or the proxy
|
||||
func (s *server) getVFS(what string, sshConn *ssh.ServerConn) (VFS *vfs.VFS) {
|
||||
if s.proxy == nil {
|
||||
return s.vfs
|
||||
if !s.provider.IsProxy() {
|
||||
return s.provider.VFS()
|
||||
}
|
||||
if sshConn.Permissions == nil || sshConn.Permissions.Extensions == nil {
|
||||
fs.Infof(what, "SSH Permissions Extensions not found")
|
||||
@@ -78,7 +74,7 @@ func (s *server) getVFS(what string, sshConn *ssh.ServerConn) (VFS *vfs.VFS) {
|
||||
fs.Infof(what, "VFS key not found")
|
||||
return nil
|
||||
}
|
||||
VFS = s.proxy.Get(key)
|
||||
VFS = s.provider.Proxy().Get(key)
|
||||
if VFS == nil {
|
||||
fs.Infof(what, "failed to read VFS from cache")
|
||||
return nil
|
||||
@@ -103,6 +99,13 @@ func (s *server) acceptConnection(nConn net.Conn) {
|
||||
// Discard all global out-of-band Requests
|
||||
go ssh.DiscardRequests(reqs)
|
||||
|
||||
if sshConn.Permissions != nil && sshConn.Permissions.Extensions != nil {
|
||||
if vfsKey := sshConn.Permissions.Extensions["_vfsKey"]; vfsKey != "" {
|
||||
s.provider.Pin(vfsKey)
|
||||
defer s.provider.Unpin(vfsKey)
|
||||
}
|
||||
}
|
||||
|
||||
c := &conn{
|
||||
what: what,
|
||||
vfs: s.getVFS(what, sshConn),
|
||||
@@ -115,7 +118,7 @@ func (s *server) acceptConnection(nConn net.Conn) {
|
||||
c.handlers = newVFSHandler(c.vfs)
|
||||
|
||||
// Accept all channels
|
||||
go c.handleChannels(chans)
|
||||
c.handleChannels(chans)
|
||||
}
|
||||
|
||||
// Accept connections and call them in a go routine
|
||||
@@ -140,12 +143,12 @@ func (s *server) configure() (err error) {
|
||||
var authorizedKeysMap map[string]struct{}
|
||||
|
||||
// ensure the user isn't trying to use conflicting flags
|
||||
if proxy.Opt.AuthProxy != "" && s.opt.AuthorizedKeys != "" && s.opt.AuthorizedKeys != Opt.AuthorizedKeys {
|
||||
if s.provider.IsProxy() && s.opt.AuthorizedKeys != "" && s.opt.AuthorizedKeys != Opt.AuthorizedKeys {
|
||||
return errors.New("--auth-proxy and --authorized-keys cannot be used at the same time")
|
||||
}
|
||||
|
||||
// Load the authorized keys
|
||||
if s.opt.AuthorizedKeys != "" && proxy.Opt.AuthProxy == "" {
|
||||
if s.opt.AuthorizedKeys != "" && !s.provider.IsProxy() {
|
||||
authKeysFile := env.ShellExpand(s.opt.AuthorizedKeys)
|
||||
authorizedKeysMap, err = loadAuthorizedKeys(authKeysFile)
|
||||
// If user set the flag away from the default then report an error
|
||||
@@ -160,7 +163,7 @@ func (s *server) configure() (err error) {
|
||||
fs.Logf(nil, "Loaded %d authorized keys from %q", len(authorizedKeysMap), authKeysFile)
|
||||
}
|
||||
|
||||
if !s.opt.NoAuth && len(authorizedKeysMap) == 0 && s.opt.User == "" && s.opt.Pass == "" && s.proxy == nil {
|
||||
if !s.opt.NoAuth && len(authorizedKeysMap) == 0 && s.opt.User == "" && s.opt.Pass == "" && !s.provider.IsProxy() {
|
||||
return errors.New("no authorization found, use --user/--pass or --authorized-keys or --no-auth or --auth-proxy")
|
||||
}
|
||||
|
||||
@@ -170,9 +173,9 @@ func (s *server) configure() (err error) {
|
||||
ServerVersion: "SSH-2.0-" + fs.GetConfig(s.ctx).UserAgent,
|
||||
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
|
||||
fs.Debugf(describeConn(c), "Password login attempt for %s", c.User())
|
||||
if s.proxy != nil {
|
||||
if s.provider.IsProxy() {
|
||||
// query the proxy for the config
|
||||
_, vfsKey, err := s.proxy.Call(c.User(), string(pass), false)
|
||||
_, vfsKey, err := s.provider.Proxy().Call(c.User(), string(pass), false, c.RemoteAddr().String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -193,12 +196,13 @@ func (s *server) configure() (err error) {
|
||||
},
|
||||
PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
|
||||
fs.Debugf(describeConn(c), "Public key login attempt for %s", c.User())
|
||||
if s.proxy != nil {
|
||||
if s.provider.IsProxy() {
|
||||
//query the proxy for the config
|
||||
_, vfsKey, err := s.proxy.Call(
|
||||
_, vfsKey, err := s.provider.Proxy().Call(
|
||||
c.User(),
|
||||
base64.StdEncoding.EncodeToString(pubKey.Marshal()),
|
||||
true,
|
||||
c.RemoteAddr().String(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -326,6 +330,7 @@ func (s *server) Shutdown() error {
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
err = nil
|
||||
}
|
||||
s.provider.Shutdown()
|
||||
s.Wait()
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -78,6 +78,32 @@ func TestSftp(t *testing.T) {
|
||||
servetest.Run(t, "sftp", start)
|
||||
}
|
||||
|
||||
// TestNewServerPerServerAuthProxy checks that a per-server proxyOpt.AuthProxy
|
||||
// enables proxy mode even when the process-global proxy.Opt.AuthProxy is empty,
|
||||
// which is the normal case when the server is configured via serve/start.
|
||||
func TestNewServerPerServerAuthProxy(t *testing.T) {
|
||||
// Ensure the global is empty so we only test the per-server option.
|
||||
assert.Equal(t, "", proxy.Opt.AuthProxy)
|
||||
|
||||
opt := Opt
|
||||
opt.ListenAddr = testBindAddress
|
||||
|
||||
proxyOpt := proxy.Opt
|
||||
proxyOpt.AuthProxy = "/path/to/auth/proxy"
|
||||
|
||||
w, err := newServer(context.Background(), nil, &opt, &vfscommon.Opt, &proxyOpt)
|
||||
require.NoError(t, err)
|
||||
// Shutdown waits for Serve to finish, so Serve must be running first.
|
||||
go func() {
|
||||
assert.NoError(t, w.Serve())
|
||||
}()
|
||||
defer func() {
|
||||
assert.NoError(t, w.Shutdown())
|
||||
}()
|
||||
assert.True(t, w.provider.IsProxy(), "expected auth proxy to be enabled by per-server option")
|
||||
assert.Nil(t, w.provider.VFS(), "expected no fixed VFS when auth proxy is in use")
|
||||
}
|
||||
|
||||
func TestRc(t *testing.T) {
|
||||
servetest.TestRc(t, rc.Params{
|
||||
"type": "sftp",
|
||||
|
||||
+16
-23
@@ -4,7 +4,6 @@ package webdav
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net"
|
||||
@@ -235,9 +234,8 @@ type WebDAV struct {
|
||||
server *libhttp.Server
|
||||
opt Options
|
||||
f fs.Fs
|
||||
_vfs *vfs.VFS // don't use directly, use getVFS
|
||||
provider *proxy.Provider
|
||||
webdavhandler *webdav.Handler
|
||||
proxy *proxy.Proxy
|
||||
ctx context.Context // for global config
|
||||
etagHashType hash.Type
|
||||
}
|
||||
@@ -265,8 +263,15 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
f: f,
|
||||
ctx: ctx,
|
||||
opt: *opt,
|
||||
provider: proxy.NewProvider(ctx, f, vfsOpt, proxyOpt),
|
||||
etagHashType: hash.None,
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
w.provider.Shutdown()
|
||||
}
|
||||
}()
|
||||
|
||||
if opt.EtagHash == "auto" {
|
||||
w.etagHashType = f.Hashes().GetOne()
|
||||
} else if opt.EtagHash != "" {
|
||||
@@ -278,12 +283,9 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
if w.etagHashType != hash.None {
|
||||
fs.Debugf(f, "Using hash %v for ETag", w.etagHashType)
|
||||
}
|
||||
if proxyOpt.AuthProxy != "" {
|
||||
w.proxy = proxy.New(ctx, proxyOpt, vfsOpt)
|
||||
// override auth
|
||||
|
||||
if w.provider.IsProxy() {
|
||||
w.opt.Auth.CustomAuthFn = w.auth
|
||||
} else {
|
||||
w._vfs = vfs.New(ctx, f, vfsOpt)
|
||||
}
|
||||
|
||||
w.server, err = libhttp.NewServer(ctx,
|
||||
@@ -335,23 +337,12 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
|
||||
|
||||
// Gets the VFS in use for this request
|
||||
func (w *WebDAV) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
|
||||
if w._vfs != nil {
|
||||
return w._vfs, nil
|
||||
}
|
||||
value := libhttp.CtxGetAuth(ctx)
|
||||
if value == nil {
|
||||
return nil, errors.New("no VFS found in context")
|
||||
}
|
||||
VFS, ok := value.(*vfs.VFS)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("context value is not VFS: %#v", value)
|
||||
}
|
||||
return VFS, nil
|
||||
return w.provider.Get(ctx)
|
||||
}
|
||||
|
||||
// auth does proxy authorization
|
||||
func (w *WebDAV) auth(user, pass string) (value any, err error) {
|
||||
VFS, _, err := w.proxy.Call(user, pass, false)
|
||||
func (w *WebDAV) auth(r *http.Request, user, pass string) (value any, err error) {
|
||||
VFS, _, err := w.provider.Proxy().Call(user, pass, false, r.RemoteAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -513,7 +504,9 @@ func (w *WebDAV) Addr() net.Addr {
|
||||
|
||||
// Shutdown the server
|
||||
func (w *WebDAV) Shutdown() error {
|
||||
return w.server.Shutdown()
|
||||
err := w.server.Shutdown()
|
||||
w.provider.Shutdown()
|
||||
return err
|
||||
}
|
||||
|
||||
// logRequest is called by the webdav module on every request
|
||||
|
||||
@@ -1059,11 +1059,12 @@ The following backends have known issues that need more investigation:
|
||||
- [`TestBisyncRemoteLocal/normalization`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncLocalRemote/ext_paths`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncLocalRemote/extended_filenames`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [4 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- [3 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- `TestPcloud` (`pcloud`)
|
||||
- [`TestBisyncRemoteRemote/check_access`](https://pub.rclone.org/integration-tests/current/pcloud-cmd.bisync-TestPcloud-1.txt)
|
||||
- [`TestBisyncRemoteRemote/rmdirs`](https://pub.rclone.org/integration-tests/current/pcloud-cmd.bisync-TestPcloud-1.txt)
|
||||
- Updated: 2026-07-31-010017
|
||||
- [`TestBisyncRemoteLocal/createemptysrcdirs`](https://pub.rclone.org/integration-tests/current/pcloud-cmd.bisync-TestPcloud-1.txt)
|
||||
- [`TestBisyncLocalRemote/resolve`](https://pub.rclone.org/integration-tests/current/pcloud-cmd.bisync-TestPcloud-1.txt)
|
||||
- [`TestBisyncRemoteRemote/createemptysrcdirs`](https://pub.rclone.org/integration-tests/current/pcloud-cmd.bisync-TestPcloud-1.txt)
|
||||
- Updated: 2026-09-04-010006
|
||||
<!--- end list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->
|
||||
|
||||
The following backends either have not been tested recently or have known issues
|
||||
@@ -1961,7 +1962,7 @@ and far less prone to critical errors / undetected changes
|
||||
- Bisync is now capable of rolling a file listing back in cases of uncertainty,
|
||||
essentially marking the file as needing to be rechecked next time.
|
||||
- A few basic terminal colors are now supported, controllable with
|
||||
[`--color`](/docs/#color) (`AUTO`|`NEVER`|`ALWAYS`)
|
||||
[`--color`](/docs/#color-autoneveralways) (`AUTO`|`NEVER`|`ALWAYS`)
|
||||
- Initial listing snapshots of Path1 and Path2 are now generated concurrently,
|
||||
using the same "march" infrastructure as `check` and `sync`,
|
||||
for performance improvements and less
|
||||
@@ -1991,7 +1992,7 @@ behavior with new [`--conflict-resolve`](#conflict-resolve),
|
||||
[`--conflict-suffix`](#conflict-suffix) flags.
|
||||
- A new [`--resync-mode`](#resync-mode) flag allows more control over which
|
||||
version of a file gets kept during a `--resync`.
|
||||
- Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time)
|
||||
- Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-duration)
|
||||
(when [`--resilient`](#resilient) is set.)
|
||||
|
||||
### `v1.64`
|
||||
|
||||
+149
-6
@@ -6,6 +6,149 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.75.1 - 2026-09-04
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.75.0...v1.75.1)
|
||||
|
||||
- Security
|
||||
- archive
|
||||
- Fix zip slip path traversal in untrusted zip files GHSA-66hp-wgxq-6f5q CVE-PENDING (Nick Craig-Wood)
|
||||
- Hide any archive entry which escapes the directory being listed GHSA-66hp-wgxq-6f5q (Nick Craig-Wood)
|
||||
- Reject unsafe entry names when mounting squashfs images GHSA-66hp-wgxq-6f5q (Nick Craig-Wood)
|
||||
- Fix zip subdirectory root matching sibling directories GHSA-66hp-wgxq-6f5q (Nick Craig-Wood)
|
||||
- Fix zip entry named "." hiding every other file GHSA-66hp-wgxq-6f5q (Nick Craig-Wood)
|
||||
- Fix "directory not found" for archive paths containing "./" or "//" GHSA-66hp-wgxq-6f5q (Nick Craig-Wood)
|
||||
- build
|
||||
- Fix multiple CVEs by upgrading to go1.26.6 (Nick Craig-Wood)
|
||||
- CVE-2026-56860: net/url: quadratic complexity in resolvePath
|
||||
- CVE-2026-56858: html/template: JavaScript regexp context tracking
|
||||
- CVE-2026-56862: crypto/tls: limit handshake messages accepted post-handshake
|
||||
- CVE-2026-56853: net/http: apply ReadHeaderTimeout to unencrypted HTTP/2 check
|
||||
- CVE-2026-56859: encoding/xml: recursion depth guard during decode
|
||||
- CVE-2026-33818: encoding/asn1: enforce maximum recursion depth
|
||||
- CVE-2026-46600: net: panic parsing an invalid SVCB or HTTPS RR in dnsmessage
|
||||
- CVE-2026-39821: net/http: reject ASCII-only Punycode-encoded labels in idna
|
||||
- Update golang.org/x/crypto to v0.56.0 to fix multiple CVEs (Nick Craig-Wood)
|
||||
- CVE-2026-56854: ssh: source-address critical option not enforced for non-public-key auth callbacks
|
||||
- CVE-2026-78662: ssh: a malicious peer could flood an undecided channel's incoming requests, deadlocking the connection
|
||||
- CVE-2026-56855: ssh: a malicious peer could send crafted messages on an established channel, deadlocking the connection
|
||||
- Update golang.org/x/image to v0.45.0 to fix CVE-2026-46603 (Nick Craig-Wood)
|
||||
- CVE-2026-46603: excessive memory allocation during VP8L decoding
|
||||
- fs: Confine directory listing entries that escape the root GHSA-3vxh-3pcx-9m8q GHSA-38xv-hf3p-h7mq CVE-PENDING (Nick Craig-Wood)
|
||||
- fshttp: Don't send `--header` values to other hosts on redirect GHSA-486v-q2wf-fp2r CVE-PENDING (Nick Craig-Wood)
|
||||
- http: Don't leak configured headers to other hosts or over plaintext on redirect GHSA-486v-q2wf-fp2r CVE-PENDING (Nick Craig-Wood)
|
||||
- lib/rest: Check HTTPS downgrades against the original request on redirect GHSA-486v-q2wf-fp2r CVE-PENDING (Nick Craig-Wood)
|
||||
- local
|
||||
- Fix dir metadata escaping the root through a planted symlink GHSA-f8g7-2xjc-7mfh CVE-PENDING (Nick Craig-Wood)
|
||||
- Fix btime escaping the root via a planted symlink GHSA-f8g7-2xjc-7mfh CVE-PENDING (Nick Craig-Wood)
|
||||
- Fix panic on Range request past the end of a symlink GHSA-p6m2-r3w9-mpxw CVE-PENDING (Nick Craig-Wood)
|
||||
- serve docker
|
||||
- Reject volume names that escape the base directory GHSA-p6vx-hf7p-98j6 (Nick Craig-Wood)
|
||||
- Reject volume names resolving to the base directory itself GHSA-p6vx-hf7p-98j6 (Nick Craig-Wood)
|
||||
- Re-derive volume mountpoint from name when restoring state GHSA-p6vx-hf7p-98j6 (Nick Craig-Wood)
|
||||
- serve ftp: Fix auth-proxy sessions sharing credentials by username GHSA-c476-6w5q-jw77 CVE-PENDING (Nick Craig-Wood)
|
||||
- serve s3
|
||||
- Fix memory exhaustion from client-declared multipart part size GHSA-2p48-j3qc-rx9f CVE-PENDING (Nick Craig-Wood)
|
||||
- Reject bogus multipart part sizes in the reorder buffer GHSA-2p48-j3qc-rx9f (Nick Craig-Wood)
|
||||
- Fix auth proxy accepting any request signed with an empty secret GHSA-xwwr-4h3p-r22c CVE-PENDING (Nick Craig-Wood)
|
||||
- **NB** the auth proxy protocol for `serve s3` has changed - the proxy program is now given the access key ID as `user` and must return the secret as `_secret_access_key`
|
||||
- Fix each server accepting the `--auth-key` credentials of all the others (Nick Craig-Wood)
|
||||
- Fix misleading anonymous access log when using an auth proxy via rc GHSA-p569-5gjg-9cmj CVE-PENDING (Nick Craig-Wood)
|
||||
- serve sftp: Fix auth proxy configured via rc being silently ignored GHSA-p569-5gjg-9cmj CVE-PENDING (Nick Craig-Wood)
|
||||
- Bug Fixes
|
||||
- accounting
|
||||
- Fix memory leak on long-running rcd (nielash)
|
||||
- Fix memory leak from stats groups on long-running rcd (nielash)
|
||||
- Fix bwlimit burst overflow (Rayan Salhab)
|
||||
- bisync
|
||||
- Fix memory leak when running via the rc (nielash)
|
||||
- Fix failed transfers of empty files being recorded as synced (Nick Craig-Wood)
|
||||
- build: Make go1.26 the minimum required version as needed by golang.org/x/crypto v0.56.0 (Nick Craig-Wood)
|
||||
- config: Redact env var config values in logs (Pastalikek65)
|
||||
- doc fixes (Anton Karpov, CAOShurong, Dean Chen, Nick Craig-Wood, Recoordinate, Rodrigo Rodrigues, Shantanav Mukherjee, shaurya)
|
||||
- lib/batcher: Prevent commits racing shutdown (Loi Nguyen)
|
||||
- lib/transform: Fix panic in `truncate_keep_extension` (VXNCXNX)
|
||||
- multipart: Fix chunked uploads storing truncated objects when the source ends early (Nick Craig-Wood)
|
||||
- operations: Fix silent truncation of streaming uploads whose source ends early (Nick Craig-Wood)
|
||||
- serve
|
||||
- Fix VFS instance leaks on server startup failures and shutdown (Hakan İSMAİL)
|
||||
- Pass the client IP address to the auth proxy (am-at-enrollvb)
|
||||
- serve http: Prevent scrolling to the top on page reload (Sune Mølgaard)
|
||||
- serve nfs: Fix EIO when creating symlinks with `--vfs-links` (SillyZir)
|
||||
- serve s3
|
||||
- Fix failed uploads deleting or corrupting the object at the key (Nick Craig-Wood)
|
||||
- Fix crash when a multipart upload is aborted while a part is uploading (Nick Craig-Wood)
|
||||
- Fix modtime not being set when only mtime metadata is supplied on PUT (Nick Craig-Wood)
|
||||
- Upload all multipart uploads via the VFS so they obey `--bwlimit` and show in stats (Nick Craig-Wood)
|
||||
- Reserve the `.rclone_temp_` prefix for temporary objects (Nick Craig-Wood)
|
||||
- Clean up abandoned multipart uploads after `--multipart-expiry` (Nick Craig-Wood)
|
||||
- vfscache
|
||||
- Fix reader deadlock when the item size drops below the read offset (Dave)
|
||||
- Fix log message growing without bound on repeated write errors (Vijay Misal)
|
||||
- walk: Stop directory traversal when the context is cancelled (Rahman Yilmaz)
|
||||
- VFS
|
||||
- Synchronize poll updates with shutdown (Loi Nguyen)
|
||||
- Make poll shutdown lifecycle deterministic (Loi Nguyen)
|
||||
- Crypt
|
||||
- Fix hash mismatches with `no_data_encryption` on backends which check upload hashes (Nick Craig-Wood)
|
||||
- Fix directory names which look like versioned file names (TowyTowy)
|
||||
- Warn about directories with legacy version-like encrypted names (Nick Craig-Wood)
|
||||
- Azure Blob
|
||||
- Fix Entra ID server-side copy source authentication (Edward Klesel)
|
||||
- Fix spurious vfs cache corruption errors during chunked reads (Nick Craig-Wood)
|
||||
- Azurefiles
|
||||
- Fix zero padded files being created when the source ends early (Nick Craig-Wood)
|
||||
- Box
|
||||
- Fix truncated files being uploaded successfully when the source ends early (Rohit Behera)
|
||||
- Compress
|
||||
- Fix corrupted objects being created when the source ends early (Nick Craig-Wood)
|
||||
- Drive
|
||||
- Don't list trashed files when removing a directory into the trash (alliasgher)
|
||||
- Dropbox
|
||||
- Preserve Paper export paths on lookup (Loi Nguyen)
|
||||
- Fix context cancellation (e.g. `--max-duration` limit) not stopping in-flight requests (debaditya)
|
||||
- Fix chunked uploads of truncated files never finishing (Nick Craig-Wood)
|
||||
- Don't retry chunked upload requests when the upload has been cancelled (Nick Craig-Wood)
|
||||
- Decode received shared-file names (Sanjay Kanth A)
|
||||
- Fix ChangeNotify when the root's case differs from Dropbox's (Loi Nguyen)
|
||||
- Filelu
|
||||
- Fix truncated files being uploaded successfully when the source ends early (Nick Craig-Wood)
|
||||
- Fix duplicate root path during multipart folder creation (kingston125)
|
||||
- Huaweidrive
|
||||
- Fix truncated files being uploaded successfully when the source ends early (Rohit Behera)
|
||||
- Iclouddrive
|
||||
- Fix uploads into an app container failing with 412 (Christian De Santis)
|
||||
- Internetarchive
|
||||
- Fix corrupted files being created when the source ends early (Nick Craig-Wood)
|
||||
- Internxt
|
||||
- Persist rotated token returned by the user info call (0rangeSeaW0lf)
|
||||
- Onedrive
|
||||
- Fix 403 Forbidden for configuration personal onedrive (machsix)
|
||||
- Fall back to manual drive ID entry when drive listing fails (SillyZir)
|
||||
- Don't retry multipart upload chunk on 404 (upload session not found) (water)
|
||||
- Overview
|
||||
- Fix "internal error: no overview data found" on 32 bit architectures (Nick Craig-Wood)
|
||||
- Pikpak
|
||||
- Fix truncated files being created when the source ends early (Nick Craig-Wood)
|
||||
- Fix truncated single part uploads reported as ok when source ends early (Nick Craig-Wood)
|
||||
- Protondrive
|
||||
- Fix files uploaded with v1.75.0 not being readable in the Proton apps (Nick Craig-Wood)
|
||||
- Fix corrupted uploads after a retried upload error (Nick Craig-Wood)
|
||||
- Quatrix
|
||||
- Fix chunk upload retries and fix memory leak (Nick Craig-Wood)
|
||||
- S3
|
||||
- Update Mega endpoints (Nick Craig-Wood)
|
||||
- Treat UploadPart success without ETag as retryable error (CAOShurong)
|
||||
- Fix server side copy failing with `--s3-no-head-object` (Anatoly Tarnavsky)
|
||||
- Sia
|
||||
- Fix corrupted files being created when the source ends early (Nick Craig-Wood)
|
||||
- Smb
|
||||
- Reuse the upload connection for SetModTime (alliasgher)
|
||||
- WebDAV
|
||||
- Fix SetModTime failing and hashes missing on Nextcloud (Nick Craig-Wood)
|
||||
- Yandex
|
||||
- Fix truncated files being uploaded successfully when the source ends early (Rohit Behera)
|
||||
|
||||
## v1.75.0 - 2026-07-31
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.0...v1.75.0)
|
||||
@@ -14,11 +157,11 @@ description: "Rclone Changelog"
|
||||
- [Scality](/s3/#scality) (RING / ARTESCA)
|
||||
- [Zero Services](/s3/#zero-z3) (ZERO-Z3)
|
||||
- Security
|
||||
- archive: Don't crash on malformed squashfs images GHSA-6jcg-q3wp-x2f4 CVE-PENDING (Nick Craig-Wood)
|
||||
- ftp: Fix ftp command injection when encoding doesn't include CRLF GHSA-8c48-q9wj-3w37 CVE-PENDING (Nick Craig-Wood)
|
||||
- archive: Don't crash on malformed squashfs images GHSA-6jcg-q3wp-x2f4 (Nick Craig-Wood)
|
||||
- ftp: Fix ftp command injection when encoding doesn't include CRLF GHSA-8c48-q9wj-3w37 CVE-2026-71311 (Nick Craig-Wood)
|
||||
- lib/http: Use TLS on all `--addr` listeners when `--cert` and `--key` are set GHSA-mfvx-7rcj-9m5g (Nick Craig-Wood)
|
||||
- lib/proxy: Fix unbounded HTTP CONNECT headers causing OOM GHSA-xhf4-832v-7xcr CVE-PENDING (Nick Craig-Wood)
|
||||
- local: Stop source file names escaping the destination directory GHSA-7p4m-qxvv-g567 CVE-PENDING (Nick Craig-Wood)
|
||||
- lib/proxy: Fix unbounded HTTP CONNECT headers causing OOM GHSA-xhf4-832v-7xcr CVE-2026-71310 (Nick Craig-Wood)
|
||||
- local: Stop source file names escaping the destination directory GHSA-7p4m-qxvv-g567 CVE-2026-71313 (Nick Craig-Wood)
|
||||
- rc
|
||||
- Don't expose pprof debug handlers on an unauthenticated server GHSA-mfvx-7rcj-9m5g CVE-PENDING (Nick Craig-Wood)
|
||||
- Require authentication to list the remotes with `--rc-serve` GHSA-mfvx-7rcj-9m5g (Nick Craig-Wood)
|
||||
@@ -27,9 +170,9 @@ description: "Rclone Changelog"
|
||||
- Fix redirect credential leaks, reject HTTPS->HTTP and strip secrets GHSA-8mxv-9xhp-86h4 (Nick Craig-Wood)
|
||||
- Strip S3 Express session token on cross-host redirects GHSA-8mxv-9xhp-86h4 (Nick Craig-Wood)
|
||||
- serve ftp: Use constant time comparison for password check GHSA-mfvx-7rcj-9m5g (Nick Craig-Wood)
|
||||
- serve restic: Fix path traversal above the served directory GHSA-45pq-889g-fcgh CVE-PENDING (Nick Craig-Wood)
|
||||
- serve restic: Fix path traversal above the served directory GHSA-45pq-889g-fcgh CVE-2026-71309 (Nick Craig-Wood)
|
||||
- serve sftp: Don't crash the whole server on a bad request GHSA-6jcg-q3wp-x2f4 (Nick Craig-Wood)
|
||||
- sftp: Fix command injection via crafted filenames on PowerShell remotes GHSA-2m8m-jhrm-w6j2 CVE-PENDING (Nick Craig-Wood)
|
||||
- sftp: Fix command injection via crafted filenames on PowerShell remotes GHSA-2m8m-jhrm-w6j2 CVE-2026-71312 (Nick Craig-Wood)
|
||||
- vfs: Don't crash the process if a backend panics on a background goroutine GHSA-6jcg-q3wp-x2f4 (Nick Craig-Wood)
|
||||
- webdav
|
||||
- Fix HTTPS to HTTP redirects leaking credentials GHSA-h4mf-4v27-hggj (Nick Craig-Wood)
|
||||
|
||||
@@ -1107,7 +1107,7 @@ rclone [flags]
|
||||
--use-json-log Use json log format
|
||||
--use-mmap Use mmap allocator (see docs)
|
||||
--use-server-modtime Use server modified time instead of object metadata
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.75.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.75.1")
|
||||
-v, --verbose count Print lots more stuff (repeat for more)
|
||||
-V, --version Print the version number
|
||||
--webdav-auth-redirect Preserve authentication on redirect
|
||||
|
||||
@@ -231,12 +231,12 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,command=e
|
||||
|
||||
```console
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20260731
|
||||
// Output: stories/The Quick Brown Fox!-20260904
|
||||
```
|
||||
|
||||
```console
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2026-07-31 0340PM
|
||||
// Output: stories/The Quick Brown Fox!-2026-09-04 0450PM
|
||||
```
|
||||
|
||||
```console
|
||||
|
||||
@@ -292,6 +292,11 @@ for all `mount` and `serve` commands on macOS. For details, see [vfs-case-sensit
|
||||
|
||||
### NFS mount
|
||||
|
||||
For macOS (and other platforms where this path is supported), prefer the dedicated
|
||||
[rclone nfsmount](/commands/rclone_nfsmount/) command. It starts the NFS server and
|
||||
performs the mount for you. `rclone mount` itself still uses FUSE (macFUSE/FUSE-T)
|
||||
and does not switch to NFS via a flag.
|
||||
|
||||
This method spins up an NFS server using [serve nfs](/commands/rclone_serve_nfs/)
|
||||
command and mounts it to the specified mountpoint. If you run this in background
|
||||
mode using |--daemon|, you will need to send SIGTERM signal to the rclone process
|
||||
@@ -629,7 +634,7 @@ at all times. The buffered data is bound to one open file and won't be
|
||||
shared.
|
||||
|
||||
This flag is a upper limit for the used memory per open file. The
|
||||
buffer will only use memory for data that is downloaded but not not
|
||||
buffer will only use memory for data that is downloaded but not
|
||||
yet read. If the buffer is empty, only a small amount of memory will
|
||||
be used.
|
||||
|
||||
@@ -684,7 +689,8 @@ longest. This cache flushing strategy is efficient and more relevant
|
||||
files are likely to remain cached.
|
||||
|
||||
The `--vfs-cache-max-age` will evict files from the cache
|
||||
after the set time since last access has passed. The default value of
|
||||
after the set time since last access has passed; it is based on access time,
|
||||
not on when the file was first added to the cache. The default value of
|
||||
1 hour will start evicting files from cache that haven't been accessed
|
||||
for 1 hour. When a cached file is accessed the 1 hour timer is reset to 0
|
||||
and will wait for 1 more hour before evicting. Specify the time with
|
||||
|
||||
@@ -293,6 +293,11 @@ for all `mount` and `serve` commands on macOS. For details, see [vfs-case-sensit
|
||||
|
||||
### NFS mount
|
||||
|
||||
For macOS (and other platforms where this path is supported), prefer the dedicated
|
||||
[rclone nfsmount](/commands/rclone_nfsmount/) command. It starts the NFS server and
|
||||
performs the mount for you. `rclone mount` itself still uses FUSE (macFUSE/FUSE-T)
|
||||
and does not switch to NFS via a flag.
|
||||
|
||||
This method spins up an NFS server using [serve nfs](/commands/rclone_serve_nfs/)
|
||||
command and mounts it to the specified mountpoint. If you run this in background
|
||||
mode using |--daemon|, you will need to send SIGTERM signal to the rclone process
|
||||
@@ -630,7 +635,7 @@ at all times. The buffered data is bound to one open file and won't be
|
||||
shared.
|
||||
|
||||
This flag is a upper limit for the used memory per open file. The
|
||||
buffer will only use memory for data that is downloaded but not not
|
||||
buffer will only use memory for data that is downloaded but not
|
||||
yet read. If the buffer is empty, only a small amount of memory will
|
||||
be used.
|
||||
|
||||
@@ -685,7 +690,8 @@ longest. This cache flushing strategy is efficient and more relevant
|
||||
files are likely to remain cached.
|
||||
|
||||
The `--vfs-cache-max-age` will evict files from the cache
|
||||
after the set time since last access has passed. The default value of
|
||||
after the set time since last access has passed; it is based on access time,
|
||||
not on when the file was first added to the cache. The default value of
|
||||
1 hour will start evicting files from cache that haven't been accessed
|
||||
for 1 hour. When a cached file is accessed the 1 hour timer is reset to 0
|
||||
and will wait for 1 more hour before evicting. Specify the time with
|
||||
|
||||
@@ -100,7 +100,7 @@ at all times. The buffered data is bound to one open file and won't be
|
||||
shared.
|
||||
|
||||
This flag is a upper limit for the used memory per open file. The
|
||||
buffer will only use memory for data that is downloaded but not not
|
||||
buffer will only use memory for data that is downloaded but not
|
||||
yet read. If the buffer is empty, only a small amount of memory will
|
||||
be used.
|
||||
|
||||
@@ -155,7 +155,8 @@ longest. This cache flushing strategy is efficient and more relevant
|
||||
files are likely to remain cached.
|
||||
|
||||
The `--vfs-cache-max-age` will evict files from the cache
|
||||
after the set time since last access has passed. The default value of
|
||||
after the set time since last access has passed; it is based on access time,
|
||||
not on when the file was first added to the cache. The default value of
|
||||
1 hour will start evicting files from cache that haven't been accessed
|
||||
for 1 hour. When a cached file is accessed the 1 hour timer is reset to 0
|
||||
and will wait for 1 more hour before evicting. Specify the time with
|
||||
|
||||
@@ -162,7 +162,7 @@ at all times. The buffered data is bound to one open file and won't be
|
||||
shared.
|
||||
|
||||
This flag is a upper limit for the used memory per open file. The
|
||||
buffer will only use memory for data that is downloaded but not not
|
||||
buffer will only use memory for data that is downloaded but not
|
||||
yet read. If the buffer is empty, only a small amount of memory will
|
||||
be used.
|
||||
|
||||
@@ -217,7 +217,8 @@ longest. This cache flushing strategy is efficient and more relevant
|
||||
files are likely to remain cached.
|
||||
|
||||
The `--vfs-cache-max-age` will evict files from the cache
|
||||
after the set time since last access has passed. The default value of
|
||||
after the set time since last access has passed; it is based on access time,
|
||||
not on when the file was first added to the cache. The default value of
|
||||
1 hour will start evicting files from cache that haven't been accessed
|
||||
for 1 hour. When a cached file is accessed the 1 hour timer is reset to 0
|
||||
and will wait for 1 more hour before evicting. Specify the time with
|
||||
|
||||
Loaded 100 of 176 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user