Compare commits

..
Author SHA1 Message Date
Nick Craig-Wood 07313554bf b2: reduce default chunk size to 16M and increase upload concurrency to 8
The 96M default chunk size dates from 2016 when 100,000,000 bytes was
the minimum part size B2 allowed. The minimum is now 5,000,000 bytes.

Measurements against B2 EU on a 1 Gbit/s link show that each part has
a fixed overhead of about a second regardless of size, so throughput
is governed by chunk size multiplied by upload concurrency rather than
by chunk size alone. 16M chunks with concurrency 8 upload a 1 GiB file
as fast as 96M chunks with concurrency 4 while using a third of the
memory per transfer, and retry much less data when a part fails.

This also means that files between --multi-thread-cutoff and the chunk
size are no longer uploaded as a single part large file, which B2
rejects with "large files must have at least 2 parts", unless the
cutoff is set below 16M.
2026-09-10 15:57:25 +01:00
Nick Craig-Wood 417ecce222 docs: update sponsors 2026-09-08 15:27:44 +01:00
353 changed files with 4882 additions and 19078 deletions

No files matched your search

-3
View File
@@ -2,9 +2,6 @@
go.mod text eol=lf
go.sum text eol=lf
# Embedded templates must render the same on every OS
lib/http/templates/** text eol=lf
# Ignore generated files in GitHub language statistics and diffs
/MANUAL.* linguist-generated=true
/rclone.1 linguist-generated=true
+9 -9
View File
@@ -37,7 +37,7 @@ jobs:
include:
- job_name: linux
os: ubuntu-latest
go: '~1.27.2'
go: '~1.27.0'
gotags: cmount
build_flags: '-include "^linux/"'
check: true
@@ -48,14 +48,14 @@ jobs:
- job_name: linux_386
os: ubuntu-latest
go: '~1.27.2'
go: '~1.27.0'
goarch: 386
gotags: cmount
quicktest: true
- job_name: mac_amd64
os: macos-latest
go: '~1.27.2'
go: '~1.27.0'
gotags: 'cmount'
build_flags: '-include "^darwin/amd64" -cgo'
quicktest: true
@@ -64,14 +64,14 @@ jobs:
- job_name: mac_arm64
os: macos-latest
go: '~1.27.2'
go: '~1.27.0'
gotags: 'cmount'
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
deploy: true
- job_name: windows
os: windows-latest
go: '~1.27.2'
go: '~1.27.0'
gotags: cmount
cgo: '0'
build_flags: '-include "^windows/"'
@@ -81,14 +81,14 @@ jobs:
- job_name: other_os
os: ubuntu-latest
go: '~1.27.2'
go: '~1.27.0'
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
compile_all: true
deploy: true
- job_name: go1.26
os: ubuntu-latest
go: '~1.26.9'
go: '~1.26.6'
quicktest: true
racequicktest: true
@@ -284,7 +284,7 @@ jobs:
id: setup-go
uses: actions/setup-go@v7
with:
go-version: '~1.27.2'
go-version: '~1.27.0'
check-latest: true
cache: false
@@ -399,7 +399,7 @@ jobs:
id: setup-go
uses: actions/setup-go@v7
with:
go-version: '~1.27.2'
go-version: '~1.27.0'
# Caching is handled explicitly below to share the module cache
# with the other jobs - see the build job for the rationale.
cache: false
-2
View File
@@ -20,8 +20,6 @@ The core parts of rclone under `fs` and `vfs` need to work with all backends and
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
## Build and Test Commands
```bash
+1 -5
View File
@@ -35,21 +35,17 @@ RUN echo "**** Print Version Binary ****" && \
# Begin final image
FROM alpine:latest
ARG RCLONE_UID=1009
ARG RCLONE_GID=1009
RUN echo "**** Install Dependencies ****" && \
apk add --no-cache \
ca-certificates \
fuse3 \
mailcap \
tzdata && \
echo "Enable user_allow_other in fuse" && \
echo "user_allow_other" >> /etc/fuse.conf
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
ENTRYPOINT [ "rclone" ]
+13 -24
View File
@@ -38,9 +38,6 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
BETA_UPLOAD_ROOT := beta.rclone.org:
BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
# Update the beta site's directory listings after an upload - add
# --changed for the uploaded directories
BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
# Pass in GOTAGS=xyz on the make command line to set build tags
ifdef GOTAGS
BUILDTAGS=-tags "$(GOTAGS)"
@@ -237,7 +234,6 @@ check_sign:
upload:
rclone -P copy build/ downloads.rclone.org:/$(TAG)
rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
upload_github:
./bin/upload-github $(TAG)
@@ -270,7 +266,6 @@ ci_upload:
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
endif
./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
@echo Beta release ready at $(BETA_URL)/testbuilds
ci_beta:
@@ -280,7 +275,6 @@ ci_beta:
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
endif
rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
@echo Beta release ready at $(BETA_URL)
# Fetch the binary builds from GitHub actions
@@ -302,26 +296,21 @@ retag:
@echo "Version is $(VERSION)"
git tag -f -s -m "Version $(VERSION)" $(VERSION)
# Start development of the next version after a release: the next
# minor version on master or the next patch version on a stable branch.
startdev:
@branch=$$(git rev-parse --abbrev-ref HEAD); \
case "$$branch" in \
master) next="$(NEXT_VERSION)" ;; \
v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
esac; \
if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
fi; \
echo "Version is $(VERSION)"; \
echo "Next version is $$next"; \
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
echo -n "$$next" > docs/layouts/partials/version.html; \
echo "$$next" > VERSION; \
git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
@echo "Version is $(VERSION)"
@echo "Next version is $(NEXT_VERSION)"
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_VERSION)" > VERSION
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
startstable: startdev
startstable:
@echo "Version is $(VERSION)"
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_PATCH_VERSION)" > VERSION
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
winzip:
zip -9 rclone-$(TAG).zip rclone.exe
-2
View File
@@ -39,7 +39,6 @@ directories to and from different cloud storage providers.
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
@@ -111,7 +110,6 @@ directories to and from different cloud storage providers.
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
+18 -69
View File
@@ -6,37 +6,27 @@ This file describes how to make the various kinds of releases
- [gh the github cli](https://github.com/cli/cli) for uploading packages
- pandoc for making the html and man pages
- hugo for building the website
- golangci-lint and docker for `make check` (markdownlint runs in docker)
- a gpg key for signing the tag and the checksums
- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
`www.rclone.org:` and `test-rclone-org:`
- the sponsor logos - see [Sponsor logos](#sponsor-logos)
## Making a release
- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
- git checkout master # see below for stable branch
- git pull # IMPORTANT
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
- git status - make sure everything is checked in
- Check GitHub actions build for master is Green
- Check the integration tests at <https://integration.rclone.org>
- cat VERSION # check this is the version being released
- make tag # signed tag on HEAD + new changelog entry + docs
- edit docs/content/changelog.md # entries already in point releases are
removed automatically
- make test # see integration test server or run locally
- make tag
- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
releases
- make tidy
- make doc # again to pick up the changelog edits
- git status - to check for new command docs in docs/content/commands - git add them
- make doc
- git status - to check for new man pages - git add them
- git commit -a -v -m "Version v1.XX.0"
- make check # if this fails, fix, git commit --amend and run it again
- make retag # moves the tag onto the Version commit
- make check
- make retag
- git push origin # without --follow-tags so it doesn't push the tag if it fails
- git push --follow-tags origin
- \# Wait for the GitHub builds to complete then...
- \# Don't commit anything until after make upload_github as these steps
use the tag on HEAD to find the version
- make fetch_binaries
- make tarball
- make vendorball
@@ -45,17 +35,9 @@ This file describes how to make the various kinds of releases
- make upload
- make upload_test_website # check sponsors are correct/unchanged
- make upload_website
- make upload_github # this triggers the docker plugin and winget workflows
- Check the docker image, docker plugin and winget GitHub actions succeeded
- make startdev # works out the next version for master or a stable branch
- git push
- Disable the release freeze rule
- Release any security advisories
- request a CVE for each advisory if not done already
- set the patched version on each advisory and publish it
- when the CVEs are assigned replace `CVE-PENDING` in the changelog on
master and the stable branch then `make upload_website`
- \# announce with forum post, twitter post, and post to announcements list
- make upload_github
- make startdev # make startstable for stable branch
- \# announce with forum post, twitter post, and post to annoucements list
## Update dependencies
@@ -208,53 +190,20 @@ First make the release branch. If this is a second point release then
this will be done already.
- git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
- make startdev # sets the version to ${BASE_TAG}.1
- git push -u origin ${BASE_TAG}-stable
- make startstable
Now
- Enable release freeze rule
- git co master
- git pull --rebase
- make fetch-gui-and-commit # update the GUI on master first
- git push
- git co ${BASE_TAG}-stable
- git pull
- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
- `git cherry-pick -x` any fixes, including the GUI update if there was one
- for security releases consider updating the Go patch version in
`.github/workflows/build.yml` and dependencies with CVE fixes
- fixes may depend on commits which weren't picked so push to CI
and check it is green before tagging
- Do the steps in "Making a release" above, except
- stay on ${BASE_TAG}-stable - don't checkout master or run
make fetch-gui-and-commit
- check the GitHub actions build for ${BASE_TAG}-stable not master
- run the integration tests on ${BASE_TAG}-stable as described below
- the release freeze rule is already enabled - leave it until the end
- `git cherry-pick -x` any fixes
- make startstable
- Do the steps as above
- git co master
- git pull --rebase
- `#` cherry pick the changes to the changelog - check the diff to make sure it
is correct
- git checkout ${BASE_TAG}-stable docs/content/changelog.md
- git diff --cached # check only the ${NEW_TAG} entry has changed
- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
- git push
- Disable release freeze rule
Point releases are only made for the latest release. Once v1.(XX+1).0
is out the ${BASE_TAG}-stable branch is abandoned.
### Running the integration tests on the stable branch
The integration tests at <https://integration.rclone.org> test master.
To test the stable branch push it to the testing server and run the
tests there in `screen` as they take a while.
```console
git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
ssh rclone@rclone-testing
screen
cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
```
## Sponsor logos
-1
View File
@@ -16,7 +16,6 @@ import (
_ "github.com/rclone/rclone/backend/compress"
_ "github.com/rclone/rclone/backend/crypt"
_ "github.com/rclone/rclone/backend/doi"
_ "github.com/rclone/rclone/backend/dosya"
_ "github.com/rclone/rclone/backend/drime"
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/dropbox"
-114
View File
@@ -12,7 +12,6 @@ import (
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
@@ -23,7 +22,6 @@ import (
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/stretchr/testify/assert"
@@ -177,18 +175,6 @@ func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T,
checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
// Now check a single file
//
// The filter below applies to both sides, so check first that the
// archive exposes only that file without it
Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
require.NoError(t, err)
entries, err := Fsingle.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
_, err = Fsingle.List(ctx, subDir)
assert.Equal(t, fs.ErrorDirNotFound, err)
_, err = Fsingle.NewObject(ctx, subDir)
assert.Equal(t, fs.ErrorObjectNotFound, err)
fiCtx, fi := filter.AddConfig(ctx)
require.NoError(t, fi.AddRule("+ "+aFile))
require.NoError(t, fi.AddRule("- *"))
@@ -417,103 +403,3 @@ func TestIsDirectChild(t *testing.T) {
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
}
}
// A symlink in a squashfs image is skipped unless -l/--links is in use,
// as it is on the other backends. With the flag it is exposed under its
// name with the link suffix and reads back as its target.
//
// Note that this uses mksquashfs as an external binary.
func TestArchiveSquashfsSymlink(t *testing.T) {
fstest.Initialise()
skipIfNoExe(t, "mksquashfs")
ctx := context.Background()
dir := t.TempDir()
input := filepath.Join(dir, "input")
require.NoError(t, os.Mkdir(input, 0777))
require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
// a regular file which happens to carry the link suffix must stay
// readable under the flag, not be taken for a symlink
require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
image := filepath.Join(dir, "test.sqfs")
run(t, "mksquashfs", input, image)
remotes := func(t *testing.T, f fs.Fs) []string {
entries, err := f.List(ctx, "")
require.NoError(t, err)
names := make([]string, 0, len(entries))
for _, entry := range entries {
names = append(names, entry.Remote())
}
sort.Strings(names)
return names
}
read := func(t *testing.T, f fs.Fs, remote string) string {
o, err := f.NewObject(ctx, remote)
require.NoError(t, err)
rc, err := o.Open(ctx)
require.NoError(t, err)
contents, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, o.Size(), int64(len(contents)))
return string(contents)
}
f, err := fs.NewFs(ctx, ":archive:"+image)
require.NoError(t, err)
assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
ctx, ci := fs.AddConfig(ctx)
ci.Links = true
f, err = fs.NewFs(ctx, ":archive:"+image)
require.NoError(t, err)
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
}
// A hostile archive must not be able to write outside the destination.
// A symlink pointing above the root with a member inside it is the
// classic way to try it, and the refusal comes from the local backend,
// so pin it from end to end here rather than trusting it stays.
func TestArchiveZipSymlinkEscape(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.Links = true
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
hdr := &zip.FileHeader{Name: "a"}
hdr.SetMode(os.ModeSymlink | 0777)
w, err := zw.CreateHeader(hdr)
require.NoError(t, err)
_, err = w.Write([]byte("../outside"))
require.NoError(t, err)
w, err = zw.Create("a/pwned.txt")
require.NoError(t, err)
_, err = w.Write([]byte("pwned"))
require.NoError(t, err)
require.NoError(t, zw.Close())
dir := t.TempDir()
zipPath := filepath.Join(dir, "evil.zip")
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
dst := filepath.Join(dir, "dst")
require.NoError(t, os.Mkdir(dst, 0777))
fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
require.NoError(t, err)
fdst, err := fs.NewFs(ctx, dst)
require.NoError(t, err)
assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
}
+2 -93
View File
@@ -42,8 +42,6 @@ type Fs struct {
prefix string // position for objects
prefixSlash string // position for objects with a slash on
root string // position to read from within the archive
file string // remote of the single file root points at, if any
links bool // read symlinks as .rclonelink files
}
// recoverParsePanic converts a panic raised while parsing a squashfs image
@@ -100,7 +98,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
c: c,
remote: remote,
root: strings.Trim(root, "/"),
links: fs.GetConfig(ctx).Links,
prefix: prefix,
prefixSlash: prefix + "/",
}
@@ -118,13 +115,10 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
_, err := f.newObjectNative(native)
if err == nil {
// If it pointed to a file, find the directory above
// and remember the file so that only it is listed
f.root = path.Dir(f.root)
if f.root == "." || f.root == "/" {
f.root = ""
}
f.file = f.prefixSlash + path.Base(native)
singleObject = true
}
}
}
@@ -156,9 +150,7 @@ func (f *Fs) Name() string {
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
// Include the file so the fs cache can tell this Fs apart from
// the one for the directory containing it
return path.Join(f.root, f.file)
return f.root
}
// Features returns the optional features of this Fs
@@ -222,30 +214,6 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
}
}
// Convert a symlink entry into an Object holding its target
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
return &Object{
fs: f,
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
size: int64(len(target)),
modTime: item.ModTime(),
isLink: true,
link: target,
}
}
// readlink returns the target of the symlink the info describes.
//
// A symlink may carry an empty target, so the caller decides what is a
// symlink from the mode rather than from the target being set.
func readlink(info os.FileInfo) (string, error) {
st, ok := info.Sys().(*squashfs.StatT)
if !ok {
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
}
return st.LinkTarget, nil
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
@@ -259,10 +227,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
defer recoverParsePanic(&err)
if f.file != "" && dir != "" {
return nil, fs.ErrorDirNotFound
}
nativeDir, err := f.toNative(dir)
if err != nil {
return nil, err
@@ -275,7 +239,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
entries = make(fs.DirEntries, 0, len(items))
skipped := 0
skippedLinks := 0
for _, item := range items {
// fs.Debugf(item.Name(), "entry = %#v", item)
// Skip entry names that aren't safe
@@ -298,27 +261,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
entry = f.objectFromFileInfo(nativeDir, info)
} else if item.Type()&os.ModeSymlink != 0 {
if !f.links {
skippedLinks++
continue
}
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
target, err := readlink(info)
if err != nil {
return nil, err
}
entry = f.objectFromLink(nativeDir, info, target)
} else {
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
continue
}
if f.file != "" && entry.Remote() != f.file {
continue
}
entries = append(entries, entry)
}
@@ -327,9 +273,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if skipped > 0 {
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
}
if skippedLinks > 0 {
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
}
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
return entries, nil
@@ -351,17 +294,7 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
}
for _, fi := range fis {
// Name the entry as List does, so that the two agree on
// which names exist
isLink := fi.Type()&os.ModeSymlink != 0
if isLink && !f.links {
continue
}
name := fi.Name()
if isLink {
name += fs.LinkSuffix
}
if name == leaf {
if fi.Name() == leaf {
if fi.IsDir() {
return nil, fs.ErrorNotAFile
}
@@ -369,14 +302,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
}
if isLink {
target, err := readlink(info)
if err != nil {
return nil, err
}
o = f.objectFromLink(dir, info, target)
break
}
o = f.objectFromFileInfo(dir, info)
break
}
@@ -391,9 +316,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
if f.file != "" && remote != f.file {
return nil, fs.ErrorObjectNotFound
}
nativePath, err := f.toNative(remote)
if err != nil {
return nil, err
@@ -455,8 +377,6 @@ type Object struct {
remote string
size int64
modTime time.Time
isLink bool // whether this is a symlink
link string // target of the symlink
}
// Fs returns read only access to the Fs that this object is part of
@@ -528,17 +448,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
}
}
if o.isLink {
// Clamp offset into range to avoid panic
if offset < 0 {
offset = 0
}
if offset > int64(len(o.link)) {
offset = int64(len(o.link))
}
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
}
remote, err := o.fs.toNative(o.remote)
if err != nil {
return nil, err
-15
View File
@@ -43,7 +43,6 @@ type Fs struct {
prefix string // position for objects
prefixSlash string // position for objects with a slash on
root string // position to read from within the archive
links bool // read symlinks as .rclonelink files
dt dirtree.DirTree // read from zipfile
}
@@ -68,7 +67,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
node: node,
remote: remote,
root: root,
links: fs.GetConfig(ctx).Links,
prefix: prefix,
prefixSlash: prefix + "/",
}
@@ -140,7 +138,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
}
dt := dirtree.New()
skipped := 0
skippedLinks := 0
for _, file := range zr.File {
// Skip entries whose name escapes the archive's own namespace
remote, err := sanitize.Path(file.Name)
@@ -155,15 +152,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
skipped++
continue
}
if !isDir && file.Mode()&os.ModeSymlink != 0 {
if !f.links {
skippedLinks++
continue
}
// The member holds the link target, so it reads back
// like a .rclonelink file
remote += fs.LinkSuffix
}
remote = path.Join(f.prefix, remote)
if f.root != "" {
// Ignore all files outside the root, requiring a path
@@ -203,9 +191,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
if skipped > 0 {
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
}
if skippedLinks > 0 {
fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
}
dt.CheckParents("")
dt.Sort()
f.dt = dt
-55
View File
@@ -5,7 +5,6 @@ import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"sort"
@@ -37,26 +36,6 @@ func writeZip(t *testing.T, dir, name string, names ...string) string {
return name
}
// writeZipSymlink builds a zip holding file and a symlink entry link
// pointing at target, and returns its leaf name.
func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, err := zw.Create(file)
require.NoError(t, err)
_, err = w.Write([]byte("data for " + file))
require.NoError(t, err)
hdr := &zip.FileHeader{Name: link}
hdr.SetMode(os.ModeSymlink | 0777)
w, err = zw.CreateHeader(hdr)
require.NoError(t, err)
_, err = w.Write([]byte(target))
require.NoError(t, err)
require.NoError(t, zw.Close())
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
return name
}
// allRemotes returns every Object remote in the mounted archive's dirtree.
func allRemotes(t *testing.T, f fs.Fs) []string {
zf, ok := f.(*Fs)
@@ -191,37 +170,3 @@ func TestListCacheNotAliased(t *testing.T) {
}
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
}
// A symlink stored in the archive is skipped unless -l/--links is in
// use, as it is on the other backends. With the flag it is exposed
// under its name with the link suffix and reads back as its target.
func TestReadZipSymlink(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "")
require.NoError(t, err)
assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
ctx, ci := fs.AddConfig(ctx)
ci.Links = true
f, err = New(ctx, localFs, name, "", "")
require.NoError(t, err)
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
require.NoError(t, err)
assert.Equal(t, int64(len("file.txt")), o.Size())
rc, err := o.Open(ctx)
require.NoError(t, err)
target, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, "file.txt", string(target))
}
+18 -11
View File
@@ -18,20 +18,24 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/fs"
"golang.org/x/sync/errgroup"
)
// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
// response, used to tell whether the service honoured an Arrow request.
const arrowContentType = "application/vnd.apache.arrow.stream"
// listArrowParallel lists (containerName, directory) by splitting the blob name
// keyspace into shards and listing them concurrently. opts must already request
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
// returns the number of raw items seen. directory/prefix are already normalised
// by the caller.
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
// Parallel listing needs the arrowlist client (server-side endBefore is
// only honoured on the Arrow path) - without it fall back to sequential.
if _, err := f.arrowCntSVC(containerName); err != nil {
fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
// Split the keyspace by a single-case character ladder (see
// arrowLadderBoundaries for why single-case matters).
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
@@ -111,13 +115,16 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
// isEndBeforeUnsupported reports whether err means the service can't honour
// the endBefore listing bound, so sharded parallel listing must not be used.
//
// The service rejects the endBefore parameter with 400
// OperationNotSupportedWithFeatureMissing "The requested operation is not
// allowed as EndBefore parameter support is missing on this account" -
// endBefore is only supported on the Arrow listing path of accounts with
// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
// accept an XML page with endBefore set, but the service never sends one.)
// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
// server answered XML with endBefore set), or the service rejecting the
// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
// requested operation is not allowed as EndBefore parameter support is
// missing on this account" - endBefore is only supported on the Arrow
// listing path of accounts with Blob Listing with Apache Arrow enabled.
func isEndBeforeUnsupported(err error) bool {
if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
return true
}
var respErr *azcore.ResponseError
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
}
+3 -1
View File
@@ -10,6 +10,7 @@ import (
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
@@ -211,8 +212,9 @@ func TestArrowLadderBoundaries(t *testing.T) {
}
func TestIsEndBeforeUnsupported(t *testing.T) {
assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
assert.False(t, isEndBeforeUnsupported(nil))
+421
View File
@@ -0,0 +1,421 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
// public type aliases so it compiles against the released azblob module.
package arrowlist
import (
"encoding/base64"
"fmt"
"io"
"net/http"
"strconv"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/ipc"
"github.com/apache/arrow-go/v18/arrow/memory"
)
const (
// ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
ArrowContentType = "application/vnd.apache.arrow.stream"
// ArrowAcceptHeader is the Accept header value to request Arrow format.
ArrowAcceptHeader = ArrowContentType
// resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
// in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
// in Arrow, all rows share the same schema and are distinguished by this field value.
resourceTypeBlobPrefix = "blobprefix"
)
// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
result := container.ListBlobsFlatResponse{}
// Extract response headers
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
// Parse Arrow IPC stream
items, nextMarker, err := parseArrowStream(resp.Body)
if err != nil {
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
}
result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
Segment: &container.BlobFlatListSegment{
BlobItems: items,
},
NextMarker: nextMarker,
}
return result, nil
}
// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
result := container.ListBlobsHierarchyResponse{}
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
items, nextMarker, err := parseArrowStream(resp.Body)
if err != nil {
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
}
// Separate blob items from blob prefixes based on ResourceType field.
var blobItems []*container.BlobItem
var blobPrefixes []*container.BlobPrefix
for _, item := range items {
if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
Name: item.Name,
Properties: item.Properties,
})
} else {
blobItems = append(blobItems, item)
}
}
result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
Segment: &container.BlobHierarchyListSegment{
BlobItems: blobItems,
BlobPrefixes: blobPrefixes,
},
NextMarker: nextMarker,
}
return result, nil
}
func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
*clientRequestID = &val
}
if val := resp.Header.Get("Content-Type"); val != "" {
*contentType = &val
}
if val := resp.Header.Get("Date"); val != "" {
if t, err := time.Parse(time.RFC1123, val); err == nil {
*date = &t
}
}
if val := resp.Header.Get("x-ms-request-id"); val != "" {
*requestID = &val
}
if val := resp.Header.Get("x-ms-version"); val != "" {
*version = &val
}
}
// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
if err != nil {
return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
}
defer reader.Release()
// Extract NextMarker from schema metadata
var nextMarker *string
var numRecords int
if md := reader.Schema().Metadata(); md.Len() > 0 {
if idx := md.FindKey("NextMarker"); idx >= 0 {
val := md.Values()[idx]
if val != "" {
nextMarker = &val
}
}
if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
numRecords = n
}
}
}
// Build column index for the schema
schema := reader.Schema()
colIndex := buildColumnIndex(schema)
// Pre-allocate if we know the count
items := make([]*container.BlobItem, 0, numRecords)
// Iterate over record batches
for reader.Next() {
rec := reader.RecordBatch()
rows := int(rec.NumRows())
for row := range rows {
item := &container.BlobItem{
Properties: &container.BlobProperties{},
}
populateBlobItem(item, rec, row, colIndex)
items = append(items, item)
}
}
if err := reader.Err(); err != nil {
return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
}
return items, nextMarker, nil
}
// columnIndex maps column names to their indices in the record batch for O(1) lookup.
type columnIndex map[string]int
func buildColumnIndex(schema *arrow.Schema) columnIndex {
idx := make(columnIndex, len(schema.Fields()))
for i, f := range schema.Fields() {
idx[f.Name] = i
}
return idx
}
// populateBlobItem fills a BlobItem from a single row across all columns.
func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
// Top-level BlobItem fields
item.Name = getStringField(rec, row, colIdx, "Name")
item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
item.VersionID = getStringField(rec, row, colIdx, "VersionId")
item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
// Metadata maps
item.Metadata = getMapField(rec, row, colIdx, "Metadata")
item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
// Tags
if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
var blobTags []*container.BlobTag
for k, v := range tags {
key := k
blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
}
item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
}
// BlobProperties fields
p := item.Properties
p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
p.ETag = getETagField(rec, row, colIdx, "Etag")
p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
p.CopyID = getStringField(rec, row, colIdx, "CopyId")
p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
p.CopySource = getStringField(rec, row, colIdx, "CopySource")
p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
// Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
// The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
// It is silently ignored for forward compatibility.
}
// Field accessor helpers that safely handle missing columns and null values.
func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.String); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Boolean); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Timestamp); ok {
val := arr.Value(row)
// Arrow timestamps are stored as int64 with a unit; convert to time.Time
dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
return &dt
}
return nil
}
func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Uint64); ok {
val := int64(arr.Value(row))
return &val
}
if arr, ok := col.(*array.Int64); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Uint64); ok {
val := int32(arr.Value(row))
return &val
}
if arr, ok := col.(*array.Int32); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
s := getStringField(rec, row, colIdx, name)
if s == nil {
return nil
}
etag := azcore.ETag(*s)
return &etag
}
func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
s := getStringField(rec, row, colIdx, name)
if s == nil {
return nil
}
decoded, err := base64.StdEncoding.DecodeString(*s)
if err != nil {
return nil
}
return decoded
}
// getEnumField returns a pointer to a string-typed enum value.
type stringEnum interface {
~string
}
func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
s := getStringField(rec, row, colIdx, name)
if s == nil {
return nil
}
val := T(*s)
return &val
}
func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
mapArr, ok := col.(*array.Map)
if !ok {
return nil
}
// Get the offsets for this row's map entries
start := int(mapArr.Offsets()[row])
end := int(mapArr.Offsets()[row+1])
if start == end {
return nil
}
keys := mapArr.Keys().(*array.String)
values := mapArr.Items().(*array.String)
result := make(map[string]*string, end-start)
for i := start; i < end; i++ {
k := keys.Value(i)
if values.IsNull(i) {
result[k] = nil
} else {
v := values.Value(i)
result[k] = &v
}
}
return result
}
+374
View File
@@ -0,0 +1,374 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
// c6fa341ca22b (branch feature/storage/bifrost).
package arrowlist
import (
"bytes"
"io"
"net/http"
"testing"
"github.com/apache/arrow-go/v18/arrow"
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/ipc"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/stretchr/testify/require"
)
// coreSchema returns a basic Arrow schema for testing with common blob fields.
func coreSchema(md *arrow.Metadata) *arrow.Schema {
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
{Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
{Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
{Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
{Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
}
return arrow.NewSchema(fields, md)
}
// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
t.Helper()
alloc := memory.DefaultAllocator
var buf bytes.Buffer
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
builder := arrowArray.NewRecordBuilder(alloc, schema)
defer builder.Release()
populate(builder)
rec := builder.NewRecordBatch()
defer rec.Release()
require.NoError(t, w.Write(rec))
require.NoError(t, w.Close())
return buf.Bytes()
}
func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
h := http.Header{}
h.Set("Content-Type", ArrowContentType)
for k, v := range headers {
h.Set(k, v)
}
return &http.Response{
StatusCode: http.StatusOK,
Header: h,
Body: io.NopCloser(bytes.NewReader(data)),
}
}
func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
schema := coreSchema(&md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
b.Field(9).(*arrowArray.StringBuilder).Append("available")
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
b.Field(9).(*arrowArray.StringBuilder).Append("available")
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.NotNil(t, result.NextMarker)
require.Equal(t, "marker123", *result.NextMarker)
require.Len(t, result.Segment.BlobItems, 2)
item := result.Segment.BlobItems[0]
require.Equal(t, "blob1.txt", *item.Name)
require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
require.Equal(t, int64(1024), *item.Properties.ContentLength)
require.Equal(t, "Hot", string(*item.Properties.AccessTier))
}
func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
schema := coreSchema(&md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
// No rows added
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Nil(t, result.NextMarker)
require.Empty(t, result.Segment.BlobItems)
}
func TestHandleFlatListResponse_NullFields(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
schema := coreSchema(&md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
b.Field(3).(*arrowArray.StringBuilder).AppendNull()
b.Field(4).(*arrowArray.StringBuilder).AppendNull()
b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
b.Field(6).(*arrowArray.StringBuilder).AppendNull()
b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
b.Field(8).(*arrowArray.StringBuilder).AppendNull()
b.Field(9).(*arrowArray.StringBuilder).AppendNull()
b.Field(10).(*arrowArray.StringBuilder).AppendNull()
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
item := result.Segment.BlobItems[0]
require.Equal(t, "blob.txt", *item.Name)
require.Nil(t, item.Properties.CreationTime)
require.Nil(t, item.Properties.BlobType)
require.Nil(t, item.Properties.ETag)
require.Nil(t, item.Properties.ContentLength)
require.Nil(t, item.Properties.ContentType)
require.Nil(t, item.Properties.ServerEncrypted)
}
func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
}
func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
alloc := memory.DefaultAllocator
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
schema := coreSchema(&md)
var buf bytes.Buffer
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
// First batch: 2 rows
builder := arrowArray.NewRecordBuilder(alloc, schema)
for range 2 {
builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
for j := 1; j < 11; j++ {
builder.Field(j).AppendNull()
}
}
rec := builder.NewRecordBatch()
require.NoError(t, w.Write(rec))
rec.Release()
builder.Release()
// Second batch: 1 row
builder = arrowArray.NewRecordBuilder(alloc, schema)
builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
for j := 1; j < 11; j++ {
builder.Field(j).AppendNull()
}
rec = builder.NewRecordBatch()
require.NoError(t, w.Write(rec))
rec.Release()
builder.Release()
require.NoError(t, w.Close())
resp := makeHTTPResponse(buf.Bytes(), nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 3)
}
func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
// A regular blob
b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
b.Field(1).(*arrowArray.StringBuilder).AppendNull()
// A blob prefix (virtual directory)
b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleHierarchyListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
require.Len(t, result.Segment.BlobPrefixes, 1)
require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
}
func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
})
resp := makeHTTPResponse(data, map[string]string{
"x-ms-request-id": "req-123",
"x-ms-version": "2026-10-06",
"x-ms-client-request-id": "client-req-456",
"Date": "Mon, 01 Jan 2024 00:00:00 GMT",
})
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Equal(t, "req-123", *result.RequestID)
require.Equal(t, "2026-10-06", *result.Version)
require.Equal(t, "client-req-456", *result.ClientRequestID)
require.NotNil(t, result.Date)
}
func TestHandleFlatListResponse_MapFields(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
{Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
metaBuilder.Append(true)
metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
tagBuilder.Append(true)
tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
item := result.Segment.BlobItems[0]
require.NotNil(t, item.Metadata)
require.Equal(t, "value1", *item.Metadata["key1"])
require.NotNil(t, item.BlobTags)
require.Len(t, item.BlobTags.BlobTagSet, 1)
}
func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
item := result.Segment.BlobItems[0]
require.True(t, *item.Deleted)
require.False(t, *item.IsCurrentVersion)
require.True(t, *item.Properties.ServerEncrypted)
}
func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
item := result.Segment.BlobItems[0]
require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
require.Equal(t, "snap-123", *item.Snapshot)
require.True(t, *item.IsCurrentVersion)
}
+259
View File
@@ -0,0 +1,259 @@
//go:build !plan9 && !js
// Package arrowlist implements the experimental "Blob Listing with Apache
// Arrow" feature on top of the released Azure azblob SDK.
//
// The Azure SDK for Go has support for Arrow listing on the unreleased
// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
// response types. Until that ships in a tagged azblob release this package
// fills the gap: it exposes the same options and pager interface as the
// experimental SDK, built only on the public API of the released SDK plus
// copies of two small internal auth policies (see sharedkey.go and
// challenge_policy.go).
//
// This package is temporary. When Arrow listing ships in a released azblob
// the whole package should be deleted and its callers pointed back at
// container.Client.NewListBlobsHierarchyPager - the option and response
// types here are deliberately source compatible with the SDK's to make that
// a mechanical change.
package arrowlist
import (
"context"
"errors"
"net/http"
"strconv"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
)
const (
// moduleName and moduleVersion identify this package in the azcore
// telemetry policy's User-Agent fragment.
moduleName = "github.com/rclone/rclone/backend/azureblob/arrowlist"
moduleVersion = "v0.0.1"
// serviceVersion is the x-ms-version sent with listing requests. Arrow
// responses and the startFrom/endBefore parameters need at least
// 2026-06-06.
serviceVersion = "2026-06-06"
// tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
// own clients.
tokenScope = "https://storage.azure.com/.default"
)
// ErrEndBeforeXMLFallback is returned by the pager when the server answered
// with XML instead of Arrow while EndBefore was set. The service only honours
// endBefore on the Arrow listing path, so an XML page could silently contain
// blobs outside the requested range and cannot be used.
var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
// ListBlobsHierarchyOptions provides the configuration for
// Client.NewListBlobsHierarchyPager.
//
// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
// the released SDK's options are embedded (providing Include, Marker,
// MaxResults, Prefix and StartFrom by promotion) with the two fields the
// released SDK lacks added alongside.
type ListBlobsHierarchyOptions struct {
container.ListBlobsHierarchyOptions
// EndBefore limits listing to blobs whose full container path is
// lexically before it (exclusive). Only honoured on Arrow responses.
EndBefore *string
// UseArrowFormat requests the response as an Apache Arrow IPC stream.
UseArrowFormat *bool
}
// ClientOptions contains the optional parameters for client creation.
type ClientOptions struct {
azcore.ClientOptions
}
// Client issues Arrow listing requests to a single container.
type Client struct {
endpoint string
azClient *azcore.Client
}
// newClient creates a Client for the container at containerURL sending
// requests through a pipeline with the given auth policy (which may be nil
// for anonymous or SAS access).
func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
if options == nil {
options = &ClientOptions{}
}
plOpts := runtime.PipelineOptions{}
if authPolicy != nil {
plOpts.PerRetry = []policy.Policy{authPolicy}
}
azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
if err != nil {
return nil, err
}
return &Client{
endpoint: containerURL,
azClient: azClient,
}, nil
}
// NewClient creates a Client authenticating with a token credential.
func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
}
// NewClientWithNoCredential creates a Client with no authentication, for
// anonymous access or a containerURL carrying a SAS token in its query.
func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
return newClient(containerURL, nil, options)
}
// NewClientWithSharedKeyCredential creates a Client signing requests with the
// account's shared key.
func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
}
// NewListBlobsHierarchyPager returns a pager over the container's blobs,
// requesting Arrow responses when o.UseArrowFormat is set and returning the
// same response type as container.Client.NewListBlobsHierarchyPager.
//
// If the service does not support Arrow listing (the feature not enabled on
// the account, or an HNS account) it answers with XML, which is decoded
// transparently - unless EndBefore is set, in which case the fetcher returns
// ErrEndBeforeXMLFallback (see there).
func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
// Copy the options so advancing the Marker doesn't mutate the caller's struct.
opts := ListBlobsHierarchyOptions{}
if o != nil {
opts = *o
}
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
More: func(page container.ListBlobsHierarchyResponse) bool {
return page.NextMarker != nil && len(*page.NextMarker) > 0
},
Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
if page != nil {
opts.Marker = page.NextMarker
}
req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
if err != nil {
return container.ListBlobsHierarchyResponse{}, err
}
resp, err := c.azClient.Pipeline().Do(req)
if err != nil {
return container.ListBlobsHierarchyResponse{}, err
}
if !runtime.HasStatusCode(resp, http.StatusOK) {
return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
}
if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
return HandleHierarchyListResponse(resp)
}
if opts.EndBefore != nil {
return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
}
return handleXMLResponse(resp)
},
})
}
// listCreateRequest builds the ListBlobs request, mirroring the generated
// request builders in the SDK.
func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
if err != nil {
return nil, err
}
// Reading the existing query first preserves any SAS token in the endpoint.
reqQP := req.Raw().URL.Query()
reqQP.Set("comp", "list")
reqQP.Set("restype", "container")
reqQP.Set("delimiter", delimiter)
if include := formatInclude(opts.Include); include != "" {
reqQP.Set("include", include)
}
if opts.Marker != nil {
reqQP.Set("marker", *opts.Marker)
}
if opts.MaxResults != nil {
reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
}
if opts.Prefix != nil {
reqQP.Set("prefix", *opts.Prefix)
}
if opts.StartFrom != nil {
reqQP.Set("startFrom", *opts.StartFrom)
}
if useArrow && opts.EndBefore != nil {
reqQP.Set("endBefore", *opts.EndBefore)
}
req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
if useArrow {
// Stream the Arrow body rather than buffering it in the pipeline.
runtime.SkipBodyDownload(req)
req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
} else {
req.Raw().Header["Accept"] = []string{"application/xml"}
}
req.Raw().Header["x-ms-version"] = []string{serviceVersion}
return req, nil
}
// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
// generated ListBlobHierarchySegmentHandleResponse.
func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
result := container.ListBlobsHierarchyResponse{}
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
return container.ListBlobsHierarchyResponse{}, err
}
return result, nil
}
// formatInclude renders the include datasets as the comma separated query
// parameter value, in the same order as the SDK's unexported
// ListBlobsInclude.format.
func formatInclude(l container.ListBlobsInclude) string {
var include []string
if l.Copy {
include = append(include, "copy")
}
if l.Deleted {
include = append(include, "deleted")
}
if l.DeletedWithVersions {
include = append(include, "deletedwithversions")
}
if l.ImmutabilityPolicy {
include = append(include, "immutabilitypolicy")
}
if l.LegalHold {
include = append(include, "legalhold")
}
if l.Metadata {
include = append(include, "metadata")
}
if l.Snapshots {
include = append(include, "snapshots")
}
if l.Tags {
include = append(include, "tags")
}
if l.UncommittedBlobs {
include = append(include, "uncommittedblobs")
}
if l.Versions {
include = append(include, "versions")
}
if l.Permissions {
include = append(include, "permissions")
}
return strings.Join(include, ",")
}
@@ -0,0 +1,198 @@
//go:build !plan9 && !js
package arrowlist
import (
"context"
"encoding/base64"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/apache/arrow-go/v18/arrow"
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// arrowPage builds a one row Arrow IPC listing page with the given blob name
// and NextMarker.
func arrowPage(t *testing.T, name, nextMarker string) []byte {
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append(name)
b.Field(1).(*arrowArray.Uint64Builder).Append(42)
})
}
func TestPagerArrow(t *testing.T) {
var requests []*http.Request
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests = append(requests, r.Clone(context.Background()))
w.Header().Set("Content-Type", ArrowContentType)
if r.URL.Query().Get("marker") == "" {
_, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
} else {
_, _ = w.Write(arrowPage(t, "blob2.txt", ""))
}
}))
defer srv.Close()
// SAS style query on the endpoint must be preserved in requests.
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
require.NoError(t, err)
opts := &ListBlobsHierarchyOptions{
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
Include: container.ListBlobsInclude{Metadata: true, Tags: true},
Prefix: new("dir/"),
MaxResults: new(int32(1000)),
StartFrom: new("testcontainer/dir/a"),
},
EndBefore: new("testcontainer/dir/n"),
UseArrowFormat: new(true),
}
pager := client.NewListBlobsHierarchyPager("/", opts)
var names []string
for pager.More() {
page, err := pager.NextPage(context.Background())
require.NoError(t, err)
assert.Equal(t, ArrowContentType, *page.ContentType)
for _, item := range page.Segment.BlobItems {
names = append(names, *item.Name)
assert.Equal(t, int64(42), *item.Properties.ContentLength)
}
}
assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
require.Len(t, requests, 2)
q := requests[0].URL.Query()
assert.Equal(t, "list", q.Get("comp"))
assert.Equal(t, "container", q.Get("restype"))
assert.Equal(t, "/", q.Get("delimiter"))
assert.Equal(t, "metadata,tags", q.Get("include"))
assert.Equal(t, "dir/", q.Get("prefix"))
assert.Equal(t, "1000", q.Get("maxresults"))
assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
assert.Equal(t, "fakesas", q.Get("sv"))
assert.Empty(t, q.Get("marker"))
assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
// The caller's options must not have been mutated by paging.
assert.Nil(t, opts.Marker)
}
const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
<Blobs>
<Blob>
<Name>file.txt</Name>
<Properties>
<Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
<Etag>0x8D1</Etag>
<Content-Length>7</Content-Length>
<BlobType>BlockBlob</BlobType>
</Properties>
</Blob>
<BlobPrefix>
<Name>subdir/</Name>
</BlobPrefix>
</Blobs>
<NextMarker />
</EnumerationResults>`
func TestPagerXMLFallback(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(xmlListResponse))
}))
defer srv.Close()
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
UseArrowFormat: new(true),
})
page, err := pager.NextPage(context.Background())
require.NoError(t, err)
assert.False(t, pager.More())
assert.Equal(t, "application/xml", *page.ContentType)
require.Len(t, page.Segment.BlobItems, 1)
assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
require.Len(t, page.Segment.BlobPrefixes, 1)
assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
}
func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(xmlListResponse))
}))
defer srv.Close()
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
EndBefore: new("testcontainer/n"),
UseArrowFormat: new(true),
})
_, err = pager.NextPage(context.Background())
require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
}
func TestPagerSharedKeyAuth(t *testing.T) {
var gotAuth, gotDate string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
gotDate = r.Header.Get("x-ms-date")
w.Header().Set("Content-Type", ArrowContentType)
_, _ = w.Write(arrowPage(t, "blob.txt", ""))
}))
defer srv.Close()
cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
require.NoError(t, err)
client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
UseArrowFormat: new(true),
})
_, err = pager.NextPage(context.Background())
require.NoError(t, err)
assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
assert.NotEmpty(t, gotDate)
}
func TestPagerResponseError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
w.WriteHeader(http.StatusNotFound)
}))
defer srv.Close()
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
UseArrowFormat: new(true),
})
_, err = pager.NextPage(context.Background())
require.Error(t, err)
assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
}
@@ -0,0 +1,121 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
// released SDK does not export its storage bearer-challenge policy, which is
// needed for token authentication with the same semantics as the SDK's own
// clients.
package arrowlist
import (
"errors"
"net/http"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
)
type storageAuthorizer struct {
scopes []string
tenantID string
}
// NewStorageChallengePolicy returns a bearer token policy for cred which
// handles the storage service's authentication challenges.
func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
s := storageAuthorizer{scopes: []string{audience}}
return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
AuthorizationHandler: policy.AuthorizationHandler{
OnRequest: s.onRequest,
OnChallenge: s.onChallenge,
},
InsecureAllowCredentialWithHTTP: allowHTTP,
})
}
func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
}
func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
// parse the challenge
err := s.parseChallenge(resp)
if err != nil {
return err
}
// TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
}
type challengePolicyError struct {
err error
}
func (c *challengePolicyError) Error() string {
return c.err.Error()
}
func (*challengePolicyError) NonRetriable() {
// marker method
}
func (c *challengePolicyError) Unwrap() error {
return c.err
}
// parses Tenant ID from auth challenge
// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
func parseTenant(url string) string {
if url == "" {
return ""
}
parts := strings.Split(url, "/")
if len(parts) >= 3 {
tenant := parts[3]
tenant = strings.ReplaceAll(tenant, ",", "")
return tenant
}
return ""
}
func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
authHeader := resp.Header.Get("WWW-Authenticate")
if authHeader == "" {
return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
}
// Strip down to auth and resource
// Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
parts := strings.Split(authHeader, " ")
vals := map[string]string{}
for _, part := range parts {
subParts := strings.Split(part, "=")
if len(subParts) == 2 {
stripped := strings.ReplaceAll(subParts[1], "\"", "")
stripped = strings.TrimSuffix(stripped, ",")
vals[subParts[0]] = stripped
}
}
s.tenantID = parseTenant(vals["authorization_uri"])
scope := vals["resource_id"]
if scope == "" {
return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
}
if !strings.HasSuffix(scope, "/.default") {
scope += "/.default"
}
s.scopes = []string{scope}
return nil
}
+318
View File
@@ -0,0 +1,318 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
// with the internal header constants inlined. The released SDK does not
// export its shared key signing policy, so a copy is needed to sign the
// Arrow listing requests this package sends through its own pipeline.
package arrowlist
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"fmt"
"net/http"
"net/url"
"sort"
"strings"
"sync/atomic"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/rclone/rclone/fs"
)
// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
// storage account's name and either its primary or secondary key.
func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
c := SharedKeyCredential{accountName: accountName}
if err := c.SetAccountKey(accountKey); err != nil {
return nil, err
}
return &c, nil
}
// SharedKeyCredential contains an account's name and its primary or secondary key.
type SharedKeyCredential struct {
// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
accountName string
accountKey atomic.Value // []byte
}
// AccountName returns the Storage account's name.
func (c *SharedKeyCredential) AccountName() string {
return c.accountName
}
// SetAccountKey replaces the existing account key with the specified account key.
func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
_bytes, err := base64.StdEncoding.DecodeString(accountKey)
if err != nil {
return fmt.Errorf("decode account key: %w", err)
}
c.accountKey.Store(_bytes)
return nil
}
// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
_, err := h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
}
func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
headers := req.Header
contentLength := getHeader("Content-Length", headers)
if contentLength == "0" {
contentLength = ""
}
canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
if err != nil {
return "", err
}
stringToSign := strings.Join([]string{
req.Method,
getHeader("Content-Encoding", headers),
getHeader("Content-Language", headers),
contentLength,
getHeader("Content-MD5", headers),
getHeader("Content-Type", headers),
"", // Empty date because x-ms-date is expected (as per web page above)
getHeader("If-Modified-Since", headers),
getHeader("If-Match", headers),
getHeader("If-None-Match", headers),
getHeader("If-Unmodified-Since", headers),
getHeader("Range", headers),
c.buildCanonicalizedHeader(headers),
canonicalizedResource,
}, "\n")
return stringToSign, nil
}
func getHeader(key string, headers map[string][]string) string {
if headers == nil {
return ""
}
if v, ok := headers[key]; ok {
if len(v) > 0 {
return v[0]
}
}
return ""
}
func getWeightTables() [][]int {
tableLv0 := [...]int{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
}
tableLv2 := [...]int{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
}
tables := [][]int{tableLv0[:], tableLv2[:]}
return tables
}
// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
func compareHeaders(lhs, rhs string, tables [][]int) int {
currLevel, i, j := 0, 0, 0
n := len(tables)
lhsLen := len(lhs)
rhsLen := len(rhs)
for currLevel < n {
if currLevel == (n-1) && i != j {
if i > j {
return -1
}
if i < j {
return 1
}
return 0
}
var w1, w2 int
// Check bounds before accessing lhs[i]
if i < lhsLen {
w1 = tables[currLevel][lhs[i]]
} else {
w1 = 0x1
}
// Check bounds before accessing rhs[j]
if j < rhsLen {
w2 = tables[currLevel][rhs[j]]
} else {
w2 = 0x1
}
if w1 == 0x1 && w2 == 0x1 {
i = 0
j = 0
currLevel++
} else if w1 == w2 {
i++
j++
} else if w1 == 0 {
i++
} else if w2 == 0 {
j++
} else {
if w1 < w2 {
return -1
}
if w1 > w2 {
return 1
}
return 0
}
}
return 0
}
func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
cm := map[string][]string{}
for k, v := range headers {
headerName := strings.TrimSpace(strings.ToLower(k))
if strings.HasPrefix(headerName, "x-ms-") {
cm[headerName] = v // NOTE: the value must not have any whitespace around it.
}
}
if len(cm) == 0 {
return ""
}
keys := make([]string, 0, len(cm))
for key := range cm {
keys = append(keys, key)
}
tables := getWeightTables()
// Sort the keys using the custom comparator
sort.Slice(keys, func(i, j int) bool {
return compareHeaders(keys[i], keys[j], tables) < 0
})
ch := bytes.NewBufferString("")
for i, key := range keys {
if i > 0 {
ch.WriteRune('\n')
}
ch.WriteString(key)
ch.WriteRune(':')
ch.WriteString(strings.Join(cm[key], ","))
}
return ch.String()
}
func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
cr := bytes.NewBufferString("/")
cr.WriteString(c.accountName)
if len(u.Path) > 0 {
// Any portion of the CanonicalizedResource string that is derived from
// the resource's URI should be encoded exactly as it is in the URI.
// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
cr.WriteString(u.EscapedPath())
} else {
// a slash is required to indicate the root path
cr.WriteString("/")
}
// params is a map[string][]string; param name is key; params values is []string
params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
if err != nil {
return "", fmt.Errorf("failed to parse query params: %w", err)
}
if len(params) > 0 { // There is at least 1 query parameter
var paramNames []string // We use this to sort the parameter key names
for paramName := range params {
paramNames = append(paramNames, paramName) // paramNames must be lowercase
}
sort.Strings(paramNames)
for _, paramName := range paramNames {
paramValues := params[paramName]
sort.Strings(paramValues)
// Join the sorted key values separated by ','
// Then prepend "keyName:"; then add this string to the buffer
cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
}
}
return cr.String(), nil
}
// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
return cred.computeHMACSHA256(message)
}
// SharedKeyCredPolicy is a pipeline policy that signs each request with the
// account's shared key.
type SharedKeyCredPolicy struct {
cred *SharedKeyCredential
}
// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
return &SharedKeyCredPolicy{cred: cred}
}
// Do signs the request and sends it via the next policy in the pipeline.
func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
// skip adding the authorization header if no SharedKeyCredential was provided.
// this prevents a panic that might be hard to diagnose and allows testing
// against http endpoints that don't require authentication.
if s.cred == nil {
return req.Next()
}
if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
}
stringToSign, err := s.cred.buildStringToSign(req.Raw())
if err != nil {
return nil, err
}
signature, err := s.cred.computeHMACSHA256(stringToSign)
if err != nil {
return nil, err
}
authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
req.Raw().Header.Set("Authorization", authHeader)
response, err := req.Next()
if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
// Service failed to authenticate request, log it
fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
}
return response, err
}
+136 -206
View File
@@ -19,7 +19,6 @@ import (
"path"
"slices"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
@@ -27,12 +26,14 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/backend/azureblob/auth"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
@@ -240,38 +241,40 @@ avoid the time out.`,
Advanced: true,
}, {
Name: "use_arrow_list",
Help: `Use the Apache Arrow listing format.
Help: `Use the experimental Apache Arrow listing format.
If set, directory listings are fetched using the ListBlobs Apache
Arrow response format instead of XML. Arrow responses are smaller and
much cheaper to parse, making listings of large containers several
times faster. Combine with "list_parallelism" for the biggest gains.
Arrow response format instead of XML. This can be faster for very
large containers.
"Blob Listing with Apache Arrow" is in public preview at Microsoft and
is only supported on flat namespace accounts. On accounts with a
hierarchical namespace (ADLS Gen2), or where the feature is otherwise
unavailable, the server returns XML and the listing transparently
falls back to the normal XML path (logged at debug level).`,
This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
preview feature to be enabled on the storage account. It is NOT
supported on accounts with a hierarchical namespace (ADLS Gen2) -
those return a 409 error. If the feature is not enabled the server
returns XML and the listing transparently falls back to the normal XML
path (logged at debug level).
Not supported with connection_string auth - falls back to normal
listing.`,
Default: false,
Advanced: true,
Hide: fs.OptionHideBoth,
}, {
Name: "list_parallelism",
Help: `Number of parallel shards to list a directory with.
If set greater than 1, the blob name keyspace of each directory is
split into ranges which are listed concurrently using the Arrow
startFrom/endBefore range parameters. This can dramatically speed up
listing containers with millions of objects, for both recursive
(ListR) and single directory listings. Speed keeps improving up to a
parallelism of around 30.
EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
directory is split into this many ranges which are listed concurrently
using the Arrow startFrom/endBefore range parameters. This can
dramatically speed up listing containers with millions of objects, for
both recursive (ListR) and single directory listings.
This has no effect unless "use_arrow_list" is also set, as Arrow is the
only listing path that supports server-side name ranges. If the
account does not support range listing (e.g. it has a hierarchical
namespace) the listing falls back to sequential. The default of 0 (or
1) lists sequentially.`,
only listing path that supports server-side name ranges. The default of
0 (or 1) lists sequentially.`,
Default: 0,
Advanced: true,
Hide: fs.OptionHideBoth,
}, {
Name: "access_tier",
Help: `Access tier of blob: hot, cool, cold or archive.
@@ -365,22 +368,7 @@ creates an empty object ending with "/", to persist the folder.
This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
the Microsoft standard.
This is not needed on storage accounts with a hierarchical namespace
(see the hns option) as these always support empty directories.
`,
}, {
Name: "hns",
Default: fs.Tristate{},
Advanced: true,
Help: `Set if the storage account has a hierarchical namespace.
Storage accounts with hierarchical namespace enabled (Azure Data Lake
Storage Gen2) have real directories which can be empty.
If this is unset then rclone will read it from the storage account
and save it in the config file.
`,
}, {
Name: "no_check_container",
Help: `If set, don't attempt to check the container exists or create it.
@@ -448,7 +436,6 @@ type Options struct {
Enc encoder.MultiEncoder `config:"encoding"`
PublicAccess string `config:"public_access"`
DirectoryMarkers bool `config:"directory_markers"`
HNS fs.Tristate `config:"hns"`
NoCheckContainer bool `config:"no_check_container"`
NoHeadObject bool `config:"no_head_object"`
DeleteSnapshots string `config:"delete_snapshots"`
@@ -462,8 +449,10 @@ type Fs struct {
opt Options // parsed config options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache and arrowCntSVCcache
cntSVCcache map[string]*container.Client // reference to containerClient per container
arrowCntSVCcache map[string]*arrowlist.Client // reference to arrowlist client per container
arrowClientOpts *arrowlist.ClientOptions // client options for the arrowlist clients
arrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retried
svc *service.Client // client to access azblob
cred azcore.TokenCredential // how to generate tokens (may be nil)
@@ -472,7 +461,6 @@ type Fs struct {
rootContainer string // container part of root (if any)
rootDirectory string // directory part of root (if any)
isLimited bool // if limited to one container
hns bool // if the storage account has a hierarchical namespace
cache *bucket.Cache // cache for container creation status
pacer *fs.Pacer // To pace and retry the API calls
uploadToken *pacer.TokenDispenser // control concurrency
@@ -539,7 +527,7 @@ func parsePath(path string) (root string) {
// relative to f.root
func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
containerPath = containerPath[:len(containerPath)-1]
}
return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
@@ -682,14 +670,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
opt: *opt,
ci: ci,
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
cache: bucket.NewCache(),
cntSVCcache: make(map[string]*container.Client, 1),
name: name,
opt: *opt,
ci: ci,
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
cache: bucket.NewCache(),
cntSVCcache: make(map[string]*container.Client, 1),
arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
}
f.publicAccess = container.PublicAccessType(opt.PublicAccess)
f.setRoot(root)
@@ -706,6 +695,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ServerSideAcrossConfigs: true,
DoubleSlash: true,
}).Fill(ctx, f)
if opt.DirectoryMarkers {
f.features.CanHaveEmptyDirectories = true
fs.Debugf(f, "Using directory markers")
}
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
DefaultBaseURL: storageDefaultBaseURL,
@@ -732,34 +725,21 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f.usingSharedKeyCred = res.UsingSharedKeyCred
f.anonymous = res.Anonymous
// Client options for the arrowlist clients, using the same transport and
// gzip policy as the SDK clients built above.
f.arrowClientOpts = &arrowlist.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: auth.Transporter(ctx),
PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
},
}
// if using Container level SAS put the container client into the cache
if opt.SASURL != "" && res.Container != "" {
_ = f.cntSVC(res.Container)
f.isLimited = true
}
if opt.HNS.Valid {
f.hns = opt.HNS.Value
} else if !f.anonymous {
containerName := f.rootContainer
if containerName == "" {
containerName = res.Container
}
f.hns, err = f.readHNS(ctx, containerName)
if err != nil {
fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
} else {
m.Set("hns", strconv.FormatBool(f.hns))
}
}
if f.hns {
fs.Debugf(f, "Storage account has a hierarchical namespace")
}
if f.dirMarkers() {
f.features.CanHaveEmptyDirectories = true
fs.Debugf(f, "Using directory markers")
}
if f.rootContainer != "" && f.rootDirectory != "" {
// Check to see if the (container,directory) is actually an existing file
oldRoot := f.root
@@ -780,39 +760,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return f, nil
}
// readHNS reads whether the storage account has a hierarchical namespace
//
// If containerName is set then the container client is used which works
// with credentials limited to that container.
func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
var isHNS *bool
err = f.pacer.Call(func() (bool, error) {
var err error
if containerName != "" {
var resp container.GetAccountInfoResponse
resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
isHNS = resp.IsHierarchicalNamespaceEnabled
} else {
var resp service.GetAccountInfoResponse
resp, err = f.svc.GetAccountInfo(ctx, nil)
isHNS = resp.IsHierarchicalNamespaceEnabled
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return false, err
}
if isHNS == nil {
return false, errors.New("not returned by server")
}
return *isHNS, nil
}
// dirMarkers returns true if directories are persisted with directory markers
func (f *Fs) dirMarkers() bool {
return f.opt.DirectoryMarkers || f.hns
}
// return the container client for the container passed in
func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
f.cntSVCcacheMu.Lock()
@@ -825,6 +772,44 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
return containerClient
}
// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
// credentials can't be used for Arrow listing.
var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
// return the arrowlist client for the container passed in
//
// Returns errArrowAuthUnsupported if the configured credentials can't be
// reused for the arrowlist client's pipeline.
func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
// The container URL includes any SAS token in its query
url := f.cntSVC(containerName).URL()
f.cntSVCcacheMu.Lock()
defer f.cntSVCcacheMu.Unlock()
if client, ok := f.arrowCntSVCcache[containerName]; ok {
return client, nil
}
switch {
case f.usingSharedKeyCred:
// Covers account+key and the emulator (auth fills in Account/Key)
var cred *arrowlist.SharedKeyCredential
cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
if err == nil {
client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
}
case f.cred != nil:
client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
case f.anonymous || f.opt.SASURL != "":
client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
default:
return nil, errArrowAuthUnsupported
}
if err != nil {
return nil, err
}
f.arrowCntSVCcache[containerName] = client
return client, nil
}
// Return an Object from a path
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -850,12 +835,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
o, err := f.newObjectWithInfo(ctx, remote, nil)
if err == fs.ErrorNotAFile {
// The blob is a directory
return nil, fs.ErrorObjectNotFound
}
return o, err
return f.newObjectWithInfo(ctx, remote, nil)
}
// getBlobSVC creates a blob client
@@ -1169,33 +1149,18 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
if endsWithSlash || remote == "" {
return true
}
return hasDirMetadata(metadata)
}
return false
}
// Returns whether the metadata marks the blob as a directory
func hasDirMetadata(metadata map[string]*string) bool {
// Note that metadata with hdi_isfolder = true seems to be a
// defacto standard for marking blobs as directories.
// Note also that the metadata hasn't been normalised to lower case yet
for k, v := range metadata {
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
return true
// Note that metadata with hdi_isfolder = true seems to be a
// defacto standard for marking blobs as directories.
// Note also that the metadata hasn't been normalised to lower case yet
for k, v := range metadata {
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
return true
}
}
}
return false
}
// Returns whether props are for a directory on a storage account with
// a hierarchical namespace
//
// These ignore a trailing "/" on the name so a file can be returned
// when looking for a directory.
func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
}
// listFn is called from list to handle an object
type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
@@ -1221,25 +1186,27 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
delimiter = "/"
}
opts := &container.ListBlobsHierarchyOptions{
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
Include: container.ListBlobsInclude{
Copy: false,
Metadata: true,
Snapshots: false,
UncommittedBlobs: false,
Deleted: false,
opts := &arrowlist.ListBlobsHierarchyOptions{
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
Include: container.ListBlobsInclude{
Copy: false,
Metadata: true,
Snapshots: false,
UncommittedBlobs: false,
Deleted: false,
},
Prefix: &directory,
MaxResults: &maxResults,
},
Prefix: &directory,
MaxResults: &maxResults,
}
// Request the Apache Arrow listing format. The SDK pager requests an
// Arrow IPC stream and decodes it, falling back to XML if the account
// doesn't have Arrow listing enabled. Skip the maxResults==1 probe
// (isEmpty) which doesn't benefit.
// Experimental: request the Apache Arrow listing format. The arrowlist
// pager requests an Arrow IPC stream and decodes it, falling back to XML
// if the account doesn't have Arrow listing enabled. Skip the
// maxResults==1 probe (isEmpty) which doesn't benefit.
useArrow := f.opt.UseArrowList && maxResults != 1
if useArrow {
opts.ResponseFormat = container.StorageResponseFormatArrow
opts.UseArrowFormat = new(true)
}
var foundItems int
@@ -1254,18 +1221,15 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
if err != nil {
return err
}
if f.dirMarkers() && foundItems == 0 && directory != "" {
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
// Determine whether the directory exists or not by whether it has a marker
props, err := f.readMetaData(ctx, containerName, directory)
_, err := f.readMetaData(ctx, containerName, directory)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
if f.hns && !isHNSDirectory(props) {
return fs.ErrorDirNotFound
}
}
return nil
}
@@ -1273,11 +1237,25 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
// listBlobsPager runs the hierarchy listing described by opts, calling fn for
// each blob and subdirectory, and returns the number of raw items seen.
// delimiter selects flat (recurse) vs hierarchical listing. If opts requests
// the Apache Arrow format and the service answers with XML (Arrow listing
// not enabled) a debug message is logged.
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
// the Apache Arrow format the listing goes through the arrowlist pager
// (falling back to the SDK pager if the credentials don't support it); if the
// service then answers with XML (Arrow listing not enabled) a debug message
// is logged.
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
if useArrow {
arrowSVC, err := f.arrowCntSVC(containerName)
if err != nil {
fs.Debugf(f, "Not using Arrow listing: %v", err)
useArrow = false
} else {
pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
}
}
if pager == nil {
pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
}
checkedArrow := false
for pager.More() {
var response container.ListBlobsHierarchyResponse
@@ -1293,7 +1271,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
// but not accelerated.
if useArrow && !checkedArrow && err == nil {
checkedArrow = true
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
}
}
@@ -1318,13 +1296,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
fs.Debugf(f, "Nil name received")
continue
}
// Storage accounts with a hierarchical namespace return
// the file "dir" when listing "dir/"
if !strings.HasPrefix(*file.Name, directory) {
fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
foundItems--
continue
}
remote := f.opt.Enc.ToStandardPath(*file.Name)
if !strings.HasPrefix(remote, prefix) {
fs.Debugf(f, "Odd name received %q", remote)
@@ -1586,7 +1557,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
func (f *Fs) listContainersToFn(fn listContainerFn) error {
max := int32(f.opt.ListChunkSize)
pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
Include: service.ListContainersInclude{Metadata: true},
Include: service.ListContainersInclude{Metadata: true, Deleted: true},
MaxResults: &max,
})
ctx := context.Background()
@@ -1636,7 +1607,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
// Create directory marker file and parents
func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
if !f.dirMarkers() || container == "" {
if !f.opt.DirectoryMarkers || container == "" {
return nil
}
@@ -1693,11 +1664,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// mkdirParent creates the parent bucket/directory if it doesn't exist
func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
if f.hns {
// Parent directories are created by the upload
container, _ := f.split(remote)
return f.makeContainer(ctx, container)
}
remote, _ = strings.CutSuffix(remote, "/")
dir := path.Dir(remote)
if dir == "/" || dir == "." {
@@ -1803,9 +1769,6 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
container, directory := f.split(dir)
if f.hns && container != "" && directory != "" {
return f.rmdirHNS(ctx, container, directory)
}
// Remove directory marker file
if f.opt.DirectoryMarkers && container != "" && directory != "" {
o := &Object{
@@ -1828,39 +1791,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return f.deleteContainer(ctx, container)
}
// rmdirHNS removes the directory on a storage account with a
// hierarchical namespace
//
// The directory must be deleted without a trailing "/" and the server
// refuses to delete it if it isn't empty.
func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
directory, _ = strings.CutSuffix(directory, "/")
// Check it is a directory as deleting a file would succeed
props, err := f.readMetaData(ctx, container, directory)
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
} else if err != nil {
return err
}
if !isHNSDirectory(props) {
return fs.ErrorIsFile
}
blb := f.getBlobSVC(container, directory)
err = f.pacer.Call(func() (bool, error) {
_, err := blb.Delete(ctx, nil)
return f.shouldRetry(ctx, err)
})
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
return fs.ErrorDirectoryNotEmpty
}
if storageErr.StatusCode == http.StatusNotFound {
return fs.ErrorDirNotFound
}
}
return err
}
// Precision of the remote
func (f *Fs) Precision() time.Duration {
return time.Nanosecond
@@ -325,14 +325,6 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
}
// helper to read blob tags for an object
// skipIfHNS skips the test if the storage account has a hierarchical
// namespace as these don't support blob tags
func skipIfHNS(t *testing.T, f *Fs) {
if f.hns {
t.Skip("blob tags not supported with hierarchical namespace")
}
}
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
ao := o.(*Object)
blb := ao.getBlobSVC()
@@ -418,7 +410,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Singlepart upload
t.Run("PutSinglepartTags", func(t *testing.T) {
skipIfHNS(t, f)
contents := random.String(int(f.opt.ChunkSize / 2))
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
meta := fs.Metadata{
@@ -434,7 +425,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Multipart upload
t.Run("PutMultipartTags", func(t *testing.T) {
skipIfHNS(t, f)
contents := random.String(int(f.opt.ChunkSize + 2048))
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
meta := fs.Metadata{
@@ -622,7 +612,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Singlepart copy
t.Run("CopySinglepartTags", func(t *testing.T) {
skipIfHNS(t, f)
// create small source
contents := random.String(int(f.opt.ChunkSize / 2))
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
@@ -648,7 +637,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Multipart copy
t.Run("CopyMultipartTags", func(t *testing.T) {
skipIfHNS(t, f)
// create large source to force multipart
contents := random.String(int(f.opt.CopyCutoff + 4096))
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+9 -28
View File
@@ -64,7 +64,7 @@ const (
maxParts = 10000
maxVersions = 100 // maximum number of versions we search in --b2-versions mode
minChunkSize = 5 * fs.Mebi
defaultChunkSize = 96 * fs.Mebi
defaultChunkSize = 16 * fs.Mebi
defaultUploadCutoff = 200 * fs.Mebi
largeFileCopyCutoff = 4 * fs.Gibi // 5E9 is the max
defaultMaxAge = 24 * time.Hour
@@ -153,7 +153,12 @@ The minimum is 0 and the maximum is 4.6 GiB.`,
When uploading large files, chunk the file into this size.
Must fit in memory. These chunks are buffered in memory and there
might a maximum of "--transfers" chunks in progress at once.
might be a maximum of "--transfers" * "--b2-upload-concurrency"
chunks in progress at once.
B2 has a fixed per part overhead of about a second, so upload speed
is roughly proportional to the chunk size multiplied by
"--b2-upload-concurrency".
5,000,000 Bytes is the minimum size.`,
Default: defaultChunkSize,
@@ -168,7 +173,7 @@ concurrently.
Note that chunks are stored in memory and there may be up to
"--transfers" * "--b2-upload-concurrency" chunks stored at once
in memory.`,
Default: 4,
Default: 8,
Advanced: true,
}, {
Name: "disable_checksum",
@@ -191,11 +196,6 @@ If the custom endpoint rewrites the requests for authentication,
e.g., in Cloudflare Workers, this header needs to be handled properly.
Leave blank if you want to use the endpoint provided by Backblaze.
Rclone will not follow a redirect from this endpoint which downgrades
HTTPS to HTTP, and will not send the "Authorization" header or the
SSE-C key headers once a redirect has left the host and port of this
endpoint.
The URL provided here SHOULD have the protocol and SHOULD NOT have
a trailing slash or specify the /file/bucket subpath as rclone will
request files with "{download_url}/file/{bucket_name}/{path}".
@@ -534,25 +534,6 @@ func (f *Fs) setRoot(root string) {
f.rootBucket, f.rootDirectory = bucket.Split(f.root)
}
// redirectSecretHeaders are the request headers carrying secrets that
// must not be forwarded when a redirect leaves the original host.
var redirectSecretHeaders = []string{
"Authorization",
sseAlgorithmHeader,
sseKeyHeader,
sseMd5Header,
}
// newClient makes the http client used for all B2 requests.
//
// The download_url option sends the requests to a host the user does
// not control so it strips headers on cross host redirects.
func newClient(ctx context.Context) *http.Client {
client := fshttp.NewClient(ctx)
client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
return client
}
// NewFs constructs an Fs from the path, bucket:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -605,7 +586,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
name: name,
opt: *opt,
ci: ci,
srv: rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
cache: bucket.NewCache(),
_bucketID: make(map[string]string, 1),
_bucketType: make(map[string]string, 1),
-59
View File
@@ -4,8 +4,6 @@ import (
"context"
"crypto/sha1"
"fmt"
"net/http"
"net/http/httptest"
"path"
"sort"
"strings"
@@ -619,60 +617,3 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
// redirectTestHeaders is a deliberately literal copy of
// redirectSecretHeaders so that a header dropped from the production
// list fails TestRedirectSecretHeaders rather than silently losing
// test coverage.
var redirectTestHeaders = map[string]string{
"Authorization": "secret-token",
"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
"X-Bz-Server-Side-Encryption-Customer-Key": "secret-key",
"X-Bz-Server-Side-Encryption-Customer-Key-Md5": "secret-md5",
}
func TestRedirectSecretHeaders(t *testing.T) {
want := make([]string, 0, len(redirectTestHeaders))
for header := range redirectTestHeaders {
want = append(want, header)
}
sort.Strings(want)
got := append([]string(nil), redirectSecretHeaders...)
sort.Strings(got)
assert.Equal(t, want, got)
}
// redirectTestRequest makes a GET request to rawURL carrying the
// secret headers plus a harmless marker header.
func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
require.NoError(t, err)
for header, value := range redirectTestHeaders {
req.Header.Set(header, value)
}
req.Header.Set("X-Bz-Test-Mode", "marker")
return req
}
func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
client := newClient(context.Background())
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for header := range redirectTestHeaders {
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
}
assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
w.WriteHeader(http.StatusOK)
}))
defer redirectServer.Close()
initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
}))
defer initialServer.Close()
resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.NoError(t, resp.Body.Close())
}
+1 -1
View File
@@ -1461,7 +1461,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
// Item in the cache has the same or newer SequenceID than
// this event. Ignore this event, it must be old.
f.itemMetaCacheMu.Unlock()
fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
continue
}
+1 -4
View File
@@ -801,12 +801,9 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
fs.Debugf(f, "skip orphan data chunk %q", remote)
break
}
if mainObject == nil && !f.useMeta && !isSpecial {
if mainObject == nil && !f.useMeta {
// this is the "nometa" case
// create dummy chunked object without metadata
//
// not for temporary or control chunks as they may be all
// that is left of an interrupted upload
mainObject = f.newObject(mainRemote, nil, nil)
byRemote[mainRemote] = mainObject
if !badEntry[mainRemote] {
-19
View File
@@ -627,22 +627,6 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
_ = file.Remove(ctx)
}
// testOrphanTempChunks checks that temporary chunks left behind by an
// interrupted upload are not listed and don't break the listing
func testOrphanTempChunks(t *testing.T, f *Fs) {
const dir = "orphan"
ctx := context.Background()
defer func() {
_ = operations.Purge(ctx, f.base, dir)
}()
modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
}
func testMetadataInput(t *testing.T, f *Fs) {
const minChunkForTest = 50
if f.opt.ChunkSize < minChunkForTest {
@@ -944,9 +928,6 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("BackwardsCompatibility", func(t *testing.T) {
testBackwardsCompatibility(t, f)
})
t.Run("OrphanTempChunks", func(t *testing.T) {
testOrphanTempChunks(t, f)
})
t.Run("ChunkerServerSideMove", func(t *testing.T) {
testChunkerServerSideMove(t, f)
})
-32
View File
@@ -1085,35 +1085,6 @@ type ObjectMetadata struct {
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
}
// validate checks metadata read from the wrapped remote before it is used to
// construct a compressed reader.
func (meta *ObjectMetadata) validate() error {
if meta.Mode != Gzip {
return nil
}
if meta.CompressionMetadataGzip == nil {
return errors.New("missing gzip metadata")
}
if meta.CompressionMetadataGzip.BlockSize <= 0 {
return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
}
if meta.Size != meta.CompressionMetadataGzip.Size {
return errors.New("gzip metadata size does not match object size")
}
if meta.Size < 0 {
return errors.New("invalid gzip object size")
}
blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
blocks := meta.Size / blockSize
if meta.Size%blockSize != 0 {
blocks++
}
if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
return errors.New("gzip block data is incomplete")
}
return nil
}
// Object with external metadata
type Object struct {
fs.Object // Wraps around data object for this object
@@ -1137,9 +1108,6 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
if err = jr.Decode(meta); err != nil {
return nil, err
}
if err = meta.validate(); err != nil {
return nil, err
}
return meta, nil
}
-89
View File
@@ -6,8 +6,6 @@ import (
"path/filepath"
"testing"
"github.com/buengese/sgzip"
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/local"
_ "github.com/rclone/rclone/backend/s3"
@@ -16,93 +14,6 @@ import (
"github.com/rclone/rclone/fstest/fstests"
)
func TestObjectMetadataValidate(t *testing.T) {
tests := []struct {
name string
meta ObjectMetadata
want bool
}{
{
name: "nil gzip metadata",
meta: ObjectMetadata{Mode: Gzip},
want: true,
},
{
name: "zero block size",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 0,
Size: 1024,
BlockData: []uint32{0},
},
},
want: true,
},
{
name: "negative block size",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: -1,
Size: 1024,
BlockData: []uint32{0},
},
},
want: true,
},
{
name: "mismatched sizes",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 2048,
BlockData: []uint32{0, 1},
},
},
want: true,
},
{
name: "short block data",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1025,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 1025,
BlockData: []uint32{0, 1},
},
},
want: true,
},
{
name: "valid",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1025,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 1025,
BlockData: []uint32{0, 1, 2},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := test.meta.validate()
if (err != nil) != test.want {
t.Fatalf("validate() error = %v, want error: %v", err, test.want)
}
})
}
}
var defaultOpt = fstests.Opt{
RemoteName: "TestCompress:",
NilObject: (*Object)(nil),
+5 -5
View File
@@ -118,8 +118,8 @@ func (mode NameEncryptionMode) String() (out string) {
return out
}
// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
type FileNameEncoding interface {
// fileNameEncoding are the encoding methods dealing with encrypted file names
type fileNameEncoding interface {
EncodeToString(src []byte) string
DecodeString(s string) ([]byte, error)
}
@@ -154,7 +154,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
}
// NewNameEncoding creates a NameEncoding from a string
func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
s = strings.ToLower(s)
switch s {
case "base32":
@@ -176,7 +176,7 @@ type Cipher struct {
nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto
block gocipher.Block
mode NameEncryptionMode
fileNameEnc FileNameEncoding
fileNameEnc fileNameEncoding
buffers sync.Pool // encrypt/decrypt buffers
cryptoRand io.Reader // read crypto random numbers from here
dirNameEncrypt bool
@@ -185,7 +185,7 @@ type Cipher struct {
}
// newCipher initialises the cipher. If salt is "" then it uses a built in salt val
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
c := &Cipher{
mode: mode,
fileNameEnc: enc,
-259
View File
@@ -1,259 +0,0 @@
// Package api has type definitions for dosya.dev
package api
import "fmt"
// Response is the base response wrapper
type Response struct {
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
}
// Error is an error response from the API.
//
// StatusCode and Status are filled in from the HTTP response. The API
// sets ErrorCode on the errors a client is expected to act on rather
// than just report, e.g. "concurrent_upload_limit".
type Error struct {
StatusCode int `json:"-"`
Status string `json:"-"`
OK bool `json:"ok"`
Message string `json:"error"`
ErrorCode string `json:"error_code,omitempty"`
MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
}
// Error satisfies the error interface
func (e *Error) Error() string {
out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
if e.Message != "" {
out += ": " + e.Message
}
if e.ErrorCode != "" {
out += " (" + e.ErrorCode + ")"
}
return out
}
// FileItem represents a file from the dosya API
type FileItem struct {
ID string `json:"id"`
Name string `json:"name"`
SizeBytes int64 `json:"size_bytes"`
MimeType string `json:"mime_type"`
Extension string `json:"extension"`
Region string `json:"region"`
CreatedAt float64 `json:"created_at"`
UpdatedAt float64 `json:"updated_at"`
FolderID *string `json:"folder_id"`
IsSynced int `json:"is_synced"`
// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
// are never re-read to hash the whole object - the same limitation S3 has).
ContentHash string `json:"content_hash"`
}
// FolderItem represents a folder from the dosya API
type FolderItem struct {
ID string `json:"id"`
Name string `json:"name"`
ParentID *string `json:"parent_id"`
CreatedAt float64 `json:"created_at"`
FileCount int `json:"file_count"`
IsSynced int `json:"is_synced"`
}
// ListResponse is the response from GET /api/files
type ListResponse struct {
Response
Files []FileItem `json:"files"`
Folders []FolderItem `json:"folders"`
Pagination Pagination `json:"pagination"`
}
// Pagination describes which page of files a ListResponse holds
type Pagination struct {
Page int `json:"page"`
PerPage int `json:"per_page"`
TotalFiles int `json:"total_files"`
TotalPages int `json:"total_pages"`
}
// CreateFolderRequest is the request body for POST /api/folders
type CreateFolderRequest struct {
WorkspaceID string `json:"workspace_id"`
ParentID *string `json:"parent_id"`
Name string `json:"name"`
}
// CreateFolderResponse is the response from POST /api/folders
type CreateFolderResponse struct {
Response
Folder struct {
ID string `json:"id"`
Name string `json:"name"`
ParentID *string `json:"parent_id"`
Workspace string `json:"workspace_id"`
} `json:"folder"`
}
// DeleteFolderResponse is the response from DELETE /api/folders/:id
type DeleteFolderResponse struct {
Response
Permanent bool `json:"permanent"`
Complete bool `json:"complete"`
Remaining int `json:"remaining"`
FilesAffected int `json:"files_affected"`
FoldersRemoved int `json:"folders_removed"`
}
// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
type RenameFolderRequest struct {
Name string `json:"name"`
}
// MoveFolderRequest is the request body for PUT /api/folders/:id/move
type MoveFolderRequest struct {
ParentID *string `json:"parent_id"`
Name string `json:"name,omitempty"`
}
// MoveFolderResponse is the response from PUT /api/folders/:id/move
type MoveFolderResponse struct {
Response
Name string `json:"name"`
}
// UploadInitRequest is the request body for POST /api/upload/init
type UploadInitRequest struct {
WorkspaceID string `json:"workspace_id"`
FileName string `json:"file_name"`
FileSize int64 `json:"file_size"`
MimeType string `json:"mime_type"`
Region string `json:"region,omitempty"`
FolderID *string `json:"folder_id"`
FileID *string `json:"file_id,omitempty"`
}
// UploadInitResponse is the response from POST /api/upload/init
type UploadInitResponse struct {
Response
SessionID string `json:"session_id"`
UploadURL string `json:"upload_url"`
WorkspaceID string `json:"workspace_id"`
FileName string `json:"file_name"`
FileSize int64 `json:"file_size"`
MimeType string `json:"mime_type"`
Extension string `json:"extension"`
Region string `json:"region"`
Resumable *struct {
PartSize int64 `json:"part_size"`
TotalParts int `json:"total_parts"`
PartUploadURL string `json:"part_upload_url"`
CompleteURL string `json:"complete_url"`
StatusURL string `json:"status_url"`
} `json:"resumable"`
}
// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
type UploadCompleteResponse struct {
Response
File struct {
ID string `json:"id"`
Name string `json:"name"`
SizeBytes int64 `json:"size_bytes"`
MimeType string `json:"mime_type"`
Extension string `json:"extension"`
Region string `json:"region"`
Version int `json:"version"`
CreatedAt float64 `json:"created_at"`
// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
// "" for a multipart upload, which the server does not hash.
ContentHash string `json:"content_hash"`
} `json:"file"`
}
// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
type UploadPartResponse struct {
Response
PartNumber int `json:"part_number"`
ETag string `json:"etag"`
BytesUploaded int64 `json:"bytes_uploaded"`
PartsCompleted int `json:"parts_completed"`
TotalParts int `json:"total_parts"`
}
// DeleteFileResponse is the response from DELETE /api/files/:id
type DeleteFileResponse struct {
Response
Permanent bool `json:"permanent"`
}
// RenameFileRequest is the request body for PUT /api/files/:id/rename
type RenameFileRequest struct {
Name string `json:"name"`
}
// RenameFileResponse is the response from PUT /api/files/:id/rename
type RenameFileResponse struct {
Response
Name string `json:"name"`
}
// MoveFileRequest is the request body for PUT /api/files/:id/move
type MoveFileRequest struct {
FolderID *string `json:"folder_id"`
Name *string `json:"name,omitempty"`
}
// CopyFileRequest is the request body for POST /api/files/:id/copy
type CopyFileRequest struct {
FolderID *string `json:"folder_id"`
Name string `json:"name,omitempty"`
}
// CopyFileResponse is the response from POST /api/files/:id/copy
type CopyFileResponse struct {
Response
FileID string `json:"file_id"`
Name string `json:"name"`
}
// ShareLinkRequest is the request body for POST /api/files/:id/share
type ShareLinkRequest struct {
ExpiresInDays *int `json:"expires_in_days,omitempty"`
}
// ShareLinkResponse is the response from POST /api/files/:id/share
type ShareLinkResponse struct {
Response
Link struct {
ID string `json:"id"`
Token string `json:"token"`
URL string `json:"url"`
ExpiresAt *int64 `json:"expires_at"`
CreatedAt float64 `json:"created_at"`
} `json:"link"`
}
// Workspace is one entry of the workspace list
type Workspace struct {
ID string `json:"id"`
Name string `json:"name"`
}
// WorkspaceListResponse is the response from GET /api/workspaces
type WorkspaceListResponse struct {
Response
Workspaces []Workspace `json:"workspaces"`
}
// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
type WorkspaceInfoResponse struct {
Response
Storage struct {
Used int64 `json:"used"`
Total int64 `json:"total"`
Free int64 `json:"free"`
} `json:"storage"`
}
-772
View File
@@ -1,772 +0,0 @@
// Package dosya provides an interface to the dosya.dev cloud storage system.
package dosya
import (
"context"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/rest"
)
const (
rootID = "root"
defaultBaseURL = "https://api.dosya.dev"
minSleep = 100 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2
attackConstant = 0
)
func init() {
fs.Register(&fs.RegInfo{
Name: "dosya",
Description: "dosya.dev",
NewFs: NewFs,
Options: []fs.Option{{
Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
Name: "api_key",
Sensitive: true,
Required: true,
}, {
Name: "workspace_id",
Help: `Your workspace ID.
Leave this blank to use the one workspace the API key can reach, which
is what a key restricted to a single workspace does. If the key can
reach several, rclone will refuse and list their IDs for you to choose
from.`,
}, {
Help: "API base URL.",
Name: "api_url",
Default: defaultBaseURL,
Advanced: true,
}, {
Name: "upload_concurrency",
Help: `Concurrency for multipart uploads.
This is the number of parts of the same file that are uploaded
concurrently. The first part is always uploaded on its own, because
the server creates the upload when it receives it.
Each part in flight is held in memory, so memory use grows with
this value times the part size the server chooses.
If you are uploading large files over high-speed links and these
uploads do not fully utilize your bandwidth, then increasing this
may help to speed up the transfers.`,
Default: 4,
Advanced: true,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeSingleQuote |
encoder.EncodeDoubleQuote |
encoder.EncodeLtGt |
encoder.EncodeCtl |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8),
}},
})
}
// Options defines the configuration for this backend
type Options struct {
APIKey string `config:"api_key"`
WorkspaceID string `config:"workspace_id"`
APIURL string `config:"api_url"`
UploadConcurrency int `config:"upload_concurrency"`
Enc encoder.MultiEncoder `config:"encoding"`
}
// Fs represents a remote dosya.dev filesystem
type Fs struct {
name string
root string
opt Options
features *fs.Features
rest *rest.Client
pacer *fs.Pacer
dirCache *dircache.DirCache
}
// Object represents a dosya.dev file
type Object struct {
fs *Fs
remote string
file api.FileItem
}
// findWorkspace fills in the workspace ID when the config leaves it blank.
//
// The ID is not shown anywhere in the dosya.dev web interface, so requiring
// it meant asking people to find it in a developer console. A key restricted
// to one workspace can only mean that one, and an account with a single
// workspace has only one possible answer; anything else genuinely has to be
// chosen, so list the choices in the error.
func (f *Fs) findWorkspace(ctx context.Context) error {
workspaces, err := f.listWorkspaces(ctx)
if err != nil {
return err
}
switch len(workspaces) {
case 0:
return errors.New("no workspaces found for this API key")
case 1:
f.opt.WorkspaceID = workspaces[0].ID
fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
return nil
}
var choices strings.Builder
for _, w := range workspaces {
_, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
}
return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
len(workspaces), choices.String())
}
// FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
listing, err := f.listFilesAndFolders(ctx, pathID)
if err != nil {
return "", false, err
}
for _, folder := range listing.Folders {
if f.opt.Enc.ToStandardName(folder.Name) == leaf {
return folder.ID, true, nil
}
}
return "", false, nil
}
// CreateDir makes a directory with pathID as parent and name leaf
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
name := f.opt.Enc.FromStandardName(leaf)
resp, err := f.createFolder(ctx, name, pathID)
if err != nil {
return "", err
}
return resp.Folder.ID, nil
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
// String returns a description of the FS
func (f *Fs) String() string {
return fmt.Sprintf("dosya.dev root '%s'", f.root)
}
// Precision of the ModTimes in this Fs
func (f *Fs) Precision() time.Duration {
// Modification times are stored and returned to the second (the API
// keeps them as Unix seconds, sent on upload via the source-mtime header).
return time.Second
}
// Hashes returns the supported hash types of the filesystem
func (f *Fs) Hashes() hash.Set {
// The server computes and stores the SHA-256 of a single-shot upload's
// bytes; multipart uploads have none (see Object.Hash).
return hash.NewHashSet(hash.SHA256)
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// NewFs makes a new Fs object from the path
func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
opt := new(Options)
err := configstruct.Set(config, opt)
if err != nil {
return nil, err
}
root = strings.Trim(root, "/")
f := &Fs{
name: name,
root: root,
opt: *opt,
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
}
// No DuplicateFiles: an upload that names no existing file but whose
// (workspace, folder, name) is already taken becomes a new VERSION of
// that file rather than a second row, so a name is unique within a
// folder and the same upload twice returns the same file id.
f.features = (&fs.Features{
CaseInsensitive: false,
CanHaveEmptyDirectories: true,
ReadMimeType: true,
}).Fill(ctx, f)
client := fshttp.NewClient(ctx)
f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
if f.opt.WorkspaceID == "" {
err = f.findWorkspace(ctx)
if err != nil {
return nil, err
}
}
f.dirCache = dircache.New(root, rootID, f)
// Find the current root
err = f.dirCache.FindRoot(ctx, false)
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
tempF := *f
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = tempF.dirCache.FindRoot(ctx, false)
if err != nil {
// No root so return old f
return f, nil
}
_, err := tempF.NewObject(ctx, remote)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
return f, nil
}
return nil, err
}
f.features.Fill(ctx, &tempF)
f.dirCache = tempF.dirCache
f.root = tempF.root
return f, fs.ErrorIsFile
}
return f, nil
}
// List the objects and directories in dir into entries
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return nil, err
}
listing, err := f.listFilesAndFolders(ctx, directoryID)
if err != nil {
return nil, err
}
for _, folder := range listing.Folders {
folderName := f.opt.Enc.ToStandardName(folder.Name)
fullPath := getRemote(dir, folderName)
modTime := time.Unix(int64(folder.CreatedAt), 0)
d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
entries = append(entries, d)
f.dirCache.Put(fullPath, folder.ID)
}
for _, file := range listing.Files {
fileName := f.opt.Enc.ToStandardName(file.Name)
remote := getRemote(dir, fileName)
o := &Object{
fs: f,
remote: remote,
file: file,
}
entries = append(entries, o)
}
return entries, nil
}
// NewObject finds the Object at remote
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
return nil, fs.ErrorObjectNotFound
}
return nil, err
}
listing, err := f.listFilesAndFolders(ctx, directoryID)
if err != nil {
return nil, err
}
for _, file := range listing.Files {
if f.opt.Enc.ToStandardName(file.Name) == leaf {
return &Object{
fs: f,
remote: remote,
file: file,
}, nil
}
}
return nil, fs.ErrorObjectNotFound
}
// Put uploads a new file
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
existingObj, err := f.NewObject(ctx, src.Remote())
switch err {
case nil:
return existingObj, existingObj.Update(ctx, in, src, options...)
case fs.ErrorObjectNotFound:
return f.put(ctx, in, src, options...)
default:
return nil, err
}
}
// put uploads src as a new file
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
remote := src.Remote()
_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}
modTime := src.ModTime(ctx)
resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
if err != nil {
return nil, err
}
return &Object{
fs: f,
remote: remote,
file: fileFromUpload(resp, modTime),
}, nil
}
// fileFromUpload builds the FileItem for the object an upload just created.
// The single-shot upload response omits updated_at, so the modification time
// is taken from what was sent (modTime) rather than defaulting to the create
// time - it is the time the server stored via the source-mtime header. The
// SHA-256 comes straight from the response (empty for a multipart upload).
func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
updatedAt := resp.File.CreatedAt
if !modTime.IsZero() {
updatedAt = float64(modTime.Unix())
}
return api.FileItem{
ID: resp.File.ID,
Name: resp.File.Name,
SizeBytes: resp.File.SizeBytes,
MimeType: resp.File.MimeType,
Extension: resp.File.Extension,
Region: resp.File.Region,
CreatedAt: resp.File.CreatedAt,
UpdatedAt: updatedAt,
ContentHash: resp.File.ContentHash,
}
}
// Mkdir makes the directory (container, bucket)
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
_, err := f.dirCache.FindDir(ctx, dir, true)
return err
}
// Rmdir removes the directory (container, bucket) if empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return err
}
// The server checks emptiness itself (?if_empty=1) and answers
// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
err = f.removeFolder(ctx, directoryID, true)
if err != nil {
return err
}
f.dirCache.FlushDir(dir)
return nil
}
// Move src to this remote using server-side move operations
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
if err != nil {
return nil, err
}
// Find destination directory
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}
needsMove := srcDirID != dstDirID
needsRename := srcLeaf != dstLeaf
// Moving and renaming in the same request (rather than a moveFile call
// followed by a separate renameFile call) matters when both are needed:
// a two-step move-then-rename can transiently place the file's CURRENT
// name into a destination folder that already holds an unrelated file
// with that name, even though the FINAL name is free - the server only
// allows one active name per folder, so that intermediate state 409s.
if needsMove || needsRename {
newName := ""
if needsRename {
newName = f.opt.Enc.FromStandardName(dstLeaf)
}
err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
if err != nil {
return nil, err
}
}
// Return new object. file.Name must reflect the destination leaf, not
// the source's - a caller that reuses this object as the src of a
// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
// recomputes srcLeaf from file.Name; a stale name there makes that
// second move see srcLeaf == dstLeaf and skip the rename entirely,
// silently leaving the object under the wrong name.
newFile := srcObj.file
newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
return &Object{
fs: f,
remote: remote,
file: newFile,
}, nil
}
// DirMove moves src, srcRemote to this remote at dstRemote
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
srcFs, ok := src.(*Fs)
if !ok {
fs.Debugf(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
if err != nil {
return err
}
srcLeaf := srcRemote
if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
srcLeaf = srcRemote[idx+1:]
}
needsRename := srcLeaf != dstLeaf
newName := ""
if needsRename {
newName = f.opt.Enc.FromStandardName(dstLeaf)
}
// Move folder to new parent only if parent changed. The new name goes
// in the same request, as for files in Move: a move followed by a
// separate rename can collide with an unrelated folder sharing the
// CURRENT name in the destination even though the final name is free.
if srcDirectoryID != dstDirectoryID {
renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
if err != nil {
return fmt.Errorf("couldn't move directory: %w", err)
}
if renamed {
needsRename = false
}
}
if needsRename {
err = f.renameFolder(ctx, srcID, newName)
if err != nil {
return fmt.Errorf("couldn't rename directory: %w", err)
}
}
srcFs.dirCache.FlushDir(srcRemote)
return nil
}
// Copy src to this remote using server-side copy operations
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}
// The copy API creates a new row named after the source file when
// copying across folders, which collides with one already at the
// destination. Copy under a temporary name, then delete the old
// destination, then move the copy onto the name - so a copy that fails
// leaves the destination as it was. Deleting it up front lost it
// (prior art: backend/box).
existingObj, err := f.NewObject(ctx, remote)
switch err {
case nil:
tempRemote := remote + "-rclone-copy-" + random.String(8)
fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
tempObj, err := f.Copy(ctx, src, tempRemote)
if err != nil {
return nil, err
}
fs.Debugf(tempRemote, "moving to real name %v", remote)
if err := existingObj.Remove(ctx); err != nil {
return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
}
return f.Move(ctx, tempObj, remote)
case fs.ErrorObjectNotFound:
// nothing at the destination, so copy straight onto the name
default:
return nil, err
}
dstName := f.opt.Enc.FromStandardName(dstLeaf)
resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
if err != nil {
return nil, fmt.Errorf("couldn't copy file: %w", err)
}
// Servers that predate the name field ignore it and name the copy after
// the source, or "Copy of <original>" in the same folder. Rename to the
// requested destination name if it differs.
if resp.Name != dstName {
err = f.renameFile(ctx, resp.FileID, dstName)
if err != nil {
return nil, fmt.Errorf("couldn't rename copied file: %w", err)
}
resp.Name = dstName
}
// A server-side copy is byte-identical to the source and the API
// preserves the source's modification time and content hash onto the
// copy, so the returned object must report the SOURCE's modtime and hash,
// not the copy's wall-clock time. Reading "now" here made the object
// rclone holds in memory disagree with the one read back from the remote
// (its Fingerprint check compares the two exactly).
return &Object{
fs: f,
remote: remote,
file: api.FileItem{
ID: resp.FileID,
Name: resp.Name,
SizeBytes: srcObj.file.SizeBytes,
MimeType: srcObj.file.MimeType,
Extension: srcObj.file.Extension,
Region: srcObj.file.Region,
CreatedAt: srcObj.file.CreatedAt,
UpdatedAt: srcObj.file.UpdatedAt,
ContentHash: srcObj.file.ContentHash,
},
}, nil
}
// Purge deletes all the files and the container
func (f *Fs) Purge(ctx context.Context, dir string) error {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return err
}
// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
err = f.removeFolder(ctx, directoryID, false)
if err != nil {
return err
}
f.dirCache.FlushDir(dir)
return nil
}
// PublicLink generates a public link to the remote path
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
o, err := f.NewObject(ctx, remote)
if err != nil {
return "", err
}
obj := o.(*Object)
resp, err := f.createShareLink(ctx, obj.file.ID, expire)
if err != nil {
return "", err
}
return resp.Link.URL, nil
}
// DirCacheFlush resets the directory cache - used in testing as an
// optional interface
func (f *Fs) DirCacheFlush() {
f.dirCache.ResetRoot()
}
// About gets quota information from the Fs
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
info, err := f.getWorkspaceInfo(ctx)
if err != nil {
return nil, err
}
usage := &fs.Usage{
Used: fs.NewUsageValue(info.Storage.Used),
Total: fs.NewUsageValue(info.Storage.Total),
Free: fs.NewUsageValue(info.Storage.Free),
}
return usage, nil
}
// ------------------------------------------------------------
// Object methods
// ------------------------------------------------------------
// String returns a description of the Object
func (o *Object) String() string {
return o.remote
}
// Remote returns the remote path
func (o *Object) Remote() string {
return o.remote
}
// ModTime returns the modification date of the file
func (o *Object) ModTime(ctx context.Context) time.Time {
if o.file.UpdatedAt > 0 {
return time.Unix(int64(o.file.UpdatedAt), 0)
}
return time.Unix(int64(o.file.CreatedAt), 0)
}
// Size returns the size of the file
func (o *Object) Size() int64 {
return o.file.SizeBytes
}
// Fs returns read only access to the Fs that this object is part of
func (o *Object) Fs() fs.Info {
return o.fs
}
// Hash returns the selected checksum of the file
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.SHA256 {
return "", hash.ErrUnsupported
}
// May be "" - a multipart-uploaded object carries no server-side hash,
// which rclone treats as "hash unavailable for this object" (as it does
// for S3 multipart objects).
return o.file.ContentHash, nil
}
// Storable says whether this object can be stored
func (o *Object) Storable() bool {
return true
}
// SetModTime sets the metadata on the object to set the modification date
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
return fs.ErrorCantSetModTime
}
// Open opens the file for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
fs.FixRangeOption(options, o.file.SizeBytes)
return o.fs.downloadFile(ctx, o.file.ID, options)
}
// Update replaces the file content
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
// Find the directory ID for this file
_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
if err != nil {
return err
}
// Upload as new version of existing file
fileID := o.file.ID
modTime := src.ModTime(ctx)
resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
if err != nil {
return err
}
// Update object metadata
o.file = fileFromUpload(resp, modTime)
return nil
}
// Remove removes this object
func (o *Object) Remove(ctx context.Context) error {
return o.fs.deleteFile(ctx, o.file.ID)
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType(ctx context.Context) string {
return o.file.MimeType
}
// ID returns the ID of the Object if known, or "" if not
func (o *Object) ID() string {
return o.file.ID
}
func getRemote(dir, fileName string) string {
if dir == "" {
return fileName
}
return dir + "/" + fileName
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.PublicLinker = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ dircache.DirCacher = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = (*Object)(nil)
_ fs.IDer = (*Object)(nil)
)
-880
View File
@@ -1,880 +0,0 @@
package dosya
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"path"
"strconv"
"strings"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"golang.org/x/sync/errgroup"
)
// sourceMTimeHeader is the request header the upload endpoints read to set a
// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
const sourceMTimeHeader = "x-dosya-source-mtime"
// modTimeHeaders returns the ExtraHeaders that carry a source modification time
// on an upload, or nil when there is nothing plausible to send. The server
// clamps implausible values itself, but a zero time is never worth a header.
func modTimeHeaders(modTime time.Time) map[string]string {
if modTime.IsZero() {
return nil
}
return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
}
// workspaceHintHeader names the workspace a by-id request is about. The API
// maps an id to its workspace through an index the workspace fills about a
// second after a write, so an id this client was handed moments ago can still
// miss; the hint lets the server look for it in the named workspace instead.
// It only locates: every route still checks membership against the workspace
// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
const workspaceHintHeader = "X-Dosya-Workspace"
// byIDHeaders returns the ExtraHeaders every /api/files/:id and
// /api/folders/:id call carries, or nil when no workspace is configured.
func (f *Fs) byIDHeaders() map[string]string {
if f.opt.WorkspaceID == "" {
return nil
}
return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
}
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
429, // Too Many Requests
500, // Internal Server Error
502, // Bad Gateway
503, // Service Unavailable
504, // Gateway Timeout
}
// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
if retry {
if d, ok := retryAfter(resp); ok {
// The API answers 503 + Retry-After while a workspace is being
// moved between storage homes, and Cloudflare sends it with 429.
// The pacer's own backoff tops out at maxSleep, far short of that.
fs.Debugf(nil, "Server asked to retry after %v", d)
err = pacer.RetryAfterError(err, d)
}
}
return retry, err
}
// maxRetryAfter caps how long a single Retry-After is honoured for
const maxRetryAfter = 5 * time.Minute
// retryAfter reads a 429 or 503 response's Retry-After header, which the API
// sends as whole seconds
func retryAfter(resp *http.Response) (time.Duration, bool) {
if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
return 0, false
}
seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
if err != nil || seconds <= 0 {
return 0, false
}
return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
}
// errorHandler decodes a non-2xx response into an *api.Error so callers can
// act on the machine-readable error_code rather than the message text.
func errorHandler(resp *http.Response) error {
body, err := rest.ReadBody(resp)
if err != nil {
return fmt.Errorf("error reading error out of body: %w", err)
}
apiErr := &api.Error{
StatusCode: resp.StatusCode,
Status: resp.Status,
}
if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
// Not the API's JSON shape (e.g. an edge response that never
// reached the API) - keep the body text so it isn't lost.
apiErr.ErrorCode = ""
apiErr.Message = strings.TrimSpace(string(body))
}
return apiErr
}
// The workspace caps the number of in-flight uploads per user. The API
// refuses an upload/init past that cap with a 400 carrying
// error_code "concurrent_upload_limit", and documents it as an error to
// wait on rather than give up on: slots free as uploads finish, and the
// server marks sessions idle for 10 minutes as failed, so waiting up to
// that long guarantees progress unless the workspace really is saturated.
// A 400 is deliberately outside the pacer's retry codes, so the wait
// lives in initUpload instead of shouldRetry.
var (
concurrentUploadRetryDelay = 5 * time.Second
concurrentUploadWaitMax = 10 * time.Minute
)
// abortUploadTimeout bounds the abort of a session the client is giving up on,
// which runs outside the transfer's context so a cancelled transfer still
// sends it
const abortUploadTimeout = 10 * time.Second
// isConcurrentUploadLimit reports whether err is the API refusing an
// upload because the workspace's concurrent upload limit is reached
func isConcurrentUploadLimit(err error) bool {
var apiErr *api.Error
return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
}
// listPageSize is the largest page GET /api/files will serve
const listPageSize = 500
// listFilesAndFolders lists files and folders in a directory
//
// The API pages files (at most listPageSize per page) but returns every
// folder on each page, so the files from all pages are gathered and the
// folders are taken from the first page only. Pages are ordered by name,
// which the API tie-breaks on id, so offsets are stable between requests.
func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
var listing *api.ListResponse
for page := 1; ; page++ {
opts := rest.Opts{
Method: "GET",
Path: "/api/files",
Parameters: map[string][]string{
"workspace_id": {f.opt.WorkspaceID},
"per_page": {strconv.Itoa(listPageSize)},
"page": {strconv.Itoa(page)},
"sort": {"name"},
"dir": {"asc"},
},
}
if folderID != "" && folderID != rootID {
opts.Parameters["folder_id"] = []string{folderID}
}
var result api.ListResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't list files: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
if listing == nil {
listing = &result
} else {
listing.Files = append(listing.Files, result.Files...)
}
if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
return listing, nil
}
}
}
// createFolder creates a folder
func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
request := api.CreateFolderRequest{
WorkspaceID: f.opt.WorkspaceID,
Name: name,
}
if parentID != "" && parentID != rootID {
request.ParentID = &parentID
}
opts := rest.Opts{
Method: "POST",
Path: "/api/folders",
}
var result api.CreateFolderResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't create folder: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// removeFolder deletes a folder
//
// ?permanent=1 trashes and purges a live folder in one request on today's
// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
// HTTP 409 "Folder is not empty" instead of purging a folder that still
// holds a live file or a live child folder; that 409 is turned into
// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
// error, and it is never retried (409 is not one of retryErrorCodes). A big
// subtree is purged in bounded passes (202, complete=false), so keep
// calling until the purge is complete. An older server that ignores
// permanent=1 only trashes on the first call (answering permanent=false),
// so the loop's second pass sends the plain DELETE that purges it.
func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
parameters := map[string][]string{"permanent": {"1"}}
if ifEmpty {
parameters["if_empty"] = []string{"1"}
}
opts := rest.Opts{
Method: "DELETE",
Path: "/api/folders/" + folderID,
Parameters: parameters,
ExtraHeaders: f.byIDHeaders(),
}
for pass := 0; ; pass++ {
var result api.DeleteFolderResponse
var statusCode int
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
if resp != nil {
statusCode = resp.StatusCode
}
return shouldRetry(ctx, resp, err)
})
if err != nil {
if ifEmpty && statusCode == http.StatusConflict {
return fs.ErrorDirectoryNotEmpty
}
if pass == 0 {
return fmt.Errorf("couldn't remove folder: %w", err)
}
return fmt.Errorf("couldn't permanently remove folder: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
if result.Permanent && result.Complete {
return nil
}
if pass >= maxFolderPurgePasses {
return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
}
}
}
// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
const maxFolderPurgePasses = 1000
// renameFolder renames a folder
func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
request := api.RenameFolderRequest{Name: newName}
opts := rest.Opts{
Method: "PUT",
Path: "/api/folders/" + folderID + "/rename",
ExtraHeaders: f.byIDHeaders(),
}
var result api.Response
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't rename folder: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
return nil
}
// moveFolder moves a folder to a new parent
func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
request := api.MoveFolderRequest{Name: newName}
if newParentID != "" && newParentID != rootID {
request.ParentID = &newParentID
}
opts := rest.Opts{
Method: "PUT",
Path: "/api/folders/" + folderID + "/move",
ExtraHeaders: f.byIDHeaders(),
}
var result api.MoveFolderResponse
err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return false, fmt.Errorf("couldn't move folder: %w", err)
}
if !result.OK {
return false, fmt.Errorf("API error: %s", result.Error)
}
// Servers that predate the name field ignore it and answer without a
// name, leaving the caller to rename separately
return newName != "" && result.Name == newName, nil
}
// deleteFile deletes a file
//
// ?permanent=1 trashes and purges it in one request on today's API. An
// older server that ignores the flag only soft-deletes it, answering
// permanent=false, so a second DELETE on the now soft-deleted file purges
// it from the database and R2, as it always has.
func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
opts := rest.Opts{
Method: "DELETE",
Path: "/api/files/" + fileID,
Parameters: map[string][]string{"permanent": {"1"}},
ExtraHeaders: f.byIDHeaders(),
}
var result api.DeleteFileResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't delete file: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
// Answered into its own value so a field the second response omits
// can't be read off the first.
if !result.Permanent {
var permanent api.DeleteFileResponse
err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't permanently delete file: %w", err)
}
if !permanent.OK {
return fmt.Errorf("API error: %s", permanent.Error)
}
}
return nil
}
// renameFile renames a file
func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
request := api.RenameFileRequest{Name: newName}
opts := rest.Opts{
Method: "PUT",
Path: "/api/files/" + fileID + "/rename",
ExtraHeaders: f.byIDHeaders(),
}
var result api.RenameFileResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't rename file: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
return nil
}
// moveFile moves a file to a new folder and, if newName is non-empty,
// renames it in the same request
func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
request := api.MoveFileRequest{}
if folderID != "" && folderID != rootID {
request.FolderID = &folderID
}
if newName != "" {
request.Name = &newName
}
opts := rest.Opts{
Method: "PUT",
Path: "/api/files/" + fileID + "/move",
ExtraHeaders: f.byIDHeaders(),
}
var result api.Response
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't move file: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
return nil
}
// copyFile copies a file to a folder
func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
request := api.CopyFileRequest{Name: name}
if folderID != "" && folderID != rootID {
request.FolderID = &folderID
}
opts := rest.Opts{
Method: "POST",
Path: "/api/files/" + fileID + "/copy",
ExtraHeaders: f.byIDHeaders(),
}
var result api.CopyFileResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't copy file: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// downloadFile downloads a file by its ID
//
// The download endpoint answers 302 with a presigned R2 URL in Location, so
// this is two calls. Both go through the rest client, which is built on
// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
// --user-agent, --bind, --dump and the proxy settings apply to downloads as
// they do to every other call, and errorHandler and shouldRetry decide the
// errors and the retries uniformly.
func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
// NoRedirect stops the client following the 302 with the API's
// Authorization header attached. A 302 is not a 2xx, so Call reports it
// as an error (having read and closed the body); the Location header
// is what actually matters, and finding it clears the error, as in
// backend/linkbox.
opts := rest.Opts{
Method: "GET",
Path: "/api/files/" + fileID + "/download",
ExtraHeaders: f.byIDHeaders(),
NoRedirect: true,
}
var downloadURL string
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.Call(ctx, &opts)
if resp != nil {
if location := resp.Header.Get("Location"); location != "" {
downloadURL, err = location, nil
}
}
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't get download URL: %w", err)
}
if downloadURL == "" {
return nil, errors.New("no download URL in redirect response")
}
// Fetch the object from the presigned URL. It carries its own
// credentials and R2 refuses a request that also has an Authorization
// header, so the client-level bearer token is removed by setting the
// header empty: the rest client skips a header with an empty value.
opts = rest.Opts{
Method: "GET",
RootURL: downloadURL,
Options: options,
ExtraHeaders: map[string]string{"Authorization": ""},
}
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
var err error
resp, err = f.rest.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't download file: %w", err)
}
return resp.Body, nil
}
// uploadSmallFile uploads a file smaller than 10MB in a single request
func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
opts := rest.Opts{
Method: "PUT",
Path: "/api/upload/" + sessionID,
Body: in,
ContentLength: &size,
ContentType: "application/octet-stream",
ExtraHeaders: modTimeHeaders(modTime),
}
// The body is a stream which can't be rewound, so a low level retry
// would resend whatever is left of it - nothing - and the server
// would store that as an empty file. Don't retry here; rclone's high
// level retries redo the whole transfer instead.
var result api.UploadCompleteResponse
err := f.pacer.CallNoRetry(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't upload file: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// uploadPart uploads a single part of a multipart upload
func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
size := int64(len(part))
opts := rest.Opts{
Method: "PUT",
Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
ContentLength: &size,
ContentType: "application/octet-stream",
}
var result api.UploadPartResponse
err := f.pacer.Call(func() (bool, error) {
// A fresh reader per attempt so a retry resends the whole part
opts.Body = bytes.NewReader(part)
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// completeUpload finalizes a multipart upload
func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
opts := rest.Opts{
Method: "POST",
Path: "/api/upload/" + sessionID + "/complete",
ExtraHeaders: modTimeHeaders(modTime),
}
var result api.UploadCompleteResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't complete upload: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// initUpload initializes a new upload session
func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
request := api.UploadInitRequest{
WorkspaceID: f.opt.WorkspaceID,
FileName: fileName,
FileSize: fileSize,
MimeType: mimeType,
FileID: fileID,
}
if folderID != "" && folderID != rootID {
request.FolderID = &folderID
}
opts := rest.Opts{
Method: "POST",
Path: "/api/upload/init",
}
var result api.UploadInitResponse
deadline := time.Now().Add(concurrentUploadWaitMax)
for {
result = api.UploadInitResponse{}
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err == nil {
break
}
if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
return nil, fmt.Errorf("couldn't init upload: %w", err)
}
fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
select {
case <-ctx.Done():
return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
case <-time.After(concurrentUploadRetryDelay):
}
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// uploadParts reads the source part by part and uploads the parts.
//
// Part 1 goes on its own: the server creates the R2 multipart upload (and a
// new file's id and key) when it receives the first part, so parts sent
// before that has finished would each start an upload of their own. The
// rest go out opt.UploadConcurrency at a time. A part is only read from the
// source once a buffer is free, so at most that many parts are held in
// memory, and a short source is still found by the reader before the part
// it could not fill is sent.
func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
readPart := func(buf []byte, partNum int) ([]byte, error) {
currentPartSize := partSize
if partNum == totalParts {
currentPartSize = size - partSize*int64(totalParts-1)
}
part := buf[:currentPartSize]
if _, err := io.ReadFull(in, part); err != nil {
if err == io.EOF || err == io.ErrUnexpectedEOF {
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
}
return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
}
return part, nil
}
concurrency := max(f.opt.UploadConcurrency, 1)
buffers := make(chan []byte, concurrency)
buffers <- make([]byte, partSize)
allocated := 1
first := <-buffers
part, err := readPart(first, 1)
if err != nil {
return err
}
if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
return err
}
buffers <- first
// A failed read has to take the parts already in flight down with it,
// or they upload to completion while the error waits on Wait.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
g, gCtx := errgroup.WithContext(ctx)
var readErr error
for partNum := 2; partNum <= totalParts; partNum++ {
// Take a free buffer, allocating one while under the limit.
var buf []byte
select {
case buf = <-buffers:
default:
if allocated < concurrency {
buf = make([]byte, partSize)
allocated++
} else {
select {
case buf = <-buffers:
case <-gCtx.Done():
}
}
}
if buf == nil {
break // a part failed; its error comes back from Wait
}
part, err := readPart(buf, partNum)
if err != nil {
readErr = err
cancel()
break
}
g.Go(func() error {
defer func() { buffers <- buf }()
_, err := f.uploadPart(gCtx, part, sessionID, partNum)
return err
})
}
err = g.Wait()
if readErr != nil {
return readErr
}
return err
}
// abortUpload retires an upload session the client is giving up on.
//
// Best effort: the server reclaims an abandoned session itself, so a failure
// here - including an older server with no such route - costs the slot the
// wait it would have cost anyway and is not worth failing an upload over.
// Sent on a context of its own so a cancelled transfer still gives its slot
// back rather than leaving it to the sweep.
func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
defer cancel()
opts := rest.Opts{
Method: "DELETE",
Path: "/api/upload/" + sessionID,
NoResponse: true,
}
err := f.pacer.CallNoRetry(func() (bool, error) {
resp, err := f.rest.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
if err != nil {
fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
}
}
// uploadFile handles the full upload flow (small or multipart)
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
// Both refusals live here so a new file and a new version of an existing
// one answer the same way: upload/init needs a byte count up front, and
// the server stores no zero-length object.
if size < 0 {
return nil, errors.New("can't upload files with unknown size")
}
if size == 0 {
return nil, fs.ErrorCantUploadEmptyFiles
}
leaf := f.opt.Enc.FromStandardName(path.Base(remote))
initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
if err != nil {
return nil, err
}
// A session abandoned here holds one of the workspace's upload slots
// until the server's sweep retires it ten minutes later, and rclone's
// high-level retry opens a fresh session rather than resuming this one -
// so without this, a few failed attempts use up every slot.
defer func() {
if err != nil {
f.abortUpload(ctx, initResp.SessionID)
}
}()
// Count what the source actually supplies, so a source which ends
// early is reported as an error rather than stored as a truncated
// file.
counter := readers.NewCountingReader(in)
// Small file upload (single PUT, no resumable)
if initResp.Resumable == nil {
resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
if err != nil {
return nil, err
}
if got := int64(counter.BytesRead()); got != size {
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
}
return resp, nil
}
// Multipart upload. Each part is read fully into memory before it is
// sent, so a short source is caught before the part goes out and a
// part which has to be retried is resent with all of its bytes.
partSize := initResp.Resumable.PartSize
totalParts := initResp.Resumable.TotalParts
// Both bounds matter. partSize*(totalParts-1) >= size means a part would
// have nothing to carry; size > partSize*totalParts means the last part
// is bigger than partSize, and readPart would slice past its buffer and
// panic.
if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
}
if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
return nil, err
}
return f.completeUpload(ctx, initResp.SessionID, modTime)
}
// createShareLink creates a public share link for a file
func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
request := api.ShareLinkRequest{}
if expire < fs.DurationOff {
days := int(time.Duration(expire).Hours() / 24)
if days < 1 {
days = 1
}
request.ExpiresInDays = &days
}
opts := rest.Opts{
Method: "POST",
Path: "/api/files/" + fileID + "/share",
ExtraHeaders: f.byIDHeaders(),
}
var result api.ShareLinkResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't create share link: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// getWorkspaceInfo fetches workspace storage info
// listWorkspaces lists the workspaces this credential can reach. An API key
// pinned to a workspace sees exactly one: its own.
func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
opts := rest.Opts{
Method: "GET",
Path: "/api/workspaces",
}
var result api.WorkspaceListResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't list workspaces: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return result.Workspaces, nil
}
func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
opts := rest.Opts{
Method: "GET",
Path: "/api/workspaces/" + f.opt.WorkspaceID,
}
var result api.WorkspaceInfoResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't get workspace info: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
-235
View File
@@ -1,235 +0,0 @@
package dosya
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
status int
header string
want time.Duration
ok bool
}{
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
{"missing header", http.StatusServiceUnavailable, "", 0, false},
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
} {
t.Run(tc.name, func(t *testing.T) {
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
if tc.header != "" {
resp.Header.Set("Retry-After", tc.header)
}
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
assert.True(t, retry)
d, ok := pacer.IsRetryAfter(err)
assert.Equal(t, tc.ok, ok)
if tc.ok {
assert.Equal(t, tc.want, d)
}
})
}
}
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
resp.Header.Set("Retry-After", "30")
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
assert.False(t, retry)
_, ok := pacer.IsRetryAfter(err)
assert.False(t, ok)
}
func TestRemoveFolderPurgesPermanently(t *testing.T) {
// a big subtree purges in bounded passes: 202 complete=false repeats
// the same request until complete=true
answers := []struct {
status int
body string
}{
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
}
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
require.Equal(t, "DELETE", r.Method)
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
a := answers[calls]
calls++
return jsonResp(a.status, a.body), nil
}))
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
assert.Equal(t, len(answers), calls)
}
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
}))
err := f.removeFolder(context.Background(), "fld_1", false)
require.Error(t, err)
assert.Contains(t, err.Error(), "locked file")
assert.Equal(t, 1, calls)
}
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
}))
err := f.removeFolder(context.Background(), "fld_1", true)
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
}
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
}))
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
assert.Equal(t, 1, calls)
}
func TestRmdirMakesOneRequest(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "DELETE", r.Method)
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
}))
f.dirCache = dircache.New("", rootID, f)
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
f.dirCache.Put("sub", "fld_1")
require.NoError(t, f.Rmdir(context.Background(), "sub"))
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
}
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
}))
f.dirCache = dircache.New("", rootID, f)
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
f.dirCache.Put("sub", "fld_1")
err := f.Rmdir(context.Background(), "sub")
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
}
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
for _, tc := range []struct {
name string
permanent bool
wantCalls int
}{
{"today's API purges in one call", true, 1},
{"an older server ignores the flag and still gets a second DELETE", false, 2},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "DELETE", r.Method)
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
}))
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
assert.Equal(t, tc.wantCalls, calls)
})
}
}
// decodeBody reads a JSON request body into a map
func decodeBody(t *testing.T, r *http.Request) map[string]any {
b, err := io.ReadAll(r.Body)
require.NoError(t, err)
var m map[string]any
require.NoError(t, json.Unmarshal(b, &m))
return m
}
func TestMoveFolderSendsName(t *testing.T) {
for _, tc := range []struct {
name string
newName string
respBody string
wantRenamed bool
}{
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
{"older server ignores name", "dst", `{"ok":true}`, false},
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
} {
t.Run(tc.name, func(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
body := decodeBody(t, r)
assert.Equal(t, "fld_parent", body["parent_id"])
if tc.newName == "" {
assert.NotContains(t, body, "name")
} else {
assert.Equal(t, tc.newName, body["name"])
}
return jsonResp(http.StatusOK, tc.respBody), nil
}))
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
require.NoError(t, err)
assert.Equal(t, tc.wantRenamed, renamed)
})
}
}
func TestCopyFileSendsName(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
body := decodeBody(t, r)
assert.Equal(t, "fld_dst", body["folder_id"])
assert.Equal(t, "final.txt", body["name"])
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
}))
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
require.NoError(t, err)
assert.Equal(t, "fil_2", resp.FileID)
assert.Equal(t, "final.txt", resp.Name)
}
-171
View File
@@ -1,171 +0,0 @@
package dosya
import (
"context"
"io"
"net/http"
"strconv"
"strings"
"testing"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
f := &Fs{}
assert.Equal(t, time.Second, f.Precision())
assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
// Exactly SHA-256, nothing else.
assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
}
func TestObjectHashReturnsContentHash(t *testing.T) {
o := &Object{file: api.FileItem{ContentHash: "abc123"}}
got, err := o.Hash(context.Background(), hash.SHA256)
require.NoError(t, err)
assert.Equal(t, "abc123", got)
// A multipart object with no server-side hash yields "" and no error,
// exactly as an S3 multipart object does.
empty := &Object{file: api.FileItem{ContentHash: ""}}
got, err = empty.Hash(context.Background(), hash.SHA256)
require.NoError(t, err)
assert.Equal(t, "", got)
// Any other algorithm is unsupported.
_, err = o.Hash(context.Background(), hash.MD5)
assert.ErrorIs(t, err, hash.ErrUnsupported)
}
func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
assert.True(t, mt.Equal(o.ModTime(context.Background())))
}
func TestModTimeHeaders(t *testing.T) {
assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
h := modTimeHeaders(mt)
require.NotNil(t, h)
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
}
func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
resp := &api.UploadCompleteResponse{}
resp.File.ID = "fil_1"
resp.File.Name = "x.txt"
resp.File.SizeBytes = 10
resp.File.CreatedAt = 1_000
resp.File.ContentHash = "deadbeef"
mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
item := fileFromUpload(resp, mt)
assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
assert.Equal(t, "deadbeef", item.ContentHash)
assert.Equal(t, float64(1_000), item.CreatedAt)
// A zero modtime falls back to the create time.
item = fileFromUpload(resp, time.Time{})
assert.Equal(t, float64(1_000), item.UpdatedAt)
}
// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
// fake transport and asserts the source-mtime header actually rides on the PUT.
func TestUploadSendsSourceMTimeHeader(t *testing.T) {
var putMTime string
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
}
switch {
case r.Method == "POST" && r.URL.Path == "/api/upload/init":
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
putMTime = r.Header.Get(sourceMTimeHeader)
return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
}
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
return nil, nil
})
f := newFakeFs(rt)
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, "h", resp.File.ContentHash)
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
}
// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
// the completion request for a multipart upload (the server stores the modtime
// there, not on the individual parts).
func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
var completeMTime string
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.ReadAll(r.Body)
}
switch {
case r.URL.Path == "/api/upload/init":
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
return jsonResp(http.StatusOK, `{"ok":true}`), nil
case r.URL.Path == "/api/upload/upl_1/complete":
completeMTime = r.Header.Get(sourceMTimeHeader)
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
}
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
return nil, nil
})
f := newFakeFs(rt)
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
}
// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
// server-side Copy returns: it must report the SOURCE's modtime and hash (the
// copy is byte-identical and the API preserves both), not the copy's wall
// clock. Reading "now" here made rclone's in-memory object disagree with the
// one read back from the remote and failed FsCopy's fingerprint check.
func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
switch {
case r.Method == "GET" && r.URL.Path == "/api/files":
// NewObject's existence probe on the destination: nothing there.
return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
}
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
return nil, nil
})
f := newFakeFs(rt)
f.dirCache = dircache.New("", rootID, f)
mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
}}
dst, err := f.Copy(context.Background(), src, "f-copy.txt")
require.NoError(t, err)
assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
assert.Equal(t, int64(100), dst.Size())
got, err := dst.Hash(context.Background(), hash.SHA256)
require.NoError(t, err)
assert.Equal(t, "srchash", got, "copy keeps the source hash")
}
-156
View File
@@ -1,156 +0,0 @@
package dosya
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
func TestErrorHandlerDecodesAPIError(t *testing.T) {
resp := &http.Response{
StatusCode: http.StatusBadRequest,
Status: "400 Bad Request",
Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),
}
err := errorHandler(resp)
var apiErr *api.Error
require.ErrorAs(t, err, &apiErr)
assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
assert.Contains(t, err.Error(), "uploads in progress")
assert.Contains(t, err.Error(), "concurrent_upload_limit")
}
func TestErrorHandlerNonJSONBody(t *testing.T) {
// e.g. an edge (Cloudflare) response that never reached the API
resp := &http.Response{
StatusCode: http.StatusTooManyRequests,
Status: "429 Too Many Requests",
Body: io.NopCloser(strings.NewReader("error code: 1015\n")),
}
err := errorHandler(resp)
var apiErr *api.Error
require.ErrorAs(t, err, &apiErr)
assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
assert.Equal(t, "", apiErr.ErrorCode)
assert.Contains(t, err.Error(), "429")
assert.Contains(t, err.Error(), "error code: 1015")
}
func TestShouldRetryAPIErrors(t *testing.T) {
ctx := context.Background()
// The concurrent upload limit is handled by initUpload's slot wait,
// not by the pacer's low level retries.
limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
assert.False(t, retry)
assert.Equal(t, limitErr, err)
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
assert.True(t, retry)
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
assert.False(t, retry)
}
// swapWaitTimings shortens the slot wait timings for a test and returns a
// function which restores them
func swapWaitTimings(delay, max time.Duration) func() {
oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
}
// newTestFs makes an Fs which talks to srv
func newTestFs(srv *httptest.Server) *Fs {
ctx := context.Background()
f := &Fs{
name: "test",
opt: Options{WorkspaceID: "ws_test"},
pacer: fs.NewPacer(ctx, pacer.NewDefault(
pacer.MinSleep(time.Millisecond),
pacer.MaxSleep(time.Millisecond),
pacer.DecayConstant(decayConstant),
pacer.AttackConstant(attackConstant),
)),
}
f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
return f
}
// limitServer answers the first n init calls with the concurrent upload
// limit error and the rest with success
func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/api/upload/init", r.URL.Path)
w.Header().Set("Content-Type", "application/json")
if calls.Add(1) <= n {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(concurrentLimitBody))
return
}
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"ok":true}`))
}))
}
func TestInitUploadWaitsForFreeSlot(t *testing.T) {
defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
var calls atomic.Int32
srv := limitServer(t, 2, &calls)
defer srv.Close()
f := newTestFs(srv)
resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
require.NoError(t, err)
assert.True(t, resp.OK)
assert.Equal(t, int32(3), calls.Load())
}
func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
var calls atomic.Int32
srv := limitServer(t, 1<<30, &calls)
defer srv.Close()
f := newTestFs(srv)
_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
require.Error(t, err)
var apiErr *api.Error
require.ErrorAs(t, err, &apiErr)
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
assert.Contains(t, err.Error(), "couldn't init upload")
assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
}
func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
defer swapWaitTimings(time.Minute, time.Hour)()
var calls atomic.Int32
srv := limitServer(t, 1<<30, &calls)
defer srv.Close()
f := newTestFs(srv)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
start := time.Now()
_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, context.DeadlineExceeded)
assert.Less(t, time.Since(start), 5*time.Second)
}
-68
View File
@@ -1,68 +0,0 @@
package dosya
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeListServer serves GET /api/files the way the API does: files paged at
// per_page, every folder repeated on each page
func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
return func(r *http.Request) (*http.Response, error) {
require.Equal(t, "/api/files", r.URL.Path)
q := r.URL.Query()
assert.Equal(t, "name", q.Get("sort"))
assert.Equal(t, "asc", q.Get("dir"))
page, _ := strconv.Atoi(q.Get("page"))
perPage, _ := strconv.Atoi(q.Get("per_page"))
require.Positive(t, page)
require.Positive(t, perPage)
*pages = append(*pages, page)
var files []string
for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
}
totalPages := max(1, (totalFiles+perPage-1)/perPage)
body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
strings.Join(files, ","), page, perPage, totalFiles, totalPages)
return jsonResp(http.StatusOK, body), nil
}
}
func TestListFilesAndFoldersFollowsPages(t *testing.T) {
var pages []int
f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
require.NoError(t, err)
assert.Equal(t, []int{1, 2, 3}, pages)
require.Len(t, listing.Files, 2*listPageSize+7)
assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
// folders repeat on every page and must not be duplicated
assert.Len(t, listing.Folders, 1)
}
func TestListFilesAndFoldersSinglePage(t *testing.T) {
for _, total := range []int{0, 1, listPageSize} {
t.Run(strconv.Itoa(total), func(t *testing.T) {
var pages []int
f := newFakeFs(fakeListServer(t, total, &pages))
listing, err := f.listFilesAndFolders(context.Background(), rootID)
require.NoError(t, err)
assert.Equal(t, []int{1}, pages)
assert.Len(t, listing.Files, total)
})
}
}
-153
View File
@@ -1,153 +0,0 @@
package dosya
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// parallelUploadServer answers the multipart endpoints without holding a
// lock across a request, so parts really do overlap, and records what the
// ordering rules below are about: how many parts were in flight at once, and
// whether any part started before part 1 had finished.
//
// Part 1 matters because the API creates the R2 multipart upload (and, for a
// new file, its id and key) on the first part it sees. Parts that race ahead
// of part 1 would each create their own upload, and only one of those would
// be completed.
type parallelUploadServer struct {
t *testing.T
resumable string
partDelay time.Duration
failPart int // answer this part with a 400, 0 for none
mu sync.Mutex
inFlight int
maxFlight int
part1Done bool
earlyParts []int // parts that started before part 1 finished
parts map[int]string
completes int
aborts int
}
func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
var body []byte
if r.Body != nil {
var err error
body, err = io.ReadAll(r.Body)
require.NoError(s.t, err)
}
path := r.URL.Path
switch {
case r.Method == "POST" && path == "/api/upload/init":
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
var n int
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
require.NoError(s.t, err)
s.mu.Lock()
if n != 1 && !s.part1Done {
s.earlyParts = append(s.earlyParts, n)
}
s.inFlight++
s.maxFlight = max(s.maxFlight, s.inFlight)
s.mu.Unlock()
time.Sleep(s.partDelay)
s.mu.Lock()
s.inFlight--
if n == 1 {
s.part1Done = true
}
s.parts[n] = string(body)
fail := n == s.failPart
s.mu.Unlock()
if fail {
return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
}
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
s.mu.Lock()
s.completes++
s.mu.Unlock()
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
case r.Method == "DELETE" && path == "/api/upload/upl_1":
s.mu.Lock()
s.aborts++
s.mu.Unlock()
return jsonResp(http.StatusOK, `{"ok":true}`), nil
}
s.t.Fatalf("unexpected request %s %s", r.Method, path)
return nil, nil
}
func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
return &parallelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
}
const tenParts = `{"part_size":2,"total_parts":10}`
func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
srv := newParallelServer(t, tenParts)
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 4
content := "aabbccddeeffgghhiijj"
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
assert.Equal(t, 1, srv.completes)
for i := 1; i <= 10; i++ {
assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
}
}
func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
srv := newParallelServer(t, tenParts)
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 1
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, 1, srv.maxFlight)
assert.Equal(t, 1, srv.completes)
}
func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
srv := newParallelServer(t, tenParts)
srv.failPart = 6
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 4
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "part 6")
assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
}
func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
// 20 bytes declared, 15 supplied: the short read is found by the
// reader, whichever parts are already in flight, and nothing completes.
srv := newParallelServer(t, tenParts)
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 4
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.Equal(t, 0, srv.completes)
assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
}
-15
View File
@@ -1,15 +0,0 @@
// Test Dosya filesystem interface
package dosya
import (
"testing"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestDosya:",
})
}
-247
View File
@@ -1,247 +0,0 @@
package dosya
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"sync"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// rtFunc adapts a function into an http.RoundTripper so the upload path
// can be driven without a network. Unlike a real server it always reads
// the whole request body and answers, which is exactly the misbehaviour
// (accepting a short body as a complete upload) these tests guard against.
type rtFunc func(*http.Request) (*http.Response, error)
func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func jsonResp(status int, body string) *http.Response {
return &http.Response{
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
Header: http.Header{"Content-Type": {"application/json"}},
Body: io.NopCloser(strings.NewReader(body)),
ContentLength: int64(len(body)),
}
}
func newFakeFs(rt http.RoundTripper) *Fs {
ctx := context.Background()
f := &Fs{
name: "test",
opt: Options{WorkspaceID: "ws_test"},
pacer: fs.NewPacer(ctx, pacer.NewDefault(
pacer.MinSleep(time.Millisecond),
pacer.MaxSleep(time.Millisecond),
pacer.DecayConstant(decayConstant),
pacer.AttackConstant(attackConstant),
)),
}
f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
return f
}
// fakeUploadServer records what an upload sends and lets a test inject
// failures per request
type fakeUploadServer struct {
t *testing.T
mu sync.Mutex
resumable string // JSON for the init "resumable" field, "" for a single PUT
puts [][]byte // bodies of PUT /api/upload/<id>
parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
completes int
aborts int // DELETE /api/upload/<id>
failFirst map[string]bool // path -> answer the first request with a 502
seenBefore map[string]bool
}
func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
return &fakeUploadServer{
t: t,
resumable: resumable,
parts: map[int][][]byte{},
failFirst: map[string]bool{},
seenBefore: map[string]bool{},
}
}
func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
s.mu.Lock()
defer s.mu.Unlock()
var body []byte
if r.Body != nil {
var err error
body, err = io.ReadAll(r.Body)
require.NoError(s.t, err)
}
path := r.URL.Path
// Bodies are recorded for every attempt, including the ones answered
// with a 502, so tests can see exactly what each attempt carried.
fail := s.failFirst[path] && !s.seenBefore[path]
s.seenBefore[path] = true
bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
switch {
case r.Method == "POST" && path == "/api/upload/init":
if s.resumable != "" {
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
}
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
case r.Method == "PUT" && path == "/api/upload/upl_1":
s.puts = append(s.puts, body)
if fail {
return bad, nil
}
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
var n int
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
require.NoError(s.t, err)
s.parts[n] = append(s.parts[n], body)
if fail {
return bad, nil
}
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
s.completes++
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
case r.Method == "DELETE" && path == "/api/upload/upl_1":
s.aborts++
return jsonResp(http.StatusOK, `{"ok":true}`), nil
}
s.t.Fatalf("unexpected request %s %s", r.Method, path)
return nil, nil
}
func TestUploadSmallSendsWholeBody(t *testing.T) {
srv := newFakeUploadServer(t, "")
f := newFakeFs(rtFunc(srv.roundTrip))
content := strings.Repeat("a", 100)
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, "fil_1", resp.File.ID)
require.Len(t, srv.puts, 1)
assert.Equal(t, content, string(srv.puts[0]))
}
func TestUploadSmallRejectsShortSource(t *testing.T) {
// The source declares 100 bytes but only supplies 60. A server which
// accepts that as a complete upload must not turn into a success.
srv := newFakeUploadServer(t, "")
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
}
func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
// A retried PUT would resend an already-consumed reader, i.e. an
// empty body, which the server would store as an empty file. The
// single PUT therefore must not be retried at the low level.
srv := newFakeUploadServer(t, "")
srv.failFirst["/api/upload/upl_1"] = true
f := newFakeFs(rtFunc(srv.roundTrip))
content := strings.Repeat("a", 100)
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
assert.Equal(t, content, string(srv.puts[0]))
}
const threeParts = `{"part_size":4,"total_parts":3}`
func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
srv := newFakeUploadServer(t, threeParts)
f := newFakeFs(rtFunc(srv.roundTrip))
content := "0123456789"
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, "fil_1", resp.File.ID)
assert.Equal(t, 1, srv.completes)
require.Len(t, srv.parts, 3)
assert.Equal(t, "0123", string(srv.parts[1][0]))
assert.Equal(t, "4567", string(srv.parts[2][0]))
assert.Equal(t, "89", string(srv.parts[3][0]))
}
func TestUploadMultipartRejectsShortSource(t *testing.T) {
// 10 bytes declared, 8 supplied: the last part must never be sent
// and the upload must not be completed.
srv := newFakeUploadServer(t, threeParts)
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
assert.Equal(t, 0, srv.completes, "upload must not be completed")
}
func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
// A part which fails transiently must be resent with all its bytes,
// not with whatever is left of a consumed reader.
srv := newFakeUploadServer(t, threeParts)
srv.failFirst["/api/upload/upl_1/part/2"] = true
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
assert.Equal(t, "4567", string(srv.parts[2][0]))
assert.Equal(t, "4567", string(srv.parts[2][1]))
assert.Equal(t, "89", string(srv.parts[3][0]))
assert.Equal(t, 1, srv.completes)
}
// An upload that fails after init has to hand its session back: the session
// holds one of the workspace's upload slots for ten minutes otherwise, and
// rclone's high-level retry opens a new one rather than resuming it.
func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
for _, tc := range []struct {
name string
resumable string
content string
size int64
}{
{"small file short source", "", "0123", 10},
{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
} {
t.Run(tc.name, func(t *testing.T) {
srv := newFakeUploadServer(t, tc.resumable)
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.Equal(t, 1, srv.aborts)
})
}
}
// A successful upload must not abort anything
func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
srv := newFakeUploadServer(t, "")
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, 0, srv.aborts)
}
@@ -1,73 +0,0 @@
package dosya
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Every by-id call names the workspace, so the API can find an id it minted
// moments ago before its file_home index row lands.
func TestByIDCallsSendWorkspaceHint(t *testing.T) {
seen := map[string]string{}
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
switch r.URL.Path {
case "/api/folders/fld_1":
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
case "/api/files/fil_1":
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
case "/api/folders/fld_1/move":
return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
default:
return jsonResp(http.StatusOK, `{"ok":true}`), nil
}
}))
ctx := context.Background()
require.NoError(t, f.removeFolder(ctx, "fld_1", false))
require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
require.NoError(t, err)
require.NoError(t, f.deleteFile(ctx, "fil_1"))
require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
require.NoError(t, err)
_, err = f.createShareLink(ctx, "fil_1", 0)
require.NoError(t, err)
for call, header := range seen {
assert.Equal(t, "ws_test", header, call)
}
assert.Len(t, seen, 8)
}
// The download's API request carries the hint; the presigned R2 fetch must not.
func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
var apiHint, blobHint string
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/files/fil_1/download" {
apiHint = r.Header.Get("X-Dosya-Workspace")
http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
return
}
blobHint = r.Header.Get("X-Dosya-Workspace")
_, _ = w.Write([]byte("bytes"))
}))
defer srv.Close()
f := newTestFs(srv)
f.opt.APIURL = srv.URL
rc, err := f.downloadFile(context.Background(), "fil_1", nil)
require.NoError(t, err)
_ = rc.Close()
assert.Equal(t, "ws_test", apiHint)
assert.Equal(t, "", blobHint)
}
-52
View File
@@ -1,52 +0,0 @@
package dosya
import (
"context"
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeWorkspaceServer answers GET /api/workspaces with the given body
func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
return func(r *http.Request) (*http.Response, error) {
require.Equal(t, "/api/workspaces", r.URL.Path)
*calls++
return jsonResp(http.StatusOK, body), nil
}
}
func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
calls := 0
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
f.opt.WorkspaceID = ""
require.NoError(t, f.findWorkspace(context.Background()))
assert.Equal(t, "ws_only", f.opt.WorkspaceID)
assert.Equal(t, 1, calls)
}
func TestFindWorkspaceListsTheChoices(t *testing.T) {
calls := 0
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
f.opt.WorkspaceID = ""
err := f.findWorkspace(context.Background())
require.Error(t, err)
// the ids have to be in the message: they are the only place the user
// can read them, which is the whole point of listing them
assert.Contains(t, err.Error(), "ws_one (One)")
assert.Contains(t, err.Error(), "ws_two (Two)")
assert.Empty(t, f.opt.WorkspaceID)
}
func TestFindWorkspaceWithNone(t *testing.T) {
calls := 0
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
f.opt.WorkspaceID = ""
require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
}
+17 -33
View File
@@ -692,27 +692,13 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// deleteObject removes an object by ID
func (f *Fs) deleteObject(ctx context.Context, id string) error {
if f.opt.HardDelete {
// Deleting forever doesn't invalidate the server's cache of
// the parent folder listing, so the entry keeps being listed
// for about a second. Moving to the trash first does.
err := f.deleteEntry(ctx, id, false)
if err != nil {
return err
}
}
return f.deleteEntry(ctx, id, f.opt.HardDelete)
}
// deleteEntry moves the entry with ID to the trash or deletes it forever
func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) error {
opts := rest.Opts{
Method: "POST",
Path: "/file-entries/delete",
}
request := api.DeleteRequest{
EntryIDs: []string{id},
DeleteForever: deleteForever,
DeleteForever: f.opt.HardDelete,
}
var result api.DeleteResponse
err := f.pacer.Call(func() (bool, error) {
@@ -1054,28 +1040,17 @@ func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err
}
// copy and rename a file or folder to directoryID with leaf
//
// If existing is set it is removed once the copy has succeeded.
func (f *Fs) copyTo(ctx context.Context, srcID, dstLeaf, dstDirectoryID string, existing fs.Object) (info *api.Item, err error) {
func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
// Can have duplicates so don't have to be careful here
// Copy to dstDirectoryID first
info, err = f.copy(ctx, srcID, dstDirectoryID)
if err != nil {
return nil, err
}
// The server names the copy "name (1)" if the destination
// directory already has an entry called "name" and refuses to
// rename it to "name" until that entry is removed.
if existing != nil {
fs.Debugf(existing, "Server side copy: removing existing object after successful copy")
err = existing.Remove(ctx)
if err != nil {
return nil, err
}
}
// Rename if required
if info.Name != f.opt.Enc.FromStandardName(dstLeaf) {
if srcLeaf != dstLeaf {
info, err = f.rename(ctx, info.ID.String(), dstLeaf)
if err != nil {
return nil, err
@@ -1099,6 +1074,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
srcLeaf := path.Base(srcObj.remote)
srcPath := srcObj.fs.rootSlash() + srcObj.remote
dstPath := f.rootSlash() + remote
if srcPath == dstPath {
@@ -1107,8 +1084,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
// Find existing object
existingObj, err := f.NewObject(ctx, remote)
if err != nil {
existingObj = nil
if err == nil {
defer func() {
// Don't remove existing object if returning an error
if err != nil {
return
}
fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
err = existingObj.Remove(ctx)
}()
}
// Create temporary object
@@ -1118,7 +1102,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
}
// Copy the object
info, err := f.copyTo(ctx, srcObj.id, dstLeaf, dstDirectoryID, existingObj)
info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
if err != nil {
return nil, err
}
+8 -11
View File
@@ -634,8 +634,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return f, nil // our root it empty so we probably want to list shared folders
}
dir := path.Dir(f.root)
if dir == "." {
dir = f.root
}
// root is not empty so we have find the right shared folder if it exists
id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
id, err := f.findSharedFolder(ctx, dir)
if err != nil {
// if we didn't find the specified shared folder we have to bail out here
return nil, err
@@ -943,21 +948,13 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
return nil
}
// sharedFolderName returns the shared folder name in root, which is its first
// path component. root must be the trimmed root as produced by setRoot (no
// leading slash).
func sharedFolderName(root string) string {
name, _, _ := strings.Cut(root, "/")
return name
}
// findSharedFolder find the id for a given shared folder name
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
// so our only option is to iterate over all shared folders
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
errFoundFile := errors.New("found file")
err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
if entry.(*fs.Dir).Remote() == name {
id = entry.(*fs.Dir).ID()
return errFoundFile
}
@@ -1039,7 +1036,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
errFoundFile := errors.New("found file")
err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
if strings.EqualFold(entry.(*Object).remote, name) {
if entry.(*Object).remote == name {
o = entry.(*Object)
return errFoundFile
}
-60
View File
@@ -188,21 +188,6 @@ func TestInternalCheckPathLength(t *testing.T) {
}
}
func TestInternalSharedFolderName(t *testing.T) {
for _, test := range []struct {
root string
want string
}{
{root: "", want: ""},
{root: "SharedFolder", want: "SharedFolder"},
{root: "SharedFolder/subdir", want: "SharedFolder"},
{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
} {
assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
}
}
func TestPaperExportRemote(t *testing.T) {
ctx := context.Background()
info := &files.FileMetadata{
@@ -490,48 +475,3 @@ func TestFindSharedFileResolvesDecodedName(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, encodedName, o.remote)
}
// sharedFoldersHandler serves a single shared folder from list_folders.
func sharedFoldersHandler(name, id string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.URL.Path, "list_folders") {
http.NotFound(w, r)
return
}
resp := sharing.ListFoldersResult{
Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(resp)
}
}
// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
// lookup must match regardless of the case of the requested name.
func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))
for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
id, err := f.findSharedFolder(context.Background(), name)
require.NoError(t, err, name)
assert.Equal(t, "folder-id", id, name)
}
_, err := f.findSharedFolder(context.Background(), "no-such-folder")
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
}
// The Dropbox backend advertises CaseInsensitive: true, so a received file
// lookup must match regardless of the case of the requested name.
func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))
for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
o, err := f.findSharedFile(context.Background(), name)
require.NoError(t, err, name)
assert.Equal(t, "TestFile.txt", o.remote, name)
}
_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}
+2 -5
View File
@@ -16,16 +16,13 @@ import (
// multipartInit starts a new multipart upload and returns server details.
func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
encodedFolderPath := f.fromStandardPath(folderPath)
encodedFilename := f.fromStandardPath(filename)
opts := rest.Opts{
Method: "GET",
Path: "/multipart/init",
Parameters: url.Values{
"key": {f.opt.Key},
"filename": {encodedFilename},
"folder_path": {encodedFolderPath},
"filename": {filename},
"folder_path": {folderPath},
},
}
+7 -16
View File
@@ -642,7 +642,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
chunkReader *pool.RW
bytesRead int64
)
chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)
chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize))
if bytesRead < int64(chunkSize) {
startMoreTransfers = false
}
@@ -869,23 +869,14 @@ func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Ac
// along with the number of bytes that have been read from reader.
// Reaching the end of reader early is not an error.
//
// If accountUpload is set, any accounting on reader is applied
// when the buffer is read instead, so the upload is what gets accounted.
//
// This must not be set for a buffer which creates or replaces a file
// in a single request. Accounting can fail part way through the upload,
// for example when --max-transfer is reached, and HiDrive keeps
// what it has received of an aborted request as the file.
//
// Any accounting on reader is applied when the buffer is read instead,
// so the upload is what gets accounted.
// The caller must Close the buffer when done with it.
func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {
func readerForChunk(reader io.Reader, length int64) (chunkReader *pool.RW, bytesRead int64, err error) {
reader, acc := unwrapAccounting(reader)
chunkReader = multipart.NewRW()
if accountUpload {
var acc *accounting.Account
reader, acc = unwrapAccounting(reader)
if acc != nil {
chunkReader.SetAccounting(acc.AccountRead)
}
if acc != nil {
chunkReader.SetAccounting(acc.AccountRead)
}
bytesRead, err = io.CopyN(chunkReader, reader, length)
+2 -2
View File
@@ -522,7 +522,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
if size := src.Size(); size >= 0 && size < prefixSize {
prefixSize = size
}
cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)
cutoffReader, bytesRead, err := readerForChunk(in, prefixSize)
if err != nil {
return nil, err
}
@@ -980,7 +980,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
} else {
var content *pool.RW
content, _, err = readerForChunk(in, src.Size(), false)
content, _, err = readerForChunk(in, src.Size())
if err == nil {
info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
_ = content.Close()
+30 -10
View File
@@ -283,7 +283,7 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
if err != nil {
return false, err
}
u, err := rest.URLJoinRoot(base, f.root)
u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))
if err != nil {
return false, err
}
@@ -526,20 +526,40 @@ func addHeaders(req *http.Request, opt *Options) {
// rest.ErrHTTPSDowngrade and strips the configured headers when the
// redirect chain has left the originally requested host at any point.
func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
headers := make([]string, 0, len(opt.Headers)/2)
for i := 0; i < len(opt.Headers); i += 2 {
headers = append(headers, opt.Headers[i])
}
stripHeaders := rest.StripHeadersOnCrossHostRedirectFn(headers...)
return func(req *http.Request, via []*http.Request) error {
err := stripHeaders(req, via)
if errors.Is(err, rest.ErrHTTPSDowngrade) {
err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
if err := rest.RefuseHTTPSDowngradeRedirectFn(req, via); err != nil {
if errors.Is(err, rest.ErrHTTPSDowngrade) {
err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
}
return err
}
return err
if redirectLeavesHost(req, via) {
for i := 0; i < len(opt.Headers); i += 2 {
req.Header.Del(opt.Headers[i])
}
}
return nil
}
}
// redirectLeavesHost reports whether any hop in the redirect chain
// via plus the pending request req is to a different host from the
// original request via[0].
//
// net/http copies the headers afresh from the original request for
// every hop, so once the chain has visited another host the headers
// must be stripped from every subsequent hop, even one back to the
// original host, as the other host chose the URL.
func redirectLeavesHost(req *http.Request, via []*http.Request) bool {
origin := via[0].URL
for _, hop := range via {
if !rest.SameHost(hop.URL, origin) {
return true
}
}
return !rest.SameHost(req.URL, origin)
}
// Adds the configured headers to the request if any
func (f *Fs) addHeaders(req *http.Request) {
addHeaders(req, &f.opt)
-31
View File
@@ -760,34 +760,3 @@ func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
}
}
func TestRootCannotChangeHost(t *testing.T) {
// A root starting with "//" is a network-path reference under the
// URL standard. It must not be able to move the remote (and the
// configured headers) to a host other than the configured one.
var requests int
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
_, _ = w.Write([]byte("hello"))
}))
defer other.Close()
m := prepareServer(t)
otherHost := strings.TrimPrefix(other.URL, "http://")
for _, root := range []string{
"//" + otherHost + "/",
"//" + otherHost + "/one%.txt",
"//" + otherHost,
"//user:pass@" + otherHost + "/",
} {
f, err := NewFs(context.Background(), remoteName, root, m)
assert.ErrorContains(t, err, "must not change the scheme, host or user", root)
assert.Nil(t, f, root)
}
assert.Equal(t, 0, requests, "request sent to the other host")
// A same-origin absolute path must still work
f, err := NewFs(context.Background(), remoteName, "/", m)
require.NoError(t, err)
assert.Equal(t, m["url"]+"/", f.String())
}
+1 -1
View File
@@ -92,7 +92,7 @@ func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, fou
item, resp, err = f.service.GetItemByPath(ctx, path.Join(f.root, dir))
return shouldRetry(ctx, resp, err)
}); err != nil {
if item == nil && resp != nil && resp.StatusCode == 404 {
if item == nil && resp.StatusCode == 404 {
return nil, false, nil
}
return nil, false, err
+2 -2
View File
@@ -477,7 +477,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
FolderPath: f.EncodePath(path.Join(f.root, dir)),
})
if res != nil && res.StatusCode == http.StatusNotFound {
if res.StatusCode == http.StatusNotFound {
return false, fs.ErrorDirNotFound
}
@@ -502,7 +502,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
FolderPath: f.EncodePath(remote),
})
if res != nil && res.StatusCode == http.StatusNotFound {
if res.StatusCode == http.StatusNotFound {
return false, fs.ErrorDirNotFound
}
+111 -238
View File
@@ -474,10 +474,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
if err != nil {
return err
}
for _, e := range childFolders {
if !recent.stale(e.UUID, id, e.PlainName) {
return fs.ErrorDirectoryNotEmpty
}
if len(childFolders) > 0 {
return fs.ErrorDirectoryNotEmpty
}
var childFiles []folders.File
@@ -489,10 +487,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
if err != nil {
return err
}
for _, e := range childFiles {
if !recent.stale(e.UUID, id, joinNameExt(e.PlainName, e.Type)) {
return fs.ErrorDirectoryNotEmpty
}
if len(childFiles) > 0 {
return fs.ErrorDirectoryNotEmpty
}
// Delete the directory
@@ -506,7 +502,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
if err != nil {
return err
}
recent.deleted(id)
f.dirCache.FlushDir(dir)
return nil
@@ -525,9 +520,6 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, e
return "", false, err
}
for _, e := range entries {
if recent.stale(e.UUID, pathID, e.PlainName) {
continue
}
if f.opt.Encoding.ToStandardName(e.PlainName) == leaf {
return e.UUID, true, nil
}
@@ -564,166 +556,46 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error)
return resp.UUID, nil
}
// splitNameExt splits a basename into the (plainName, type) pair that Internxt
// stores a file under.
func splitNameExt(baseName string) (name, ext string) {
ext = path.Ext(baseName)
if ext == baseName {
return baseName, ""
// preUploadCheck checks if a file exists in the given directory
// Returns the file metadata if it exists, nil if not
func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
// Parse name and extension from the leaf
baseName := f.opt.Encoding.FromStandardName(leaf)
name := strings.TrimSuffix(baseName, path.Ext(baseName))
ext := strings.TrimPrefix(path.Ext(baseName), ".")
checkResult, err := files.CheckFilesExistence(ctx, f.cfg, directoryID, []files.FileExistenceCheck{
{
PlainName: name,
Type: ext,
OriginalFile: struct{}{},
},
})
if err != nil {
// If existence check fails, assume file doesn't exist to allow upload to proceed
return nil, nil
}
return strings.TrimSuffix(baseName, ext), strings.TrimPrefix(ext, ".")
}
// joinNameExt reassembles the basename a stored (plainName, type) pair
// displays as, the way List builds its entries.
func joinNameExt(name, ext string) string {
if ext == "" {
return name
}
return name + "." + ext
}
if len(checkResult.Files) > 0 && checkResult.Files[0].FileExists() {
result := checkResult.Files[0]
if result.Type != ext {
return nil, nil
}
// legacyNameExt splits a basename at the final dot, the way this backend used
// to before it adopted the convention the other clients share. It differs from
// splitNameExt only for a name whose sole dot leads it: ".bashrc" splits into
// an empty name of type "bashrc".
func legacyNameExt(baseName string) (name, ext string) {
return strings.TrimSuffix(baseName, path.Ext(baseName)), strings.TrimPrefix(path.Ext(baseName), ".")
}
existingUUID := result.UUID
if existingUUID != "" {
fileMeta, err := files.GetFileMeta(ctx, f.cfg, existingUUID)
if err == nil && fileMeta != nil {
return convertFileMetaToFile(fileMeta), nil
}
// existenceCheck builds a lookup criterion for one (plainName, type) spelling.
func existenceCheck(name, ext string) files.FileExistenceCheck {
return files.FileExistenceCheck{
PlainName: name,
Type: ext,
OriginalFile: struct{}{},
}
}
// staleWindow is how long the API may carry on returning an item from
// where it was before this process moved or deleted it.
const staleWindow = time.Minute
// location is where this process last put an item. An empty parentUUID
// means the item was deleted.
type location struct {
parentUUID string
name string // encoded name, including any extension
expires time.Time
}
// recentChanges holds the files and folders this process has moved or
// deleted recently, keyed by UUID.
//
// The API serves lookups and listings from read replicas which lag behind
// writes, so an item can be returned from its old location for a while after
// it was moved or deleted. Acting on such a stale entry can be destructive,
// eg overwriting it would delete the file from where it was moved to.
type recentChanges struct {
mu sync.Mutex
items map[string]location
lastPrune time.Time
}
// recent is shared by all the Fs in the process as UUIDs are global.
var recent = recentChanges{items: map[string]location{}}
// set records the new location of uuid, pruning expired entries.
func (r *recentChanges) set(uuid string, loc location) {
r.mu.Lock()
defer r.mu.Unlock()
now := time.Now()
if now.Sub(r.lastPrune) > staleWindow {
for k, v := range r.items {
if now.After(v.expires) {
delete(r.items, k)
if err != nil {
return nil, err
}
}
r.lastPrune = now
}
loc.expires = now.Add(staleWindow)
r.items[uuid] = loc
}
// moved records that uuid was moved to name in parentUUID.
func (r *recentChanges) moved(uuid, parentUUID, name string) {
r.set(uuid, location{parentUUID: parentUUID, name: name})
}
// deleted records that uuid was deleted.
func (r *recentChanges) deleted(uuid string) {
r.set(uuid, location{})
}
// stale reports whether an API result saying uuid is called name in
// parentUUID is out of date because of a change made by this process.
func (r *recentChanges) stale(uuid, parentUUID, name string) bool {
r.mu.Lock()
defer r.mu.Unlock()
loc, ok := r.items[uuid]
if !ok || time.Now().After(loc.expires) {
return false
}
return loc.parentUUID != parentUUID || loc.name != name
}
// findFile looks up a single file by name within directoryID. Returns
// (nil, nil) when the file does not exist; surfaces transport/API errors.
func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
encodedName := f.opt.Encoding.FromStandardName(leaf)
name, ext := splitNameExt(encodedName)
checks := []files.FileExistenceCheck{existenceCheck(name, ext)}
if encodedName != name {
checks = append(checks, existenceCheck(encodedName, ""))
}
if legacyName, legacyExt := legacyNameExt(encodedName); legacyExt != "" && legacyName != name {
checks = append(checks, existenceCheck(legacyName, legacyExt))
}
var checkResult *files.CheckFilesExistenceResponse
err := f.pacer.Call(func() (bool, error) {
var err error
checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, checks)
if isNotFoundError(err) {
return true, err
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
var uuid string
for _, result := range checkResult.Files {
if !result.FileExists() || result.UUID == "" {
continue
}
if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {
fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)
continue
}
if joinNameExt(result.PlainName, result.Type) == encodedName {
uuid = result.UUID
break
}
}
if uuid == "" {
return nil, nil
}
var fileMeta *files.FileMeta
err = f.pacer.Call(func() (bool, error) {
var err error
fileMeta, err = files.GetFileMeta(ctx, f.cfg, uuid)
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
if fileMeta == nil {
return nil, nil
}
return convertFileMetaToFile(fileMeta), nil
return nil, nil
}
// convertFileMetaToFile converts files.FileMeta to folders.File
@@ -761,9 +633,6 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
return nil, err
}
for _, e := range foldersList {
if recent.stale(e.UUID, dirID, e.PlainName) {
continue
}
remote := path.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
out = append(out, fs.NewDir(remote, e.ModificationTime))
}
@@ -781,9 +650,6 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
if len(e.Type) > 0 {
remote += "." + e.Type
}
if recent.stale(e.UUID, dirID, remote) {
continue
}
remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))
out = append(out, newObjectWithFile(f, remote, &e))
}
@@ -808,7 +674,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
return nil, err
}
existingFile, err := f.findFile(ctx, leaf, directoryID)
// Check if file already exists
existingFile, err := f.preUploadCheck(ctx, leaf, directoryID)
if err != nil {
return nil, err
}
@@ -856,28 +723,44 @@ func (f *Fs) Remove(ctx context.Context, remote string) error {
if err != nil {
return err
}
recent.deleted(dirID)
f.dirCache.FlushDir(remote)
return nil
}
// NewObject creates a new object by looking up a single file's metadata.
// NewObject creates a new object
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
return nil, fs.ErrorObjectNotFound
}
return nil, err
parentDir := path.Dir(remote)
if parentDir == "." {
parentDir = ""
}
file, err := f.findFile(ctx, leaf, directoryID)
dirID, err := f.dirCache.FindDir(ctx, parentDir, false)
if err != nil {
return nil, err
}
if file == nil {
return nil, fs.ErrorObjectNotFound
}
return newObjectWithFile(f, remote, file), nil
var files []folders.File
err = f.pacer.Call(func() (bool, error) {
var err error
files, err = folders.ListAllFiles(ctx, f.cfg, dirID)
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
targetName := path.Base(remote)
for _, e := range files {
name := e.PlainName
if len(e.Type) > 0 {
name += "." + e.Type
}
decodedName := f.opt.Encoding.ToStandardName(name)
if decodedName == targetName {
return newObjectWithFile(f, remote, &e), nil
}
}
return nil, fs.ErrorObjectNotFound
}
// newObjectWithFile returns a new object by file info
@@ -981,23 +864,19 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return nil, err
}
newName, newType := splitNameExt(f.opt.Encoding.FromStandardName(leaf))
// Parse name and extension from the leaf
baseName := f.opt.Encoding.FromStandardName(leaf)
newName := strings.TrimSuffix(baseName, path.Ext(baseName))
newType := strings.TrimPrefix(path.Ext(baseName), ".")
// Move the file server-side
err = f.pacer.Call(func() (bool, error) {
err := files.MoveFile(ctx, f.cfg, srcObj.uuid, directoryID, newName, newType)
// The move is checked against lagging read replicas so it can
// fail with 404 if the destination directory was just created
// or 409 if a file at the destination was just deleted.
if err != nil && isStaleReadError(err, true) {
return true, err
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))
dstObj := &Object{
f: f,
@@ -1033,17 +912,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
encodedLeaf := f.opt.Encoding.FromStandardName(dstLeaf)
err = f.pacer.Call(func() (bool, error) {
err := folders.MoveFolder(ctx, f.cfg, srcID, dstDirectoryID, encodedLeaf)
// Moving a large folder can outlast the gateway timeout (520/502)
// yet still complete, in which case the retry reports this.
if err != nil && isConflictError(err) && strings.Contains(err.Error(), "already moved to that location") {
return false, nil
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
recent.moved(srcID, dstDirectoryID, encodedLeaf)
srcFs.dirCache.FlushDir(srcRemote)
return nil
@@ -1091,7 +964,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
remote := o.remote
origName, origType := splitNameExt(path.Base(remote))
origBaseName := path.Base(remote)
origName := strings.TrimSuffix(origBaseName, path.Ext(origBaseName))
origType := strings.TrimPrefix(path.Ext(origBaseName), ".")
// Create directory if it doesn't exist
_, dirID, err := o.f.dirCache.FindPath(ctx, remote, true)
@@ -1108,7 +983,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Step 1: If file exists, rename to backup (preserves old file during upload)
if oldUUID != "" {
name, ext := splitNameExt(path.Base(remote))
// Generate unique backup name
baseName := path.Base(remote)
name := strings.TrimSuffix(baseName, path.Ext(baseName))
ext := strings.TrimPrefix(path.Ext(baseName), ".")
backupSuffix := fmt.Sprintf(".rclone-backup-%s", random.String(8))
backupName = o.f.opt.Encoding.FromStandardName(name + backupSuffix)
backupType = ext
@@ -1181,11 +1060,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return o.f.tooLargeError(remote, tooLarge)
}
if err != nil && isStaleReadError(err, backupUUID != "") {
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return fserrors.RetryError(err)
}
if err != nil {
meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
}
@@ -1223,7 +1097,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
backupName, backupType, backupUUID, err)
// Don't fail the upload just because backup deletion failed
} else {
recent.deleted(backupUUID)
fs.Debugf(o.f, "Successfully deleted backup file")
}
}
@@ -1243,19 +1116,6 @@ func isTimeoutError(err error) bool {
return false
}
// isNotFoundError reports whether err is a 404 from the API.
func isNotFoundError(err error) bool {
var httpErr *sdkerrors.HTTPError
return errors.As(err, &httpErr) && httpErr.StatusCode() == 404
}
// isStaleReadError reports whether err is the API contradicting a write this
// backend has already made, which it does because lookups are served by read
// replicas that lag behind the primary.
func isStaleReadError(err error, renamed bool) bool {
return isNotFoundError(err) || (renamed && isConflictError(err))
}
// isConflictError checks if an error indicates a file conflict (409)
func isConflictError(err error) bool {
errMsg := err.Error()
@@ -1296,22 +1156,40 @@ func (o *Object) recoverFromTimeoutConflict(ctx context.Context, uploadErr error
}
baseName := path.Base(remote)
encodedName := o.f.opt.Encoding.FromStandardName(baseName)
existingFile, err := o.f.findFile(ctx, baseName, dirID)
if err != nil || existingFile == nil {
var meta *buckets.CreateMetaResponse
checkErr := o.f.pacer.Call(func() (bool, error) {
existingFile, err := o.f.preUploadCheck(ctx, encodedName, dirID)
if err != nil {
return o.f.shouldRetry(ctx, err)
}
if existingFile != nil {
name := strings.TrimSuffix(baseName, path.Ext(baseName))
ext := strings.TrimPrefix(path.Ext(baseName), ".")
meta = &buckets.CreateMetaResponse{
UUID: existingFile.UUID,
FileID: existingFile.FileID,
Name: name,
PlainName: name,
Type: ext,
Size: existingFile.Size,
}
o.id = existingFile.FileID
}
return false, nil
})
if checkErr != nil {
return nil, uploadErr
}
name, ext := splitNameExt(baseName)
o.id = existingFile.FileID
return &buckets.CreateMetaResponse{
UUID: existingFile.UUID,
FileID: existingFile.FileID,
Name: name,
PlainName: name,
Type: ext,
Size: existingFile.Size,
}, nil
if meta != nil {
return meta, nil
}
return nil, uploadErr
}
// restoreBackupFile restores a backup file after upload failure
@@ -1329,13 +1207,8 @@ func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, or
// Remove deletes a file
func (o *Object) Remove(ctx context.Context) error {
err := o.f.pacer.Call(func() (bool, error) {
return o.f.pacer.Call(func() (bool, error) {
err := files.DeleteFile(ctx, o.f.cfg, o.uuid)
return o.f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
recent.deleted(o.uuid)
return nil
}
-191
View File
@@ -1,191 +0,0 @@
package internxt
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/internxt/rclone-adapter/config"
"github.com/internxt/rclone-adapter/endpoints"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// storedFile is a file as the service holds it: a (plainName, type) pair
// split however the client that uploaded it chose to split it.
type storedFile struct {
uuid string
plainName string
fileType string
}
// wireType renders a stored type the way the service does: a file with no
// extension comes back as JSON null, not an empty string.
func wireType(fileType string) any {
if fileType == "" {
return nil
}
return fileType
}
// existenceServer answers the existence check and file meta endpoints from
// files, matching plainName exactly and only constraining type when the
// criterion supplies one, as the service does.
func existenceServer(t *testing.T, stored []storedFile) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/meta") {
uuid := strings.Split(strings.TrimPrefix(r.URL.Path, "/drive/files/"), "/")[0]
for _, f := range stored {
if f.uuid == uuid {
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
}))
return
}
}
w.WriteHeader(http.StatusNotFound)
return
}
var req struct {
Files []struct {
PlainName string `json:"plainName"`
Type string `json:"type"`
} `json:"files"`
}
require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
var existent []map[string]any
for _, f := range stored {
for _, c := range req.Files {
if c.PlainName == f.plainName && (c.Type == "" || c.Type == f.fileType) {
existent = append(existent, map[string]any{
"exists": true, "status": "EXISTS",
"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
})
break
}
}
}
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"existentFiles": existent}))
}))
}
func testFs(t *testing.T, server *httptest.Server) *Fs {
t.Helper()
f := &Fs{
opt: Options{Encoding: encoder.EncodeInvalidUtf8},
cfg: &config.Config{
HTTPClient: server.Client(),
Endpoints: endpoints.NewConfig(server.URL),
},
}
f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
return f
}
func TestSplitJoinNameExt(t *testing.T) {
for _, test := range []struct {
baseName string
name string
ext string
}{
{"foo.txt", "foo", "txt"},
{"foo.tar.gz", "foo.tar", "gz"},
{"README", "README", ""},
{".bashrc", ".bashrc", ""},
{"..hidden", ".", "hidden"},
{"", "", ""},
} {
name, ext := splitNameExt(test.baseName)
assert.Equal(t, test.name, name, test.baseName)
assert.Equal(t, test.ext, ext, test.baseName)
assert.Equal(t, test.baseName, joinNameExt(name, ext), test.baseName)
}
}
func TestFindFile(t *testing.T) {
for _, test := range []struct {
what string
stored []storedFile
leaf string
wantUUID string
}{{
what: "split as rclone splits it",
stored: []storedFile{{"uuid-1", "foo.tar", "gz"}},
leaf: "foo.tar.gz",
wantUUID: "uuid-1",
}, {
what: "whole name stored as plainName by another client",
stored: []storedFile{{"uuid-2", "foo.tar.gz", ""}},
leaf: "foo.tar.gz",
wantUUID: "uuid-2",
}, {
what: "extensionless name is not confused with an extended one",
stored: []storedFile{{"uuid-3", "README", "md"}, {"uuid-4", "README", ""}},
leaf: "README",
wantUUID: "uuid-4",
}, {
what: "dotfile stored the way the web client splits it",
stored: []storedFile{{"uuid-6", ".bashrc", ""}},
leaf: ".bashrc",
wantUUID: "uuid-6",
}, {
what: "dotfile stored the way older rclone versions split it",
stored: []storedFile{{"uuid-7", "", "bashrc"}},
leaf: ".bashrc",
wantUUID: "uuid-7",
}, {
what: "trailing dot kept in plainName by a rename",
stored: []storedFile{{"uuid-8", "trailing.", ""}},
leaf: "trailing.",
wantUUID: "uuid-8",
}, {
what: "missing file",
stored: []storedFile{{"uuid-5", "other", "txt"}},
leaf: "foo.txt",
}} {
t.Run(test.what, func(t *testing.T) {
server := existenceServer(t, test.stored)
defer server.Close()
file, err := testFs(t, server).findFile(context.Background(), test.leaf, "dir-uuid")
require.NoError(t, err)
if test.wantUUID == "" {
assert.Nil(t, file)
return
}
require.NotNil(t, file)
assert.Equal(t, test.wantUUID, file.UUID)
})
}
}
func TestFindFileIgnoresStale(t *testing.T) {
ctx := context.Background()
server := existenceServer(t, []storedFile{{"uuid-moved", "foo", "txt"}, {"uuid-deleted", "bar", "txt"}, {"uuid-renamed", "baz", "txt"}})
defer server.Close()
f := testFs(t, server)
recent.moved("uuid-moved", "other-dir-uuid", "foo.txt.bak")
recent.deleted("uuid-deleted")
recent.moved("uuid-renamed", "dir-uuid", "baz.txt")
file, err := f.findFile(ctx, "foo.txt", "dir-uuid")
require.NoError(t, err)
assert.Nil(t, file, "moved away")
file, err = f.findFile(ctx, "bar.txt", "dir-uuid")
require.NoError(t, err)
assert.Nil(t, file, "deleted")
file, err = f.findFile(ctx, "baz.txt", "dir-uuid")
require.NoError(t, err)
require.NotNil(t, file, "moved here")
assert.Equal(t, "uuid-renamed", file.UUID)
}
+4 -1
View File
@@ -7,6 +7,7 @@ import (
"io"
"path"
"sort"
"strings"
"sync"
"github.com/internxt/rclone-adapter/buckets"
@@ -262,7 +263,9 @@ func (w *internxtChunkWriter) Close(ctx context.Context) error {
}
// Create file metadata in Internxt Drive
name, ext := splitNameExt(w.f.opt.Encoding.FromStandardName(path.Base(w.remote)))
baseName := w.f.opt.Encoding.FromStandardName(path.Base(w.remote))
name := strings.TrimSuffix(baseName, path.Ext(baseName))
ext := strings.TrimPrefix(path.Ext(baseName), ".")
var meta *buckets.CreateMetaResponse
err = w.f.pacer.Call(func() (bool, error) {
+35 -123
View File
@@ -12,7 +12,6 @@ import (
"path"
"path/filepath"
"runtime"
"slices"
"strings"
"sync"
"sync/atomic"
@@ -37,10 +36,6 @@ import (
const (
devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
// maxLinkTargetSize is the largest symlink target accepted when
// translating a .rclonelink object, comfortably above any OS path
// limit (Windows allows 32767 UTF-16 units).
maxLinkTargetSize = 128 * 1024
)
// timeType allows the user to choose what exactly ModTime() returns
@@ -316,10 +311,10 @@ set this flag.`,
Name: "fatal_if_no_space",
Help: `Make out-of-space errors fatal during transfers.
When enabled, an out of space error while writing, creating a directory,
or creating a file returns a fatal error so that rclone aborts rather
than retrying the operation. Useful for backup scripts that should halt
loudly on a full disk rather than spin retrying.`,
When enabled, an ENOSPC error during a write returns a fatal error so
that rclone aborts rather than retrying the operation. Useful for
backup scripts that should halt loudly on a full disk rather than spin
retrying.`,
Default: false,
Advanced: true,
},
@@ -443,9 +438,6 @@ var (
errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
errLinksNeedsSuffix = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
errPathEscapes = errors.New("file name is not a path within the local root - check the encoding")
errSymlinkLoop = errors.New("loop detected: points to a parent directory")
errLinkTargetTooLong = errors.New("symlink target is too long to be a path")
errLinkNameInvalid = errors.New("translated link needs a file name before the \"" + fs.LinkSuffix + "\" suffix")
)
// NewFs constructs an Fs from the path
@@ -573,26 +565,30 @@ func (f *Fs) caseInsensitive() bool {
return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
}
// translateLink checks whether the remote is a translated link
// and returns a new path, removing the suffix as needed,
// It also returns whether this is a translated link at all
//
// for regular files, localPath is returned unchanged
func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
isTranslatedLink = strings.HasSuffix(remote, fs.LinkSuffix)
newLocalPath = strings.TrimSuffix(localPath, fs.LinkSuffix)
return newLocalPath, isTranslatedLink
}
// newObject makes a half completed Object
func (f *Fs) newObject(remote string) (*Object, error) {
name, translatedLink := remote, false
if f.opt.TranslateSymlinks {
name, translatedLink = strings.CutSuffix(remote, fs.LinkSuffix)
}
if translatedLink {
// Check this makes a valid native file name
native := f.nativePath(name)
switch native[strings.LastIndex(native, string(filepath.Separator))+1:] {
case "", ".", "..":
return nil, fserrors.NoRetryError(fmt.Errorf("%q: %w", remote, errLinkNameInvalid))
}
}
// Check the name after any suffix removal
localPath, err := f.localPath(name)
translatedLink := false
localPath, err := f.localPath(remote)
if err != nil {
return nil, err
}
if f.opt.TranslateSymlinks {
// Possibly receive a new name for localPath
localPath, translatedLink = translateLink(remote, localPath)
}
return &Object{
fs: f,
remote: remote,
@@ -641,23 +637,15 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
}
// Create new directory object from the info passed in
//
// Directories are never translated links, so the name is used as is
// whatever suffix it has.
func (f *Fs) newDirectory(dir string, fi os.FileInfo) (*Directory, error) {
localPath, err := f.localPath(dir)
o, err := f.newObject(dir)
if err != nil {
return nil, err
}
d := &Directory{
Object: Object{
fs: f,
remote: dir,
path: localPath,
},
}
d.setMetadata(fi)
return d, nil
o.setMetadata(fi)
return &Directory{
Object: *o,
}, nil
}
// List the objects and directories in dir into entries. The
@@ -680,7 +668,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if err != nil {
return nil, fs.ErrorDirNotFound
}
var parents []os.FileInfo // fsDirPath and its parents, read on first use
fd, err := os.Open(fsDirPath)
if err != nil {
@@ -761,28 +748,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if f.opt.FollowSymlinks && (mode&symlinkFlag) != 0 {
localPath := filepath.Join(fsDirPath, name)
fi, err = os.Stat(localPath)
// A directory symlink pointing to a parent would be followed
// until the OS returns ELOOP, which can take a very long time
if err == nil && fi.IsDir() {
if parents == nil {
parents = statParents(fsDirPath)
}
if slices.ContainsFunc(parents, func(parent os.FileInfo) bool { return os.SameFile(fi, parent) }) {
// Quietly skip loops the directory filters exclude as
// the layer above wouldn't recurse into them
if useFilter {
if include, dirErr := filter.IncludeDirectory(ctx, f)(newRemote); dirErr == nil && !include {
continue
}
}
err = errSymlinkLoop
}
}
// Quietly skip errors on excluded files and directories
if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
continue
}
if os.IsNotExist(err) || isCircularSymlinkError(err) || errors.Is(err, errSymlinkLoop) {
if os.IsNotExist(err) || isCircularSymlinkError(err) {
// Skip bad symlinks and circular symlinks
err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
fs.Errorf(newRemote, "Listing error: %v", err)
@@ -815,12 +785,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
continue
}
fso, err := f.newObjectWithInfo(newRemote, fi)
if errors.Is(err, errLinkNameInvalid) {
// Skip a badly named ".rclonelink" file
fs.Errorf(newRemote, "Listing error: %v", err)
_ = accounting.Stats(ctx).Error(err)
continue
}
if err != nil {
return nil, err
}
@@ -833,21 +797,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return entries, nil
}
// statParents returns the info for dirPath and each of its parents up
// to the root of the file system, skipping any which can't be read.
func statParents(dirPath string) (parents []os.FileInfo) {
for {
if fi, err := os.Stat(dirPath); err == nil {
parents = append(parents, fi)
}
parent := filepath.Dir(dirPath)
if parent == dirPath {
return parents
}
dirPath = parent
}
}
func (f *Fs) cleanRemote(dir, filename string) (remote string) {
if f.opt.UTFNorm {
filename = norm.NFC.String(filename)
@@ -865,19 +814,14 @@ func (f *Fs) cleanRemote(dir, filename string) (remote string) {
return
}
// nativePath returns name in the OS encoding with OS path separators,
// relative to f.root, without cleaning it.
func (f *Fs) nativePath(name string) string {
return filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
}
// localPath returns the OS path for the object called name, which is
// always underneath f.root.
//
// It returns errPathEscapes if name resolves outside f.root which can
// happen depending on the encoding.
func (f *Fs) localPath(name string) (string, error) {
localPath := filepath.Join(f.root, f.nativePath(name))
native := filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
localPath := filepath.Join(f.root, native)
rel, err := filepath.Rel(f.root, localPath)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return "", fserrors.NoRetryError(fmt.Errorf("%q: %w", name, errPathEscapes))
@@ -1690,17 +1634,12 @@ func isDiskFullError(err error) bool {
return errors.Is(err, file.ErrDiskFull) || fserrors.IsErrNoSpace(err)
}
func wrapFatalIfNoSpace(err error, enabled bool) error {
if err != nil && enabled && isDiskFullError(err) {
return fserrors.FatalError(err)
}
return err
}
// Update the object from in with modTime and size
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
defer func() {
err = wrapFatalIfNoSpace(err, o.fs.opt.FatalIfNoSpace)
if err != nil && o.fs.opt.FatalIfNoSpace && isDiskFullError(err) {
err = fserrors.FatalError(err)
}
}()
var out io.WriteCloser
var hasher *hash.MultiHasher
@@ -1757,9 +1696,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
out = f
} else {
// The target is buffered in memory, so stop reading just past
// the longest acceptable one rather than trust the source
in = io.LimitReader(in, maxLinkTargetSize+1)
out = nopWriterCloser{&symlinkData}
}
@@ -1776,9 +1712,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
if o.translatedLink {
if err == nil && symlinkData.Len() > maxLinkTargetSize {
err = fserrors.NoRetryError(errLinkTargetTooLong)
}
if err == nil {
// Use the contents of the copied object to create a symlink,
// without following or creating it through a planted symlink
@@ -1829,30 +1762,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
var sparseWarning sync.Once
type fatalIfNoSpaceWriterAt struct {
fs.WriterAtCloser
enabled bool
}
func (w *fatalIfNoSpaceWriterAt) WriteAt(p []byte, off int64) (n int, err error) {
n, err = w.WriterAtCloser.WriteAt(p, off)
return n, wrapFatalIfNoSpace(err, w.enabled)
}
func (w *fatalIfNoSpaceWriterAt) Close() error {
return wrapFatalIfNoSpace(w.WriterAtCloser.Close(), w.enabled)
}
// OpenWriterAt opens with a handle for random access writes
//
// Pass in the remote desired and the size if known.
//
// It truncates any existing object
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (writer fs.WriterAtCloser, err error) {
defer func() {
err = wrapFatalIfNoSpace(err, f.opt.FatalIfNoSpace)
}()
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
// Temporary Object under construction
o, err := f.newObject(remote)
if err != nil {
@@ -1890,10 +1805,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (write
}
}
return &fatalIfNoSpaceWriterAt{
WriterAtCloser: out,
enabled: f.opt.FatalIfNoSpace,
}, nil
return out, nil
}
// setMetadata sets the file info from the os.FileInfo passed in
+1 -85
View File
@@ -60,7 +60,7 @@ func TestIsDiskFullError(t *testing.T) {
// FatalIfNoSpace setting, returning the error from Update.
func updateWithReader(t *testing.T, fatalIfNoSpace bool, readerErr error) error {
t.Helper()
r := fstest.NewRunIndividual(t)
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
f.opt.FatalIfNoSpace = fatalIfNoSpace
@@ -96,87 +96,3 @@ func TestUpdateFatalIfNoSpaceOnButNotDiskFull(t *testing.T) {
require.Error(t, err)
assert.False(t, fserrors.IsFatalError(err), "non-disk-full errors must not be fatal regardless of option")
}
type diskFullWriterAtCloser struct {
writeErr error
closeErr error
n int
data []byte
offset int64
closed bool
}
func (w *diskFullWriterAtCloser) WriteAt(p []byte, off int64) (int, error) {
w.data = append([]byte(nil), p...)
w.offset = off
return w.n, w.writeErr
}
func (w *diskFullWriterAtCloser) Close() error {
w.closed = true
return w.closeErr
}
func testOpenWriterAtError(t *testing.T, injected error, diskFull bool) {
t.Helper()
for _, enabled := range []bool{false, true} {
t.Run(fmt.Sprintf("enabled=%v", enabled), func(t *testing.T) {
r := fstest.NewRunIndividual(t)
f := r.Flocal.(*Fs)
f.opt.FatalIfNoSpace = enabled
f.opt.NoPreAllocate = true
writer, err := f.OpenWriterAt(context.Background(), "test.txt", 0)
require.NoError(t, err)
wrapped, ok := writer.(*fatalIfNoSpaceWriterAt)
require.True(t, ok)
require.NoError(t, wrapped.WriterAtCloser.Close())
wantN := 2
if injected == nil {
wantN = 4
}
underlying := &diskFullWriterAtCloser{writeErr: injected, closeErr: injected, n: wantN}
wrapped.WriterAtCloser = underlying
n, err := writer.WriteAt([]byte("data"), 17)
assert.Equal(t, wantN, n)
assert.Equal(t, []byte("data"), underlying.data)
assert.Equal(t, int64(17), underlying.offset)
assert.ErrorIs(t, err, injected)
assert.Equal(t, enabled && diskFull, fserrors.IsFatalError(err))
err = writer.Close()
assert.True(t, underlying.closed)
assert.ErrorIs(t, err, injected)
assert.Equal(t, enabled && diskFull, fserrors.IsFatalError(err))
})
}
}
func TestOpenWriterAtFatalIfNoSpace(t *testing.T) {
for _, test := range []struct {
name string
err error
diskFull bool
}{
{"success", nil, false},
{"unrelated", syscall.EPERM, false},
{"ENOSPC", syscall.ENOSPC, true},
{"ErrDiskFull", file.ErrDiskFull, true},
{"wrapped", fmt.Errorf("write: %w", syscall.ENOSPC), true},
} {
t.Run(test.name, func(t *testing.T) { testOpenWriterAtError(t, test.err, test.diskFull) })
}
}
func TestOpenWriterAtSetupError(t *testing.T) {
for _, enabled := range []bool{false, true} {
t.Run(fmt.Sprintf("enabled=%v", enabled), func(t *testing.T) {
r := fstest.NewRunIndividual(t)
f := r.Flocal.(*Fs)
f.opt.FatalIfNoSpace = enabled
require.NoError(t, os.WriteFile(filepath.Join(r.LocalName, "parent"), nil, 0600))
writer, err := f.OpenWriterAt(context.Background(), "parent/child", 0)
require.Error(t, err)
assert.Nil(t, writer)
assert.False(t, fserrors.IsFatalError(err))
})
}
}
@@ -1,52 +0,0 @@
//go:build windows
package local
import (
"os"
"testing"
"github.com/rclone/rclone/fs/fserrors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/windows"
)
func TestUpdateFatalIfNoSpaceWindows(t *testing.T) {
tests := []struct {
name string
err error
}{
{"ERROR_DISK_FULL", windows.ERROR_DISK_FULL},
{"openat PathError", &os.PathError{Op: "openat", Path: "test.txt", Err: windows.ERROR_DISK_FULL}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Run("off", func(t *testing.T) {
err := updateWithReader(t, false, test.err)
require.Error(t, err)
assert.False(t, fserrors.IsFatalError(err))
})
t.Run("on", func(t *testing.T) {
err := updateWithReader(t, true, test.err)
require.Error(t, err)
assert.True(t, fserrors.IsFatalError(err))
})
})
}
}
func TestOpenWriterAtFatalIfNoSpaceWindows(t *testing.T) {
for _, test := range []struct {
name string
err error
diskFull bool
}{
{"ERROR_DISK_FULL", windows.ERROR_DISK_FULL, true},
{"ERROR_HANDLE_DISK_FULL", windows.ERROR_HANDLE_DISK_FULL, true},
{"PathError", &os.PathError{Op: "write", Path: "test.txt", Err: windows.ERROR_DISK_FULL}, true},
{"ERROR_ACCESS_DENIED", windows.ERROR_ACCESS_DENIED, false},
} {
t.Run(test.name, func(t *testing.T) { testOpenWriterAtError(t, test.err, test.diskFull) })
}
}
-291
View File
@@ -3,7 +3,6 @@ package local
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"math"
@@ -24,7 +23,6 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/file"
@@ -272,29 +270,6 @@ func linksMode(f *Fs) {
f.lstat = os.Lstat
}
// TestSymlinkTargetTooLong checks that a .rclonelink object whose body is
// far bigger than any path is refused without being buffered in memory,
// and that nothing is created at the destination.
func TestSymlinkTargetTooLong(t *testing.T) {
skipIfNoSymlinks(t)
ctx := context.Background()
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
// A source which never ends, so the read must be bounded
src := object.NewStaticObjectInfo("big"+fs.LinkSuffix, fstest.Time("2001-02-03T04:05:10Z"), -1, true, nil, nil)
in := readers.NewCountingReader(readers.NewPatternReader(1 << 40))
_, err := f.Put(ctx, in, src)
require.ErrorIs(t, err, errLinkTargetTooLong)
assert.True(t, fserrors.IsNoRetryError(err))
assert.LessOrEqual(t, in.BytesRead(), uint64(maxLinkTargetSize+1), "read more of the body than needed")
_, err = os.Lstat(filepath.Join(f.root, "big"))
assert.True(t, os.IsNotExist(err), "nothing should be created for a refused symlink")
}
// TestSymlinkEscapeWriteThroughBlocked mirrors the GHSA-cf44-9pgv-m4xc PoC: a
// malicious --links source serves "pwn.rclonelink" whose body is a path outside
// the destination, plus a sibling "pwn/authkeys" that sorts after it and would
@@ -646,154 +621,6 @@ func TestLocalPath(t *testing.T) {
}
}
// TestLinkSuffixEscapeBlocked checks that in --links mode a name which
// passes the root check with its ".rclonelink" suffix can't escape the
// destination once the suffix is removed: "...rclonelink" must not
// become "..", the parent of the root.
func TestLinkSuffixEscapeBlocked(t *testing.T) {
skipIfNoSymlinks(t)
ctx := context.Background()
outer := t.TempDir()
newLinksFs := func(dir, encoding string) *Fs {
fRaw, err := NewFs(ctx, "local", filepath.Join(outer, dir), configmap.Simple{"encoding": encoding})
require.NoError(t, err)
f := fRaw.(*Fs)
linksMode(f)
require.NoError(t, f.Mkdir(ctx, ""))
return f
}
f := newLinksFs("dst", encoder.OS.String())
s := newLinksFs("slash", "Slash")
// Stamp the parent of the destinations so a write to it shows up.
before := fstest.Time("2011-12-25T12:59:59Z")
require.NoError(t, os.Chtimes(outer, before, before))
parentUnchanged := func() {
t.Helper()
fi, err := os.Lstat(outer)
require.NoError(t, err)
assert.True(t, fi.ModTime().Equal(before), "the parent of the destination was modified")
}
// A directory called "...rclonelink" is created under its own name
// and its metadata goes there, not to the parent of the root which
// trimming the suffix off its path would address.
remote := ".." + fs.LinkSuffix
metadata := fs.Metadata{"mtime": "2001-02-03T04:05:06Z"}
_, err := f.MkdirMetadata(ctx, remote, metadata)
require.NoError(t, err)
parentUnchanged()
// A file called "...rclonelink" is a link called "..", which the
// default encoding stores as fullwidth dots like any other name.
require.NoError(t, putFile(ctx, f, remote, "target"))
fi, err := os.Lstat(filepath.Join(f.root, ".."))
require.NoError(t, err)
assert.NotZero(t, fi.Mode()&os.ModeSymlink)
parentUnchanged()
// Ordinary translated links still resolve to the file without the suffix.
o, err := f.newObject("sub/link" + fs.LinkSuffix)
require.NoError(t, err)
assert.True(t, o.translatedLink)
assert.Equal(t, filepath.Join(f.root, "sub", "link"), o.path)
// A name which merely contains the suffix is a regular file.
o, err = f.newObject("sub/link" + fs.LinkSuffix + ".txt")
require.NoError(t, err)
assert.False(t, o.translatedLink)
assert.Equal(t, filepath.Join(f.root, "sub", "link"+fs.LinkSuffix+".txt"), o.path)
// An encoding without Dot leaves ".." as a real ".." once the suffix
// is gone, so the name is refused instead.
require.ErrorIs(t, putFile(ctx, s, remote, "PWNED"), errLinkNameInvalid)
_, err = s.MkdirMetadata(ctx, remote, metadata)
require.NoError(t, err)
parentUnchanged()
// As is any other name which doesn't leave a file name once the
// suffix is removed and the encoding applied.
for _, name := range []string{
remote,
"sub/.." + fs.LinkSuffix,
"sub/." + fs.LinkSuffix,
"sub/.." + fs.LinkSuffix,
"sub/sub2/.." + fs.LinkSuffix,
"sub/." + fs.LinkSuffix,
"sub/" + fs.LinkSuffix,
fs.LinkSuffix,
} {
_, err = s.newObject(name)
assert.ErrorIs(t, err, errLinkNameInvalid, name)
assert.True(t, fserrors.IsNoRetryError(err), name)
}
// The root check still applies to the name of the link.
_, err = s.newObject("../link" + fs.LinkSuffix)
require.ErrorIs(t, err, errPathEscapes)
if runtime.GOOS == "windows" {
// A backslash is a path separator on Windows, so an encoding
// without BackSlash can hide ".." behind one.
w := newLinksFs("win", "Slash,Dot")
_, err = w.newObject(`sub\..` + fs.LinkSuffix)
assert.ErrorIs(t, err, errLinkNameInvalid)
}
}
// TestLinkSuffixListSkipsInvalidName checks that with --links a regular
// file whose name is just the suffix is reported as an error and skipped
// rather than failing the listing of its whole directory.
func TestLinkSuffixListSkipsInvalidName(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
when := fstest.Time("2001-02-03T04:05:06Z")
r.WriteFile("sub/"+fs.LinkSuffix, "not a link", when)
r.WriteFile("sub/file.txt", "hello", when)
accounting.Stats(ctx).ResetErrors()
entries, err := f.List(ctx, "sub")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, "sub/file.txt", entries[0].Remote())
assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors())
accounting.Stats(ctx).ResetErrors()
}
// TestLinkSuffixDirectory checks that in --links mode a directory whose
// name ends in ".rclonelink" is an ordinary directory. Only files are
// translated into links, so the suffix must stay in its path rather
// than making it address a sibling.
func TestLinkSuffixDirectory(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
f := r.Flocal.(*Fs)
linksMode(f)
dir := "sub" + fs.LinkSuffix
want := filepath.Join(f.root, dir)
mtime := fstest.Time("2001-02-03T04:05:06Z")
d, err := f.MkdirMetadata(ctx, dir, fs.Metadata{"mtime": mtime.Format(time.RFC3339Nano)})
require.NoError(t, err)
assert.Equal(t, want, d.(*Directory).path)
fi, err := os.Lstat(want)
require.NoError(t, err)
assert.True(t, fi.IsDir())
assert.True(t, fi.ModTime().Equal(mtime), "metadata was not written to the directory")
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
got, ok := entries[0].(*Directory)
require.True(t, ok)
assert.Equal(t, dir, got.Remote())
assert.Equal(t, want, got.path)
assert.False(t, got.translatedLink)
}
func TestHashWithTypeNone(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
@@ -1349,121 +1176,3 @@ func TestCopySymlink(t *testing.T) {
want = fstest.NewItem("dst2/file.txt", "hello world", when)
fstest.CompareItems(t, []fs.DirEntry{dst}, []fstest.Item{want}, nil, f.precision, "")
}
// TestCopyLinksLoop checks that -L/--copy-links follows a symlink to a
// sibling directory but skips symlinks which point to the directory
// being listed or one of its parents, as these would loop forever.
func TestCopyLinksLoop(t *testing.T) {
skipIfNoSymlinks(t)
dir := t.TempDir()
// folder-A/link-to-B and folder-B/link-to-A point at each other
// and folder-A/link-to-parent points at dir
require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-A"), 0700))
require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-B"), 0700))
require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-A", "file-A"), []byte("A"), 0600))
require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-B", "file-B"), []byte("B"), 0600))
require.NoError(t, os.Symlink(filepath.Join("..", "folder-B"), filepath.Join(dir, "folder-A", "link-to-B")))
require.NoError(t, os.Symlink(filepath.Join("..", "folder-A"), filepath.Join(dir, "folder-B", "link-to-A")))
require.NoError(t, os.Symlink("..", filepath.Join(dir, "folder-A", "link-to-parent")))
wantRoot := []string{
"folder-A",
"folder-A/file-A",
"folder-A/link-to-B",
"folder-A/link-to-B/file-B",
"folder-B",
"folder-B/file-B",
"folder-B/link-to-A",
"folder-B/link-to-A/file-A",
}
for _, test := range []struct {
name string
root string
exclude string
want []string
wantErrors int64
}{{
name: "Root",
root: dir,
want: wantRoot,
// folder-A/link-to-parent, folder-A/link-to-B/link-to-A,
// folder-B/link-to-A/link-to-B, folder-B/link-to-A/link-to-parent
wantErrors: 4,
}, {
// link-to-parent points above the root of the Fs
name: "SubDir",
root: filepath.Join(dir, "folder-A"),
want: []string{
"file-A",
"link-to-B",
"link-to-B/file-B",
},
// link-to-parent, link-to-B/link-to-A
wantErrors: 2,
}, {
// Excluded loops aren't reported
name: "Excluded",
root: dir,
exclude: "link-to-parent/**",
want: wantRoot,
// folder-A/link-to-B/link-to-A, folder-B/link-to-A/link-to-B
wantErrors: 2,
}} {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
if test.exclude != "" {
var fi *filter.Filter
ctx, fi = filter.AddConfig(ctx)
require.NoError(t, fi.AddRule("- "+test.exclude))
}
f, err := NewFs(ctx, "local", test.root, configmap.Simple{
"copy_links": "true",
})
require.NoError(t, err)
accounting.Stats(ctx).ResetErrors()
defer accounting.Stats(ctx).ResetErrors()
var got []string
err = walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error {
for _, entry := range entries {
got = append(got, entry.Remote())
}
// Stop a runaway listing rather than waiting for ELOOP
if len(got) > 100 {
return errors.New("too many entries - symlink loop not detected")
}
return nil
})
require.NoError(t, err)
sort.Strings(got)
assert.Equal(t, test.want, got)
assert.Equal(t, test.wantErrors, accounting.Stats(ctx).GetErrors(), "global errors found")
})
}
// A single List reports and skips the link to a parent, but still
// returns the link to a sibling
t.Run("List", func(t *testing.T) {
ctx := context.Background()
f, err := NewFs(ctx, "local", dir, configmap.Simple{
"copy_links": "true",
})
require.NoError(t, err)
accounting.Stats(ctx).ResetErrors()
defer accounting.Stats(ctx).ResetErrors()
entries, err := f.List(ctx, "folder-A")
require.NoError(t, err)
var got []string
for _, entry := range entries {
got = append(got, entry.Remote())
}
sort.Strings(got)
assert.Equal(t, []string{"folder-A/file-A", "folder-A/link-to-B"}, got)
assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
})
}
+5 -7
View File
@@ -236,9 +236,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, fmt.Errorf("couldn't parse URL %q: %w", opt.Endpoint, err)
}
u, err := rest.URLJoinRoot(base, root)
u, err := rest.URLJoin(base, rest.URLPathEscape(root))
if err != nil {
return nil, err
return nil, fmt.Errorf("couldn't join URL %q and %q: %w", base.String(), root, err)
}
client := fshttp.NewClient(ctx)
@@ -573,12 +573,10 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
fs.Errorf(nil, "Unable to parse URL %q: %v", f.endpointURL, err)
return fs.ErrorDirNotFound
}
// A bad resume start must fail the listing rather than
// return fs.ErrorDirNotFound which sync treats as an empty
// directory
resumeURL, err := rest.URLJoinRoot(u, resumeStart)
resumeURL, err := rest.URLJoin(u, rest.URLPathEscape(resumeStart))
if err != nil {
return fmt.Errorf("unable to join URL %q for resumeStart %q: %w", f.endpointURL, resumeStart, err)
fs.Errorf(nil, "Unable to join URL %q for resumeStart %s: %v", f.endpointURL, resumeStart, err)
return fs.ErrorDirNotFound
}
URL = resumeURL.String()
}
@@ -1,61 +0,0 @@
package netstorage
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestListResumeCannotChangeHost(t *testing.T) {
// The resume start returned by a list request comes from the
// server. It must not be able to move the signed continuation
// request to another host.
var requests int
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
http.Error(w, "not found", http.StatusNotFound)
}))
defer other.Close()
otherHost := strings.TrimPrefix(other.URL, "http://")
var listRequests int
configured := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
action := r.Header.Get("X-Akamai-ACS-Action")
switch {
case strings.Contains(action, "action=stat"):
_, _ = w.Write([]byte(`<stat directory="/123456/dir"><file type="dir" name="dir" mtime="0"/></stat>`))
case strings.Contains(action, "action=list"):
listRequests++
_, _ = w.Write([]byte(`<list><file type="file" name="123456/dir/a.txt" size="1" mtime="0"/><resume start="//` + otherHost + `/123456/dir/b.txt"/></list>`))
default:
http.Error(w, "unexpected action "+action, http.StatusBadRequest)
}
}))
defer configured.Close()
m := configmap.Simple{
"type": "netstorage",
"protocol": "http",
"host": strings.TrimPrefix(configured.URL, "http://") + "/123456/",
"account": "account",
"secret": obscure.MustObscure("secret"),
}
f, err := NewFs(context.Background(), "TestNetStorage", "dir", m)
require.NoError(t, err)
// The listing must fail loudly rather than with
// fs.ErrorDirNotFound which sync treats as an empty directory
err = f.(*Fs).ListR(context.Background(), "", func(entries fs.DirEntries) error { return nil })
assert.ErrorContains(t, err, "must not change the scheme, host or user")
assert.NotErrorIs(t, err, fs.ErrorDirNotFound)
assert.Equal(t, 1, listRequests, "list request sent to the configured host")
assert.Equal(t, 0, requests, "request sent to the other host")
}
+5 -5
View File
@@ -20,7 +20,7 @@ import (
const (
dirMimeType = "inode/directory"
timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.999Z" // keeps mS if present, though OneDrive only stores S
timeFormatOut = "2006-01-02T15:04:05.999Z" // mS for OneDrive Personal, otherwise only S
)
// system metadata keys which this backend owns
@@ -32,17 +32,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
ReadOnly: true,
},
"mtime": {
Help: "Time of last modification with S accuracy.",
Help: "Time of last modification with S accuracy (mS for OneDrive Personal).",
Type: "RFC 3339",
Example: "2006-01-02T15:04:05Z",
},
"btime": {
Help: "Time of file birth (creation) with S accuracy.",
Help: "Time of file birth (creation) with S accuracy (mS for OneDrive Personal).",
Type: "RFC 3339",
Example: "2006-01-02T15:04:05Z",
},
"utime": {
Help: "Time of upload with S accuracy.",
Help: "Time of upload with S accuracy (mS for OneDrive Personal).",
Type: "RFC 3339",
Example: "2006-01-02T15:04:05Z",
ReadOnly: true,
@@ -113,7 +113,7 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
ReadOnly: true,
},
"shared-time": {
Help: "Time when the item was shared, with S accuracy.",
Help: "Time when the item was shared, with S accuracy (mS for OneDrive Personal).",
Type: "RFC 3339",
Example: "2006-01-02T15:04:05Z",
ReadOnly: true,
+89 -152
View File
@@ -56,16 +56,7 @@ const (
driveTypeSharepoint = "documentLibrary"
defaultChunkSize = 10 * fs.Mebi
chunkSizeMultiple = 320 * fs.Kibi
defaultTenantAPIVersion = "v2.0"
// maxSinglePartSize is the size at which Graph stops accepting an upload in a
// single request (PUT /items/{id}/content). Microsoft documents this as
// "250 MB", see
// https://learn.microsoft.com/en-us/graph/api/driveitem-put-content
// Measured against SharePoint Online the figure is binary and exclusive: a
// body of 262143999 bytes is accepted, one of 262144000 is not. That matches
// upload_cutoff being an exclusive threshold in Update below, so a cutoff of
// exactly 250Mi sends everything the server would refuse to multipart.
maxSinglePartSize = 250 * fs.Mebi
maxSinglePartSize = 4 * fs.Mebi
regionGlobal = "global"
regionUS = "us"
@@ -167,7 +158,7 @@ See: https://github.com/rclone/rclone/issues/1716
Name: "tenant_url",
Help: `The tenant URL for non-admin OneDrive access.
Set this to your SharePoint tenant URL to use the SharePoint API
Set this to your SharePoint tenant URL to use the SharePoint v2.0 API
endpoint instead of the standard Microsoft Graph API. This allows
accessing business OneDrive without admin consent.
@@ -178,16 +169,6 @@ for "driveAccessToken" in the network requests. Look for the
Example: https://your-tenant.sharepoint.com/_api`,
Default: "",
Advanced: true,
}, {
Name: "tenant_api_version",
Help: `The SharePoint API version to use with tenant_url.
Set this to the SharePoint API version matching the browser-extracted
access token. For example, use v2.1 with a driveAccessTokenV21 token.
This only applies when tenant_url is set.`,
Default: defaultTenantAPIVersion,
Advanced: true,
}, {
Name: "chunk_size",
Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
@@ -294,13 +275,16 @@ cases, rclone will fall back to normal copy (which will be slightly slower).`,
Default: false,
Help: `Remove all versions on modifying operations.
Onedrive creates versions when rclone uploads new files
Onedrive for business creates versions when rclone uploads new files
overwriting an existing one and when it sets the modification time.
These versions take up space out of the quota.
This flag checks for versions after file upload and setting
modification time and removes all but the last version.
**NB** Onedrive personal can't currently delete versions so don't use
this flag there.
`,
Advanced: true,
}, {
@@ -310,7 +294,8 @@ modification time and removes all but the last version.
Normally files will get sent to the recycle bin on deletion. Setting
this flag causes them to be permanently deleted. Use with care.
This works with OneDrive for Business, SharePoint document libraries, and OneDrive personal accounts, including free accounts.
OneDrive personal accounts do not support the permanentDelete API,
it only applies to OneDrive for Business and SharePoint document libraries.
`,
Advanced: true,
Default: false,
@@ -339,18 +324,14 @@ This works with OneDrive for Business, SharePoint document libraries, and OneDri
Help: "Creates a read-write link to the item.",
}, {
Value: "embed",
Help: "Creates an embeddable link to the item.\nOnly available in OneDrive personal.",
Help: "Creates an embeddable link to the item.",
}},
}, {
Name: "link_password",
Default: "",
Help: `Set the password for links created by the link command.
At the time of writing this works with OneDrive for Business and
OneDrive personal paid accounts.
OneDrive personal free accounts can't set a password or an expiry time
(with --expire) on links.
At the time of writing this only works with OneDrive personal paid accounts.
`,
Advanced: true,
Sensitive: true,
@@ -438,12 +419,21 @@ Setting this flag speeds up these things greatly:
rclone size onedrive:
rclone rc vfs/refresh recursive=true
Rclone asks for the delta listing of the directory being listed. If
the drive only supports delta listings at the root of the drive (as
some older OneDrive for Business and SharePoint drives do) then
rclone lists from the root and discards all the data that is not
under the directory you asked for. So it will be correct but may not
be very efficient.
**However** the delta listing API **only** works at the root of the
drive. If you use it not at the root then it recurses from the root
and discards all the data that is not under the directory you asked
for. So it will be correct but may not be very efficient.
This is why this flag is not set as the default.
As a rule of thumb if nearly all of your data is under rclone's root
directory (the |root/directory| in |onedrive:root/directory|) then
using this flag will be a big performance win. If your data is
mostly not under the root then using this flag will be a big
performance loss.
It is recommended if you are mounting your onedrive at the root
(or near the root when using crypt) and using rclone |rc vfs/refresh|.
`, "|", "`"),
Advanced: true,
}, {
@@ -512,20 +502,12 @@ func getRegionURL(m configmap.Mapper) (region, graphURL string) {
// Check if tenant_url is provided for non-admin mode
tenantURL, _ := m.Get("tenant_url")
if tenantURL != "" {
tenantAPIVersion, _ := m.Get("tenant_api_version")
graphURL = tenantAPIEndpoint(tenantURL, tenantAPIVersion)
graphURL = tenantURL + "/v2.0"
}
return region, graphURL
}
func tenantAPIEndpoint(tenantURL, tenantAPIVersion string) string {
if tenantAPIVersion == "" {
tenantAPIVersion = defaultTenantAPIVersion
}
return strings.TrimRight(tenantURL, "/") + "/" + strings.TrimLeft(tenantAPIVersion, "/")
}
// Config for chooseDrive
type chooseDriveOpt struct {
opts rest.Opts
@@ -819,7 +801,6 @@ type Options struct {
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
TenantURL string `config:"tenant_url"`
TenantAPIVersion string `config:"tenant_api_version"`
DriveID string `config:"drive_id"`
DriveType string `config:"drive_type"`
RootFolderID string `config:"root_folder_id"`
@@ -1127,7 +1108,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID
if opt.TenantURL != "" {
rootURL = tenantAPIEndpoint(opt.TenantURL, opt.TenantAPIVersion) + "/drives/" + opt.DriveID
rootURL = opt.TenantURL + "/v2.0" + "/drives/" + opt.DriveID
}
oauthConfig, err := makeOauthConfig(ctx, opt)
@@ -1414,8 +1395,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, dir string, info *api.Item) (en
id := info.GetID()
f.dirCache.Put(remote, id)
d := f.newDir(id, remote)
// OneDrive reports the total size of a folder's contents
d.size = info.GetSize()
d.items = folder.ChildCount
f.setSystemMetadata(info, d.meta, remote, dirMimeType)
entry = d
@@ -1504,6 +1483,10 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return err
}
// ListR only works at the root of a onedrive, not on a folder
// So we have to filter things outside of the root which is
// inefficient.
list := list.NewHelper(callback)
// list a folder conventionally - used for shared folders
@@ -1579,26 +1562,17 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return nil
}
listDelta := func(opts rest.Opts) error {
opts.Parameters = url.Values{
opts := rest.Opts{
Method: "GET",
Path: "/root/delta",
Parameters: map[string][]string{
// "token": {token},
"$top": {fmt.Sprintf("%d", f.opt.ListChunk)},
}
var result api.DeltaResponse
return f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
},
}
err = listDelta(f.newOptsCall(directoryID, "GET", "/delta"))
// Some drives only support delta listings at the root of the
// drive, in which case list the whole drive and filter out the
// items outside dir.
var apiErr *api.Error
if err != nil && len(seen) == 0 && errors.As(err, &apiErr) {
fs.Debugf(f, "Delta listing of directory failed, listing from the root of the drive instead: %v", err)
err = listDelta(rest.Opts{
Method: "GET",
Path: "/root/delta",
})
}
var result api.DeltaResponse
err = f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
if err != nil {
return err
}
@@ -2812,13 +2786,13 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
return info, o.setMetaData(info)
}
// Update the content of a remote file smaller than maxSinglePartSize in one single request
// Update the content of a remote file within 4 MiB size in one single request
// (currently only used when size is exactly 0)
// This function will set modtime and metadata after uploading, which will create a new version for the remote file
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
size := src.Size()
if size < 0 || size >= int64(maxSinglePartSize) {
return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and < %v", maxSinglePartSize)
if size < 0 || size > int64(maxSinglePartSize) {
return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)
}
fs.Debugf(o, "Starting singlepart upload")
@@ -2937,7 +2911,7 @@ func (o *Object) ID() string {
func (f *Fs) parseNormalizedID(ID string) (string, string, string) {
var rootURL string
if f.opt.TenantURL != "" {
rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
rootURL = f.opt.TenantURL + "/v2.0/drives"
} else {
rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
}
@@ -3105,15 +3079,9 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
}
case <-tickerC:
fs.Debugf(f, "Checking for changes on remote")
var newDeltaToken string
newDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
if err != nil {
// Keep the token we already have: replacing it with the
// empty token from a failed poll would reset the delta
// listing to the start and lose every change in between.
fs.Infof(f, "Change notify listener failure: %s", err)
} else {
nextDeltaToken = newDeltaToken
}
}
}
@@ -3121,7 +3089,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
}
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
delta, err := f.changeNotifyNextChange(ctx, "latest", "")
delta, err := f.changeNotifyNextChange(ctx, "latest")
if err != nil {
return
}
@@ -3133,37 +3101,18 @@ func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken str
return
}
// changeNotifyNextChange fetches one page of the drive delta listing.
//
// Pass the delta token to resume from in token, or the @odata.nextLink URL
// returned by the previous page in nextLink. Exactly one of the two should
// be set.
func (f *Fs) changeNotifyNextChange(ctx context.Context, token, nextLink string) (delta api.DeltaResponse, err error) {
opts := f.buildDriveDeltaOpts(token, nextLink)
func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta api.DeltaResponse, err error) {
opts := f.buildDriveDeltaOpts(token)
err = f.pacer.Call(func() (bool, error) {
var resp *http.Response
resp, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
return shouldRetry(ctx, resp, err)
})
_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
return
}
// buildDriveDeltaOpts builds the request for one page of the drive delta
// listing. nextLink, when set, is the complete @odata.nextLink URL from the
// previous page and is used as-is, overriding the drive root.
func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
if nextLink != "" {
return rest.Opts{
Method: "GET",
RootURL: nextLink,
}
}
func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
var rootURL string
if f.opt.TenantURL != "" {
rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
rootURL = f.opt.TenantURL + "/v2.0/drives"
} else {
rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
}
@@ -3176,58 +3125,46 @@ func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
}
}
// changeNotifyRunner polls for changes and calls notifyFunc for each one.
//
// It walks every page of the change set and returns the token to resume from
// next time. The returned token is only valid when err is nil: the caller
// must keep the token it passed in on error, otherwise a failed poll would
// reset the delta listing to the start and lose the changes in between.
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (string, error) {
token, nextLink := deltaToken, ""
for {
delta, err := f.changeNotifyNextChange(ctx, token, nextLink)
if err != nil {
return "", err
}
for _, item := range delta.Value {
isDriveRootFolder := item.GetParentReference().ID == ""
if isDriveRootFolder {
continue
}
fullPath, err := getItemFullPath(&item)
if err != nil {
fs.Errorf(f, "Could not get item full path: %s", err)
continue
}
if fullPath == f.root {
continue
}
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
if !insideRoot {
continue
}
if item.GetFile() != nil {
notifyFunc(relName, fs.EntryObject)
} else if item.GetFolder() != nil {
notifyFunc(relName, fs.EntryDirectory)
}
}
if delta.NextLink == "" {
// end of the change set - the deltaLink carries the resume token
parsedURL, err := url.Parse(delta.DeltaLink)
if err != nil {
return "", err
}
return parsedURL.Query().Get("token"), nil
}
nextLink = delta.NextLink
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
delta, err := f.changeNotifyNextChange(ctx, deltaToken)
if err != nil {
return
}
parsedURL, err := url.Parse(delta.DeltaLink)
if err != nil {
return
}
nextDeltaToken = parsedURL.Query().Get("token")
for _, item := range delta.Value {
isDriveRootFolder := item.GetParentReference().ID == ""
if isDriveRootFolder {
continue
}
fullPath, err := getItemFullPath(&item)
if err != nil {
fs.Errorf(f, "Could not get item full path: %s", err)
continue
}
if fullPath == f.root {
continue
}
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
if !insideRoot {
continue
}
if item.GetFile() != nil {
notifyFunc(relName, fs.EntryObject)
} else if item.GetFolder() != nil {
notifyFunc(relName, fs.EntryDirectory)
}
}
return
}
func getItemFullPath(item *api.Item) (fullPath string, err error) {
-220
View File
@@ -4,11 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
@@ -18,10 +15,7 @@ import (
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -48,40 +42,6 @@ func TestMain(m *testing.M) {
fstest.TestMain(m)
}
func TestTenantAPIEndpoint(t *testing.T) {
for _, tc := range []struct {
name string
tenantURL string
tenantAPIVersion string
want string
}{
{
name: "default",
tenantURL: "https://example-my.sharepoint.com/_api",
want: "https://example-my.sharepoint.com/_api/v2.0",
}, {
name: "explicit v2.1",
tenantURL: "https://example-my.sharepoint.com/_api",
tenantAPIVersion: "v2.1",
want: "https://example-my.sharepoint.com/_api/v2.1",
}, {
name: "future version",
tenantURL: "https://example-my.sharepoint.com/_api",
tenantAPIVersion: "v2.2",
want: "https://example-my.sharepoint.com/_api/v2.2",
}, {
name: "trailing slash",
tenantURL: "https://example-my.sharepoint.com/_api/",
tenantAPIVersion: "/v2.1",
want: "https://example-my.sharepoint.com/_api/v2.1",
},
} {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, tenantAPIEndpoint(tc.tenantURL, tc.tenantAPIVersion))
})
}
}
// sharingRefused caches whether the remote refuses sharing invitations.
var sharingRefused *bool
@@ -556,30 +516,6 @@ func (f *Fs) resetTestDefaults(r *fstest.Run) {
r.Finalise()
}
// TestCheckUploadCutoff checks the bounds of the --onedrive-upload-cutoff
// validation, in particular that the documented API maximum is accepted.
func TestCheckUploadCutoff(t *testing.T) {
for _, test := range []struct {
in fs.SizeSuffix
wantErr bool
}{
{in: 0, wantErr: false},
{in: 4 * fs.Mebi, wantErr: false},
// maxSinglePartSize is a legal cutoff even though the server refuses a
// body of that size, because the cutoff is an exclusive threshold: at
// this setting a file of exactly maxSinglePartSize goes multipart.
{in: maxSinglePartSize, wantErr: false},
{in: maxSinglePartSize + 1, wantErr: true},
} {
err := checkUploadCutoff(test.in)
if test.wantErr {
assert.Error(t, err, test.in.String())
} else {
assert.NoError(t, err, test.in.String())
}
}
}
// InternalTest dispatches all internal tests
func (f *Fs) InternalTest(t *testing.T) {
newTestF := func() (*Fs, *fstest.Run) {
@@ -614,159 +550,3 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
// newChangeNotifyTestFs returns an Fs whose delta listing is served by
// handler on a local test server, plus the server itself. The tenant URL
// is pointed at the server so no request leaves the test.
func newChangeNotifyTestFs(t *testing.T, handler http.Handler) (*Fs, *httptest.Server) {
t.Helper()
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
ctx := context.Background()
f := &Fs{
ci: fs.GetConfig(ctx),
root: "",
driveID: "testdrive",
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
}
f.opt.TenantURL = server.URL
f.opt.TenantAPIVersion = defaultTenantAPIVersion
return f, server
}
// deltaItem builds a delta entry for a file named name inside a folder.
func deltaItem(name, folder string) api.Item {
return api.Item{
ID: name,
Name: name,
File: &api.FileFacet{},
ParentReference: &api.ItemReference{
ID: "parentid",
Path: "driveid:/" + folder,
},
}
}
// TestChangeNotifyRetriesAndKeepsToken covers a transient failure of the
// delta poll: the request must go through the pacer so it is retried, and
// a poll which fails outright must not hand back an empty token, since the
// caller stores whatever it returns. An empty token would reset the delta
// listing to the start and silently lose every change in between.
func TestChangeNotifyRetriesAndKeepsToken(t *testing.T) {
ctx := context.Background()
var attempts atomic.Int32
var tokens []string
fail := true
f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
attempts.Add(1)
tokens = append(tokens, r.URL.Query().Get("token"))
if fail {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
"code": "serviceNotAvailable",
"message": "Service unavailable",
}})
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
Value: []api.Item{deltaItem("file1.txt", "folder")},
DeltaLink: "https://example.com/delta?token=newtoken",
})
}))
var notified []string
notifyFunc := func(path string, entryType fs.EntryType) {
notified = append(notified, path)
}
// The poll fails, but is retried by the pacer first
token, err := f.changeNotifyRunner(ctx, notifyFunc, "tok1")
require.Error(t, err)
assert.Empty(t, token, "a failed poll must not return a token")
assert.Equal(t, fs.GetConfig(ctx).LowLevelRetries, int(attempts.Load()), "the failed poll should have been retried by the pacer")
assert.Equal(t, slices.Repeat([]string{"tok1"}, int(attempts.Load())), tokens, "every retry should resume from the same token")
assert.Empty(t, notified)
// Keeping the previous token (as ChangeNotify does) resumes correctly
fail = false
attempts.Store(0)
tokens = nil
token, err = f.changeNotifyRunner(ctx, notifyFunc, "tok1")
require.NoError(t, err)
assert.Equal(t, "newtoken", token)
assert.Equal(t, []string{"tok1"}, tokens, "the second poll should resume from the retained token")
assert.Equal(t, []string{"folder/file1.txt"}, notified)
}
// TestChangeNotifyFollowsNextLink covers a change set which spans more than
// one page. Graph returns @odata.nextLink and no @odata.deltaLink until the
// last page, so a runner which stops after the first page both loses the
// remaining changes and ends up with an empty resume token.
func TestChangeNotifyFollowsNextLink(t *testing.T) {
ctx := context.Background()
var urls []string
var server *httptest.Server
f, server := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
urls = append(urls, r.URL.RequestURI())
w.Header().Set("Content-Type", "application/json")
switch len(urls) {
case 1:
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
Value: []api.Item{deltaItem("page1.txt", "folder")},
NextLink: server.URL + "/nextpage?$skiptoken=abc",
})
case 2:
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
Value: []api.Item{deltaItem("page2.txt", "folder")},
DeltaLink: "https://example.com/delta?token=lasttoken",
})
default:
t.Errorf("unexpected extra request to %s", r.URL)
}
}))
var notified []string
token, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
notified = append(notified, path)
}, "tok1")
require.NoError(t, err)
assert.Equal(t, "lasttoken", token, "the resume token should come from the last page")
assert.Equal(t, []string{"folder/page1.txt", "folder/page2.txt"}, notified, "changes from every page should be notified")
require.Len(t, urls, 2, "the nextLink page should have been fetched")
assert.Contains(t, urls[0], "/testdrive/root/delta")
assert.Contains(t, urls[0], "token=tok1")
assert.Equal(t, "/nextpage?$skiptoken=abc", urls[1], "the nextLink should be used as-is, without the delta token")
}
// TestListDirectorySize checks that a listed directory carries the size
// OneDrive reports for it, the total size of its contents, instead of -1.
func TestListDirectorySize(t *testing.T) {
ctx := context.Background()
f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/items/rootid/children", r.URL.Path)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(api.ListChildrenResponse{
Value: []api.Item{{
ID: "dirid",
Name: "photos",
Size: 2646204760,
Folder: &api.FolderFacet{ChildCount: 3},
}},
})
}))
f.dirCache = dircache.New("", "rootid", f)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
dir, ok := entries[0].(fs.Directory)
require.True(t, ok, "entry should be a directory")
assert.Equal(t, "photos", dir.Remote())
assert.Equal(t, int64(2646204760), dir.Size())
assert.Equal(t, int64(3), dir.Items())
}
-3
View File
@@ -115,14 +115,11 @@ func useBYOKCopyObject(fs *Fs, request *objectstorage.CopyObjectRequest) {
}
if fs.opt.SSECustomerAlgorithm != "" {
request.OpcSseCustomerAlgorithm = new(fs.opt.SSECustomerAlgorithm)
request.OpcSourceSseCustomerAlgorithm = new(fs.opt.SSECustomerAlgorithm)
}
if fs.opt.SSECustomerKey != "" {
request.OpcSseCustomerKey = new(fs.opt.SSECustomerKey)
request.OpcSourceSseCustomerKey = new(fs.opt.SSECustomerKey)
}
if fs.opt.SSECustomerKeySha256 != "" {
request.OpcSseCustomerKeySha256 = new(fs.opt.SSECustomerKeySha256)
request.OpcSourceSseCustomerKeySha256 = new(fs.opt.SSECustomerKeySha256)
}
}
-25
View File
@@ -1,25 +0,0 @@
//go:build !plan9 && !solaris && !js
package oracleobjectstorage
import (
"testing"
"github.com/oracle/oci-go-sdk/v65/objectstorage"
"github.com/stretchr/testify/assert"
)
func TestUseBYOKCopyObject(t *testing.T) {
f := &Fs{opt: Options{
SSECustomerAlgorithm: "AES256",
SSECustomerKey: "customer-key",
SSECustomerKeySha256: "customer-key-sha256",
}}
req := &objectstorage.CopyObjectRequest{}
useBYOKCopyObject(f, req)
assert.Equal(t, &f.opt.SSECustomerAlgorithm, req.OpcSourceSseCustomerAlgorithm)
assert.Equal(t, &f.opt.SSECustomerKey, req.OpcSourceSseCustomerKey)
assert.Equal(t, &f.opt.SSECustomerKeySha256, req.OpcSourceSseCustomerKeySha256)
}
+17 -1
View File
@@ -11,6 +11,7 @@ import (
"sync"
"time"
"github.com/oracle/oci-go-sdk/v65/common"
"github.com/oracle/oci-go-sdk/v65/objectstorage"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
@@ -196,7 +197,22 @@ func (f *Fs) rename(ctx context.Context, remote, newName string) (any, error) {
}
return nil, fs.ErrorNotAFile
}
err = f.renameObject(ctx, bucketName, objectPath, newName)
details := objectstorage.RenameObjectDetails{
SourceName: new(objectPath),
NewName: new(newName),
}
request := objectstorage.RenameObjectRequest{
NamespaceName: new(f.opt.Namespace),
BucketName: new(bucketName),
RenameObjectDetails: details,
OpcClientRequestId: nil,
RequestMetadata: common.RequestMetadata{},
}
var response objectstorage.RenameObjectResponse
err = f.pacer.Call(func() (bool, error) {
response, err = f.srv.RenameObject(ctx, request)
return shouldRetry(ctx, response.HTTPResponse(), err)
})
if err != nil {
return nil, err
}
-49
View File
@@ -40,55 +40,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
return f.NewObject(ctx, remote)
}
// ------------------------------------------------------------
// Implement Mover is an optional interface for Fs
//------------------------------------------------------------
// Move src to this remote using server-side rename operations.
//
// This is stored with the remote path given.
// It returns the destination Object and a possible error.
// Will only be called if src.Fs().Name() == f.Name().
// If it isn't possible then return fs.ErrorCantMove.
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
return nil, fs.ErrorCantMove
}
srcBucket, srcPath := srcObj.split()
dstBucket, dstPath := f.split(remote)
if srcBucket != dstBucket {
return nil, fs.ErrorCantMove
}
err := f.renameObject(ctx, srcBucket, srcPath, dstPath)
if err != nil {
return nil, err
}
return f.NewObject(ctx, remote)
}
// renameObject renames sourceName to newName in bucketName.
func (f *Fs) renameObject(ctx context.Context, bucketName, sourceName, newName string) error {
req := objectstorage.RenameObjectRequest{
NamespaceName: new(f.opt.Namespace),
BucketName: new(bucketName),
RenameObjectDetails: objectstorage.RenameObjectDetails{
SourceName: new(sourceName),
NewName: new(newName),
},
}
var resp objectstorage.RenameObjectResponse
err := f.pacer.Call(func() (bool, error) {
var err error
resp, err = f.srv.RenameObject(ctx, req)
return shouldRetry(ctx, resp.HTTPResponse(), err)
})
if err != nil {
return err
}
return nil
}
// copy does a server-side copy from dstObj <- srcObj
//
// If newInfo is nil then the metadata will be copied otherwise it
-4
View File
@@ -3,7 +3,6 @@
package oracleobjectstorage
import (
"bufio"
"bytes"
"context"
"encoding/base64"
@@ -397,9 +396,6 @@ func isZeroLength(streamReader io.Reader) bool {
return v.Len() == 0
case *strings.Reader:
return v.Len() == 0
case *bufio.Reader:
_, err := v.Peek(1)
return err == io.EOF
case *os.File:
fi, err := v.Stat()
if err != nil {
+1 -2
View File
@@ -257,8 +257,7 @@ to start uploading.`,
// so that OSS keys are always valid file names
Default: encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeDot |
encoder.EncodeCrLf,
encoder.EncodeDot,
}, {
Name: "leave_parts_on_error",
Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts for manual recovery.
@@ -4,9 +4,7 @@
package oracleobjectstorage
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net/http"
@@ -304,28 +302,17 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
// listFn is called from list to handle an object.
type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error
// list calls fn for objects in bucket and directory.
//
// ctx controls the API requests.
// bucket is the bucket to list.
// directory is the path to list within bucket.
// list the objects into the function supplied from
// the bucket and root supplied
// (bucket, directory) is the starting directory
// If prefix is set then it is removed from all file names
// If addBucket is set then it adds the bucket to the start of the remotes generated
// If recurse is set the function will recursively list
// If limit is > 0 then it limits to that many files (must be less than 1000)
// includeDirectoryMarkers includes zero-length objects with names ending in a slash.
// fn receives each listed object.
func (f *Fs) list(
ctx context.Context,
bucket string,
directory string,
prefix string,
addBucket bool,
recurse bool,
limit int,
includeDirectoryMarkers bool,
fn listFn,
) (err error) {
// If hidden is set then it will list the hidden (deleted) files too.
// if findFile is set it will look for files called (bucket, directory)
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int,
fn listFn) (err error) {
if prefix != "" {
prefix += "/"
}
@@ -420,7 +407,7 @@ func (f *Fs) list(
remote = path.Join(bucket, remote)
}
// is this a directory marker?
if isDirectory && object.Size != nil && *object.Size == 0 && !includeDirectoryMarkers {
if isDirectory && object.Size != nil && *object.Size == 0 {
continue // skip directory marker
}
if isDirectory && len(remote) > 1 {
@@ -469,7 +456,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
}
return nil
}
err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, false, fn)
err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn)
if err != nil {
return err
}
@@ -566,7 +553,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
// PutStream uploads to the remote path with the modTime given of indeterminate size
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return f.Put(ctx, bufio.NewReader(in), src, options...)
return f.Put(ctx, in, src, options...)
}
// Mkdir creates the bucket if it doesn't exist
@@ -654,66 +641,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
})
}
// ------------------------------------------------------------
// Implement Purger is an optional interfaces for Fs
//------------------------------------------------------------
// Purge deletes all the files in dir.
//
// It implements fs.Purger, using OCI batch requests rather than removing each
// object returned by List. It also removes directory markers and, when dir is
// a bucket root, removes the empty bucket.
func (f *Fs) Purge(ctx context.Context, dir string) error {
bucketName, directory := f.split(dir)
if bucketName == "" {
return fs.ErrorListBucketRequired
}
const batchSize = 1000
objects := make([]objectstorage.BatchDeleteObjectIdentifier, 0, batchSize)
deleteObjects := func() error {
if len(objects) == 0 {
return nil
}
req := objectstorage.BatchDeleteObjectsRequest{
NamespaceName: new(f.opt.Namespace),
BucketName: new(bucketName),
BatchDeleteObjectsDetails: objectstorage.BatchDeleteObjectsDetails{
Objects: objects,
IsSkipDeletedResult: new(true),
},
}
var resp objectstorage.BatchDeleteObjectsResponse
err := f.pacer.Call(func() (bool, error) {
var err error
resp, err = f.srv.BatchDeleteObjects(ctx, req)
return shouldRetry(ctx, resp.HTTPResponse(), err)
})
if err != nil {
return err
}
var errs []error
for _, failed := range resp.Failed {
errs = append(errs, fmt.Errorf("failed to delete object %q: %s", *failed.ObjectName, *failed.ErrorMessage))
}
objects = objects[:0]
return errors.Join(errs...)
}
err := f.list(ctx, bucketName, directory, f.rootDirectory, false, true, 0, true, func(_ string, object *objectstorage.ObjectSummary, _ bool) error {
objects = append(objects, objectstorage.BatchDeleteObjectIdentifier{ObjectName: object.Name})
if len(objects) == batchSize {
return deleteObjects()
}
return nil
})
if err != nil {
return err
}
if err = deleteObjects(); err != nil {
return err
}
return f.Rmdir(ctx, dir)
}
func (f *Fs) abortMultiPartUpload(ctx context.Context, bucketName, bucketPath, uploadID *string) (err error) {
if uploadID == nil || *uploadID == "" {
return nil
@@ -801,7 +728,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
bucketName, directory := f.split(dir)
list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, false, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
if err != nil {
return err
@@ -874,13 +801,11 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
var (
_ fs.Fs = &Fs{}
_ fs.Copier = &Fs{}
_ fs.Mover = &Fs{}
_ fs.PutStreamer = &Fs{}
_ fs.ListRer = &Fs{}
_ fs.ListPer = &Fs{}
_ fs.Commander = &Fs{}
_ fs.CleanUpper = &Fs{}
_ fs.Purger = &Fs{}
_ fs.OpenChunkWriter = &Fs{}
_ fs.Object = &Object{}
@@ -3,28 +3,18 @@
package oracleobjectstorage
import (
"bufio"
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"errors"
"fmt"
"io"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/oracle/oci-go-sdk/v65/objectstorage"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -41,51 +31,6 @@ func TestIntegration(t *testing.T) {
})
}
func TestIsZeroLength(t *testing.T) {
newFile := func(contents string) *os.File {
file, err := os.CreateTemp(t.TempDir(), "stream")
require.NoError(t, err)
_, err = file.WriteString(contents)
require.NoError(t, err)
_, err = file.Seek(0, io.SeekStart)
require.NoError(t, err)
t.Cleanup(func() { _ = file.Close() })
return file
}
closedFile := newFile("")
require.NoError(t, closedFile.Close())
for _, test := range []struct {
name string
in io.Reader
want bool
wantContents string
}{
{name: "EmptyBuffer", in: bytes.NewBuffer(nil), want: true},
{name: "NonEmptyBuffer", in: bytes.NewBufferString("contents"), want: false},
{name: "EmptyBytesReader", in: bytes.NewReader(nil), want: true},
{name: "NonEmptyBytesReader", in: bytes.NewReader([]byte("contents")), want: false},
{name: "EmptyStringsReader", in: strings.NewReader(""), want: true},
{name: "NonEmptyStringsReader", in: strings.NewReader("contents"), want: false},
{name: "EmptyBufferedStream", in: bufio.NewReader(io.TeeReader(bytes.NewReader(nil), io.Discard)), want: true},
{name: "NonEmptyBufferedStream", in: bufio.NewReader(io.TeeReader(bytes.NewBufferString("contents"), io.Discard)), want: false, wantContents: "contents"},
{name: "BufferedStreamError", in: bufio.NewReader(readers.ErrorReader{Err: errors.New("stream error")}), want: false},
{name: "EmptyFile", in: newFile(""), want: true},
{name: "NonEmptyFile", in: newFile("contents"), want: false},
{name: "ClosedFile", in: closedFile, want: false},
{name: "OpaqueEmptyStream", in: io.TeeReader(bytes.NewReader(nil), io.Discard), want: false},
} {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.want, isZeroLength(test.in))
if test.wantContents != "" {
contents, err := io.ReadAll(test.in)
require.NoError(t, err)
assert.Equal(t, test.wantContents, string(contents))
}
})
}
}
func gz(t *testing.T, s string) string {
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
@@ -136,99 +81,9 @@ func (f *Fs) InternalTestGzipEncoding(t *testing.T) {
})
}
// InternalTestPurgeBatches tests purging more objects than fit in one batch.
func (f *Fs) InternalTestPurgeBatches(t *testing.T) {
ctx := context.Background()
const (
dir = "purge-batches"
objectCount = 1001
uploadConcurrency = 16
)
defer func() { _ = f.Purge(ctx, dir) }()
jobs := make(chan int)
errs := make(chan error, objectCount)
var wg sync.WaitGroup
for range uploadConcurrency {
wg.Go(func() {
for i := range jobs {
remote := fmt.Sprintf("%s/%04d", dir, i)
src := object.NewStaticObjectInfo(remote, time.Time{}, 0, true, nil, f)
_, err := f.Put(ctx, bytes.NewReader(nil), src)
if err != nil {
errs <- err
}
}
})
}
for i := range objectCount {
jobs <- i
}
close(jobs)
wg.Wait()
close(errs)
for err := range errs {
require.NoError(t, err)
}
require.NoError(t, f.Purge(ctx, dir))
entries, err := f.List(ctx, dir)
require.NoError(t, err)
assert.Empty(t, entries)
}
// InternalTestPurgeDirectoryMarkers tests purging directory marker objects.
func (f *Fs) InternalTestPurgeDirectoryMarkers(t *testing.T) {
ctx := context.Background()
const dir = "purge-directory-markers"
bucketName, directory := f.split(dir)
objectNames := []string{
directory + "/",
directory + "/child/",
directory + "/child/object",
}
defer func() { _ = f.Purge(ctx, dir) }()
for _, objectName := range objectNames {
req := objectstorage.PutObjectRequest{
NamespaceName: new(f.opt.Namespace),
BucketName: new(bucketName),
ObjectName: new(objectName),
PutObjectBody: io.NopCloser(bytes.NewReader(nil)),
}
err := f.pacer.Call(func() (bool, error) {
resp, err := f.srv.PutObject(ctx, req)
return shouldRetry(ctx, resp.HTTPResponse(), err)
})
require.NoError(t, err)
}
// Use the SDK directly because normal OOS listings hide directory markers.
listObjectNames := func() []string {
req := objectstorage.ListObjectsRequest{
NamespaceName: new(f.opt.Namespace),
BucketName: new(bucketName),
Prefix: new(directory + "/"),
}
var resp objectstorage.ListObjectsResponse
err := f.pacer.Call(func() (bool, error) {
var err error
resp, err = f.srv.ListObjects(ctx, req)
return shouldRetry(ctx, resp.HTTPResponse(), err)
})
require.NoError(t, err)
names := make([]string, 0, len(resp.Objects))
for _, object := range resp.Objects {
names = append(names, *object.Name)
}
return names
}
assert.ElementsMatch(t, objectNames, listObjectNames())
require.NoError(t, f.Purge(ctx, dir))
assert.Empty(t, listObjectNames())
}
// InternalTest is called by fstests.Run to extra tests
func (f *Fs) InternalTest(t *testing.T) {
t.Run("GzipEncoding", f.InternalTestGzipEncoding)
t.Run("PurgeBatches", f.InternalTestPurgeBatches)
t.Run("PurgeDirectoryMarkers", f.InternalTestPurgeDirectoryMarkers)
}
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+4 -21
View File
@@ -14,7 +14,6 @@ import (
"net/http"
"net/url"
"path"
"regexp"
"strconv"
"strings"
"time"
@@ -57,10 +56,6 @@ var (
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
RedirectURL: oauthutil.RedirectLocalhostURL,
}
// hostnameRe matches a lower case API hostname within pCloud's
// domain, eg api.pcloud.com or eapi.pcloud.com
hostnameRe = regexp.MustCompile(`^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+pcloud\.com$`)
)
// Update the TokenURL with the actual hostname
@@ -68,19 +63,6 @@ func updateTokenURL(oauthConfig *oauthutil.Config, hostname string) {
oauthConfig.TokenURL = "https://" + hostname + "/oauth2_token"
}
// callbackHostname checks and returns the API hostname pCloud sent back in the
// OAuth callback, lower cased, or defaultHostname if it sent none.
func callbackHostname(hostname string) (string, error) {
if hostname == "" {
return defaultHostname, nil
}
hostname = strings.ToLower(hostname)
if !hostnameRe.MatchString(hostname) {
return "", fmt.Errorf("invalid hostname %q in oauth response", hostname)
}
return hostname, nil
}
// Register with Fs
func init() {
updateTokenURL(oauthConfig, defaultHostname)
@@ -99,9 +81,9 @@ func init() {
if auth == nil || auth.Form == nil {
return errors.New("form not found in response")
}
hostname, err := callbackHostname(auth.Form.Get("hostname"))
if err != nil {
return err
hostname := auth.Form.Get("hostname")
if hostname == "" {
hostname = defaultHostname
}
// Save the hostname in the config
m.Set("hostname", hostname)
@@ -113,6 +95,7 @@ func init() {
return oauthutil.ConfigOut("", &oauthutil.Options{
OAuth2Config: oauthConfig,
CheckAuth: checkAuth,
StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
})
},
Options: append(oauthutil.SharedOptions, []fs.Option{{
-46
View File
@@ -1,46 +0,0 @@
package pcloud
import (
"testing"
"github.com/stretchr/testify/assert"
)
// TestCallbackHostname checks that only hosts within pCloud's domain
// are accepted.
func TestCallbackHostname(t *testing.T) {
for _, test := range []struct {
in string
want string // "" means an error is expected
}{
{"", defaultHostname},
{"api.pcloud.com", "api.pcloud.com"},
{"eapi.pcloud.com", "eapi.pcloud.com"},
{"EAPI.pCloud.COM", "eapi.pcloud.com"},
{"new-region.api.pcloud.com", "new-region.api.pcloud.com"},
{"pcloud.com", ""},
{".pcloud.com", ""},
{"-.pcloud.com", ""},
{"a-.pcloud.com", ""},
{"a..pcloud.com", ""},
{"api.pcloud.com.", ""},
{"api.pclоud.com", ""}, // Cyrillic о
{"127.0.0.1:18443", ""},
{"evil.example.com", ""},
{"api.pcloud.com.evil.example.com", ""},
{"api.pcloud.com:443", ""},
{"api.pcloud.com/path", ""},
{"user@evil.example.com#.pcloud.com", ""},
{"evil.example.com?.pcloud.com", ""},
} {
t.Run("["+test.in+"]", func(t *testing.T) {
got, err := callbackHostname(test.in)
if test.want == "" {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
assert.Equal(t, test.want, got)
})
}
}
+2 -32
View File
@@ -993,13 +993,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}()
}
// The upload server truncates the file name at the first ";" so
// upload under a temporary name and rename it afterwards.
uploadLeaf := leaf
if strings.Contains(leaf, ";") {
uploadLeaf = "rclone-upload-" + random.String(16)
}
opts = rest.Opts{
Method: "POST",
RootURL: info.URL,
@@ -1007,8 +1000,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
MultipartParams: url.Values{
"token": {info.Token},
},
MultipartContentName: "file", // ..name of the parameter which is the attached file
MultipartFileName: uploadLeaf, // ..name of the file for the attached file
MultipartContentName: "file", // ..name of the parameter which is the attached file
MultipartFileName: leaf, // ..name of the file for the attached file
ContentLength: &size,
}
var result api.Response
@@ -1023,29 +1016,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return fmt.Errorf("upload file: %w", err)
}
if uploadLeaf != leaf {
var newID string
var found bool
_, found, err = o.fs.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {
if item.Name == uploadLeaf {
newID = item.ID
return true
}
return false
})
if err != nil {
return fmt.Errorf("upload find temporary file: %w", err)
}
if !found {
return fmt.Errorf("upload find temporary file %q: %w", uploadLeaf, fs.ErrorObjectNotFound)
}
err = o.fs.renameLeaf(ctx, true, newID, leaf)
if err != nil {
_ = o.fs.remove(ctx, newID)
return fmt.Errorf("upload rename temporary file: %w", err)
}
}
// on successful upload, remove old file if it exists
uploaded = true
if o.hasMetaData {
-4
View File
@@ -19,7 +19,3 @@ quirks:
force_path_style: true
list_url_encode: false
use_already_exists: false
# Ceph RGW (and reverse proxies such as Cloudflare in front of it) can
# mutate Accept-Encoding and break SigV4 when that header is signed.
# See: https://github.com/rclone/rclone/issues/8206
sign_accept_encoding: false
-2
View File
@@ -3,7 +3,6 @@ description: Fastly Object Storage
region:
au-east-1: AU East 1
eu-central: EU Central
eu-central-1: EU Central 1
eu-south-1: EU South 1
eu-west-1: EU West 1
jp-central-1: JP Central 1
@@ -16,7 +15,6 @@ region:
endpoint:
au-east-1.object.fastlystorage.app: AU East 1
eu-central.object.fastlystorage.app: EU Central
eu-central-1.object.fastlystorage.app: EU Central 1
eu-south-1.object.fastlystorage.app: EU South 1
eu-west-1.object.fastlystorage.app: EU West 1
jp-central-1.object.fastlystorage.app: JP Central 1
-5
View File
@@ -24,8 +24,3 @@ endpoint:
us-iad-10.linodeobjects.com: Washington, DC, US (us-iad-10)
acl: {}
bucket_acl: true
quirks:
# Linode Object Storage is Ceph-backed; signing Accept-Encoding causes
# SignatureDoesNotMatch with rclone 1.68+. Match the Ceph quirk.
# See: https://github.com/rclone/rclone/issues/8206
sign_accept_encoding: false
-7
View File
@@ -1,7 +0,0 @@
name: RelAix
description: RelAix.Storage
endpoint:
s3-gw1.relaix.net: Aachen-1, Germany
s3-gw2.relaix.net: Aachen-2, Germany
acl: {}
bucket_acl: true
-2
View File
@@ -4,12 +4,10 @@ region:
nl-ams: Amsterdam, The Netherlands
fr-par: Paris, France
pl-waw: Warsaw, Poland
it-mil: Milan, Italy
endpoint:
s3.nl-ams.scw.cloud: Amsterdam Endpoint
s3.fr-par.scw.cloud: Paris Endpoint
s3.pl-waw.scw.cloud: Warsaw Endpoint
s3.it-mil.scw.cloud: Milan Endpoint
acl: {}
storage_class:
'': Default.
+44 -148
View File
@@ -1355,10 +1355,10 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
}
// s3RedirectSecretHeaders are the request headers carrying origin-bound
// secrets that must not be forwarded when a redirect leaves the original
// host. Go strips Authorization on a hostname change but keeps it for a
// subdomain or a different port, and has no knowledge that the SSE-C
// headers hold raw encryption keys, so we strip them all ourselves.
// secrets that must not be forwarded when a redirect crosses a host or
// downgrades the scheme. Go strips Authorization on a hostname change but not
// on a scheme downgrade, and has no knowledge that the SSE-C headers hold raw
// encryption keys, so we strip them all ourselves.
var s3RedirectSecretHeaders = []string{
"X-Amz-Security-Token", // AWS STS session token
"X-Amz-S3session-Token", // S3 Express (directory bucket) session token
@@ -1373,10 +1373,36 @@ var s3RedirectSecretHeaders = []string{
"Referer", // may be a presigned request
}
// s3CheckRedirect is the http.Client CheckRedirect policy for S3. It
// refuses HTTPS to HTTP downgrades and strips s3RedirectSecretHeaders
// once a redirect has left the original host.
var s3CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(s3RedirectSecretHeaders...)
func s3CheckRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
// Never follow a redirect that downgrades the transport from HTTPS to
// HTTP. An S3 endpoint has no legitimate reason to do this, and replaying
// the request over plaintext would expose whatever credential it carries.
if via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme == "http" {
return fmt.Errorf("refusing to follow insecure redirect from HTTPS to HTTP: %s", req.URL.Redacted())
}
if s3RedirectCrossesHost(req, via) {
for _, header := range s3RedirectSecretHeaders {
req.Header.Del(header)
}
}
return nil
}
func s3RedirectCrossesHost(req *http.Request, via []*http.Request) bool {
if len(via) == 0 {
return false
}
scheme, host := via[0].URL.Scheme, via[0].URL.Host
for _, redirect := range via[1:] {
if redirect.URL.Host != host || redirect.URL.Scheme != scheme {
return true
}
}
return host != req.URL.Host || scheme != req.URL.Scheme
}
// Fixup the request if needed.
//
@@ -2289,31 +2315,7 @@ func versionLess(a, b *types.ObjectVersion) bool {
// types.ObjectVersion with Size = isDeleteMarker to tell them apart
//
// We then merge them back into the Versions in the correct order
func mergeDeleteMarkers(oldVersions []types.ObjectVersion, deleteMarkers []types.DeleteMarkerEntry, urlEncoded bool) (newVersions []types.ObjectVersion) {
encodedKeys := make(map[string]string)
if urlEncoded {
// URL encoding can change key order, so compare decoded keys and restore the encoded keys for the caller.
oldVersions = append([]types.ObjectVersion(nil), oldVersions...)
deleteMarkers = append([]types.DeleteMarkerEntry(nil), deleteMarkers...)
decodeKey := func(key **string) {
if *key == nil {
return
}
encodedKey := **key
decodedKey, err := url.QueryUnescape(encodedKey)
if err != nil {
return
}
encodedKeys[decodedKey] = encodedKey
*key = &decodedKey
}
for i := range oldVersions {
decodeKey(&oldVersions[i].Key)
}
for i := range deleteMarkers {
decodeKey(&deleteMarkers[i].Key)
}
}
func mergeDeleteMarkers(oldVersions []types.ObjectVersion, deleteMarkers []types.DeleteMarkerEntry) (newVersions []types.ObjectVersion) {
newVersions = make([]types.ObjectVersion, 0, len(oldVersions)+len(deleteMarkers))
for _, deleteMarker := range deleteMarkers {
var obj types.ObjectVersion
@@ -2328,11 +2330,6 @@ func mergeDeleteMarkers(oldVersions []types.ObjectVersion, deleteMarkers []types
}
// Merge any remaining versions
newVersions = append(newVersions, oldVersions...)
for i := range newVersions {
if encodedKey, ok := encodedKeys[deref(newVersions[i].Key)]; ok {
newVersions[i].Key = &encodedKey
}
}
return newVersions
}
@@ -2372,7 +2369,7 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
// Merge in delete Markers as types.ObjectVersion if we need them
if ls.hidden || ls.usingVersionAt {
respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers, ls.req.EncodingType == types.EncodingTypeUrl)
respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers)
}
// Convert the Versions and the DeleteMarkers into an array of types.Object
@@ -2399,8 +2396,6 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
var obj types.Object
//structs.SetFrom(obj, objVersion)
setFrom_typesObject_typesObjectVersion(&obj, &objVersion)
// StorageClass has a different type in ObjectVersion so isn't copied by setFrom
obj.StorageClass = types.ObjectStorageClass(objVersion.StorageClass)
// Adjust the file names
if !ls.usingVersionAt && (!deref(objVersion.IsLatest) || objVersion.Size == isDeleteMarker) {
if obj.Key != nil && objVersion.LastModified != nil {
@@ -3308,10 +3303,6 @@ func (f *Fs) Hashes() hash.Set {
// PublicLink generates a public link to the remote path (usually readable by anyone)
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
return f.publicLink(ctx, remote, expire, &s3.GetObjectInput{})
}
func (f *Fs) publicLink(ctx context.Context, remote string, expire fs.Duration, req *s3.GetObjectInput) (link string, err error) {
if strings.HasSuffix(remote, "/") {
return "", fs.ErrorCantShareDirectories
}
@@ -3325,10 +3316,11 @@ func (f *Fs) publicLink(ctx context.Context, remote string, expire fs.Duration,
expire = maxExpireDuration
}
bucket, bucketPath := f.split(remote)
req.Bucket = &bucket
req.Key = &bucketPath
req.VersionId = o.versionID
httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, req, s3.WithPresignExpires(time.Duration(expire)))
httpReq, err := s3.NewPresignClient(f.c).PresignGetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &bucketPath,
VersionId: o.versionID,
}, s3.WithPresignExpires(time.Duration(expire)))
if err != nil {
return "", err
}
@@ -3336,40 +3328,6 @@ func (f *Fs) publicLink(ctx context.Context, remote string, expire fs.Duration,
}
var commandHelp = []fs.CommandHelp{{
Name: "link",
Short: "Generate a signed link with response header overrides.",
Long: `This command generates a signed download link for one file, with optional
overrides for the HTTP response headers. Pass the file path as a separate
argument, relative to the remote path.
Usage examples:
` + "```console" + `
rclone backend link s3:bucket path/to/file -o expire=1h
rclone backend link s3:bucket path/to/file -o response-expires="Thu, 01 Jan 1970 00:00:00 GMT"
rclone backend link s3:bucket path/to/file -o response-content-disposition='attachment; filename="download.txt"'
` + "```" + `
The link expires after 7 days by default. Use ` + "`-o expire=1h`" + ` to change
its lifetime. Durations must be at least 1 second; values above 7 days are
limited to 7 days, as with ` + "`rclone link`" + `.
The ` + "`response-expires`" + ` option sets the HTTP Expires response header,
not the lifetime of the signed link. It must be an HTTP date, such as
` + "`Thu, 01 Jan 1970 00:00:00 GMT`" + `.
The overrides are included in the signature and must not be changed in the
returned URL. They do not modify the object's stored metadata.`,
Opts: map[string]string{
"expire": "How long the link will be valid (default 7d, maximum 7d).",
"response-cache-control": "Set the Cache-Control response header.",
"response-content-disposition": "Set the Content-Disposition response header.",
"response-content-encoding": "Set the Content-Encoding response header.",
"response-content-language": "Set the Content-Language response header.",
"response-content-type": "Set the Content-Type response header.",
"response-expires": "Set the Expires response header to an HTTP date.",
},
}, {
Name: "restore",
Short: "Restore objects from GLACIER or INTELLIGENT-TIERING archive tier.",
Long: `This command can be used to restore one or more objects from GLACIER to normal
@@ -3597,44 +3555,6 @@ It doesn't return anything.`,
// otherwise it will be JSON encoded and shown to the user like that
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
switch name {
case "link":
if len(arg) != 1 || arg[0] == "" {
return nil, errors.New("link requires exactly one file path argument")
}
req := s3.GetObjectInput{}
expire := maxExpireDuration
for key, value := range opt {
switch key {
case "expire":
duration, err := fs.ParseDuration(value)
if err != nil {
return nil, fmt.Errorf("invalid expire: %w", err)
}
if duration < time.Second {
return nil, errors.New("expire must be at least 1 second")
}
expire = fs.Duration(duration)
case "response-cache-control":
req.ResponseCacheControl = aws.String(value)
case "response-content-disposition":
req.ResponseContentDisposition = aws.String(value)
case "response-content-encoding":
req.ResponseContentEncoding = aws.String(value)
case "response-content-language":
req.ResponseContentLanguage = aws.String(value)
case "response-content-type":
req.ResponseContentType = aws.String(value)
case "response-expires":
date, err := http.ParseTime(value)
if err != nil {
return nil, fmt.Errorf("invalid response-expires: %w", err)
}
req.ResponseExpires = &date
default:
return nil, fmt.Errorf("unknown link option %q", key)
}
}
return f.publicLink(ctx, arg[0], expire, &req)
case "restore":
req := s3.RestoreObjectInput{
//Bucket: &f.rootBucket,
@@ -4503,16 +4423,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
})
if awsError, ok := errors.AsType[smithy.APIError](err); ok {
if awsError.ErrorCode() == "InvalidObjectState" {
storageClass := "GLACIER"
if stateErr, ok := errors.AsType[*types.InvalidObjectState](err); ok && stateErr.StorageClass != "" {
storageClass = string(stateErr.StorageClass)
if stateErr.AccessTier != "" {
storageClass += " (" + string(stateErr.AccessTier) + " tier)"
}
} else if o.storageClass != nil && *o.storageClass != "" {
storageClass = *o.storageClass
}
return nil, fmt.Errorf("Object in %s, restore first: bucket=%q, key=%q", storageClass, bucket, bucketPath)
return nil, fmt.Errorf("Object in GLACIER, restore first: bucket=%q, key=%q", bucket, bucketPath)
}
}
if err != nil {
@@ -4701,23 +4612,8 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
// create checksum of buffer for integrity checking
// currently there is no way to calculate the md5 without reading the chunk a 2nd time (1st read is in uploadMultipart)
// possible in AWS SDK v2 with trailers?
//
// io.Copy reads with a 32 KiB buffer from a reader that implements
// neither WriterTo nor a memory backed Read - which is what a local
// file opened by a multi-thread copy is - so give it a bigger one to
// keep the source reads a useful size. It isn't taken from the pool,
// where it could wait for ever on --max-buffer-memory. A reader with
// WriterTo, such as a pool.RW, doesn't need one.
m := md5.New()
var (
currentChunkSize int64
err error
)
if _, ok := reader.(io.WriterTo); ok {
currentChunkSize, err = io.Copy(m, reader)
} else {
currentChunkSize, err = io.CopyBuffer(m, reader, make([]byte, multipart.BufferSize))
}
currentChunkSize, err := io.Copy(m, reader)
if err != nil {
return -1, err
}
@@ -4784,7 +4680,7 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
w.addCompletedPart(s3PartNumber, uout.ETag)
fs.Debugf(w.o, "multipart upload wrote chunk %d with %v bytes, etag %v and md5 %v", chunkNumber+1, currentChunkSize, *uout.ETag, hex.EncodeToString(md5sumBinary))
fs.Debugf(w.o, "multipart upload wrote chunk %d with %v bytes and etag %v", chunkNumber+1, currentChunkSize, *uout.ETag)
return currentChunkSize, err
}
+1 -56
View File
@@ -9,8 +9,6 @@ import (
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"path"
"strings"
"testing"
@@ -307,64 +305,11 @@ func TestMergeDeleteMarkers(t *testing.T) {
},
},
} {
got := mergeDeleteMarkers(test.versions, test.markers, false)
got := mergeDeleteMarkers(test.versions, test.markers)
assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test))
}
}
func TestMergeDeleteMarkersWithURLEncodedKeys(t *testing.T) {
plainKey := "images/reservations/photo.png"
encodedKey := "images/%D0%9F%D1%80%D0%B8%D0%B2%D0%B5%D1%82-01.jpg"
t1 := fstest.Time("2022-01-21T12:00:00+01:00")
t2 := fstest.Time("2022-01-21T12:00:01+01:00")
versions := []types.ObjectVersion{
{Key: &plainKey, LastModified: &t2},
{Key: &encodedKey, LastModified: &t1},
}
markers := []types.DeleteMarkerEntry{
{Key: &encodedKey, LastModified: &t2},
}
got := mergeDeleteMarkers(versions, markers, true)
want := []types.ObjectVersion{
{Key: &plainKey, LastModified: &t2},
{Key: &encodedKey, LastModified: &t2, Size: isDeleteMarker},
{Key: &encodedKey, LastModified: &t1},
}
assert.Equal(t, want, got)
}
func TestVersionsListStorageClass(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, `<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bucket</Name>
<IsTruncated>false</IsTruncated>
<Version>
<Key>archived.bin</Key>
<VersionId>v1</VersionId>
<IsLatest>true</IsLatest>
<LastModified>2026-09-26T12:00:00.000Z</LastModified>
<ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
<Size>0</Size>
<StorageClass>DEEP_ARCHIVE</StorageClass>
</Version>
</ListVersionsResult>`)
}))
defer srv.Close()
f := &Fs{c: s3.New(s3.Options{
Region: "us-east-1",
BaseEndpoint: aws.String(srv.URL),
UsePathStyle: true,
Credentials: aws.AnonymousCredentials{},
})}
ls := f.newVersionsList(&s3.ListObjectsV2Input{Bucket: aws.String("bucket")}, false, time.Time{})
resp, _, err := ls.List(context.Background())
require.NoError(t, err)
require.Len(t, resp.Contents, 1)
assert.Equal(t, types.ObjectStorageClassDeepArchive, resp.Contents[0].StorageClass)
}
func TestRemoveAWSChunked(t *testing.T) {
ps := func(s string) *string {
return &s
-227
View File
@@ -1,227 +0,0 @@
package s3
import (
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newLinkTestFs(t *testing.T) *Fs {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Has("versions") {
assert.Equal(t, http.MethodGet, r.Method)
assert.Equal(t, "prefix/file", r.URL.Query().Get("prefix"))
_, err := io.WriteString(w, `<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<IsTruncated>false</IsTruncated>
<Version><Key>prefix/file</Key><VersionId>version+id/1</VersionId><IsLatest>false</IsLatest>
<LastModified>2024-01-01T00:00:00Z</LastModified><Size>1</Size></Version>
</ListVersionsResult>`)
assert.NoError(t, err)
return
}
assert.Equal(t, http.MethodHead, r.Method)
if strings.HasSuffix(r.URL.Path, "/missing") {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Length", "1")
w.Header().Set("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT")
}))
t.Cleanup(server.Close)
ctx, opt, client := SetupS3Test(t)
opt.Endpoint = server.URL
opt.ForcePathStyle = true
opt.Region = "us-east-1"
opt.AccessKeyID = "test-access-key"
opt.SecretAccessKey = "test-secret-key"
opt.SessionToken = "test-token+/="
c, _, err := s3Connection(ctx, opt, client)
require.NoError(t, err)
f := &Fs{
name: "s3test",
opt: *opt,
ctx: ctx,
c: c,
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
cache: bucket.NewCache(),
}
f.setRoot("bucket/prefix")
return f
}
func linkSignature(t *testing.T, u *url.URL) string {
t.Helper()
q := u.Query()
signingTime, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date"))
require.NoError(t, err)
q.Del("X-Amz-Signature")
unsigned := *u
unsigned.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, unsigned.String(), nil)
require.NoError(t, err)
signed, _, err := v4.NewSigner().PresignHTTP(context.Background(), aws.Credentials{
AccessKeyID: "test-access-key",
SecretAccessKey: "test-secret-key",
SessionToken: "test-token+/=",
}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", signingTime, func(opt *v4.SignerOptions) {
opt.DisableURIPathEscaping = true
})
require.NoError(t, err)
result, err := url.Parse(signed)
require.NoError(t, err)
return result.Query().Get("X-Amz-Signature")
}
func TestCommandLink(t *testing.T) {
f := newLinkTestFs(t)
remote := "dir/a file +&?%\u2603.txt"
overrides := map[string]string{
"response-cache-control": "private, max-age=0",
"response-content-disposition": `attachment; filename="a +&?.txt"`,
"response-content-encoding": "identity",
"response-content-language": "en-US",
"response-content-type": "text/plain; charset=utf-8",
"response-expires": "Thu, 01 Jan 1970 00:00:00 GMT",
}
opts := map[string]string{"expire": "1h"}
for k, v := range overrides {
opts[k] = v
}
out, err := f.Command(context.Background(), "link", []string{remote}, opts)
require.NoError(t, err)
link, ok := out.(string)
require.True(t, ok)
u, err := url.Parse(link)
require.NoError(t, err)
assert.Equal(t, "/bucket/prefix/"+remote, u.Path)
assert.NotContains(t, u.RawQuery, " ")
q := u.Query()
assert.Equal(t, "3600", q.Get("X-Amz-Expires"))
assert.Equal(t, "test-token+/=", q.Get("X-Amz-Security-Token"))
assert.Equal(t, "AWS4-HMAC-SHA256", q.Get("X-Amz-Algorithm"))
assert.Equal(t, "host", q.Get("X-Amz-SignedHeaders"))
for k, v := range overrides {
assert.Equal(t, v, q.Get(k), k)
}
signature := q.Get("X-Amz-Signature")
require.NotEmpty(t, signature)
assert.Equal(t, signature, linkSignature(t, u))
for k := range overrides {
t.Run("Signed/"+k, func(t *testing.T) {
tampered := *u
changed := u.Query()
changed.Set(k, "changed")
tampered.RawQuery = changed.Encode()
assert.NotEqual(t, signature, linkSignature(t, &tampered))
})
}
}
func TestCommandLinkExpire(t *testing.T) {
f := newLinkTestFs(t)
for _, test := range []struct {
name string
opts map[string]string
expire string
}{
{"Default", nil, "604800"},
{"Minimum", map[string]string{"expire": "1s"}, "1"},
{"Maximum", map[string]string{"expire": "7d"}, "604800"},
{"Clamped", map[string]string{"expire": "8d"}, "604800"},
{"Off", map[string]string{"expire": "off"}, "604800"},
} {
t.Run(test.name, func(t *testing.T) {
out, err := f.Command(context.Background(), "link", []string{"file"}, test.opts)
require.NoError(t, err)
u, err := url.Parse(out.(string))
require.NoError(t, err)
assert.Equal(t, test.expire, u.Query().Get("X-Amz-Expires"))
for key := range u.Query() {
assert.False(t, strings.HasPrefix(key, "response-"))
}
assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
})
}
}
func TestCommandLinkErrors(t *testing.T) {
f := newLinkTestFs(t)
for _, test := range []struct {
name string
args []string
opts map[string]string
}{
{"NoPath", nil, nil},
{"TooManyPaths", []string{"one", "two"}, nil},
{"EmptyPath", []string{""}, nil},
{"Directory", []string{"dir/"}, nil},
{"Missing", []string{"missing"}, nil},
{"UnknownOption", []string{"file"}, map[string]string{"response-unknown": "value"}},
{"InvalidDate", []string{"file"}, map[string]string{"response-expires": "tomorrow"}},
{"InvalidExpire", []string{"file"}, map[string]string{"expire": "invalid"}},
{"EmptyExpire", []string{"file"}, map[string]string{"expire": ""}},
{"ZeroExpire", []string{"file"}, map[string]string{"expire": "0"}},
{"NegativeExpire", []string{"file"}, map[string]string{"expire": "-1s"}},
{"SubsecondExpire", []string{"file"}, map[string]string{"expire": "500ms"}},
} {
t.Run(test.name, func(t *testing.T) {
_, err := f.Command(context.Background(), "link", test.args, test.opts)
require.Error(t, err)
assert.NotErrorIs(t, err, fs.ErrorCommandNotFound)
})
}
}
func TestCommandLinkVersionAt(t *testing.T) {
f := newLinkTestFs(t)
f.opt.VersionAt = fs.Time(time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC))
for _, command := range []bool{false, true} {
var link string
if command {
out, err := f.Command(context.Background(), "link", []string{"file"}, map[string]string{
"response-content-type": "text/plain",
})
require.NoError(t, err)
link = out.(string)
} else {
var err error
link, err = f.PublicLink(context.Background(), "file", fs.DurationOff, false)
require.NoError(t, err)
}
u, err := url.Parse(link)
require.NoError(t, err)
assert.Equal(t, "/bucket/prefix/file", u.Path)
assert.Equal(t, "version+id/1", u.Query().Get("versionId"))
assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
}
}
func TestPublicLink(t *testing.T) {
f := newLinkTestFs(t)
link, err := f.PublicLink(context.Background(), "file", fs.DurationOff, false)
require.NoError(t, err)
u, err := url.Parse(link)
require.NoError(t, err)
assert.Equal(t, "/bucket/prefix/file", u.Path)
assert.Equal(t, "604800", u.Query().Get("X-Amz-Expires"))
assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
_, err = f.PublicLink(context.Background(), "dir/", fs.DurationOff, false)
assert.ErrorIs(t, err, fs.ErrorCantShareDirectories)
_, err = f.PublicLink(context.Background(), "missing", fs.DurationOff, false)
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}
+48 -60
View File
@@ -3,10 +3,8 @@ package s3
import (
"context"
"io"
"net/http"
"net/http/httptest"
"path"
"testing"
"time"
@@ -230,6 +228,54 @@ func TestS3CheckRedirectRejectsSchemeDowngrade(t *testing.T) {
}
}
func TestRedirectCrossesHost(t *testing.T) {
mustReq := func(method, url string) *http.Request {
req, err := http.NewRequest(method, url, nil)
require.NoError(t, err)
return req
}
for _, test := range []struct {
name string
via []string
req string
want bool
}{
{
name: "SameHost",
via: []string{"https://bucket.example.com/"},
req: "https://bucket.example.com/redirected",
want: false,
},
{
name: "DifferentHost",
via: []string{"https://bucket.example.com/"},
req: "https://evil.example.com/redirected",
want: true,
},
{
name: "SchemeDowngradeSameHost",
via: []string{"https://bucket.example.com/"},
req: "http://bucket.example.com/redirected",
want: true,
},
{
name: "SchemeDowngradeMidChain",
via: []string{"https://bucket.example.com/", "http://bucket.example.com/middle"},
req: "http://bucket.example.com/final",
want: true,
},
} {
t.Run(test.name, func(t *testing.T) {
via := make([]*http.Request, len(test.via))
for i, url := range test.via {
via[i] = mustReq(http.MethodGet, url)
}
got := s3RedirectCrossesHost(mustReq(http.MethodGet, test.req), via)
assert.Equal(t, test.want, got)
})
}
}
func TestClientStopsAfterTenRedirects(t *testing.T) {
_, _, client := SetupS3Test(t)
@@ -452,61 +498,3 @@ func TestParseRetainUntilDate(t *testing.T) {
})
}
}
func TestOpenRestoreFirstError(t *testing.T) {
errorBodies := map[string]string{
"DeepArchive": `<Error><Code>InvalidObjectState</Code><StorageClass>DEEP_ARCHIVE</StorageClass></Error>`,
"IntelligentTiering": `<Error><Code>InvalidObjectState</Code><StorageClass>INTELLIGENT_TIERING</StorageClass><AccessTier>DEEP_ARCHIVE_ACCESS</AccessTier></Error>`,
"FromObject": `<Error><Code>InvalidObjectState</Code></Error>`,
"Unknown": `<Error><Code>InvalidObjectState</Code></Error>`,
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
_, _ = io.WriteString(w, errorBodies[path.Base(r.URL.Path)])
}))
defer server.Close()
ctx, opt, client := SetupS3Test(t)
opt.Endpoint = server.URL
opt.ForcePathStyle = true
opt.Region = "us-east-1"
opt.AccessKeyID = "test-access-key"
opt.SecretAccessKey = "test-secret-key"
c, _, err := s3Connection(ctx, opt, client)
require.NoError(t, err)
f := &Fs{
name: "s3test",
opt: *opt,
ctx: ctx,
c: c,
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
cache: bucket.NewCache(),
}
f.setRoot("bucket")
for _, test := range []struct {
name string
storageClass *string
want string
}{{
name: "DeepArchive",
want: "Object in DEEP_ARCHIVE, restore first",
}, {
name: "IntelligentTiering",
want: "Object in INTELLIGENT_TIERING (DEEP_ARCHIVE_ACCESS tier), restore first",
}, {
name: "FromObject",
storageClass: aws.String("DEEP_ARCHIVE"),
want: "Object in DEEP_ARCHIVE, restore first",
}, {
name: "Unknown",
want: "Object in GLACIER, restore first",
}} {
t.Run(test.name, func(t *testing.T) {
o := &Object{fs: f, remote: test.name, storageClass: test.storageClass}
_, err := o.Open(ctx)
require.Error(t, err)
assert.Contains(t, err.Error(), test.want)
})
}
}
-2
View File
@@ -2832,8 +2832,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
if err != nil {
// Close the handle otherwise the server keeps the removed file's space allocated
_ = file.Close()
o.fs.putSftpConnection(&c, err)
remove()
return fmt.Errorf("Update ReadFrom failed: %w", err)
+3 -5
View File
@@ -380,11 +380,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
//Need to check if destination exists
fullPath := f.buildFullPath(dstRemote)
var response api.ListDirResponse
res, err := f.callAPI(ctx, "GET", fmt.Sprintf("/%s/fs/attr?path=%s", f.drive, fullPath), &response)
if res == nil || res.StatusCode != http.StatusNotFound {
if err != nil {
return fmt.Errorf("failed to check destination directory: %w", err)
}
res, _ := f.callAPI(ctx, "GET", fmt.Sprintf("/%s/fs/attr?path=%s", f.drive, fullPath), &response)
if res.StatusCode != http.StatusNotFound {
return fs.ErrorDirExists
}
+4 -5
View File
@@ -60,11 +60,10 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (c *conn, err error
}
} else {
d.Initiator = &smb2.NTLMInitiator{
User: f.opt.User,
Password: pass,
Domain: f.opt.Domain,
Workstation: f.opt.Workstation,
TargetSPN: f.opt.SPN,
User: f.opt.User,
Password: pass,
Domain: f.opt.Domain,
TargetSPN: f.opt.SPN,
}
}
+2 -16
View File
@@ -51,7 +51,8 @@ func init() {
Sensitive: true,
}, {
Name: "user",
Help: "SMB username.\n\nLeave blank to use the current user name.",
Help: "SMB username.",
Default: currentUser,
Sensitive: true,
}, {
Name: "port",
@@ -78,17 +79,6 @@ authentication, and it often needs to be set for clusters. For example:
Leave blank if not sure.
`,
Sensitive: true,
}, {
Name: "workstation",
Help: `Workstation name sent in NTLM authentication.
The NetBIOS name sent as the client workstation in the NTLM AUTHENTICATE
message. Some servers (e.g. domain-joined Windows servers with per-account
workstation restrictions or NTLM allow-lists) require this to match a
specific name. Leave blank to send no workstation name (the default).
`,
Advanced: true,
Sensitive: true,
}, {
Name: "use_kerberos",
Help: `Use Kerberos authentication.
@@ -163,7 +153,6 @@ type Options struct {
Pass string `config:"pass"`
Domain string `config:"domain"`
SPN string `config:"spn"`
Workstation string `config:"workstation"`
UseKerberos bool `config:"use_kerberos"`
KerberosCCache string `config:"kerberos_ccache"`
HideSpecial bool `config:"hide_special_share"`
@@ -204,9 +193,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, err
}
if opt.User == "" {
opt.User = currentUser
}
// if root is empty or ends with / (must be a directory)
isRootDir := isPathDir(root)
-59
View File
@@ -12,8 +12,6 @@ import (
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
@@ -95,33 +93,6 @@ func TestUploadConnectionReuse(t *testing.T) {
assert.Equal(t, 1, pooled, "upload should leave exactly one connection in the pool")
}
// TestOptionsWorkstationConfig verifies the workstation config key is backwards compatible
func TestOptionsWorkstationConfig(t *testing.T) {
tests := []struct {
name string
workstation string
}{
{"named workstation", "MYWORKSTATION"},
{"empty workstation sends no name", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := configmap.Simple{
"host": "example.com",
"workstation": tt.workstation,
}
opt := new(Options)
if err := configstruct.Set(m, opt); err != nil {
t.Fatalf("configstruct.Set failed: %v", err)
}
if opt.Workstation != tt.workstation {
t.Errorf("opt.Workstation = %q, want %q", opt.Workstation, tt.workstation)
}
})
}
}
// TestIsPathDir tests the isPathDir function logic
func TestIsPathDir(t *testing.T) {
tests := []struct {
@@ -158,33 +129,3 @@ func TestIsPathDir(t *testing.T) {
})
}
}
func TestNewFsUser(t *testing.T) {
ctx := context.Background()
for _, test := range []struct {
user string
want string
}{
{user: "", want: currentUser},
{user: currentUser, want: currentUser},
{user: "someone", want: "someone"},
} {
m := configmap.Simple{"host": "localhost", "user": test.user}
f, err := NewFs(ctx, "TestSMB", "", m)
require.NoError(t, err)
assert.Equal(t, test.want, f.(*Fs).opt.User, "user=%q", test.user)
}
// No user in the config at all
f, err := NewFs(ctx, "TestSMB", "", configmap.Simple{"host": "localhost"})
require.NoError(t, err)
assert.Equal(t, currentUser, f.(*Fs).opt.User)
}
// The default is blank so that a user name which happens to match the
// current user is still written to the config file.
func TestUserDefault(t *testing.T) {
ri, err := fs.Find("smb")
require.NoError(t, err)
assert.Equal(t, "", ri.Options.Get("user").Default)
}
+1 -1
View File
@@ -860,7 +860,7 @@ type RenamingObjectInfoProxy struct {
remote string
}
// String implements fs.ObjectInfo.String by delegating to the wrapped instance.
// Remote implements fs.ObjectInfo.Remote by delegating to the wrapped instance.
func (s *RenamingObjectInfoProxy) String() string {
return s.delegate.String()
}
+1 -21
View File
@@ -17,7 +17,6 @@ import (
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
)
@@ -134,26 +133,7 @@ func (o *Object) uploadChunks(ctx context.Context, in0 io.Reader, size int64, pa
return io.NopCloser(in), nil
}
// Retry this chunk in place so a transient failure does not make the caller
// restart the whole upload and resend chunks that were already accepted.
maxTries := max(fs.GetConfig(ctx).LowLevelRetries, 1)
var err error
for try := 1; try <= maxTries; try++ {
retryBody, bodyErr := getBody()
if bodyErr != nil {
return fmt.Errorf("rewinding chunk for retry failed: %w", bodyErr)
}
err = partObj.updateSimple(ctx, retryBody, getBody, partObj.remote, contentLength, "application/x-www-form-urlencoded", nil, o.fs.chunksUploadURL, options...)
_ = retryBody.Close()
if err == nil || (!fserrors.IsRetryError(err) && !fserrors.ShouldRetry(err)) {
break
}
if try == maxTries {
// The chunk retry budget is exhausted. Do not let the outer
// low-level retry restart the upload from its first chunk.
err = fserrors.NoLowLevelRetryError(errors.New(err.Error()))
}
}
err := partObj.updateSimple(ctx, in, getBody, partObj.remote, contentLength, "application/x-www-form-urlencoded", nil, o.fs.chunksUploadURL, options...)
if err != nil {
return fmt.Errorf("uploading chunk failed: %w", err)
}
+5 -80
View File
@@ -135,13 +135,6 @@ The input format is comma separated list of key,value pairs. Standard
For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
The headers are only sent to the host in the configured URL. If the
server redirects to another host (including a subdomain or a different
port) the headers are not sent to it, or to any further hop in that
redirect chain. The exception is the directory listing and file
download requests when auth_redirect is set, as that keeps all
credentials across redirects.
`,
Default: fs.CommaSepList{},
Advanced: true,
@@ -242,8 +235,6 @@ type Fs struct {
canChunk bool // set if nextcloud and nextcloud_chunk_size is set
canRecalcHash bool // set if the server can recalculate checksums with PATCH (nextcloud)
authSingleflight *singleflight.Group
// redirect policy which keeps opt.Headers on the configured host
checkRedirect func(req *http.Request, via []*http.Request) error
}
// Object describes a webdav object
@@ -354,41 +345,6 @@ func itemIsDir(item *api.Response) bool {
return false
}
// sabreNotFound is the exception SabreDAV based servers report for a
// path which doesn't exist
var sabreNotFound = []byte(`Sabre\DAV\Exception\NotFound`)
// callPropfind does the PROPFIND in opts and decodes the response into result
//
// ownCloud 10.16 starts a 207 Multi-Status response before it finds
// that the path doesn't exist, then appends a Sabre NotFound error to
// it which makes it invalid XML. This is returned as the 404 *api.Error
// which servers normally give for a path which doesn't exist.
func (f *Fs) callPropfind(ctx context.Context, opts *rest.Opts, result *api.Multistatus) (resp *http.Response, err error) {
resp, err = f.srv.Call(ctx, opts)
if err != nil {
return resp, err
}
// Keep the start of the body to look for the error in
start := make([]byte, 4096)
n, _ := io.ReadFull(resp.Body, start)
start = start[:n]
body := resp.Body
resp.Body = struct {
io.Reader
io.Closer
}{io.MultiReader(bytes.NewReader(start), body), body}
err = rest.DecodeXML(resp, result)
if err != nil && bytes.Contains(start, sabreNotFound) {
return resp, &api.Error{
Exception: string(sabreNotFound),
Status: http.StatusText(http.StatusNotFound),
StatusCode: http.StatusNotFound,
}
}
return resp, err
}
// readMetaDataForPath reads the metadata from the path
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Prop, err error) {
// FIXME how do we read back additional properties?
@@ -398,7 +354,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Pr
ExtraHeaders: map[string]string{
"Depth": "0",
},
CheckRedirect: f.preserveMethodRedirect,
CheckRedirect: rest.PreserveMethodRedirectFn,
}
if f.hasOCMD5 || f.hasOCSHA1 {
opts.Body = bytes.NewBuffer(owncloudProps)
@@ -409,11 +365,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Pr
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
var attempt api.Multistatus
resp, err = f.callPropfind(ctx, &opts, &attempt)
if err == nil {
result = attempt
}
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return f.shouldRetry(ctx, resp, err)
})
if apiErr, ok := err.(*api.Error); ok {
@@ -564,17 +516,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
rt: ntlmssp.Negotiator{RoundTripper: t},
}
}
// Refuse redirects that downgrade HTTPS to plaintext HTTP and
// don't send the configured headers to another host.
//
// The auth_redirect option bypasses this for the requests it applies
// to as it is an explicit opt-in to keep credentials across redirects.
headerNames := make([]string, 0, len(opt.Headers)/2)
for i := 0; i < len(opt.Headers); i += 2 {
headerNames = append(headerNames, opt.Headers[i])
}
f.checkRedirect = rest.StripHeadersOnCrossHostRedirectFn(headerNames...)
client.CheckRedirect = f.checkRedirect
// Refuse redirects that downgrade HTTPS to plaintext HTTP.
client.CheckRedirect = rest.RefuseHTTPSDowngradeRedirectFn
f.srv = rest.NewClient(client).SetRoot(u.String())
f.features = (&fs.Features{
@@ -601,10 +544,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if !f.findHeader(opt.Headers, "Referer") {
f.srv.SetHeader("Referer", u.String())
}
// Nextcloud public shares reject requests without this header
if opt.Vendor == "nextcloud" && strings.Contains(u.Path, "public.php/dav/files") && !f.findHeader(opt.Headers, "X-Requested-With") {
f.srv.SetHeader("X-Requested-With", "XMLHttpRequest")
}
if root != "" && !rootIsDir {
// Check to see if the root actually an existing file
@@ -657,16 +596,6 @@ func (f *Fs) fetchBearerToken(cmd fs.SpaceSepList) (string, error) {
return stdoutString, nil
}
// preserveMethodRedirect is an http.Client.CheckRedirect function
// which applies f.checkRedirect and then preserves the original HTTP
// method like rest.PreserveMethodRedirectFn.
func (f *Fs) preserveMethodRedirect(req *http.Request, via []*http.Request) error {
if err := f.checkRedirect(req, via); err != nil {
return err
}
return rest.PreserveMethodRedirectFn(req, via)
}
// Adds the configured headers to the request if any
func (f *Fs) addHeaders(headers fs.CommaSepList) {
for i := 0; i < len(headers); i += 2 {
@@ -886,11 +815,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
var attempt api.Multistatus
resp, err = f.callPropfind(ctx, &opts, &attempt)
if err == nil {
result = attempt
}
resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
return f.shouldRetry(ctx, resp, err)
})
if err != nil {
-332
View File
@@ -1,17 +1,14 @@
package webdav_test
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/backend/webdav"
@@ -19,93 +16,11 @@ import (
"github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestChunkedUploadRetriesOnlyFailedChunk(t *testing.T) {
tests := []struct {
name string
failChunk1 bool
wantError bool
wantChunk2 int32
}{
{name: "retry failed chunk", wantChunk2: 1},
{name: "stop after chunk retry budget", failChunk1: true, wantError: true},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var chunk0, chunk1, chunk2 atomic.Int32
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/dav/uploads/alice/"):
switch {
case strings.HasSuffix(r.URL.Path, "000000000000000-000000000000003"):
chunk0.Add(1)
case strings.HasSuffix(r.URL.Path, "000000000000004-000000000000007"):
if test.failChunk1 || chunk1.Add(1) == 1 {
if test.failChunk1 {
chunk1.Add(1)
}
w.WriteHeader(http.StatusServiceUnavailable)
return
}
case strings.HasSuffix(r.URL.Path, "000000000000008-000000000000009"):
chunk2.Add(1)
default:
t.Errorf("unexpected chunk path %q", r.URL.Path)
}
w.WriteHeader(http.StatusCreated)
case r.Method == "MKCOL":
w.WriteHeader(http.StatusCreated)
case r.Method == "MOVE":
w.WriteHeader(http.StatusCreated)
case r.Method == "DELETE":
w.WriteHeader(http.StatusNotFound)
case r.Method == "PROPFIND":
w.WriteHeader(http.StatusMultiStatus)
_, _ = io.WriteString(w, `<d:multistatus xmlns:d="DAV:"><d:response><d:href>/file</d:href><d:propstat><d:prop><d:getcontentlength>10</d:getcontentlength><d:resourcetype/></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>`)
case r.Method == "PATCH":
w.Header().Set("OC-Checksum", "SHA1:0000000000000000000000000000000000000000")
w.WriteHeader(http.StatusOK)
default:
t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
}
})
ts := httptest.NewServer(handler)
defer ts.Close()
configfile.Install()
ctx, ci := fs.AddConfig(context.Background())
ci.LowLevelRetries = 2
m := configmap.Simple{
"type": "webdav",
"url": ts.URL + "/remote.php/dav/files/alice",
"vendor": "nextcloud",
"nextcloud_chunk_size": "4B",
}
f, err := webdav.NewFs(ctx, remoteName, "", m)
require.NoError(t, err)
src := object.NewStaticObjectInfo("file", time.Now(), 10, true, nil, f)
body := []byte("abcdefghij")
err = operations.Retry(ctx, src, ci.LowLevelRetries, func() error {
_, err := f.Put(ctx, bytes.NewReader(body), src)
return err
})
if test.wantError {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.EqualValues(t, 1, chunk0.Load(), "successful first chunk should not be uploaded again")
assert.EqualValues(t, 2, chunk1.Load(), "only the failed chunk should be retried")
assert.EqualValues(t, test.wantChunk2, chunk2.Load(), "later chunks should not upload after failure")
})
}
}
var (
remoteName = "TestWebDAV"
headers = []string{"X-Potato", "sausage", "X-Rhubarb", "cucumber"}
@@ -209,100 +124,6 @@ func TestListAllAuthRedirect(t *testing.T) {
assert.NotEmpty(t, targetAuth, "Authorization header should be preserved across redirect")
}
// redirectTestHeaders are the headers option for the redirect tests.
// The one starting with "*" is sent without canonicalising.
var redirectTestHeaders = []string{"X-Potato", "sausage", "*x-marrow", "pea"}
// redirectHeaders makes a webdav remote with redirectTestHeaders
// pointing at a server which redirects every request to a second
// server on hostname, or back to itself if hostname is empty, and
// exercises each of the backend's redirect policies against it: the
// PROPFIND used to look up an object, the GET used to read it and the
// PROPFIND used to list a directory. It returns the headers the
// redirect target saw on each of those requests.
func redirectHeaders(t *testing.T, hostname string) []http.Header {
var target string
var mu sync.Mutex
var seen []http.Header
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/moved/") {
http.Redirect(w, r, target+"/moved"+r.URL.Path, http.StatusTemporaryRedirect)
return
}
mu.Lock()
seen = append(seen, r.Header.Clone())
mu.Unlock()
switch r.Method {
case "PROPFIND":
w.WriteHeader(http.StatusMultiStatus)
_, err := fmt.Fprint(w, fileInfoResponse)
assert.NoError(t, err)
case http.MethodGet:
_, err := fmt.Fprint(w, "hello")
assert.NoError(t, err)
default:
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
}
})
ts := httptest.NewServer(handler)
defer ts.Close()
if hostname == "" {
target = ts.URL
} else {
// Serve the final response from a second server on a different host
other := httptest.NewServer(handler)
defer other.Close()
target = strings.Replace(other.URL, "127.0.0.1", hostname, 1)
}
configfile.Install()
m := configmap.Simple{
"type": "webdav",
"url": ts.URL,
"headers": strings.Join(redirectTestHeaders, ","),
}
ctx := context.Background()
f, err := webdav.NewFs(ctx, remoteName, "", m)
require.NoError(t, err)
// PROPFIND with Depth 0 uses rest.PreserveMethodRedirectFn
o, err := f.NewObject(ctx, "file.txt")
require.NoError(t, err)
// GET uses the client's default redirect policy
fd, err := o.Open(ctx)
require.NoError(t, err)
data, err := io.ReadAll(fd)
require.NoError(t, err)
require.NoError(t, fd.Close())
assert.Equal(t, "hello", string(data))
// PROPFIND with Depth 1 uses the client's default redirect policy
_, err = f.List(ctx, "")
require.NoError(t, err)
mu.Lock()
defer mu.Unlock()
require.Len(t, seen, 3, "expected the redirect target to see all three requests")
return seen
}
func TestRedirectKeepsHeadersOnSameHost(t *testing.T) {
for _, got := range redirectHeaders(t, "") {
for i := 0; i < len(redirectTestHeaders); i += 2 {
assert.Equal(t, redirectTestHeaders[i+1], got.Get(strings.TrimPrefix(redirectTestHeaders[i], "*")))
}
}
}
func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
// Redirect to a different host name and port. net/http strips
// Authorization on its own here so this checks the custom headers.
for _, got := range redirectHeaders(t, "localhost") {
for i := 0; i < len(redirectTestHeaders); i += 2 {
assert.Empty(t, got.Values(strings.TrimPrefix(redirectTestHeaders[i], "*")), "header %q leaked to redirect target", redirectTestHeaders[i])
}
}
}
// TestReservedCharactersInPathAreEscaped verifies that reserved characters
// like semicolons and equals signs in file paths are percent-encoded in
// HTTP requests to the WebDAV server (RFC 3986 compliance).
@@ -463,156 +284,3 @@ func TestCopyFallsBackWhenRangeIgnored(t *testing.T) {
assert.LessOrEqual(t, rangeRequests.Load(), int32(3))
assert.Equal(t, int32(1), fullRequests.Load())
}
// TestListAllRetryDoesNotConcatenate verifies that when a listing PROPFIND
// fails after having partially decoded and is retried, the final listing is
// exactly the retried response, with no entries carried over from the
// failed attempt.
func TestListAllRetryDoesNotConcatenate(t *testing.T) {
entryXML := func(i int) string {
name := fmt.Sprintf("file-%03d.bin", i)
return "<d:response><d:href>/" + name + "</d:href><d:propstat><d:prop>" +
"<d:displayname>" + name + "</d:displayname><d:getcontentlength>1024</d:getcontentlength>" +
"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>\n"
}
head := `<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:">`
self := head + `<d:response><d:href>/</d:href><d:propstat><d:prop><d:resourcetype><d:collection xmlns:d="DAV:"/></d:resourcetype></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>`
writeEntries := func(w http.ResponseWriter, from, to int) {
_, err := fmt.Fprint(w, head)
require.NoError(t, err)
for i := from; i <= to; i++ {
_, err := fmt.Fprint(w, entryXML(i))
require.NoError(t, err)
}
}
var listCalls atomic.Int64
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, "PROPFIND", r.Method)
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
if r.Header.Get("Depth") == "0" {
w.WriteHeader(207)
_, err := fmt.Fprint(w, self)
require.NoError(t, err)
return
}
if listCalls.Add(1) == 1 {
// return file-000..file-010 but kill the response before it
// completes, so the client has parsed those entries when it
// hits an unexpected EOF
w.Header().Set("Content-Length", "1000000")
w.WriteHeader(207)
writeEntries(w, 0, 10)
return
}
// return the disjoint file-011..file-030 as a complete listing
w.WriteHeader(207)
writeEntries(w, 11, 30)
_, err := fmt.Fprint(w, "</d:multistatus>")
require.NoError(t, err)
}))
defer ts.Close()
configfile.Install()
m := configmap.Simple{
"type": "webdav",
"url": ts.URL,
}
f, err := webdav.NewFs(context.Background(), remoteName, "", m)
require.NoError(t, err)
entries, err := f.List(context.Background(), "")
require.NoError(t, err)
var remotes []string
for _, e := range entries {
remotes = append(remotes, e.Remote())
}
var want []string
for i := 11; i <= 30; i++ {
want = append(want, fmt.Sprintf("file-%03d.bin", i))
}
assert.ElementsMatch(t, want, remotes)
}
// TestNotFoundInMultistatus checks that a missing path is detected when the
// server reports it by appending a Sabre NotFound error to a 207 response it
// has already started, as ownCloud 10.16 does, rather than by returning 404.
func TestNotFoundInMultistatus(t *testing.T) {
head := `<?xml version="1.0"?>` + "\n" + `<d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns"`
root := head + `><d:response><d:href>/</d:href><d:propstat><d:prop><d:resourcetype><d:collection/></d:resourcetype></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>`
notFound := head + `<?xml version="1.0" encoding="utf-8"?>
<d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
<s:exception>Sabre\DAV\Exception\NotFound</s:exception>
<s:message>File with name missing could not be located</s:message>
</d:error>
`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, "PROPFIND", r.Method)
w.Header().Set("Content-Type", "application/xml; charset=utf-8")
w.WriteHeader(207)
body := notFound
if r.URL.Path == "/" {
body = root
}
_, err := fmt.Fprint(w, body)
require.NoError(t, err)
}))
defer ts.Close()
configfile.Install()
m := configmap.Simple{
"type": "webdav",
"url": ts.URL,
}
ctx := context.Background()
// A root which doesn't exist yet
_, err := webdav.NewFs(ctx, remoteName, "missing", m)
require.NoError(t, err)
f, err := webdav.NewFs(ctx, remoteName, "", m)
require.NoError(t, err)
_, err = f.NewObject(ctx, "missing")
assert.Equal(t, fs.ErrorObjectNotFound, err)
_, err = f.List(ctx, "missing")
assert.Equal(t, fs.ErrorDirNotFound, err)
}
// TestNextcloudPublicShareHeader checks X-Requested-With is sent for
// Nextcloud public shares unless the user overrides it.
func TestNextcloudPublicShareHeader(t *testing.T) {
for _, test := range []struct {
name string
path string
headers string
want string
}{
{"public share", "/public.php/dav/files/token", "", "XMLHttpRequest"},
{"user override", "/public.php/dav/files/token", "X-Requested-With,custom", "custom"},
{"regular nextcloud", "/remote.php/dav/files/user", "", ""},
} {
t.Run(test.name, func(t *testing.T) {
var got atomic.Value
got.Store("")
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got.Store(r.Header.Get("X-Requested-With"))
w.WriteHeader(http.StatusMultiStatus)
_, _ = io.WriteString(w, `<d:multistatus xmlns:d="DAV:"></d:multistatus>`)
}))
defer ts.Close()
configfile.Install()
m := configmap.Simple{
"type": "webdav",
"url": ts.URL + test.path,
"vendor": "nextcloud",
"headers": test.headers,
}
f, err := webdav.NewFs(context.Background(), remoteName, "", m)
require.NoError(t, err)
_, _ = f.List(context.Background(), "")
assert.Equal(t, test.want, got.Load().(string))
})
}
}
+17 -45
View File
@@ -100,20 +100,6 @@ normally enough to stop them, at the cost of slowing down uploads.
Yandex support recommend a value of 1.5s - 3s.`,
Default: fs.Duration(0),
Advanced: true,
}, {
Name: "app_folder",
Help: `Use the application folder as the root.
If you registered your own OAuth application with Yandex with only
the "Application Folder" permission (cloud_api:disk.app_folder)
instead of full disk access, then rclone needs to address paths with
the "app:/" prefix instead of the usual "disk:/" prefix, otherwise
all requests fail with a 403 Forbidden error.
Set this to true if your OAuth token only has access to the
application folder.`,
Default: false,
Advanced: true,
}}...),
})
}
@@ -125,20 +111,18 @@ type Options struct {
Enc encoder.MultiEncoder `config:"encoding"`
SpoofUserAgent bool `config:"spoof_ua"`
UploadWait fs.Duration `config:"upload_wait"`
AppFolder bool `config:"app_folder"`
}
// Fs represents a remote yandex
type Fs struct {
name string
root string // root path
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv *rest.Client // the connection to the yandex server
pacer *fs.Pacer // pacer for API calls
diskRoot string // root path with the container prefix, e.g. "disk:/" or "app:/"
container string // container prefix in use, e.g. "disk:" or "app:"
name string
root string // root path
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv *rest.Client // the connection to the yandex server
pacer *fs.Pacer // pacer for API calls
diskRoot string // root path with "disk:/" container name
}
// Object describes a swift object
@@ -225,21 +209,13 @@ func errorHandler(resp *http.Response) error {
func (f *Fs) setRoot(root string) {
//Set root path
f.root = strings.Trim(root, "/")
//Set the container prefix. This is "disk:" normally, or "app:" if the
//OAuth token only has access to the application folder (see the
//app_folder option).
if f.opt.AppFolder {
f.container = "app:"
} else {
f.container = "disk:"
}
//Set disk root path.
//Adding the container prefix to root path as all paths on disk start with it
//Adding "disk:" to root path as all paths on disk start with it
var diskRoot string
if f.root == "" {
diskRoot = f.container + "/"
diskRoot = "disk:/"
} else {
diskRoot = f.container + "/" + f.root + "/"
diskRoot = "disk:/" + f.root + "/"
}
f.diskRoot = diskRoot
}
@@ -518,13 +494,9 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
NoResponse: true,
}
if f.opt.AppFolder {
// Bare relative paths are not resolved under the app folder root,
// unlike the disk root, so the "app:" prefix is always required.
path = f.container + path
} else if strings.ContainsRune(path, ':') {
// If creating a directory with a : use (undocumented) disk: prefix
path = f.container + path
// If creating a directory with a : use (undocumented) disk: prefix
if strings.ContainsRune(path, ':') {
path = "disk:" + path
}
opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(path))
@@ -546,8 +518,8 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
func (f *Fs) mkDirs(ctx context.Context, path string) (err error) {
//trim filename from path
//dirString := strings.TrimSuffix(path, filepath.Base(path))
//trim the container prefix, e.g. "disk:" or "app:", from path
dirString := strings.TrimPrefix(path, f.container)
//trim "disk:" from path
dirString := strings.TrimPrefix(path, "disk:")
if dirString == "" {
return nil
}
@@ -836,7 +808,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
//fmt.Printf("Move src: %s (FullPath: %s), dst: %s (FullPath: %s)\n", srcRemote, srcPath, dstRemote, dstPath)
// Refuse to move to or from the root
if srcPath == srcFs.container+"/" || dstPath == f.container+"/" {
if srcPath == "disk:/" || dstPath == "disk:/" {
fs.Debugf(src, "DirMove error: Can't move root")
return errors.New("can't move root directory")
}
-88
View File
@@ -1,88 +0,0 @@
package yandex
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestSetRoot checks that setRoot addresses paths with the "disk:/" prefix
// by default, and with the "app:/" prefix when app_folder is set.
func TestSetRoot(t *testing.T) {
for _, test := range []struct {
appFolder bool
root string
wantDiskRoot string
wantFilePath string
}{
{false, "", "disk:/", "disk:/file.txt"},
{false, "backups", "disk:/backups/", "disk:/backups/file.txt"},
{true, "", "app:/", "app:/file.txt"},
{true, "backups", "app:/backups/", "app:/backups/file.txt"},
} {
f := &Fs{opt: Options{AppFolder: test.appFolder}}
f.setRoot(test.root)
assert.Equal(t, test.wantDiskRoot, f.diskRoot, "diskRoot for root=%q app_folder=%v", test.root, test.appFolder)
assert.Equal(t, test.wantFilePath, f.filePath("file.txt"), "filePath for root=%q app_folder=%v", test.root, test.appFolder)
}
}
// TestCreateDir checks the path CreateDir sends to the API, both for the
// default disk:/ root and for the app:/ root used when app_folder is set.
// Yandex Disk does not resolve bare relative paths under the app folder
// root, so those must be sent with an explicit "app:/" prefix, but the
// default disk:/ behaviour (relying on the API's implicit disk root for
// bare relative paths, only prefixing when the path itself contains a ':')
// must stay unchanged for existing users.
func TestCreateDir(t *testing.T) {
for _, test := range []struct {
appFolder bool
path string
wantPath string
}{
{false, "/backups/", "/backups/"},
{false, "/foo:bar/", "disk:/foo:bar/"},
{true, "/backups/", "app:/backups/"},
{true, "/foo:bar/", "app:/foo:bar/"},
} {
var gotPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Query().Get("path")
w.WriteHeader(http.StatusCreated)
}))
ctx := context.Background()
f := &Fs{
opt: Options{AppFolder: test.appFolder},
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
}
f.setRoot("")
err := f.CreateDir(ctx, test.path)
require.NoError(t, err, "path=%q app_folder=%v", test.path, test.appFolder)
assert.Equal(t, test.wantPath, gotPath, "path=%q app_folder=%v", test.path, test.appFolder)
server.Close()
}
}
// TestDirMoveRoot checks that DirMove refuses to move a directory to the
// container root, both for the default disk:/ root and for the app:/ root
// used when app_folder is set.
func TestDirMoveRoot(t *testing.T) {
for _, appFolder := range []bool{false, true} {
f := &Fs{opt: Options{AppFolder: appFolder}}
f.setRoot("")
err := f.DirMove(context.Background(), f, "somedir", "")
assert.EqualError(t, err, "can't move root directory", "app_folder=%v", appFolder)
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
# Encrypted rclone configuration File
RCLONE_ENCRYPT_V0:
FdQuiNBDdjD+k7BlbjpZMYbk9Schl+pKJvxwGDtq/Qcu1BDQEP/s6RBqLQWps4Y7p72nD1S5g0rv/b1JBnlHtekJ3+l73H1es/MpIab+fBX78xZTuGdQ+VO43j65FhQ6azChbaeGoeCuoZS6OLYEpLvCn/JB4w+CZlqc5a6/IRmcbFRIG+Sy9bFEHnxmTZLAGIyCIZXyWDbnazR+Tx9f9LUepPQYWKaGV51UENZY8ZX+YvnwfB7FYX4abQt4Zmpe2hczBWbCg3SNECwjrUZL6+FsEY/e209eqLPPtDrH+11e77wOWGmXIbr/ROCVxcPnpir/CP9WFWle73RYRGeiZOE8/s6HVDxRGFqHgQ5ZPR1HwMtX2KuWFogB/Rp8jJkeCXnH+Jh+mxjAvsdAdUcQqF72wtOaMiEVW4QVxa10VjU8NVVjldj2zO8GHyTU94R6j54skVHxgJzqK1MkCRXdoA7DK70vfKQu/aCXffkPPzomenKP+ILz4QmkdTb+GxxG/Y2B1mj4o5eMzBYk6wKTkFKjzCT5AVL8RwsO4yuIxlv2gSbHMZtX7wCs0NY++Pp31A==
H/3G07cjuVyWvg8Ce4dkNzXDRAr37SBHa8cxkPQFtfwKOFDkEmSecuCYun0j/0XtQl9EluGssBMgdztL9sErIsber+QFi27VUWTCXaLnhj940ylSbjI7DnOXtnQGi+WWjdWlW4e8IEbVg44h1Z/iU4D0pyMzMLaEpGHUtnhab+/dBNI5QTtXS+2W1kvDau0C/C3CJULoyMBGC63NMIFl1ks15j67CyiGspCGGbGqo9UlaoMvnK1YlMQX0zIDD4YeNU8M8puxteVHWX6MKGCsd2xY1l7hd47DDIQ1cDIFI5oJWSE2lFGhUs3F5xU4gIV9dOQHrsKom7PCwKaEfLXBJToBMLKphMtElkucEb7DTDdDGNyOyOQ8GSwbkDnsBWydE5S0YQEAt893kZ+QisQsWqfNBcKOLatbuUyq4RBvuNxyyXzC9sVYdA2qgJ+BlZvEihd76i/ZwXnyVpQc+LemgGYFjz0QkVfAd0X63oUkn5GGMpw8eL0w7LcIUan4z//x3jUUwV5muGegC4LUWTBBkgDnkrzuAQ==
-1
View File
@@ -44,7 +44,6 @@ docs = [
"compress.md",
"combine.md",
"doi.md",
"dosya.md",
"drime.md",
"dropbox.md",
"filefabric.md",
+6 -22
View File
@@ -17,28 +17,15 @@ def load(filename):
authors = fd.read()
return set(re.findall(r"<(.*?)>", authors))
def add_email(added, name, email):
def add_email(name, email):
"""
adds the email passed in to the end of authors.md and records it in added
adds the email passed in to the end of authors.md
"""
print("Adding %s <%s>" % (name, email))
with open(AUTHORS, "a+") as fd:
print("- %s <%s>" % (name, email), file=fd)
added.append((name, email))
def commit(added):
"""
commits authors.md with a message describing the names in added
"""
if not added:
return
if len(added) == 1:
message = "Add %s to contributors" % added[0][0]
else:
message = "Add %d new contributors\n\n" % len(added)
message += "".join("- %s\n" % name for name, _ in added)
subprocess.check_call(["git", "commit", "-m", message, AUTHORS])
subprocess.check_call(["git", "commit", "-m", "Add %s to contributors" % name, AUTHORS])
def main():
# Add emails from authors
out = subprocess.check_output(["git", "log", '--reverse', '--format=%an|%ae', "master"])
@@ -47,7 +34,6 @@ def main():
ignored = load(IGNORE)
previous = load(AUTHORS)
previous.update(ignored)
added = []
for line in out.split("\n"):
line = line.strip()
if line == "":
@@ -56,7 +42,7 @@ def main():
if email in previous:
continue
previous.add(email)
add_email(added, name, email)
add_email(name, email)
# Add emails from Co-authored-by: lines
out = subprocess.check_output(["git", "log", '-i', '--grep', 'Co-authored-by:', "master"])
@@ -74,9 +60,7 @@ def main():
if email in previous:
continue
previous.add(email)
add_email(added, name, email)
commit(added)
add_email(name, email)
if __name__ == "__main__":
main()
-1
View File
@@ -33,7 +33,6 @@ import (
_ "github.com/rclone/rclone/cmd/gitannex"
_ "github.com/rclone/rclone/cmd/gui"
_ "github.com/rclone/rclone/cmd/hashsum"
_ "github.com/rclone/rclone/cmd/index"
_ "github.com/rclone/rclone/cmd/link"
_ "github.com/rclone/rclone/cmd/listremotes"
_ "github.com/rclone/rclone/cmd/ls"
-43
View File
@@ -4,8 +4,6 @@ import (
"archive/zip"
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
@@ -136,47 +134,6 @@ func testArchiveRemote(t *testing.T, fromLocal bool, subDir string, extension st
fstest.CheckListingWithPrecision(t, src, items, nil, fs.ModTimeNotSupported)
}
// test archiving to stdout when no destination is given
func TestArchiveStdout(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
f1 := r.WriteObject(ctx, "file1.txt", "content 1", t1)
f2 := r.WriteObject(ctx, "dir1/sub1.txt", "sub content 1", t1)
fstest.CheckItems(t, r.Fremote, f1, f2)
// redirect stdout into a file on the local fs
archiveName := "stdout.zip"
require.NoError(t, r.Flocal.Mkdir(ctx, ""))
out, err := os.Create(filepath.Join(r.LocalName, archiveName))
require.NoError(t, err)
oldStdout := os.Stdout
os.Stdout = out
err = create.ArchiveCreate(ctx, nil, "", r.Fremote, "zip", "")
os.Stdout = oldStdout
require.NoError(t, out.Close())
require.NoError(t, err)
// check the archive written to stdout is readable
expected := map[string]bool{
"file1.txt": true,
"dir1/": true,
"dir1/sub1.txt": true,
}
listFile := func(ctx context.Context, f archives.FileInfo) error {
name := f.NameInArchive
if f.IsDir() && !strings.HasSuffix(name, "/") {
name += "/"
}
assert.True(t, expected[name], name)
delete(expected, name)
return nil
}
err = list.ArchiveList(ctx, r.Flocal, archiveName, listFile)
require.NoError(t, err)
assert.Equal(t, 0, len(expected), expected)
}
func testArchive(t *testing.T) {
var extensions = []string{
"zip",
Loaded 100 of 353 files, more files were not shown because too many files have changed in this diff. Show more