Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
687d264b68 | ||
|
|
4158f63d2f | ||
|
|
c47832ad34 | ||
|
|
60e96416df | ||
|
|
c841c3bba3 | ||
|
|
79fbc0842f | ||
|
|
22859b7e69 | ||
|
|
925fb4fb21 | ||
|
|
96f298bdec | ||
|
|
60fb55dc6a | ||
|
|
181ef190ce | ||
|
|
ad8cd41c49 | ||
|
|
5dae3adbf5 | ||
|
|
6507e13d5a | ||
|
|
e4913c9e91 | ||
|
|
28bf49d66f | ||
|
|
17b0c03338 | ||
|
|
a7ab39d3d1 | ||
|
|
7a594b09c6 | ||
|
|
f376c64784 | ||
|
|
afeb26ae74 | ||
|
|
c6af0b57c2 | ||
|
|
90595f34f2 | ||
|
|
b3cf0444a2 | ||
|
|
11ff6e49bb | ||
|
|
bcef98db3f | ||
|
|
f18aacf2c7 | ||
|
|
ab1f458013 | ||
|
|
7c1dfd99f3 | ||
|
|
57842c5ee4 | ||
|
|
739403963a | ||
|
|
1bec2133c5 | ||
|
|
64de81e6a0 | ||
|
|
0b187f3d7a | ||
|
|
0e5100203c | ||
|
|
ba3d34b9f8 | ||
|
|
77e6390f5b | ||
|
|
f190b34d95 | ||
|
|
49e7016d12 | ||
|
|
6240cbb694 | ||
|
|
66bc465d1a | ||
|
|
b576cbdf40 | ||
|
|
66dba8239f | ||
|
|
92ef010fd2 | ||
|
|
feb0664b64 | ||
|
|
61427e9729 | ||
|
|
3d1fd51f61 | ||
|
|
6c44400bf7 | ||
|
|
041b766428 | ||
|
|
3c505b1e99 | ||
|
|
6317faccc0 | ||
|
|
5a490a31c5 | ||
|
|
b8470e4cac | ||
|
|
e7ae39f42d | ||
|
|
43f107e049 | ||
|
|
748eaf9b28 | ||
|
|
11dd394670 | ||
|
|
21483522a6 | ||
|
|
f92262d48c | ||
|
|
ad87bf6c93 | ||
|
|
f1c06a047e | ||
|
|
d6fb148d9b | ||
|
|
930de88f69 | ||
|
|
fad0359e6b | ||
|
|
c14e507609 | ||
|
|
0720a194cf | ||
|
|
194d22ce71 | ||
|
|
7995d87cd5 | ||
|
|
2d261879dd | ||
|
|
2e0718167f | ||
|
|
9718ed3fc2 | ||
|
|
645e1ea5be | ||
|
|
40dab7ecdd | ||
|
|
43a2a92950 | ||
|
|
62f8f944e2 | ||
|
|
841ff2d418 | ||
|
|
33ab81ce6d | ||
|
|
632ff74375 | ||
|
|
39b926a2ce | ||
|
|
31f4c78c5e | ||
|
|
fcb317e226 | ||
|
|
82a4a63b92 | ||
|
|
d407018995 | ||
|
|
b5f0ea9b7b | ||
|
|
d166eaf635 | ||
|
|
62a1df7271 | ||
|
|
e171f996d6 | ||
|
|
2bf9e0d20e | ||
|
|
afb1daa8fa | ||
|
|
5775f0430b | ||
|
|
d0f00a22d5 | ||
|
|
ad7e0c64af | ||
|
|
674d7a8a70 | ||
|
|
1274301521 | ||
|
|
840755543f | ||
|
|
2472718b7c | ||
|
|
a4f1c1558e | ||
|
|
76d5c49ae6 | ||
|
|
9fec073ab7 | ||
|
|
a9c4548d92 | ||
|
|
d038b8b1c9 | ||
|
|
3ceea42329 | ||
|
|
6beb4a5bd2 | ||
|
|
7d70921bf5 | ||
|
|
76c6992260 | ||
|
|
f1297d6298 | ||
|
|
9a112f40c1 | ||
|
|
a3dcf7f0ad | ||
|
|
3084177049 | ||
|
|
e4b778b5e3 | ||
|
|
ee363b1826 | ||
|
|
e7f9c7bc10 | ||
|
|
10eb7b5b10 | ||
|
|
e0faeeb5d6 | ||
|
|
dcf15b5f9c | ||
|
|
c93cc1e1ae | ||
|
|
21c98ff5ff | ||
|
|
5fbc677ed4 | ||
|
|
00347ecf36 |
No files matched your search
@@ -2,9 +2,6 @@
|
||||
go.mod text eol=lf
|
||||
go.sum text eol=lf
|
||||
|
||||
# Embedded templates must render the same on every OS
|
||||
lib/http/templates/** text eol=lf
|
||||
|
||||
# Ignore generated files in GitHub language statistics and diffs
|
||||
/MANUAL.* linguist-generated=true
|
||||
/rclone.1 linguist-generated=true
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.6'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +48,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.6'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.6'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +64,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.6'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.6'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,14 +81,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.6'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.26
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.9'
|
||||
go: '~1.26.6'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -284,7 +284,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
go-version: '~1.26.6'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
@@ -399,7 +399,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
go-version: '~1.26.6'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
@@ -20,8 +20,6 @@ The core parts of rclone under `fs` and `vfs` need to work with all backends and
|
||||
|
||||
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
|
||||
|
||||
Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
|
||||
|
||||
## Build and Test Commands
|
||||
|
||||
```bash
|
||||
|
||||
+1
-5
@@ -35,21 +35,17 @@ RUN echo "**** Print Version Binary ****" && \
|
||||
# Begin final image
|
||||
FROM alpine:latest
|
||||
|
||||
ARG RCLONE_UID=1009
|
||||
ARG RCLONE_GID=1009
|
||||
|
||||
RUN echo "**** Install Dependencies ****" && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
fuse3 \
|
||||
mailcap \
|
||||
tzdata && \
|
||||
echo "Enable user_allow_other in fuse" && \
|
||||
echo "user_allow_other" >> /etc/fuse.conf
|
||||
|
||||
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
|
||||
|
||||
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
|
||||
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
|
||||
|
||||
ENTRYPOINT [ "rclone" ]
|
||||
|
||||
|
||||
Generated
+2233
-1480
File diff suppressed because it is too large.
Load diff
Generated
+1489
-813
File diff suppressed because it is too large.
Load diff
@@ -38,9 +38,6 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
|
||||
BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
|
||||
BETA_UPLOAD_ROOT := beta.rclone.org:
|
||||
BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
|
||||
# Update the beta site's directory listings after an upload - add
|
||||
# --changed for the uploaded directories
|
||||
BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
# Pass in GOTAGS=xyz on the make command line to set build tags
|
||||
ifdef GOTAGS
|
||||
BUILDTAGS=-tags "$(GOTAGS)"
|
||||
@@ -237,7 +234,6 @@ check_sign:
|
||||
upload:
|
||||
rclone -P copy build/ downloads.rclone.org:/$(TAG)
|
||||
rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
|
||||
rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
|
||||
upload_github:
|
||||
./bin/upload-github $(TAG)
|
||||
@@ -270,7 +266,6 @@ ci_upload:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
|
||||
endif
|
||||
./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
|
||||
@echo Beta release ready at $(BETA_URL)/testbuilds
|
||||
|
||||
ci_beta:
|
||||
@@ -280,7 +275,6 @@ ci_beta:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
|
||||
endif
|
||||
rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
|
||||
@echo Beta release ready at $(BETA_URL)
|
||||
|
||||
# Fetch the binary builds from GitHub actions
|
||||
@@ -302,26 +296,21 @@ retag:
|
||||
@echo "Version is $(VERSION)"
|
||||
git tag -f -s -m "Version $(VERSION)" $(VERSION)
|
||||
|
||||
# Start development of the next version after a release: the next
|
||||
# minor version on master or the next patch version on a stable branch.
|
||||
startdev:
|
||||
@branch=$$(git rev-parse --abbrev-ref HEAD); \
|
||||
case "$$branch" in \
|
||||
master) next="$(NEXT_VERSION)" ;; \
|
||||
v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
|
||||
*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
|
||||
echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
|
||||
fi; \
|
||||
echo "Version is $(VERSION)"; \
|
||||
echo "Next version is $$next"; \
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
|
||||
echo -n "$$next" > docs/layouts/partials/version.html; \
|
||||
echo "$$next" > VERSION; \
|
||||
git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next version is $(NEXT_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
startstable: startdev
|
||||
startstable:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_PATCH_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
winzip:
|
||||
zip -9 rclone-$(TAG).zip rclone.exe
|
||||
|
||||
@@ -39,7 +39,6 @@ directories to and from different cloud storage providers.
|
||||
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
|
||||
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
|
||||
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
|
||||
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
|
||||
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
@@ -111,7 +110,6 @@ directories to and from different cloud storage providers.
|
||||
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
|
||||
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
|
||||
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
|
||||
- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
|
||||
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
|
||||
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
|
||||
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
|
||||
|
||||
+18
-69
@@ -6,37 +6,27 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
- [gh the github cli](https://github.com/cli/cli) for uploading packages
|
||||
- pandoc for making the html and man pages
|
||||
- hugo for building the website
|
||||
- golangci-lint and docker for `make check` (markdownlint runs in docker)
|
||||
- a gpg key for signing the tag and the checksums
|
||||
- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
|
||||
`www.rclone.org:` and `test-rclone-org:`
|
||||
- the sponsor logos - see [Sponsor logos](#sponsor-logos)
|
||||
|
||||
## Making a release
|
||||
|
||||
- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
|
||||
- git checkout master # see below for stable branch
|
||||
- git pull # IMPORTANT
|
||||
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
|
||||
- git status - make sure everything is checked in
|
||||
- Check GitHub actions build for master is Green
|
||||
- Check the integration tests at <https://integration.rclone.org>
|
||||
- cat VERSION # check this is the version being released
|
||||
- make tag # signed tag on HEAD + new changelog entry + docs
|
||||
- edit docs/content/changelog.md # entries already in point releases are
|
||||
removed automatically
|
||||
- make test # see integration test server or run locally
|
||||
- make tag
|
||||
- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
|
||||
releases
|
||||
- make tidy
|
||||
- make doc # again to pick up the changelog edits
|
||||
- git status - to check for new command docs in docs/content/commands - git add them
|
||||
- make doc
|
||||
- git status - to check for new man pages - git add them
|
||||
- git commit -a -v -m "Version v1.XX.0"
|
||||
- make check # if this fails, fix, git commit --amend and run it again
|
||||
- make retag # moves the tag onto the Version commit
|
||||
- make check
|
||||
- make retag
|
||||
- git push origin # without --follow-tags so it doesn't push the tag if it fails
|
||||
- git push --follow-tags origin
|
||||
- \# Wait for the GitHub builds to complete then...
|
||||
- \# Don't commit anything until after make upload_github as these steps
|
||||
use the tag on HEAD to find the version
|
||||
- make fetch_binaries
|
||||
- make tarball
|
||||
- make vendorball
|
||||
@@ -45,17 +35,9 @@ This file describes how to make the various kinds of releases
|
||||
- make upload
|
||||
- make upload_test_website # check sponsors are correct/unchanged
|
||||
- make upload_website
|
||||
- make upload_github # this triggers the docker plugin and winget workflows
|
||||
- Check the docker image, docker plugin and winget GitHub actions succeeded
|
||||
- make startdev # works out the next version for master or a stable branch
|
||||
- git push
|
||||
- Disable the release freeze rule
|
||||
- Release any security advisories
|
||||
- request a CVE for each advisory if not done already
|
||||
- set the patched version on each advisory and publish it
|
||||
- when the CVEs are assigned replace `CVE-PENDING` in the changelog on
|
||||
master and the stable branch then `make upload_website`
|
||||
- \# announce with forum post, twitter post, and post to announcements list
|
||||
- make upload_github
|
||||
- make startdev # make startstable for stable branch
|
||||
- \# announce with forum post, twitter post, and post to annoucements list
|
||||
|
||||
## Update dependencies
|
||||
|
||||
@@ -208,53 +190,20 @@ First make the release branch. If this is a second point release then
|
||||
this will be done already.
|
||||
|
||||
- git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
|
||||
- make startdev # sets the version to ${BASE_TAG}.1
|
||||
- git push -u origin ${BASE_TAG}-stable
|
||||
- make startstable
|
||||
|
||||
Now
|
||||
|
||||
- Enable release freeze rule
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- make fetch-gui-and-commit # update the GUI on master first
|
||||
- git push
|
||||
- git co ${BASE_TAG}-stable
|
||||
- git pull
|
||||
- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
|
||||
- `git cherry-pick -x` any fixes, including the GUI update if there was one
|
||||
- for security releases consider updating the Go patch version in
|
||||
`.github/workflows/build.yml` and dependencies with CVE fixes
|
||||
- fixes may depend on commits which weren't picked so push to CI
|
||||
and check it is green before tagging
|
||||
- Do the steps in "Making a release" above, except
|
||||
- stay on ${BASE_TAG}-stable - don't checkout master or run
|
||||
make fetch-gui-and-commit
|
||||
- check the GitHub actions build for ${BASE_TAG}-stable not master
|
||||
- run the integration tests on ${BASE_TAG}-stable as described below
|
||||
- the release freeze rule is already enabled - leave it until the end
|
||||
- `git cherry-pick -x` any fixes
|
||||
- make startstable
|
||||
- Do the steps as above
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- `#` cherry pick the changes to the changelog - check the diff to make sure it
|
||||
is correct
|
||||
- git checkout ${BASE_TAG}-stable docs/content/changelog.md
|
||||
- git diff --cached # check only the ${NEW_TAG} entry has changed
|
||||
- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
|
||||
- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
|
||||
- git push
|
||||
- Disable release freeze rule
|
||||
|
||||
Point releases are only made for the latest release. Once v1.(XX+1).0
|
||||
is out the ${BASE_TAG}-stable branch is abandoned.
|
||||
|
||||
### Running the integration tests on the stable branch
|
||||
|
||||
The integration tests at <https://integration.rclone.org> test master.
|
||||
To test the stable branch push it to the testing server and run the
|
||||
tests there in `screen` as they take a while.
|
||||
|
||||
```console
|
||||
git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
|
||||
ssh rclone@rclone-testing
|
||||
screen
|
||||
cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
|
||||
```
|
||||
|
||||
## Sponsor logos
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/compress"
|
||||
_ "github.com/rclone/rclone/backend/crypt"
|
||||
_ "github.com/rclone/rclone/backend/doi"
|
||||
_ "github.com/rclone/rclone/backend/dosya"
|
||||
_ "github.com/rclone/rclone/backend/drime"
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/dropbox"
|
||||
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -530,14 +529,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// isDirectChild reports whether remote names an entry directly in dir
|
||||
// ("" being the root), with no ".." or other components in between.
|
||||
func isDirectChild(dir, remote string) bool {
|
||||
if dir != "" {
|
||||
var ok bool
|
||||
remote, ok = strings.CutPrefix(remote, dir+"/")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if remote == "" || remote == "." || remote == ".." || strings.HasSuffix(remote, "/") {
|
||||
return false
|
||||
}
|
||||
return sanitize.Leaf(remote) == nil
|
||||
parent := path.Dir(remote)
|
||||
if parent == "." || parent == "/" {
|
||||
parent = ""
|
||||
}
|
||||
return parent == dir && path.Clean(remote) == remote
|
||||
}
|
||||
|
||||
// NewObject creates a new remote archive file object
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -23,7 +22,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -177,18 +175,6 @@ func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T,
|
||||
checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
|
||||
|
||||
// Now check a single file
|
||||
//
|
||||
// The filter below applies to both sides, so check first that the
|
||||
// archive exposes only that file without it
|
||||
Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
|
||||
require.NoError(t, err)
|
||||
entries, err := Fsingle.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
_, err = Fsingle.List(ctx, subDir)
|
||||
assert.Equal(t, fs.ErrorDirNotFound, err)
|
||||
_, err = Fsingle.NewObject(ctx, subDir)
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
fiCtx, fi := filter.AddConfig(ctx)
|
||||
require.NoError(t, fi.AddRule("+ "+aFile))
|
||||
require.NoError(t, fi.AddRule("- *"))
|
||||
@@ -393,127 +379,3 @@ func TestArchiveEscapingArchiver(t *testing.T) {
|
||||
_, err = f.NewObject(ctx, "test.escaping/file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
|
||||
// TestIsDirectChild checks the guard which decides whether an entry
|
||||
// returned by an archiver belongs directly in the directory listed.
|
||||
func TestIsDirectChild(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
dir, remote string
|
||||
want bool
|
||||
}{
|
||||
{"", "a.txt", true},
|
||||
{"", "/a.txt", false},
|
||||
{"", "a.txt/", false},
|
||||
{"", "../a.txt", false},
|
||||
{"", "sub/a.txt", false},
|
||||
{"d", "d/a.txt", true},
|
||||
{"d", "d", false},
|
||||
{"d", "d/", false},
|
||||
{"d", "d//a.txt", false},
|
||||
{"d", "d/../a.txt", false},
|
||||
{"d", "dd/a.txt", false},
|
||||
{"d", "a.txt", false},
|
||||
} {
|
||||
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
|
||||
}
|
||||
}
|
||||
|
||||
// A symlink in a squashfs image is skipped unless -l/--links is in use,
|
||||
// as it is on the other backends. With the flag it is exposed under its
|
||||
// name with the link suffix and reads back as its target.
|
||||
//
|
||||
// Note that this uses mksquashfs as an external binary.
|
||||
func TestArchiveSquashfsSymlink(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "mksquashfs")
|
||||
ctx := context.Background()
|
||||
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "input")
|
||||
require.NoError(t, os.Mkdir(input, 0777))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
|
||||
require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
|
||||
// a regular file which happens to carry the link suffix must stay
|
||||
// readable under the flag, not be taken for a symlink
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
|
||||
image := filepath.Join(dir, "test.sqfs")
|
||||
run(t, "mksquashfs", input, image)
|
||||
|
||||
remotes := func(t *testing.T, f fs.Fs) []string {
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
names = append(names, entry.Remote())
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
read := func(t *testing.T, f fs.Fs, remote string) string {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
require.NoError(t, err)
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
contents, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, o.Size(), int64(len(contents)))
|
||||
return string(contents)
|
||||
}
|
||||
|
||||
f, err := fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
|
||||
assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
|
||||
}
|
||||
|
||||
// A hostile archive must not be able to write outside the destination.
|
||||
// A symlink pointing above the root with a member inside it is the
|
||||
// classic way to try it, and the refusal comes from the local backend,
|
||||
// so pin it from end to end here rather than trusting it stays.
|
||||
func TestArchiveZipSymlinkEscape(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
hdr := &zip.FileHeader{Name: "a"}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err := zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("../outside"))
|
||||
require.NoError(t, err)
|
||||
w, err = zw.Create("a/pwned.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("pwned"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
zipPath := filepath.Join(dir, "evil.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
dst := filepath.Join(dir, "dst")
|
||||
require.NoError(t, os.Mkdir(dst, 0777))
|
||||
|
||||
fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
|
||||
require.NoError(t, err)
|
||||
fdst, err := fs.NewFs(ctx, dst)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
|
||||
|
||||
_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
|
||||
assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
|
||||
}
|
||||
@@ -42,8 +42,6 @@ type Fs struct {
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
file string // remote of the single file root points at, if any
|
||||
links bool // read symlinks as .rclonelink files
|
||||
}
|
||||
|
||||
// recoverParsePanic converts a panic raised while parsing a squashfs image
|
||||
@@ -100,7 +98,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
|
||||
c: c,
|
||||
remote: remote,
|
||||
root: strings.Trim(root, "/"),
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -118,13 +115,10 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
|
||||
_, err := f.newObjectNative(native)
|
||||
if err == nil {
|
||||
// If it pointed to a file, find the directory above
|
||||
// and remember the file so that only it is listed
|
||||
f.root = path.Dir(f.root)
|
||||
if f.root == "." || f.root == "/" {
|
||||
f.root = ""
|
||||
}
|
||||
f.file = f.prefixSlash + path.Base(native)
|
||||
singleObject = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -156,9 +150,7 @@ func (f *Fs) Name() string {
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
// Include the file so the fs cache can tell this Fs apart from
|
||||
// the one for the directory containing it
|
||||
return path.Join(f.root, f.file)
|
||||
return f.root
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
@@ -222,30 +214,6 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
|
||||
}
|
||||
}
|
||||
|
||||
// Convert a symlink entry into an Object holding its target
|
||||
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
|
||||
size: int64(len(target)),
|
||||
modTime: item.ModTime(),
|
||||
isLink: true,
|
||||
link: target,
|
||||
}
|
||||
}
|
||||
|
||||
// readlink returns the target of the symlink the info describes.
|
||||
//
|
||||
// A symlink may carry an empty target, so the caller decides what is a
|
||||
// symlink from the mode rather than from the target being set.
|
||||
func readlink(info os.FileInfo) (string, error) {
|
||||
st, ok := info.Sys().(*squashfs.StatT)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
|
||||
}
|
||||
return st.LinkTarget, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
@@ -259,10 +227,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
||||
defer recoverParsePanic(&err)
|
||||
|
||||
if f.file != "" && dir != "" {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
nativeDir, err := f.toNative(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -275,7 +239,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, item := range items {
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
// Skip entry names that aren't safe
|
||||
@@ -298,27 +261,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = f.objectFromFileInfo(nativeDir, info)
|
||||
} else if item.Type()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry = f.objectFromLink(nativeDir, info, target)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
}
|
||||
if f.file != "" && entry.Remote() != f.file {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
@@ -327,9 +273,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
@@ -351,17 +294,7 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
// Name the entry as List does, so that the two agree on
|
||||
// which names exist
|
||||
isLink := fi.Type()&os.ModeSymlink != 0
|
||||
if isLink && !f.links {
|
||||
continue
|
||||
}
|
||||
name := fi.Name()
|
||||
if isLink {
|
||||
name += fs.LinkSuffix
|
||||
}
|
||||
if name == leaf {
|
||||
if fi.Name() == leaf {
|
||||
if fi.IsDir() {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
@@ -369,14 +302,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
|
||||
}
|
||||
if isLink {
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o = f.objectFromLink(dir, info, target)
|
||||
break
|
||||
}
|
||||
o = f.objectFromFileInfo(dir, info)
|
||||
break
|
||||
}
|
||||
@@ -391,9 +316,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
|
||||
|
||||
if f.file != "" && remote != f.file {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
nativePath, err := f.toNative(remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -455,8 +377,6 @@ type Object struct {
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
isLink bool // whether this is a symlink
|
||||
link string // target of the symlink
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
@@ -528,17 +448,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
}
|
||||
}
|
||||
|
||||
if o.isLink {
|
||||
// Clamp offset into range to avoid panic
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset > int64(len(o.link)) {
|
||||
offset = int64(len(o.link))
|
||||
}
|
||||
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
|
||||
}
|
||||
|
||||
remote, err := o.fs.toNative(o.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
+10
-27
@@ -9,7 +9,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -43,7 +42,6 @@ type Fs struct {
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
links bool // read symlinks as .rclonelink files
|
||||
dt dirtree.DirTree // read from zipfile
|
||||
}
|
||||
|
||||
@@ -68,7 +66,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -140,7 +137,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
}
|
||||
dt := dirtree.New()
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, file := range zr.File {
|
||||
// Skip entries whose name escapes the archive's own namespace
|
||||
remote, err := sanitize.Path(file.Name)
|
||||
@@ -148,22 +144,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
isDir := strings.HasSuffix(file.Name, "/")
|
||||
// A file entry whose last component is "", "." or ".." names
|
||||
// a directory rather than a file, so skip
|
||||
if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !isDir && file.Mode()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
// The member holds the link target, so it reads back
|
||||
// like a .rclonelink file
|
||||
remote += fs.LinkSuffix
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root, requiring a path
|
||||
@@ -178,12 +158,19 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
remote = strings.TrimPrefix(remote, f.root+"/")
|
||||
}
|
||||
}
|
||||
if isDir {
|
||||
if strings.HasSuffix(file.Name, "/") {
|
||||
dir := fs.NewDir(remote, file.Modified)
|
||||
dt.AddDir(dir)
|
||||
} else {
|
||||
if remote == "" {
|
||||
// A file at the root is the archive member f.root points at
|
||||
// A file at the root itself can only be the
|
||||
// archive member f.root points at - with no root
|
||||
// it is a crafted name for the archive's own
|
||||
// directory, which can't be a file
|
||||
if f.root == "" {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
remote = path.Base(f.root)
|
||||
singleObject = true
|
||||
dt = dirtree.New()
|
||||
@@ -203,9 +190,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
@@ -233,8 +217,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
// Return a copy as callers may filter the listing in place
|
||||
return slices.Clone(entries), nil
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -37,26 +35,6 @@ func writeZip(t *testing.T, dir, name string, names ...string) string {
|
||||
return name
|
||||
}
|
||||
|
||||
// writeZipSymlink builds a zip holding file and a symlink entry link
|
||||
// pointing at target, and returns its leaf name.
|
||||
func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create(file)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data for " + file))
|
||||
require.NoError(t, err)
|
||||
hdr := &zip.FileHeader{Name: link}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err = zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(target))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// allRemotes returns every Object remote in the mounted archive's dirtree.
|
||||
func allRemotes(t *testing.T, f fs.Fs) []string {
|
||||
zf, ok := f.(*Fs)
|
||||
@@ -127,12 +105,10 @@ func TestReadZipRootBoundary(t *testing.T) {
|
||||
assert.Equal(t, []string{"a.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A file entry whose name refers to a directory (".", "/", "./", "" or
|
||||
// "sub/.") must be skipped, not turn the archive or the directory
|
||||
// mounted as its root into a single file which hides every other
|
||||
// entry, or become an object named for the archive itself when the
|
||||
// archive is found by listing its directory.
|
||||
func TestReadZipDirNamedFileEntry(t *testing.T) {
|
||||
// A file entry whose name refers to the archive's own root (".", "/",
|
||||
// "./" or "") must be skipped, not turn the whole archive into a single
|
||||
// file which hides every other entry.
|
||||
func TestReadZipRootNamedEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "dot.zip",
|
||||
@@ -141,87 +117,14 @@ func TestReadZipDirNamedFileEntry(t *testing.T) {
|
||||
"./",
|
||||
"",
|
||||
"good.txt",
|
||||
"sub/",
|
||||
"sub/.",
|
||||
"sub/good.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, test := range []struct {
|
||||
prefix, root string
|
||||
want []string
|
||||
}{
|
||||
{"", "", []string{"good.txt", "sub/good.txt"}},
|
||||
{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
|
||||
{"", "sub", []string{"good.txt"}},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
|
||||
f, err := New(ctx, localFs, name, test.prefix, test.root)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, allRemotes(t, f))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Listings are served from a cache which must survive callers
|
||||
// filtering the returned slice in place, as fs/list does.
|
||||
func TestListCacheNotAliased(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 3)
|
||||
// Compact in place, dropping the first entry
|
||||
copy(entries, entries[1:])
|
||||
|
||||
entries, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A symlink stored in the archive is skipped unless -l/--links is in
|
||||
// use, as it is on the other backends. With the flag it is exposed
|
||||
// under its name with the link suffix and reads back as its target.
|
||||
func TestReadZipSymlink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
|
||||
|
||||
o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len("file.txt")), o.Size())
|
||||
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
target, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, "file.txt", string(target))
|
||||
remotes := allRemotes(t, f)
|
||||
assert.Equal(t, []string{"good.txt"}, remotes)
|
||||
}
|
||||
+22
-15
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Parallel listing support for azureblob, built on the Apache Arrow
|
||||
// listing format.
|
||||
@@ -18,20 +18,24 @@ import (
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
|
||||
// response, used to tell whether the service honoured an Arrow request.
|
||||
const arrowContentType = "application/vnd.apache.arrow.stream"
|
||||
|
||||
// listArrowParallel lists (containerName, directory) by splitting the blob name
|
||||
// keyspace into shards and listing them concurrently. opts must already request
|
||||
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
|
||||
// returns the number of raw items seen. directory/prefix are already normalised
|
||||
// by the caller.
|
||||
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
|
||||
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
|
||||
// Parallel listing needs the arrowlist client (server-side endBefore is
|
||||
// only honoured on the Arrow path) - without it fall back to sequential.
|
||||
if _, err := f.arrowCntSVC(containerName); err != nil {
|
||||
fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
|
||||
// Split the keyspace by a single-case character ladder (see
|
||||
// arrowLadderBoundaries for why single-case matters).
|
||||
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
|
||||
@@ -62,7 +66,7 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
|
||||
|
||||
// Schedule every shard; SetLimit bounds concurrency, so more shards than
|
||||
// workers gives free load balancing as workers pick up the next shard.
|
||||
var total atomic.Int64
|
||||
var total int64
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(f.opt.ListParallelism)
|
||||
for _, sh := range shards {
|
||||
@@ -85,12 +89,12 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
|
||||
shardOpts.EndBefore = nil
|
||||
}
|
||||
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
|
||||
total.Add(int64(n))
|
||||
atomic.AddInt64(&total, int64(n))
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := g.Wait()
|
||||
found := int(total.Load())
|
||||
found := int(atomic.LoadInt64(&total))
|
||||
if isEndBeforeUnsupported(err) {
|
||||
// The account can't honour the shards' endBefore bounds, so parallel
|
||||
// listing can't be used on it.
|
||||
@@ -111,13 +115,16 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
|
||||
// isEndBeforeUnsupported reports whether err means the service can't honour
|
||||
// the endBefore listing bound, so sharded parallel listing must not be used.
|
||||
//
|
||||
// The service rejects the endBefore parameter with 400
|
||||
// OperationNotSupportedWithFeatureMissing "The requested operation is not
|
||||
// allowed as EndBefore parameter support is missing on this account" -
|
||||
// endBefore is only supported on the Arrow listing path of accounts with
|
||||
// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
|
||||
// accept an XML page with endBefore set, but the service never sends one.)
|
||||
// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
|
||||
// server answered XML with endBefore set), or the service rejecting the
|
||||
// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
|
||||
// requested operation is not allowed as EndBefore parameter support is
|
||||
// missing on this account" - endBefore is only supported on the Arrow
|
||||
// listing path of accounts with Blob Listing with Apache Arrow enabled.
|
||||
func isEndBeforeUnsupported(err error) bool {
|
||||
if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
|
||||
return true
|
||||
}
|
||||
var respErr *azcore.ResponseError
|
||||
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
@@ -211,8 +212,9 @@ func TestArrowLadderBoundaries(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIsEndBeforeUnsupported(t *testing.T) {
|
||||
assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
|
||||
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
|
||||
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
|
||||
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
|
||||
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
|
||||
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
|
||||
assert.False(t, isEndBeforeUnsupported(nil))
|
||||
|
||||
@@ -0,0 +1,421 @@
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
|
||||
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
|
||||
// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
|
||||
// public type aliases so it compiles against the released azblob module.
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
"github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/apache/arrow-go/v18/arrow/ipc"
|
||||
"github.com/apache/arrow-go/v18/arrow/memory"
|
||||
)
|
||||
|
||||
const (
|
||||
// ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
|
||||
ArrowContentType = "application/vnd.apache.arrow.stream"
|
||||
|
||||
// ArrowAcceptHeader is the Accept header value to request Arrow format.
|
||||
ArrowAcceptHeader = ArrowContentType
|
||||
|
||||
// resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
|
||||
// in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
|
||||
// in Arrow, all rows share the same schema and are distinguished by this field value.
|
||||
resourceTypeBlobPrefix = "blobprefix"
|
||||
)
|
||||
|
||||
// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
|
||||
func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
|
||||
result := container.ListBlobsFlatResponse{}
|
||||
|
||||
// Extract response headers
|
||||
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
|
||||
|
||||
// Parse Arrow IPC stream
|
||||
items, nextMarker, err := parseArrowStream(resp.Body)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
|
||||
}
|
||||
|
||||
result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
|
||||
Segment: &container.BlobFlatListSegment{
|
||||
BlobItems: items,
|
||||
},
|
||||
NextMarker: nextMarker,
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
|
||||
func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
|
||||
result := container.ListBlobsHierarchyResponse{}
|
||||
|
||||
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
|
||||
|
||||
items, nextMarker, err := parseArrowStream(resp.Body)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
|
||||
}
|
||||
|
||||
// Separate blob items from blob prefixes based on ResourceType field.
|
||||
var blobItems []*container.BlobItem
|
||||
var blobPrefixes []*container.BlobPrefix
|
||||
for _, item := range items {
|
||||
if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
|
||||
blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
|
||||
Name: item.Name,
|
||||
Properties: item.Properties,
|
||||
})
|
||||
} else {
|
||||
blobItems = append(blobItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
|
||||
Segment: &container.BlobHierarchyListSegment{
|
||||
BlobItems: blobItems,
|
||||
BlobPrefixes: blobPrefixes,
|
||||
},
|
||||
NextMarker: nextMarker,
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
|
||||
if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
|
||||
*clientRequestID = &val
|
||||
}
|
||||
if val := resp.Header.Get("Content-Type"); val != "" {
|
||||
*contentType = &val
|
||||
}
|
||||
if val := resp.Header.Get("Date"); val != "" {
|
||||
if t, err := time.Parse(time.RFC1123, val); err == nil {
|
||||
*date = &t
|
||||
}
|
||||
}
|
||||
if val := resp.Header.Get("x-ms-request-id"); val != "" {
|
||||
*requestID = &val
|
||||
}
|
||||
if val := resp.Header.Get("x-ms-version"); val != "" {
|
||||
*version = &val
|
||||
}
|
||||
}
|
||||
|
||||
// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
|
||||
func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
|
||||
reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
|
||||
}
|
||||
defer reader.Release()
|
||||
|
||||
// Extract NextMarker from schema metadata
|
||||
var nextMarker *string
|
||||
var numRecords int
|
||||
if md := reader.Schema().Metadata(); md.Len() > 0 {
|
||||
if idx := md.FindKey("NextMarker"); idx >= 0 {
|
||||
val := md.Values()[idx]
|
||||
if val != "" {
|
||||
nextMarker = &val
|
||||
}
|
||||
}
|
||||
if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
|
||||
if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
|
||||
numRecords = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build column index for the schema
|
||||
schema := reader.Schema()
|
||||
colIndex := buildColumnIndex(schema)
|
||||
|
||||
// Pre-allocate if we know the count
|
||||
items := make([]*container.BlobItem, 0, numRecords)
|
||||
|
||||
// Iterate over record batches
|
||||
for reader.Next() {
|
||||
rec := reader.RecordBatch()
|
||||
rows := int(rec.NumRows())
|
||||
|
||||
for row := 0; row < rows; row++ {
|
||||
item := &container.BlobItem{
|
||||
Properties: &container.BlobProperties{},
|
||||
}
|
||||
populateBlobItem(item, rec, row, colIndex)
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
|
||||
if err := reader.Err(); err != nil {
|
||||
return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
|
||||
}
|
||||
|
||||
return items, nextMarker, nil
|
||||
}
|
||||
|
||||
// columnIndex maps column names to their indices in the record batch for O(1) lookup.
|
||||
type columnIndex map[string]int
|
||||
|
||||
func buildColumnIndex(schema *arrow.Schema) columnIndex {
|
||||
idx := make(columnIndex, len(schema.Fields()))
|
||||
for i, f := range schema.Fields() {
|
||||
idx[f.Name] = i
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// populateBlobItem fills a BlobItem from a single row across all columns.
|
||||
func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
|
||||
// Top-level BlobItem fields
|
||||
item.Name = getStringField(rec, row, colIdx, "Name")
|
||||
item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
|
||||
item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
|
||||
item.VersionID = getStringField(rec, row, colIdx, "VersionId")
|
||||
item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
|
||||
item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
|
||||
|
||||
// Metadata maps
|
||||
item.Metadata = getMapField(rec, row, colIdx, "Metadata")
|
||||
item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
|
||||
|
||||
// Tags
|
||||
if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
|
||||
var blobTags []*container.BlobTag
|
||||
for k, v := range tags {
|
||||
key := k
|
||||
blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
|
||||
}
|
||||
item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
|
||||
}
|
||||
|
||||
// BlobProperties fields
|
||||
p := item.Properties
|
||||
p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
|
||||
p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
|
||||
p.ETag = getETagField(rec, row, colIdx, "Etag")
|
||||
p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
|
||||
p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
|
||||
p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
|
||||
p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
|
||||
p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
|
||||
p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
|
||||
p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
|
||||
p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
|
||||
p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
|
||||
p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
|
||||
p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
|
||||
p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
|
||||
p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
|
||||
p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
|
||||
p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
|
||||
p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
|
||||
p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
|
||||
p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
|
||||
p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
|
||||
p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
|
||||
p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
|
||||
p.CopyID = getStringField(rec, row, colIdx, "CopyId")
|
||||
p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
|
||||
p.CopySource = getStringField(rec, row, colIdx, "CopySource")
|
||||
p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
|
||||
p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
|
||||
p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
|
||||
p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
|
||||
p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
|
||||
p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
|
||||
p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
|
||||
p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
|
||||
p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
|
||||
p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
|
||||
p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
|
||||
p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
|
||||
p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
|
||||
|
||||
// Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
|
||||
// The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
|
||||
// It is silently ignored for forward compatibility.
|
||||
}
|
||||
|
||||
// Field accessor helpers that safely handle missing columns and null values.
|
||||
|
||||
func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.String); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Boolean); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Timestamp); ok {
|
||||
val := arr.Value(row)
|
||||
// Arrow timestamps are stored as int64 with a unit; convert to time.Time
|
||||
dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
|
||||
return &dt
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Uint64); ok {
|
||||
val := int64(arr.Value(row))
|
||||
return &val
|
||||
}
|
||||
if arr, ok := col.(*array.Int64); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Uint64); ok {
|
||||
val := int32(arr.Value(row))
|
||||
return &val
|
||||
}
|
||||
if arr, ok := col.(*array.Int32); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
|
||||
s := getStringField(rec, row, colIdx, name)
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
etag := azcore.ETag(*s)
|
||||
return &etag
|
||||
}
|
||||
|
||||
func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
|
||||
s := getStringField(rec, row, colIdx, name)
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(*s)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return decoded
|
||||
}
|
||||
|
||||
// getEnumField returns a pointer to a string-typed enum value.
|
||||
type stringEnum interface {
|
||||
~string
|
||||
}
|
||||
|
||||
func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
|
||||
s := getStringField(rec, row, colIdx, name)
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
val := T(*s)
|
||||
return &val
|
||||
}
|
||||
|
||||
func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
mapArr, ok := col.(*array.Map)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the offsets for this row's map entries
|
||||
start := int(mapArr.Offsets()[row])
|
||||
end := int(mapArr.Offsets()[row+1])
|
||||
if start == end {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := mapArr.Keys().(*array.String)
|
||||
values := mapArr.Items().(*array.String)
|
||||
|
||||
result := make(map[string]*string, end-start)
|
||||
for i := start; i < end; i++ {
|
||||
k := keys.Value(i)
|
||||
if values.IsNull(i) {
|
||||
result[k] = nil
|
||||
} else {
|
||||
v := values.Value(i)
|
||||
result[k] = &v
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
|
||||
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
|
||||
// c6fa341ca22b (branch feature/storage/bifrost).
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/apache/arrow-go/v18/arrow/ipc"
|
||||
"github.com/apache/arrow-go/v18/arrow/memory"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// coreSchema returns a basic Arrow schema for testing with common blob fields.
|
||||
func coreSchema(md *arrow.Metadata) *arrow.Schema {
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
|
||||
{Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
|
||||
{Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
|
||||
{Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
{Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
}
|
||||
return arrow.NewSchema(fields, md)
|
||||
}
|
||||
|
||||
// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
|
||||
func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
|
||||
t.Helper()
|
||||
alloc := memory.DefaultAllocator
|
||||
|
||||
var buf bytes.Buffer
|
||||
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
|
||||
|
||||
builder := arrowArray.NewRecordBuilder(alloc, schema)
|
||||
defer builder.Release()
|
||||
|
||||
populate(builder)
|
||||
|
||||
rec := builder.NewRecordBatch()
|
||||
defer rec.Release()
|
||||
|
||||
require.NoError(t, w.Write(rec))
|
||||
require.NoError(t, w.Close())
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
|
||||
h := http.Header{}
|
||||
h.Set("Content-Type", ArrowContentType)
|
||||
for k, v := range headers {
|
||||
h.Set(k, v)
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: h,
|
||||
Body: io.NopCloser(bytes.NewReader(data)),
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
|
||||
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
|
||||
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
|
||||
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
|
||||
b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
|
||||
b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
|
||||
b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
|
||||
b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
|
||||
b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
|
||||
b.Field(9).(*arrowArray.StringBuilder).Append("available")
|
||||
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
|
||||
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
|
||||
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
|
||||
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
|
||||
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
|
||||
b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
|
||||
b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
|
||||
b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
|
||||
b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
|
||||
b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
|
||||
b.Field(9).(*arrowArray.StringBuilder).Append("available")
|
||||
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, result.NextMarker)
|
||||
require.Equal(t, "marker123", *result.NextMarker)
|
||||
require.Len(t, result.Segment.BlobItems, 2)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.Equal(t, "blob1.txt", *item.Name)
|
||||
require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
|
||||
require.Equal(t, int64(1024), *item.Properties.ContentLength)
|
||||
require.Equal(t, "Hot", string(*item.Properties.AccessTier))
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
// No rows added
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, result.NextMarker)
|
||||
require.Empty(t, result.Segment.BlobItems)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_NullFields(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
|
||||
b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
|
||||
b.Field(3).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(4).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
|
||||
b.Field(6).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
|
||||
b.Field(8).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(9).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(10).(*arrowArray.StringBuilder).AppendNull()
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.Equal(t, "blob.txt", *item.Name)
|
||||
require.Nil(t, item.Properties.CreationTime)
|
||||
require.Nil(t, item.Properties.BlobType)
|
||||
require.Nil(t, item.Properties.ETag)
|
||||
require.Nil(t, item.Properties.ContentLength)
|
||||
require.Nil(t, item.Properties.ContentType)
|
||||
require.Nil(t, item.Properties.ServerEncrypted)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
|
||||
alloc := memory.DefaultAllocator
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
var buf bytes.Buffer
|
||||
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
|
||||
|
||||
// First batch: 2 rows
|
||||
builder := arrowArray.NewRecordBuilder(alloc, schema)
|
||||
for i := 0; i < 2; i++ {
|
||||
builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
|
||||
for j := 1; j < 11; j++ {
|
||||
builder.Field(j).AppendNull()
|
||||
}
|
||||
}
|
||||
rec := builder.NewRecordBatch()
|
||||
require.NoError(t, w.Write(rec))
|
||||
rec.Release()
|
||||
builder.Release()
|
||||
|
||||
// Second batch: 1 row
|
||||
builder = arrowArray.NewRecordBuilder(alloc, schema)
|
||||
builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
|
||||
for j := 1; j < 11; j++ {
|
||||
builder.Field(j).AppendNull()
|
||||
}
|
||||
rec = builder.NewRecordBatch()
|
||||
require.NoError(t, w.Write(rec))
|
||||
rec.Release()
|
||||
builder.Release()
|
||||
|
||||
require.NoError(t, w.Close())
|
||||
|
||||
resp := makeHTTPResponse(buf.Bytes(), nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 3)
|
||||
}
|
||||
|
||||
func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
// A regular blob
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
|
||||
b.Field(1).(*arrowArray.StringBuilder).AppendNull()
|
||||
|
||||
// A blob prefix (virtual directory)
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
|
||||
b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleHierarchyListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
|
||||
|
||||
require.Len(t, result.Segment.BlobPrefixes, 1)
|
||||
require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, map[string]string{
|
||||
"x-ms-request-id": "req-123",
|
||||
"x-ms-version": "2026-10-06",
|
||||
"x-ms-client-request-id": "client-req-456",
|
||||
"Date": "Mon, 01 Jan 2024 00:00:00 GMT",
|
||||
})
|
||||
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "req-123", *result.RequestID)
|
||||
require.Equal(t, "2026-10-06", *result.Version)
|
||||
require.Equal(t, "client-req-456", *result.ClientRequestID)
|
||||
require.NotNil(t, result.Date)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_MapFields(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
|
||||
{Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
|
||||
metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
|
||||
metaBuilder.Append(true)
|
||||
metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
|
||||
metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
|
||||
|
||||
tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
|
||||
tagBuilder.Append(true)
|
||||
tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
|
||||
tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.NotNil(t, item.Metadata)
|
||||
require.Equal(t, "value1", *item.Metadata["key1"])
|
||||
|
||||
require.NotNil(t, item.BlobTags)
|
||||
require.Len(t, item.BlobTags.BlobTagSet, 1)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
|
||||
b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
|
||||
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.True(t, *item.Deleted)
|
||||
require.False(t, *item.IsCurrentVersion)
|
||||
require.True(t, *item.Properties.ServerEncrypted)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
|
||||
b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
|
||||
b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
|
||||
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
|
||||
require.Equal(t, "snap-123", *item.Snapshot)
|
||||
require.True(t, *item.IsCurrentVersion)
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Package arrowlist implements the experimental "Blob Listing with Apache
|
||||
// Arrow" feature on top of the released Azure azblob SDK.
|
||||
//
|
||||
// The Azure SDK for Go has support for Arrow listing on the unreleased
|
||||
// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
|
||||
// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
|
||||
// response types. Until that ships in a tagged azblob release this package
|
||||
// fills the gap: it exposes the same options and pager interface as the
|
||||
// experimental SDK, built only on the public API of the released SDK plus
|
||||
// copies of two small internal auth policies (see sharedkey.go and
|
||||
// challenge_policy.go).
|
||||
//
|
||||
// This package is temporary. When Arrow listing ships in a released azblob
|
||||
// the whole package should be deleted and its callers pointed back at
|
||||
// container.Client.NewListBlobsHierarchyPager - the option and response
|
||||
// types here are deliberately source compatible with the SDK's to make that
|
||||
// a mechanical change.
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
)
|
||||
|
||||
const (
|
||||
// moduleName and moduleVersion identify this package in the azcore
|
||||
// telemetry policy's User-Agent fragment.
|
||||
moduleName = "github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
moduleVersion = "v0.0.1"
|
||||
|
||||
// serviceVersion is the x-ms-version sent with listing requests. Arrow
|
||||
// responses and the startFrom/endBefore parameters need at least
|
||||
// 2026-06-06.
|
||||
serviceVersion = "2026-06-06"
|
||||
|
||||
// tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
|
||||
// own clients.
|
||||
tokenScope = "https://storage.azure.com/.default"
|
||||
)
|
||||
|
||||
// ErrEndBeforeXMLFallback is returned by the pager when the server answered
|
||||
// with XML instead of Arrow while EndBefore was set. The service only honours
|
||||
// endBefore on the Arrow listing path, so an XML page could silently contain
|
||||
// blobs outside the requested range and cannot be used.
|
||||
var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
|
||||
|
||||
// ListBlobsHierarchyOptions provides the configuration for
|
||||
// Client.NewListBlobsHierarchyPager.
|
||||
//
|
||||
// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
|
||||
// the released SDK's options are embedded (providing Include, Marker,
|
||||
// MaxResults, Prefix and StartFrom by promotion) with the two fields the
|
||||
// released SDK lacks added alongside.
|
||||
type ListBlobsHierarchyOptions struct {
|
||||
container.ListBlobsHierarchyOptions
|
||||
// EndBefore limits listing to blobs whose full container path is
|
||||
// lexically before it (exclusive). Only honoured on Arrow responses.
|
||||
EndBefore *string
|
||||
// UseArrowFormat requests the response as an Apache Arrow IPC stream.
|
||||
UseArrowFormat *bool
|
||||
}
|
||||
|
||||
// ClientOptions contains the optional parameters for client creation.
|
||||
type ClientOptions struct {
|
||||
azcore.ClientOptions
|
||||
}
|
||||
|
||||
// Client issues Arrow listing requests to a single container.
|
||||
type Client struct {
|
||||
endpoint string
|
||||
azClient *azcore.Client
|
||||
}
|
||||
|
||||
// newClient creates a Client for the container at containerURL sending
|
||||
// requests through a pipeline with the given auth policy (which may be nil
|
||||
// for anonymous or SAS access).
|
||||
func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
|
||||
if options == nil {
|
||||
options = &ClientOptions{}
|
||||
}
|
||||
plOpts := runtime.PipelineOptions{}
|
||||
if authPolicy != nil {
|
||||
plOpts.PerRetry = []policy.Policy{authPolicy}
|
||||
}
|
||||
azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{
|
||||
endpoint: containerURL,
|
||||
azClient: azClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewClient creates a Client authenticating with a token credential.
|
||||
func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
|
||||
return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
|
||||
}
|
||||
|
||||
// NewClientWithNoCredential creates a Client with no authentication, for
|
||||
// anonymous access or a containerURL carrying a SAS token in its query.
|
||||
func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
|
||||
return newClient(containerURL, nil, options)
|
||||
}
|
||||
|
||||
// NewClientWithSharedKeyCredential creates a Client signing requests with the
|
||||
// account's shared key.
|
||||
func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
|
||||
return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
|
||||
}
|
||||
|
||||
// NewListBlobsHierarchyPager returns a pager over the container's blobs,
|
||||
// requesting Arrow responses when o.UseArrowFormat is set and returning the
|
||||
// same response type as container.Client.NewListBlobsHierarchyPager.
|
||||
//
|
||||
// If the service does not support Arrow listing (the feature not enabled on
|
||||
// the account, or an HNS account) it answers with XML, which is decoded
|
||||
// transparently - unless EndBefore is set, in which case the fetcher returns
|
||||
// ErrEndBeforeXMLFallback (see there).
|
||||
func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
|
||||
// Copy the options so advancing the Marker doesn't mutate the caller's struct.
|
||||
opts := ListBlobsHierarchyOptions{}
|
||||
if o != nil {
|
||||
opts = *o
|
||||
}
|
||||
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
|
||||
return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
|
||||
More: func(page container.ListBlobsHierarchyResponse) bool {
|
||||
return page.NextMarker != nil && len(*page.NextMarker) > 0
|
||||
},
|
||||
Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
|
||||
if page != nil {
|
||||
opts.Marker = page.NextMarker
|
||||
}
|
||||
req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
|
||||
if err != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, err
|
||||
}
|
||||
resp, err := c.azClient.Pipeline().Do(req)
|
||||
if err != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, err
|
||||
}
|
||||
if !runtime.HasStatusCode(resp, http.StatusOK) {
|
||||
return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
|
||||
}
|
||||
if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
|
||||
return HandleHierarchyListResponse(resp)
|
||||
}
|
||||
if opts.EndBefore != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
|
||||
}
|
||||
return handleXMLResponse(resp)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// listCreateRequest builds the ListBlobs request, mirroring the generated
|
||||
// request builders in the SDK.
|
||||
func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
|
||||
req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reading the existing query first preserves any SAS token in the endpoint.
|
||||
reqQP := req.Raw().URL.Query()
|
||||
reqQP.Set("comp", "list")
|
||||
reqQP.Set("restype", "container")
|
||||
reqQP.Set("delimiter", delimiter)
|
||||
if include := formatInclude(opts.Include); include != "" {
|
||||
reqQP.Set("include", include)
|
||||
}
|
||||
if opts.Marker != nil {
|
||||
reqQP.Set("marker", *opts.Marker)
|
||||
}
|
||||
if opts.MaxResults != nil {
|
||||
reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
|
||||
}
|
||||
if opts.Prefix != nil {
|
||||
reqQP.Set("prefix", *opts.Prefix)
|
||||
}
|
||||
if opts.StartFrom != nil {
|
||||
reqQP.Set("startFrom", *opts.StartFrom)
|
||||
}
|
||||
if useArrow && opts.EndBefore != nil {
|
||||
reqQP.Set("endBefore", *opts.EndBefore)
|
||||
}
|
||||
req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
|
||||
if useArrow {
|
||||
// Stream the Arrow body rather than buffering it in the pipeline.
|
||||
runtime.SkipBodyDownload(req)
|
||||
req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
|
||||
} else {
|
||||
req.Raw().Header["Accept"] = []string{"application/xml"}
|
||||
}
|
||||
req.Raw().Header["x-ms-version"] = []string{serviceVersion}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
|
||||
// generated ListBlobHierarchySegmentHandleResponse.
|
||||
func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
|
||||
result := container.ListBlobsHierarchyResponse{}
|
||||
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
|
||||
if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// formatInclude renders the include datasets as the comma separated query
|
||||
// parameter value, in the same order as the SDK's unexported
|
||||
// ListBlobsInclude.format.
|
||||
func formatInclude(l container.ListBlobsInclude) string {
|
||||
var include []string
|
||||
if l.Copy {
|
||||
include = append(include, "copy")
|
||||
}
|
||||
if l.Deleted {
|
||||
include = append(include, "deleted")
|
||||
}
|
||||
if l.DeletedWithVersions {
|
||||
include = append(include, "deletedwithversions")
|
||||
}
|
||||
if l.ImmutabilityPolicy {
|
||||
include = append(include, "immutabilitypolicy")
|
||||
}
|
||||
if l.LegalHold {
|
||||
include = append(include, "legalhold")
|
||||
}
|
||||
if l.Metadata {
|
||||
include = append(include, "metadata")
|
||||
}
|
||||
if l.Snapshots {
|
||||
include = append(include, "snapshots")
|
||||
}
|
||||
if l.Tags {
|
||||
include = append(include, "tags")
|
||||
}
|
||||
if l.UncommittedBlobs {
|
||||
include = append(include, "uncommittedblobs")
|
||||
}
|
||||
if l.Versions {
|
||||
include = append(include, "versions")
|
||||
}
|
||||
if l.Permissions {
|
||||
include = append(include, "permissions")
|
||||
}
|
||||
return strings.Join(include, ",")
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// arrowPage builds a one row Arrow IPC listing page with the given blob name
|
||||
// and NextMarker.
|
||||
func arrowPage(t *testing.T, name, nextMarker string) []byte {
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append(name)
|
||||
b.Field(1).(*arrowArray.Uint64Builder).Append(42)
|
||||
})
|
||||
}
|
||||
|
||||
func TestPagerArrow(t *testing.T) {
|
||||
var requests []*http.Request
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests = append(requests, r.Clone(context.Background()))
|
||||
w.Header().Set("Content-Type", ArrowContentType)
|
||||
if r.URL.Query().Get("marker") == "" {
|
||||
_, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
|
||||
} else {
|
||||
_, _ = w.Write(arrowPage(t, "blob2.txt", ""))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// SAS style query on the endpoint must be preserved in requests.
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &ListBlobsHierarchyOptions{
|
||||
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
|
||||
Include: container.ListBlobsInclude{Metadata: true, Tags: true},
|
||||
Prefix: to.Ptr("dir/"),
|
||||
MaxResults: to.Ptr(int32(1000)),
|
||||
StartFrom: to.Ptr("testcontainer/dir/a"),
|
||||
},
|
||||
EndBefore: to.Ptr("testcontainer/dir/n"),
|
||||
UseArrowFormat: to.Ptr(true),
|
||||
}
|
||||
pager := client.NewListBlobsHierarchyPager("/", opts)
|
||||
|
||||
var names []string
|
||||
for pager.More() {
|
||||
page, err := pager.NextPage(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ArrowContentType, *page.ContentType)
|
||||
for _, item := range page.Segment.BlobItems {
|
||||
names = append(names, *item.Name)
|
||||
assert.Equal(t, int64(42), *item.Properties.ContentLength)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
|
||||
|
||||
require.Len(t, requests, 2)
|
||||
q := requests[0].URL.Query()
|
||||
assert.Equal(t, "list", q.Get("comp"))
|
||||
assert.Equal(t, "container", q.Get("restype"))
|
||||
assert.Equal(t, "/", q.Get("delimiter"))
|
||||
assert.Equal(t, "metadata,tags", q.Get("include"))
|
||||
assert.Equal(t, "dir/", q.Get("prefix"))
|
||||
assert.Equal(t, "1000", q.Get("maxresults"))
|
||||
assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
|
||||
assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
|
||||
assert.Equal(t, "fakesas", q.Get("sv"))
|
||||
assert.Empty(t, q.Get("marker"))
|
||||
assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
|
||||
assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
|
||||
assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
|
||||
|
||||
// The caller's options must not have been mutated by paging.
|
||||
assert.Nil(t, opts.Marker)
|
||||
}
|
||||
|
||||
const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
|
||||
<Blobs>
|
||||
<Blob>
|
||||
<Name>file.txt</Name>
|
||||
<Properties>
|
||||
<Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
|
||||
<Etag>0x8D1</Etag>
|
||||
<Content-Length>7</Content-Length>
|
||||
<BlobType>BlockBlob</BlobType>
|
||||
</Properties>
|
||||
</Blob>
|
||||
<BlobPrefix>
|
||||
<Name>subdir/</Name>
|
||||
</BlobPrefix>
|
||||
</Blobs>
|
||||
<NextMarker />
|
||||
</EnumerationResults>`
|
||||
|
||||
func TestPagerXMLFallback(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(xmlListResponse))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
UseArrowFormat: to.Ptr(true),
|
||||
})
|
||||
page, err := pager.NextPage(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.False(t, pager.More())
|
||||
assert.Equal(t, "application/xml", *page.ContentType)
|
||||
require.Len(t, page.Segment.BlobItems, 1)
|
||||
assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
|
||||
assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
|
||||
require.Len(t, page.Segment.BlobPrefixes, 1)
|
||||
assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
|
||||
}
|
||||
|
||||
func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(xmlListResponse))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
EndBefore: to.Ptr("testcontainer/n"),
|
||||
UseArrowFormat: to.Ptr(true),
|
||||
})
|
||||
_, err = pager.NextPage(context.Background())
|
||||
require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
|
||||
}
|
||||
|
||||
func TestPagerSharedKeyAuth(t *testing.T) {
|
||||
var gotAuth, gotDate string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
gotDate = r.Header.Get("x-ms-date")
|
||||
w.Header().Set("Content-Type", ArrowContentType)
|
||||
_, _ = w.Write(arrowPage(t, "blob.txt", ""))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
|
||||
require.NoError(t, err)
|
||||
client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
UseArrowFormat: to.Ptr(true),
|
||||
})
|
||||
_, err = pager.NextPage(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
|
||||
assert.NotEmpty(t, gotDate)
|
||||
}
|
||||
|
||||
func TestPagerResponseError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
UseArrowFormat: to.Ptr(true),
|
||||
})
|
||||
_, err = pager.NextPage(context.Background())
|
||||
require.Error(t, err)
|
||||
assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
|
||||
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
|
||||
// released SDK does not export its storage bearer-challenge policy, which is
|
||||
// needed for token authentication with the same semantics as the SDK's own
|
||||
// clients.
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
|
||||
)
|
||||
|
||||
type storageAuthorizer struct {
|
||||
scopes []string
|
||||
tenantID string
|
||||
}
|
||||
|
||||
// NewStorageChallengePolicy returns a bearer token policy for cred which
|
||||
// handles the storage service's authentication challenges.
|
||||
func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
|
||||
s := storageAuthorizer{scopes: []string{audience}}
|
||||
return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
|
||||
AuthorizationHandler: policy.AuthorizationHandler{
|
||||
OnRequest: s.onRequest,
|
||||
OnChallenge: s.onChallenge,
|
||||
},
|
||||
InsecureAllowCredentialWithHTTP: allowHTTP,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
|
||||
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
|
||||
}
|
||||
|
||||
func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
|
||||
// parse the challenge
|
||||
err := s.parseChallenge(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
|
||||
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
|
||||
}
|
||||
|
||||
type challengePolicyError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *challengePolicyError) Error() string {
|
||||
return c.err.Error()
|
||||
}
|
||||
|
||||
func (*challengePolicyError) NonRetriable() {
|
||||
// marker method
|
||||
}
|
||||
|
||||
func (c *challengePolicyError) Unwrap() error {
|
||||
return c.err
|
||||
}
|
||||
|
||||
// parses Tenant ID from auth challenge
|
||||
// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
|
||||
func parseTenant(url string) string {
|
||||
if url == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(url, "/")
|
||||
if len(parts) >= 3 {
|
||||
tenant := parts[3]
|
||||
tenant = strings.ReplaceAll(tenant, ",", "")
|
||||
return tenant
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
|
||||
authHeader := resp.Header.Get("WWW-Authenticate")
|
||||
if authHeader == "" {
|
||||
return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
|
||||
}
|
||||
|
||||
// Strip down to auth and resource
|
||||
// Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
|
||||
authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
|
||||
|
||||
parts := strings.Split(authHeader, " ")
|
||||
|
||||
vals := map[string]string{}
|
||||
for _, part := range parts {
|
||||
subParts := strings.Split(part, "=")
|
||||
if len(subParts) == 2 {
|
||||
stripped := strings.ReplaceAll(subParts[1], "\"", "")
|
||||
stripped = strings.TrimSuffix(stripped, ",")
|
||||
vals[subParts[0]] = stripped
|
||||
}
|
||||
}
|
||||
|
||||
s.tenantID = parseTenant(vals["authorization_uri"])
|
||||
|
||||
scope := vals["resource_id"]
|
||||
if scope == "" {
|
||||
return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(scope, "/.default") {
|
||||
scope += "/.default"
|
||||
}
|
||||
s.scopes = []string{scope}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
|
||||
// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
|
||||
// with the internal header constants inlined. The released SDK does not
|
||||
// export its shared key signing policy, so a copy is needed to sign the
|
||||
// Arrow listing requests this package sends through its own pipeline.
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
|
||||
// storage account's name and either its primary or secondary key.
|
||||
func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
|
||||
c := SharedKeyCredential{accountName: accountName}
|
||||
if err := c.SetAccountKey(accountKey); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// SharedKeyCredential contains an account's name and its primary or secondary key.
|
||||
type SharedKeyCredential struct {
|
||||
// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
|
||||
accountName string
|
||||
accountKey atomic.Value // []byte
|
||||
}
|
||||
|
||||
// AccountName returns the Storage account's name.
|
||||
func (c *SharedKeyCredential) AccountName() string {
|
||||
return c.accountName
|
||||
}
|
||||
|
||||
// SetAccountKey replaces the existing account key with the specified account key.
|
||||
func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
|
||||
_bytes, err := base64.StdEncoding.DecodeString(accountKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode account key: %w", err)
|
||||
}
|
||||
c.accountKey.Store(_bytes)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
|
||||
func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
|
||||
h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
|
||||
_, err := h.Write([]byte(message))
|
||||
return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
|
||||
}
|
||||
|
||||
func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
|
||||
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
|
||||
headers := req.Header
|
||||
contentLength := getHeader("Content-Length", headers)
|
||||
if contentLength == "0" {
|
||||
contentLength = ""
|
||||
}
|
||||
|
||||
canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
stringToSign := strings.Join([]string{
|
||||
req.Method,
|
||||
getHeader("Content-Encoding", headers),
|
||||
getHeader("Content-Language", headers),
|
||||
contentLength,
|
||||
getHeader("Content-MD5", headers),
|
||||
getHeader("Content-Type", headers),
|
||||
"", // Empty date because x-ms-date is expected (as per web page above)
|
||||
getHeader("If-Modified-Since", headers),
|
||||
getHeader("If-Match", headers),
|
||||
getHeader("If-None-Match", headers),
|
||||
getHeader("If-Unmodified-Since", headers),
|
||||
getHeader("Range", headers),
|
||||
c.buildCanonicalizedHeader(headers),
|
||||
canonicalizedResource,
|
||||
}, "\n")
|
||||
return stringToSign, nil
|
||||
}
|
||||
|
||||
func getHeader(key string, headers map[string][]string) string {
|
||||
if headers == nil {
|
||||
return ""
|
||||
}
|
||||
if v, ok := headers[key]; ok {
|
||||
if len(v) > 0 {
|
||||
return v[0]
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func getWeightTables() [][]int {
|
||||
tableLv0 := [...]int{
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
|
||||
0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
|
||||
0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
|
||||
0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
|
||||
0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
|
||||
0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
|
||||
0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
|
||||
}
|
||||
tableLv2 := [...]int{
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
}
|
||||
tables := [][]int{tableLv0[:], tableLv2[:]}
|
||||
return tables
|
||||
}
|
||||
|
||||
// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
|
||||
func compareHeaders(lhs, rhs string, tables [][]int) int {
|
||||
currLevel, i, j := 0, 0, 0
|
||||
n := len(tables)
|
||||
lhsLen := len(lhs)
|
||||
rhsLen := len(rhs)
|
||||
|
||||
for currLevel < n {
|
||||
if currLevel == (n-1) && i != j {
|
||||
if i > j {
|
||||
return -1
|
||||
}
|
||||
if i < j {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var w1, w2 int
|
||||
|
||||
// Check bounds before accessing lhs[i]
|
||||
if i < lhsLen {
|
||||
w1 = tables[currLevel][lhs[i]]
|
||||
} else {
|
||||
w1 = 0x1
|
||||
}
|
||||
|
||||
// Check bounds before accessing rhs[j]
|
||||
if j < rhsLen {
|
||||
w2 = tables[currLevel][rhs[j]]
|
||||
} else {
|
||||
w2 = 0x1
|
||||
}
|
||||
|
||||
if w1 == 0x1 && w2 == 0x1 {
|
||||
i = 0
|
||||
j = 0
|
||||
currLevel++
|
||||
} else if w1 == w2 {
|
||||
i++
|
||||
j++
|
||||
} else if w1 == 0 {
|
||||
i++
|
||||
} else if w2 == 0 {
|
||||
j++
|
||||
} else {
|
||||
if w1 < w2 {
|
||||
return -1
|
||||
}
|
||||
if w1 > w2 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
|
||||
cm := map[string][]string{}
|
||||
for k, v := range headers {
|
||||
headerName := strings.TrimSpace(strings.ToLower(k))
|
||||
if strings.HasPrefix(headerName, "x-ms-") {
|
||||
cm[headerName] = v // NOTE: the value must not have any whitespace around it.
|
||||
}
|
||||
}
|
||||
if len(cm) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(cm))
|
||||
for key := range cm {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
tables := getWeightTables()
|
||||
// Sort the keys using the custom comparator
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
return compareHeaders(keys[i], keys[j], tables) < 0
|
||||
})
|
||||
ch := bytes.NewBufferString("")
|
||||
for i, key := range keys {
|
||||
if i > 0 {
|
||||
ch.WriteRune('\n')
|
||||
}
|
||||
ch.WriteString(key)
|
||||
ch.WriteRune(':')
|
||||
ch.WriteString(strings.Join(cm[key], ","))
|
||||
}
|
||||
return ch.String()
|
||||
}
|
||||
|
||||
func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
|
||||
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
|
||||
cr := bytes.NewBufferString("/")
|
||||
cr.WriteString(c.accountName)
|
||||
|
||||
if len(u.Path) > 0 {
|
||||
// Any portion of the CanonicalizedResource string that is derived from
|
||||
// the resource's URI should be encoded exactly as it is in the URI.
|
||||
// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
|
||||
cr.WriteString(u.EscapedPath())
|
||||
} else {
|
||||
// a slash is required to indicate the root path
|
||||
cr.WriteString("/")
|
||||
}
|
||||
|
||||
// params is a map[string][]string; param name is key; params values is []string
|
||||
params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse query params: %w", err)
|
||||
}
|
||||
|
||||
if len(params) > 0 { // There is at least 1 query parameter
|
||||
var paramNames []string // We use this to sort the parameter key names
|
||||
for paramName := range params {
|
||||
paramNames = append(paramNames, paramName) // paramNames must be lowercase
|
||||
}
|
||||
sort.Strings(paramNames)
|
||||
|
||||
for _, paramName := range paramNames {
|
||||
paramValues := params[paramName]
|
||||
sort.Strings(paramValues)
|
||||
|
||||
// Join the sorted key values separated by ','
|
||||
// Then prepend "keyName:"; then add this string to the buffer
|
||||
cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
|
||||
}
|
||||
}
|
||||
return cr.String(), nil
|
||||
}
|
||||
|
||||
// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
|
||||
func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
|
||||
return cred.computeHMACSHA256(message)
|
||||
}
|
||||
|
||||
// SharedKeyCredPolicy is a pipeline policy that signs each request with the
|
||||
// account's shared key.
|
||||
type SharedKeyCredPolicy struct {
|
||||
cred *SharedKeyCredential
|
||||
}
|
||||
|
||||
// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
|
||||
func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
|
||||
return &SharedKeyCredPolicy{cred: cred}
|
||||
}
|
||||
|
||||
// Do signs the request and sends it via the next policy in the pipeline.
|
||||
func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
|
||||
// skip adding the authorization header if no SharedKeyCredential was provided.
|
||||
// this prevents a panic that might be hard to diagnose and allows testing
|
||||
// against http endpoints that don't require authentication.
|
||||
if s.cred == nil {
|
||||
return req.Next()
|
||||
}
|
||||
|
||||
if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
|
||||
req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
|
||||
}
|
||||
stringToSign, err := s.cred.buildStringToSign(req.Raw())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signature, err := s.cred.computeHMACSHA256(stringToSign)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
|
||||
req.Raw().Header.Set("Authorization", authHeader)
|
||||
|
||||
response, err := req.Next()
|
||||
if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
|
||||
// Service failed to authenticate request, log it
|
||||
fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
|
||||
}
|
||||
return response, err
|
||||
}
|
||||
+158
-220
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -27,12 +26,15 @@ import (
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
|
||||
"github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
"github.com/rclone/rclone/backend/azureblob/auth"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
@@ -240,38 +242,40 @@ avoid the time out.`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_arrow_list",
|
||||
Help: `Use the Apache Arrow listing format.
|
||||
Help: `Use the experimental Apache Arrow listing format.
|
||||
|
||||
If set, directory listings are fetched using the ListBlobs Apache
|
||||
Arrow response format instead of XML. Arrow responses are smaller and
|
||||
much cheaper to parse, making listings of large containers several
|
||||
times faster. Combine with "list_parallelism" for the biggest gains.
|
||||
Arrow response format instead of XML. This can be faster for very
|
||||
large containers.
|
||||
|
||||
"Blob Listing with Apache Arrow" is in public preview at Microsoft and
|
||||
is only supported on flat namespace accounts. On accounts with a
|
||||
hierarchical namespace (ADLS Gen2), or where the feature is otherwise
|
||||
unavailable, the server returns XML and the listing transparently
|
||||
falls back to the normal XML path (logged at debug level).`,
|
||||
This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
|
||||
preview feature to be enabled on the storage account. It is NOT
|
||||
supported on accounts with a hierarchical namespace (ADLS Gen2) -
|
||||
those return a 409 error. If the feature is not enabled the server
|
||||
returns XML and the listing transparently falls back to the normal XML
|
||||
path (logged at debug level).
|
||||
|
||||
Not supported with connection_string auth - falls back to normal
|
||||
listing.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: "list_parallelism",
|
||||
Help: `Number of parallel shards to list a directory with.
|
||||
|
||||
If set greater than 1, the blob name keyspace of each directory is
|
||||
split into ranges which are listed concurrently using the Arrow
|
||||
startFrom/endBefore range parameters. This can dramatically speed up
|
||||
listing containers with millions of objects, for both recursive
|
||||
(ListR) and single directory listings. Speed keeps improving up to a
|
||||
parallelism of around 30.
|
||||
EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
|
||||
directory is split into this many ranges which are listed concurrently
|
||||
using the Arrow startFrom/endBefore range parameters. This can
|
||||
dramatically speed up listing containers with millions of objects, for
|
||||
both recursive (ListR) and single directory listings.
|
||||
|
||||
This has no effect unless "use_arrow_list" is also set, as Arrow is the
|
||||
only listing path that supports server-side name ranges. If the
|
||||
account does not support range listing (e.g. it has a hierarchical
|
||||
namespace) the listing falls back to sequential. The default of 0 (or
|
||||
1) lists sequentially.`,
|
||||
only listing path that supports server-side name ranges. The default of
|
||||
0 (or 1) lists sequentially.`,
|
||||
Default: 0,
|
||||
Advanced: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: "access_tier",
|
||||
Help: `Access tier of blob: hot, cool, cold or archive.
|
||||
@@ -365,22 +369,7 @@ creates an empty object ending with "/", to persist the folder.
|
||||
|
||||
This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
|
||||
the Microsoft standard.
|
||||
|
||||
This is not needed on storage accounts with a hierarchical namespace
|
||||
(see the hns option) as these always support empty directories.
|
||||
`,
|
||||
}, {
|
||||
Name: "hns",
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
Help: `Set if the storage account has a hierarchical namespace.
|
||||
|
||||
Storage accounts with hierarchical namespace enabled (Azure Data Lake
|
||||
Storage Gen2) have real directories which can be empty.
|
||||
|
||||
If this is unset then rclone will read it from the storage account
|
||||
and save it in the config file.
|
||||
`,
|
||||
}, {
|
||||
Name: "no_check_container",
|
||||
Help: `If set, don't attempt to check the container exists or create it.
|
||||
@@ -448,7 +437,6 @@ type Options struct {
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
PublicAccess string `config:"public_access"`
|
||||
DirectoryMarkers bool `config:"directory_markers"`
|
||||
HNS fs.Tristate `config:"hns"`
|
||||
NoCheckContainer bool `config:"no_check_container"`
|
||||
NoHeadObject bool `config:"no_head_object"`
|
||||
DeleteSnapshots string `config:"delete_snapshots"`
|
||||
@@ -462,8 +450,10 @@ type Fs struct {
|
||||
opt Options // parsed config options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache
|
||||
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache and arrowCntSVCcache
|
||||
cntSVCcache map[string]*container.Client // reference to containerClient per container
|
||||
arrowCntSVCcache map[string]*arrowlist.Client // reference to arrowlist client per container
|
||||
arrowClientOpts *arrowlist.ClientOptions // client options for the arrowlist clients
|
||||
arrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retried
|
||||
svc *service.Client // client to access azblob
|
||||
cred azcore.TokenCredential // how to generate tokens (may be nil)
|
||||
@@ -472,7 +462,6 @@ type Fs struct {
|
||||
rootContainer string // container part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
isLimited bool // if limited to one container
|
||||
hns bool // if the storage account has a hierarchical namespace
|
||||
cache *bucket.Cache // cache for container creation status
|
||||
pacer *fs.Pacer // To pace and retry the API calls
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
@@ -539,7 +528,7 @@ func parsePath(path string) (root string) {
|
||||
// relative to f.root
|
||||
func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
|
||||
containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
|
||||
if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
|
||||
if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
|
||||
containerPath = containerPath[:len(containerPath)-1]
|
||||
}
|
||||
return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
|
||||
@@ -587,7 +576,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
// General errors from:
|
||||
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
|
||||
// Blob specific errors from:
|
||||
@@ -682,14 +672,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
ci: ci,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
|
||||
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
|
||||
cache: bucket.NewCache(),
|
||||
cntSVCcache: make(map[string]*container.Client, 1),
|
||||
name: name,
|
||||
opt: *opt,
|
||||
ci: ci,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
|
||||
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
|
||||
cache: bucket.NewCache(),
|
||||
cntSVCcache: make(map[string]*container.Client, 1),
|
||||
arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
|
||||
}
|
||||
f.publicAccess = container.PublicAccessType(opt.PublicAccess)
|
||||
f.setRoot(root)
|
||||
@@ -706,6 +697,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ServerSideAcrossConfigs: true,
|
||||
DoubleSlash: true,
|
||||
}).Fill(ctx, f)
|
||||
if opt.DirectoryMarkers {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
|
||||
DefaultBaseURL: storageDefaultBaseURL,
|
||||
@@ -732,34 +727,21 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.usingSharedKeyCred = res.UsingSharedKeyCred
|
||||
f.anonymous = res.Anonymous
|
||||
|
||||
// Client options for the arrowlist clients, using the same transport and
|
||||
// gzip policy as the SDK clients built above.
|
||||
f.arrowClientOpts = &arrowlist.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Transport: auth.Transporter(ctx),
|
||||
PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
|
||||
},
|
||||
}
|
||||
|
||||
// if using Container level SAS put the container client into the cache
|
||||
if opt.SASURL != "" && res.Container != "" {
|
||||
_ = f.cntSVC(res.Container)
|
||||
f.isLimited = true
|
||||
}
|
||||
|
||||
if opt.HNS.Valid {
|
||||
f.hns = opt.HNS.Value
|
||||
} else if !f.anonymous {
|
||||
containerName := f.rootContainer
|
||||
if containerName == "" {
|
||||
containerName = res.Container
|
||||
}
|
||||
f.hns, err = f.readHNS(ctx, containerName)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
|
||||
} else {
|
||||
m.Set("hns", strconv.FormatBool(f.hns))
|
||||
}
|
||||
}
|
||||
if f.hns {
|
||||
fs.Debugf(f, "Storage account has a hierarchical namespace")
|
||||
}
|
||||
if f.dirMarkers() {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
if f.rootContainer != "" && f.rootDirectory != "" {
|
||||
// Check to see if the (container,directory) is actually an existing file
|
||||
oldRoot := f.root
|
||||
@@ -780,39 +762,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// readHNS reads whether the storage account has a hierarchical namespace
|
||||
//
|
||||
// If containerName is set then the container client is used which works
|
||||
// with credentials limited to that container.
|
||||
func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
|
||||
var isHNS *bool
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
if containerName != "" {
|
||||
var resp container.GetAccountInfoResponse
|
||||
resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
} else {
|
||||
var resp service.GetAccountInfoResponse
|
||||
resp, err = f.svc.GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if isHNS == nil {
|
||||
return false, errors.New("not returned by server")
|
||||
}
|
||||
return *isHNS, nil
|
||||
}
|
||||
|
||||
// dirMarkers returns true if directories are persisted with directory markers
|
||||
func (f *Fs) dirMarkers() bool {
|
||||
return f.opt.DirectoryMarkers || f.hns
|
||||
}
|
||||
|
||||
// return the container client for the container passed in
|
||||
func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
|
||||
f.cntSVCcacheMu.Lock()
|
||||
@@ -825,6 +774,44 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
|
||||
return containerClient
|
||||
}
|
||||
|
||||
// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
|
||||
// credentials can't be used for Arrow listing.
|
||||
var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
|
||||
|
||||
// return the arrowlist client for the container passed in
|
||||
//
|
||||
// Returns errArrowAuthUnsupported if the configured credentials can't be
|
||||
// reused for the arrowlist client's pipeline.
|
||||
func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
|
||||
// The container URL includes any SAS token in its query
|
||||
url := f.cntSVC(containerName).URL()
|
||||
f.cntSVCcacheMu.Lock()
|
||||
defer f.cntSVCcacheMu.Unlock()
|
||||
if client, ok := f.arrowCntSVCcache[containerName]; ok {
|
||||
return client, nil
|
||||
}
|
||||
switch {
|
||||
case f.usingSharedKeyCred:
|
||||
// Covers account+key and the emulator (auth fills in Account/Key)
|
||||
var cred *arrowlist.SharedKeyCredential
|
||||
cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
|
||||
if err == nil {
|
||||
client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
|
||||
}
|
||||
case f.cred != nil:
|
||||
client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
|
||||
case f.anonymous || f.opt.SASURL != "":
|
||||
client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
|
||||
default:
|
||||
return nil, errArrowAuthUnsupported
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.arrowCntSVCcache[containerName] = client
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Return an Object from a path
|
||||
//
|
||||
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
||||
@@ -850,12 +837,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
o, err := f.newObjectWithInfo(ctx, remote, nil)
|
||||
if err == fs.ErrorNotAFile {
|
||||
// The blob is a directory
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, err
|
||||
return f.newObjectWithInfo(ctx, remote, nil)
|
||||
}
|
||||
|
||||
// getBlobSVC creates a blob client
|
||||
@@ -924,15 +906,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
|
||||
lowerKey := strings.ToLower(k)
|
||||
switch lowerKey {
|
||||
case "cache-control":
|
||||
headers.BlobCacheControl = new(v)
|
||||
headers.BlobCacheControl = pString(v)
|
||||
case "content-disposition":
|
||||
headers.BlobContentDisposition = new(v)
|
||||
headers.BlobContentDisposition = pString(v)
|
||||
case "content-encoding":
|
||||
headers.BlobContentEncoding = new(v)
|
||||
headers.BlobContentEncoding = pString(v)
|
||||
case "content-language":
|
||||
headers.BlobContentLanguage = new(v)
|
||||
headers.BlobContentLanguage = pString(v)
|
||||
case "content-type":
|
||||
headers.BlobContentType = new(v)
|
||||
headers.BlobContentType = pString(v)
|
||||
case "x-ms-tags":
|
||||
parsed, perr := parseXMsTags(v)
|
||||
if perr != nil {
|
||||
@@ -1169,33 +1151,18 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
|
||||
if endsWithSlash || remote == "" {
|
||||
return true
|
||||
}
|
||||
return hasDirMetadata(metadata)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether the metadata marks the blob as a directory
|
||||
func hasDirMetadata(metadata map[string]*string) bool {
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether props are for a directory on a storage account with
|
||||
// a hierarchical namespace
|
||||
//
|
||||
// These ignore a trailing "/" on the name so a file can be returned
|
||||
// when looking for a directory.
|
||||
func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
|
||||
return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
|
||||
}
|
||||
|
||||
// listFn is called from list to handle an object
|
||||
type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
|
||||
|
||||
@@ -1221,25 +1188,27 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
delimiter = "/"
|
||||
}
|
||||
|
||||
opts := &container.ListBlobsHierarchyOptions{
|
||||
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
|
||||
Include: container.ListBlobsInclude{
|
||||
Copy: false,
|
||||
Metadata: true,
|
||||
Snapshots: false,
|
||||
UncommittedBlobs: false,
|
||||
Deleted: false,
|
||||
opts := &arrowlist.ListBlobsHierarchyOptions{
|
||||
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
|
||||
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
|
||||
Include: container.ListBlobsInclude{
|
||||
Copy: false,
|
||||
Metadata: true,
|
||||
Snapshots: false,
|
||||
UncommittedBlobs: false,
|
||||
Deleted: false,
|
||||
},
|
||||
Prefix: &directory,
|
||||
MaxResults: &maxResults,
|
||||
},
|
||||
Prefix: &directory,
|
||||
MaxResults: &maxResults,
|
||||
}
|
||||
// Request the Apache Arrow listing format. The SDK pager requests an
|
||||
// Arrow IPC stream and decodes it, falling back to XML if the account
|
||||
// doesn't have Arrow listing enabled. Skip the maxResults==1 probe
|
||||
// (isEmpty) which doesn't benefit.
|
||||
// Experimental: request the Apache Arrow listing format. The arrowlist
|
||||
// pager requests an Arrow IPC stream and decodes it, falling back to XML
|
||||
// if the account doesn't have Arrow listing enabled. Skip the
|
||||
// maxResults==1 probe (isEmpty) which doesn't benefit.
|
||||
useArrow := f.opt.UseArrowList && maxResults != 1
|
||||
if useArrow {
|
||||
opts.ResponseFormat = container.StorageResponseFormatArrow
|
||||
opts.UseArrowFormat = to.Ptr(true)
|
||||
}
|
||||
|
||||
var foundItems int
|
||||
@@ -1254,18 +1223,15 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f.dirMarkers() && foundItems == 0 && directory != "" {
|
||||
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
|
||||
// Determine whether the directory exists or not by whether it has a marker
|
||||
props, err := f.readMetaData(ctx, containerName, directory)
|
||||
_, err := f.readMetaData(ctx, containerName, directory)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
if f.hns && !isHNSDirectory(props) {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1273,11 +1239,25 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
// listBlobsPager runs the hierarchy listing described by opts, calling fn for
|
||||
// each blob and subdirectory, and returns the number of raw items seen.
|
||||
// delimiter selects flat (recurse) vs hierarchical listing. If opts requests
|
||||
// the Apache Arrow format and the service answers with XML (Arrow listing
|
||||
// not enabled) a debug message is logged.
|
||||
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
|
||||
useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
|
||||
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
|
||||
// the Apache Arrow format the listing goes through the arrowlist pager
|
||||
// (falling back to the SDK pager if the credentials don't support it); if the
|
||||
// service then answers with XML (Arrow listing not enabled) a debug message
|
||||
// is logged.
|
||||
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
|
||||
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
|
||||
var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
|
||||
if useArrow {
|
||||
arrowSVC, err := f.arrowCntSVC(containerName)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Not using Arrow listing: %v", err)
|
||||
useArrow = false
|
||||
} else {
|
||||
pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
|
||||
}
|
||||
}
|
||||
if pager == nil {
|
||||
pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
|
||||
}
|
||||
checkedArrow := false
|
||||
for pager.More() {
|
||||
var response container.ListBlobsHierarchyResponse
|
||||
@@ -1293,7 +1273,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
// but not accelerated.
|
||||
if useArrow && !checkedArrow && err == nil {
|
||||
checkedArrow = true
|
||||
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
|
||||
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
|
||||
fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
|
||||
}
|
||||
}
|
||||
@@ -1318,13 +1298,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
fs.Debugf(f, "Nil name received")
|
||||
continue
|
||||
}
|
||||
// Storage accounts with a hierarchical namespace return
|
||||
// the file "dir" when listing "dir/"
|
||||
if !strings.HasPrefix(*file.Name, directory) {
|
||||
fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
|
||||
foundItems--
|
||||
continue
|
||||
}
|
||||
remote := f.opt.Enc.ToStandardPath(*file.Name)
|
||||
if !strings.HasPrefix(remote, prefix) {
|
||||
fs.Debugf(f, "Odd name received %q", remote)
|
||||
@@ -1586,7 +1559,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
|
||||
func (f *Fs) listContainersToFn(fn listContainerFn) error {
|
||||
max := int32(f.opt.ListChunkSize)
|
||||
pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
|
||||
Include: service.ListContainersInclude{Metadata: true},
|
||||
Include: service.ListContainersInclude{Metadata: true, Deleted: true},
|
||||
MaxResults: &max,
|
||||
})
|
||||
ctx := context.Background()
|
||||
@@ -1636,7 +1609,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
|
||||
// Create directory marker file and parents
|
||||
func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
|
||||
if !f.dirMarkers() || container == "" {
|
||||
if !f.opt.DirectoryMarkers || container == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1693,11 +1666,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
|
||||
// mkdirParent creates the parent bucket/directory if it doesn't exist
|
||||
func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
|
||||
if f.hns {
|
||||
// Parent directories are created by the upload
|
||||
container, _ := f.split(remote)
|
||||
return f.makeContainer(ctx, container)
|
||||
}
|
||||
remote, _ = strings.CutSuffix(remote, "/")
|
||||
dir := path.Dir(remote)
|
||||
if dir == "/" || dir == "." {
|
||||
@@ -1803,9 +1771,6 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
|
||||
// Returns an error if it isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
container, directory := f.split(dir)
|
||||
if f.hns && container != "" && directory != "" {
|
||||
return f.rmdirHNS(ctx, container, directory)
|
||||
}
|
||||
// Remove directory marker file
|
||||
if f.opt.DirectoryMarkers && container != "" && directory != "" {
|
||||
o := &Object{
|
||||
@@ -1828,39 +1793,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return f.deleteContainer(ctx, container)
|
||||
}
|
||||
|
||||
// rmdirHNS removes the directory on a storage account with a
|
||||
// hierarchical namespace
|
||||
//
|
||||
// The directory must be deleted without a trailing "/" and the server
|
||||
// refuses to delete it if it isn't empty.
|
||||
func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
|
||||
directory, _ = strings.CutSuffix(directory, "/")
|
||||
// Check it is a directory as deleting a file would succeed
|
||||
props, err := f.readMetaData(ctx, container, directory)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isHNSDirectory(props) {
|
||||
return fs.ErrorIsFile
|
||||
}
|
||||
blb := f.getBlobSVC(container, directory)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err := blb.Delete(ctx, nil)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if storageErr.StatusCode == http.StatusNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Precision of the remote
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Nanosecond
|
||||
@@ -2699,6 +2631,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return downloadResponse.Body, nil
|
||||
}
|
||||
|
||||
// Converts a string into a pointer to a string
|
||||
func pString(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
|
||||
type readSeekCloser struct {
|
||||
io.Reader
|
||||
@@ -2853,7 +2790,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
// isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
|
||||
// returning true if it is found
|
||||
func isInvalidBlockOrBlob(err error) bool {
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
|
||||
}
|
||||
return false
|
||||
@@ -3225,7 +3163,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
|
||||
// Start with default content-type based on source
|
||||
ui.httpHeaders = blob.HTTPHeaders{
|
||||
BlobContentType: new(fs.MimeType(ctx, src)),
|
||||
BlobContentType: pString(fs.MimeType(ctx, src)),
|
||||
}
|
||||
|
||||
// Apply mapped metadata/headers/tags if requested
|
||||
@@ -3271,15 +3209,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
o.tags[parts[0]] = parts[1]
|
||||
}
|
||||
case "cache-control":
|
||||
ui.httpHeaders.BlobCacheControl = new(value)
|
||||
ui.httpHeaders.BlobCacheControl = pString(value)
|
||||
case "content-disposition":
|
||||
ui.httpHeaders.BlobContentDisposition = new(value)
|
||||
ui.httpHeaders.BlobContentDisposition = pString(value)
|
||||
case "content-encoding":
|
||||
ui.httpHeaders.BlobContentEncoding = new(value)
|
||||
ui.httpHeaders.BlobContentEncoding = pString(value)
|
||||
case "content-language":
|
||||
ui.httpHeaders.BlobContentLanguage = new(value)
|
||||
ui.httpHeaders.BlobContentLanguage = pString(value)
|
||||
case "content-type":
|
||||
ui.httpHeaders.BlobContentType = new(value)
|
||||
ui.httpHeaders.BlobContentType = pString(value)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
@@ -325,14 +325,6 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
|
||||
}
|
||||
|
||||
// helper to read blob tags for an object
|
||||
// skipIfHNS skips the test if the storage account has a hierarchical
|
||||
// namespace as these don't support blob tags
|
||||
func skipIfHNS(t *testing.T, f *Fs) {
|
||||
if f.hns {
|
||||
t.Skip("blob tags not supported with hierarchical namespace")
|
||||
}
|
||||
}
|
||||
|
||||
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
||||
ao := o.(*Object)
|
||||
blb := ao.getBlobSVC()
|
||||
@@ -418,7 +410,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart upload
|
||||
t.Run("PutSinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -434,7 +425,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart upload
|
||||
t.Run("PutMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize + 2048))
|
||||
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -622,7 +612,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart copy
|
||||
t.Run("CopySinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
@@ -648,7 +637,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart copy
|
||||
t.Run("CopyMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
||||
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || solaris || js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -650,6 +650,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
|
||||
@@ -813,8 +818,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
|
||||
}
|
||||
opt := file.RenameOptions{
|
||||
IgnoreReadOnly: new(true),
|
||||
ReplaceIfExists: new(true),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
ReplaceIfExists: ptr(true),
|
||||
}
|
||||
dstAbsPath := f.absPath(remote)
|
||||
fc := srcObj.fileClient()
|
||||
@@ -859,8 +864,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
opt := directory.RenameOptions{
|
||||
IgnoreReadOnly: new(false),
|
||||
ReplaceIfExists: new(false),
|
||||
IgnoreReadOnly: ptr(false),
|
||||
ReplaceIfExists: ptr(false),
|
||||
}
|
||||
dstAbsPath := dstFs.absPath(dstRemote)
|
||||
dirClient := srcFs.dirClient(srcRemote)
|
||||
@@ -899,8 +904,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
ChangeTime: file.SourceCopyFileChangeTime{},
|
||||
CreationTime: file.SourceCopyFileCreationTime{},
|
||||
LastWriteTime: file.SourceCopyFileLastWriteTime{},
|
||||
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: new(true),
|
||||
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
},
|
||||
}
|
||||
srcURL := srcObj.fileClient().URL()
|
||||
|
||||
+1
-25
@@ -191,11 +191,6 @@ If the custom endpoint rewrites the requests for authentication,
|
||||
e.g., in Cloudflare Workers, this header needs to be handled properly.
|
||||
Leave blank if you want to use the endpoint provided by Backblaze.
|
||||
|
||||
Rclone will not follow a redirect from this endpoint which downgrades
|
||||
HTTPS to HTTP, and will not send the "Authorization" header or the
|
||||
SSE-C key headers once a redirect has left the host and port of this
|
||||
endpoint.
|
||||
|
||||
The URL provided here SHOULD have the protocol and SHOULD NOT have
|
||||
a trailing slash or specify the /file/bucket subpath as rclone will
|
||||
request files with "{download_url}/file/{bucket_name}/{path}".
|
||||
@@ -534,25 +529,6 @@ func (f *Fs) setRoot(root string) {
|
||||
f.rootBucket, f.rootDirectory = bucket.Split(f.root)
|
||||
}
|
||||
|
||||
// redirectSecretHeaders are the request headers carrying secrets that
|
||||
// must not be forwarded when a redirect leaves the original host.
|
||||
var redirectSecretHeaders = []string{
|
||||
"Authorization",
|
||||
sseAlgorithmHeader,
|
||||
sseKeyHeader,
|
||||
sseMd5Header,
|
||||
}
|
||||
|
||||
// newClient makes the http client used for all B2 requests.
|
||||
//
|
||||
// The download_url option sends the requests to a host the user does
|
||||
// not control so it strips headers on cross host redirects.
|
||||
func newClient(ctx context.Context) *http.Client {
|
||||
client := fshttp.NewClient(ctx)
|
||||
client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
|
||||
return client
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, bucket:path
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
@@ -605,7 +581,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
name: name,
|
||||
opt: *opt,
|
||||
ci: ci,
|
||||
srv: rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
|
||||
srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
|
||||
cache: bucket.NewCache(),
|
||||
_bucketID: make(map[string]string, 1),
|
||||
_bucketType: make(map[string]string, 1),
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -619,60 +617,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// redirectTestHeaders is a deliberately literal copy of
|
||||
// redirectSecretHeaders so that a header dropped from the production
|
||||
// list fails TestRedirectSecretHeaders rather than silently losing
|
||||
// test coverage.
|
||||
var redirectTestHeaders = map[string]string{
|
||||
"Authorization": "secret-token",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Key": "secret-key",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Key-Md5": "secret-md5",
|
||||
}
|
||||
|
||||
func TestRedirectSecretHeaders(t *testing.T) {
|
||||
want := make([]string, 0, len(redirectTestHeaders))
|
||||
for header := range redirectTestHeaders {
|
||||
want = append(want, header)
|
||||
}
|
||||
sort.Strings(want)
|
||||
got := append([]string(nil), redirectSecretHeaders...)
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
// redirectTestRequest makes a GET request to rawURL carrying the
|
||||
// secret headers plus a harmless marker header.
|
||||
func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
|
||||
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
|
||||
require.NoError(t, err)
|
||||
for header, value := range redirectTestHeaders {
|
||||
req.Header.Set(header, value)
|
||||
}
|
||||
req.Header.Set("X-Bz-Test-Mode", "marker")
|
||||
return req
|
||||
}
|
||||
|
||||
func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
|
||||
client := newClient(context.Background())
|
||||
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for header := range redirectTestHeaders {
|
||||
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
|
||||
}
|
||||
assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer initialServer.Close()
|
||||
|
||||
resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
}
|
||||
+1
-1
@@ -1461,7 +1461,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
// Item in the cache has the same or newer SequenceID than
|
||||
// this event. Ignore this event, it must be old.
|
||||
f.itemMetaCacheMu.Unlock()
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
+15
-23
@@ -3,6 +3,7 @@
|
||||
package box
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
@@ -19,7 +20,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
|
||||
}
|
||||
|
||||
// uploadPart uploads a part in an upload session
|
||||
//
|
||||
// chunk must contain exactly the part's data and is read from the start
|
||||
// on each attempt.
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
hasher := sha1.New()
|
||||
chunkSize, err := io.Copy(hasher, chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
chunkSize := int64(len(chunk))
|
||||
sha1sum := sha1.Sum(chunk)
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/files/upload_sessions/" + SessionID,
|
||||
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
|
||||
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Digest": sha1Digest(hasher.Sum(nil)),
|
||||
"Digest": sha1Digest(sha1sum[:]),
|
||||
},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = chunk.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = wrap(chunk)
|
||||
opts.Body = wrap(bytes.NewReader(chunk))
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -223,25 +213,27 @@ outer:
|
||||
|
||||
reqSize := min(remaining, chunkSize)
|
||||
|
||||
// Read the chunk into memory from the global pool, hashing
|
||||
// it for the whole file hash (must be done sequentially)
|
||||
rw := multipart.NewRW().Reserve(reqSize)
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
|
||||
// Make a block of memory
|
||||
buf := make([]byte, reqSize)
|
||||
|
||||
// Read the chunk
|
||||
_, err = io.ReadFull(in, buf)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
err = fmt.Errorf("multipart upload failed to read source: %w", err)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Make the global hash (must be done sequentially)
|
||||
_, _ = hash.Write(buf)
|
||||
|
||||
// Transfer the chunk
|
||||
wg.Add(1)
|
||||
o.fs.uploadToken.Get()
|
||||
go func(part int, position int64) {
|
||||
defer wg.Done()
|
||||
defer o.fs.uploadToken.Put()
|
||||
defer func() { _ = rw.Close() }()
|
||||
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
|
||||
select {
|
||||
|
||||
@@ -801,12 +801,9 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
|
||||
fs.Debugf(f, "skip orphan data chunk %q", remote)
|
||||
break
|
||||
}
|
||||
if mainObject == nil && !f.useMeta && !isSpecial {
|
||||
if mainObject == nil && !f.useMeta {
|
||||
// this is the "nometa" case
|
||||
// create dummy chunked object without metadata
|
||||
//
|
||||
// not for temporary or control chunks as they may be all
|
||||
// that is left of an interrupted upload
|
||||
mainObject = f.newObject(mainRemote, nil, nil)
|
||||
byRemote[mainRemote] = mainObject
|
||||
if !badEntry[mainRemote] {
|
||||
|
||||
@@ -43,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
@@ -627,22 +627,6 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
|
||||
_ = file.Remove(ctx)
|
||||
}
|
||||
|
||||
// testOrphanTempChunks checks that temporary chunks left behind by an
|
||||
// interrupted upload are not listed and don't break the listing
|
||||
func testOrphanTempChunks(t *testing.T, f *Fs) {
|
||||
const dir = "orphan"
|
||||
ctx := context.Background()
|
||||
defer func() {
|
||||
_ = operations.Purge(ctx, f.base, dir)
|
||||
}()
|
||||
|
||||
modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
|
||||
item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
|
||||
fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
|
||||
|
||||
fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
|
||||
}
|
||||
|
||||
func testMetadataInput(t *testing.T, f *Fs) {
|
||||
const minChunkForTest = 50
|
||||
if f.opt.ChunkSize < minChunkForTest {
|
||||
@@ -944,9 +928,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("BackwardsCompatibility", func(t *testing.T) {
|
||||
testBackwardsCompatibility(t, f)
|
||||
})
|
||||
t.Run("OrphanTempChunks", func(t *testing.T) {
|
||||
testOrphanTempChunks(t, f)
|
||||
})
|
||||
t.Run("ChunkerServerSideMove", func(t *testing.T) {
|
||||
testChunkerServerSideMove(t, f)
|
||||
})
|
||||
|
||||
@@ -472,8 +472,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
if updateOptions, ok := option.(*api.UpdateOptions); ok {
|
||||
if updateOptions.PublicID != "" {
|
||||
updateObject = true
|
||||
params.Overwrite = new(true)
|
||||
params.Invalidate = new(true)
|
||||
params.Overwrite = SDKApi.Bool(true)
|
||||
params.Invalidate = SDKApi.Bool(true)
|
||||
params.PublicID = updateOptions.PublicID
|
||||
params.ResourceType = updateOptions.ResourceType
|
||||
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -564,19 +563,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
|
||||
|
||||
// cache small files in memory and do normal upload
|
||||
rw := multipart.NewRW()
|
||||
defer fs.CheckClose(rw, &err)
|
||||
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
|
||||
if err == io.EOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, rw, src, options...)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
buf := make([]byte, f.opt.RAMCacheLimit)
|
||||
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
|
||||
}
|
||||
|
||||
// Need to include what we already read
|
||||
in = &ReadCloserWrapper{
|
||||
Reader: io.MultiReader(rw, in),
|
||||
Reader: io.MultiReader(bytes.NewReader(buf), in),
|
||||
Closer: in,
|
||||
}
|
||||
|
||||
@@ -1085,35 +1080,6 @@ type ObjectMetadata struct {
|
||||
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
|
||||
}
|
||||
|
||||
// validate checks metadata read from the wrapped remote before it is used to
|
||||
// construct a compressed reader.
|
||||
func (meta *ObjectMetadata) validate() error {
|
||||
if meta.Mode != Gzip {
|
||||
return nil
|
||||
}
|
||||
if meta.CompressionMetadataGzip == nil {
|
||||
return errors.New("missing gzip metadata")
|
||||
}
|
||||
if meta.CompressionMetadataGzip.BlockSize <= 0 {
|
||||
return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
|
||||
}
|
||||
if meta.Size != meta.CompressionMetadataGzip.Size {
|
||||
return errors.New("gzip metadata size does not match object size")
|
||||
}
|
||||
if meta.Size < 0 {
|
||||
return errors.New("invalid gzip object size")
|
||||
}
|
||||
blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
|
||||
blocks := meta.Size / blockSize
|
||||
if meta.Size%blockSize != 0 {
|
||||
blocks++
|
||||
}
|
||||
if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
|
||||
return errors.New("gzip block data is incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Object with external metadata
|
||||
type Object struct {
|
||||
fs.Object // Wraps around data object for this object
|
||||
@@ -1137,9 +1103,6 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
|
||||
if err = jr.Decode(meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = meta.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
@@ -1157,15 +1120,6 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return objErr
|
||||
}
|
||||
|
||||
// countingDiscard is an io.Writer which discards its input and counts
|
||||
// the bytes written
|
||||
type countingDiscard int64
|
||||
|
||||
func (c *countingDiscard) Write(p []byte) (int, error) {
|
||||
*c += countingDiscard(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
|
||||
type ReadCloserWrapper struct {
|
||||
io.Reader
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/buengese/sgzip"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
@@ -16,93 +14,6 @@ import (
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
func TestObjectMetadataValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
meta ObjectMetadata
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "nil gzip metadata",
|
||||
meta: ObjectMetadata{Mode: Gzip},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "zero block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 0,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "negative block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: -1,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "mismatched sizes",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 2048,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "short block data",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1, 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := test.meta.validate()
|
||||
if (err != nil) != test.want {
|
||||
t.Fatalf("validate() error = %v, want error: %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var defaultOpt = fstests.Opt{
|
||||
RemoteName: "TestCompress:",
|
||||
NilObject: (*Object)(nil),
|
||||
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type gzipModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
|
||||
// PutStream. It reads each body twice when it is seekable to model a
|
||||
// backend retrying an upload.
|
||||
type putRecorder struct {
|
||||
fs.Fs
|
||||
t *testing.T
|
||||
stream bool // expose PutStream from the wrapped Fs
|
||||
size int64
|
||||
seekable bool
|
||||
body []byte
|
||||
streamed bool
|
||||
}
|
||||
|
||||
func (r *putRecorder) Features() *fs.Features {
|
||||
features := *r.Fs.Features()
|
||||
features.PutStream = nil
|
||||
if r.stream {
|
||||
features.PutStream = r.PutStream
|
||||
}
|
||||
return &features
|
||||
}
|
||||
|
||||
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
|
||||
r.size = src.Size()
|
||||
body, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
r.seekable = true
|
||||
_, err = seeker.Seek(0, io.SeekStart)
|
||||
require.NoError(r.t, err)
|
||||
again, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
assert.Equal(r.t, body, again, "retried body should be identical")
|
||||
}
|
||||
r.body = body
|
||||
return body
|
||||
}
|
||||
|
||||
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
body := r.record(in, src)
|
||||
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
|
||||
}
|
||||
|
||||
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
r.streamed = true
|
||||
return r.Put(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// TestRcat checks the three upload paths rcat takes: small files buffered
|
||||
// in memory and uploaded with a known size and a seekable body, large
|
||||
// files streamed when the wrapped remote supports it, and large files
|
||||
// spooled to a temporary file when it doesn't. Pool pages taken for the
|
||||
// buffer must be returned on every path.
|
||||
func TestRcat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const limit = 1024
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
stream bool
|
||||
wantSize int64
|
||||
}{
|
||||
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
|
||||
{name: "Empty", size: 0, stream: true, wantSize: 0},
|
||||
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
|
||||
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
|
||||
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
|
||||
require.NoError(t, err)
|
||||
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
|
||||
f := &Fs{Fs: rec}
|
||||
f.opt.RAMCacheLimit = limit
|
||||
content := []byte(random.String(test.size))
|
||||
inUse := pool.Global().InUse()
|
||||
|
||||
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
|
||||
|
||||
assert.Equal(t, test.wantSize, rec.size)
|
||||
assert.Equal(t, content, rec.body)
|
||||
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
|
||||
if test.size < limit {
|
||||
assert.True(t, rec.seekable, "small file body should be seekable")
|
||||
}
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, content, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsCompressible checks the compressibility heuristic on data at
|
||||
// both ends of the scale for every compression mode which has one.
|
||||
func TestIsCompressible(t *testing.T) {
|
||||
compressible := bytes.Repeat([]byte("compress me "), 4096)
|
||||
incompressible := make([]byte, len(compressible))
|
||||
_, err := io.ReadFull(rand.Reader, incompressible)
|
||||
require.NoError(t, err)
|
||||
for name, handler := range map[string]compressionModeHandler{
|
||||
"gzip": &gzipModeHandler{},
|
||||
"zstd": &zstdModeHandler{},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type zstdModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -118,8 +118,8 @@ func (mode NameEncryptionMode) String() (out string) {
|
||||
return out
|
||||
}
|
||||
|
||||
// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
|
||||
type FileNameEncoding interface {
|
||||
// fileNameEncoding are the encoding methods dealing with encrypted file names
|
||||
type fileNameEncoding interface {
|
||||
EncodeToString(src []byte) string
|
||||
DecodeString(s string) ([]byte, error)
|
||||
}
|
||||
@@ -154,7 +154,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
|
||||
}
|
||||
|
||||
// NewNameEncoding creates a NameEncoding from a string
|
||||
func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
|
||||
func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
|
||||
s = strings.ToLower(s)
|
||||
switch s {
|
||||
case "base32":
|
||||
@@ -176,7 +176,7 @@ type Cipher struct {
|
||||
nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto
|
||||
block gocipher.Block
|
||||
mode NameEncryptionMode
|
||||
fileNameEnc FileNameEncoding
|
||||
fileNameEnc fileNameEncoding
|
||||
buffers sync.Pool // encrypt/decrypt buffers
|
||||
cryptoRand io.Reader // read crypto random numbers from here
|
||||
dirNameEncrypt bool
|
||||
@@ -185,7 +185,7 @@ type Cipher struct {
|
||||
}
|
||||
|
||||
// newCipher initialises the cipher. If salt is "" then it uses a built in salt val
|
||||
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
|
||||
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
|
||||
c := &Cipher{
|
||||
mode: mode,
|
||||
fileNameEnc: enc,
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
// Package api has type definitions for dosya.dev
|
||||
package api
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Response is the base response wrapper
|
||||
type Response struct {
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Error is an error response from the API.
|
||||
//
|
||||
// StatusCode and Status are filled in from the HTTP response. The API
|
||||
// sets ErrorCode on the errors a client is expected to act on rather
|
||||
// than just report, e.g. "concurrent_upload_limit".
|
||||
type Error struct {
|
||||
StatusCode int `json:"-"`
|
||||
Status string `json:"-"`
|
||||
OK bool `json:"ok"`
|
||||
Message string `json:"error"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
|
||||
}
|
||||
|
||||
// Error satisfies the error interface
|
||||
func (e *Error) Error() string {
|
||||
out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
|
||||
if e.Message != "" {
|
||||
out += ": " + e.Message
|
||||
}
|
||||
if e.ErrorCode != "" {
|
||||
out += " (" + e.ErrorCode + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileItem represents a file from the dosya API
|
||||
type FileItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
UpdatedAt float64 `json:"updated_at"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
|
||||
// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
|
||||
// are never re-read to hash the whole object - the same limitation S3 has).
|
||||
ContentHash string `json:"content_hash"`
|
||||
}
|
||||
|
||||
// FolderItem represents a folder from the dosya API
|
||||
type FolderItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
FileCount int `json:"file_count"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
}
|
||||
|
||||
// ListResponse is the response from GET /api/files
|
||||
type ListResponse struct {
|
||||
Response
|
||||
Files []FileItem `json:"files"`
|
||||
Folders []FolderItem `json:"folders"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
// Pagination describes which page of files a ListResponse holds
|
||||
type Pagination struct {
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest is the request body for POST /api/folders
|
||||
type CreateFolderRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse is the response from POST /api/folders
|
||||
type CreateFolderResponse struct {
|
||||
Response
|
||||
Folder struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Workspace string `json:"workspace_id"`
|
||||
} `json:"folder"`
|
||||
}
|
||||
|
||||
// DeleteFolderResponse is the response from DELETE /api/folders/:id
|
||||
type DeleteFolderResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
Complete bool `json:"complete"`
|
||||
Remaining int `json:"remaining"`
|
||||
FilesAffected int `json:"files_affected"`
|
||||
FoldersRemoved int `json:"folders_removed"`
|
||||
}
|
||||
|
||||
// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
|
||||
type RenameFolderRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFolderRequest is the request body for PUT /api/folders/:id/move
|
||||
type MoveFolderRequest struct {
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// MoveFolderResponse is the response from PUT /api/folders/:id/move
|
||||
type MoveFolderResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// UploadInitRequest is the request body for POST /api/upload/init
|
||||
type UploadInitRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Region string `json:"region,omitempty"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
FileID *string `json:"file_id,omitempty"`
|
||||
}
|
||||
|
||||
// UploadInitResponse is the response from POST /api/upload/init
|
||||
type UploadInitResponse struct {
|
||||
Response
|
||||
SessionID string `json:"session_id"`
|
||||
UploadURL string `json:"upload_url"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Resumable *struct {
|
||||
PartSize int64 `json:"part_size"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
PartUploadURL string `json:"part_upload_url"`
|
||||
CompleteURL string `json:"complete_url"`
|
||||
StatusURL string `json:"status_url"`
|
||||
} `json:"resumable"`
|
||||
}
|
||||
|
||||
// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
|
||||
type UploadCompleteResponse struct {
|
||||
Response
|
||||
File struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Version int `json:"version"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
|
||||
// "" for a multipart upload, which the server does not hash.
|
||||
ContentHash string `json:"content_hash"`
|
||||
} `json:"file"`
|
||||
}
|
||||
|
||||
// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
|
||||
type UploadPartResponse struct {
|
||||
Response
|
||||
PartNumber int `json:"part_number"`
|
||||
ETag string `json:"etag"`
|
||||
BytesUploaded int64 `json:"bytes_uploaded"`
|
||||
PartsCompleted int `json:"parts_completed"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
}
|
||||
|
||||
// DeleteFileResponse is the response from DELETE /api/files/:id
|
||||
type DeleteFileResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
}
|
||||
|
||||
// RenameFileRequest is the request body for PUT /api/files/:id/rename
|
||||
type RenameFileRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// RenameFileResponse is the response from PUT /api/files/:id/rename
|
||||
type RenameFileResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFileRequest is the request body for PUT /api/files/:id/move
|
||||
type MoveFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest is the request body for POST /api/files/:id/copy
|
||||
type CopyFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileResponse is the response from POST /api/files/:id/copy
|
||||
type CopyFileResponse struct {
|
||||
Response
|
||||
FileID string `json:"file_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// ShareLinkRequest is the request body for POST /api/files/:id/share
|
||||
type ShareLinkRequest struct {
|
||||
ExpiresInDays *int `json:"expires_in_days,omitempty"`
|
||||
}
|
||||
|
||||
// ShareLinkResponse is the response from POST /api/files/:id/share
|
||||
type ShareLinkResponse struct {
|
||||
Response
|
||||
Link struct {
|
||||
ID string `json:"id"`
|
||||
Token string `json:"token"`
|
||||
URL string `json:"url"`
|
||||
ExpiresAt *int64 `json:"expires_at"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
} `json:"link"`
|
||||
}
|
||||
|
||||
// Workspace is one entry of the workspace list
|
||||
type Workspace struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// WorkspaceListResponse is the response from GET /api/workspaces
|
||||
type WorkspaceListResponse struct {
|
||||
Response
|
||||
Workspaces []Workspace `json:"workspaces"`
|
||||
}
|
||||
|
||||
// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
|
||||
type WorkspaceInfoResponse struct {
|
||||
Response
|
||||
Storage struct {
|
||||
Used int64 `json:"used"`
|
||||
Total int64 `json:"total"`
|
||||
Free int64 `json:"free"`
|
||||
} `json:"storage"`
|
||||
}
|
||||
@@ -1,772 +0,0 @@
|
||||
// Package dosya provides an interface to the dosya.dev cloud storage system.
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
rootID = "root"
|
||||
defaultBaseURL = "https://api.dosya.dev"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2
|
||||
attackConstant = 0
|
||||
)
|
||||
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "dosya",
|
||||
Description: "dosya.dev",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
|
||||
Name: "api_key",
|
||||
Sensitive: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "workspace_id",
|
||||
Help: `Your workspace ID.
|
||||
|
||||
Leave this blank to use the one workspace the API key can reach, which
|
||||
is what a key restricted to a single workspace does. If the key can
|
||||
reach several, rclone will refuse and list their IDs for you to choose
|
||||
from.`,
|
||||
}, {
|
||||
Help: "API base URL.",
|
||||
Name: "api_url",
|
||||
Default: defaultBaseURL,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
Help: `Concurrency for multipart uploads.
|
||||
|
||||
This is the number of parts of the same file that are uploaded
|
||||
concurrently. The first part is always uploaded on its own, because
|
||||
the server creates the upload when it receives it.
|
||||
|
||||
Each part in flight is held in memory, so memory use grows with
|
||||
this value times the part size the server chooses.
|
||||
|
||||
If you are uploading large files over high-speed links and these
|
||||
uploads do not fully utilize your bandwidth, then increasing this
|
||||
may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeSingleQuote |
|
||||
encoder.EncodeDoubleQuote |
|
||||
encoder.EncodeLtGt |
|
||||
encoder.EncodeCtl |
|
||||
encoder.EncodeLeftSpace |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
APIKey string `config:"api_key"`
|
||||
WorkspaceID string `config:"workspace_id"`
|
||||
APIURL string `config:"api_url"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dosya.dev filesystem
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
features *fs.Features
|
||||
rest *rest.Client
|
||||
pacer *fs.Pacer
|
||||
dirCache *dircache.DirCache
|
||||
}
|
||||
|
||||
// Object represents a dosya.dev file
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
file api.FileItem
|
||||
}
|
||||
|
||||
// findWorkspace fills in the workspace ID when the config leaves it blank.
|
||||
//
|
||||
// The ID is not shown anywhere in the dosya.dev web interface, so requiring
|
||||
// it meant asking people to find it in a developer console. A key restricted
|
||||
// to one workspace can only mean that one, and an account with a single
|
||||
// workspace has only one possible answer; anything else genuinely has to be
|
||||
// chosen, so list the choices in the error.
|
||||
func (f *Fs) findWorkspace(ctx context.Context) error {
|
||||
workspaces, err := f.listWorkspaces(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch len(workspaces) {
|
||||
case 0:
|
||||
return errors.New("no workspaces found for this API key")
|
||||
case 1:
|
||||
f.opt.WorkspaceID = workspaces[0].ID
|
||||
fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
|
||||
return nil
|
||||
}
|
||||
var choices strings.Builder
|
||||
for _, w := range workspaces {
|
||||
_, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
|
||||
}
|
||||
return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
|
||||
len(workspaces), choices.String())
|
||||
}
|
||||
|
||||
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
||||
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
|
||||
listing, err := f.listFilesAndFolders(ctx, pathID)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, folder := range listing.Folders {
|
||||
if f.opt.Enc.ToStandardName(folder.Name) == leaf {
|
||||
return folder.ID, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// CreateDir makes a directory with pathID as parent and name leaf
|
||||
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
|
||||
name := f.opt.Enc.FromStandardName(leaf)
|
||||
resp, err := f.createFolder(ctx, name, pathID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.Folder.ID, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("dosya.dev root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
// Modification times are stored and returned to the second (the API
|
||||
// keeps them as Unix seconds, sent on upload via the source-mtime header).
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash types of the filesystem
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
// The server computes and stores the SHA-256 of a single-shot upload's
|
||||
// bytes; multipart uploads have none (see Object.Hash).
|
||||
return hash.NewHashSet(hash.SHA256)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(config, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
|
||||
}
|
||||
|
||||
// No DuplicateFiles: an upload that names no existing file but whose
|
||||
// (workspace, folder, name) is already taken becomes a new VERSION of
|
||||
// that file rather than a second row, so a name is unique within a
|
||||
// folder and the same upload twice returns the same file id.
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
ReadMimeType: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
|
||||
f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
|
||||
|
||||
if f.opt.WorkspaceID == "" {
|
||||
err = f.findWorkspace(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, rootID, f)
|
||||
|
||||
// Find the current root
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// Assume it is a file
|
||||
newRoot, remote := dircache.SplitPath(root)
|
||||
tempF := *f
|
||||
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
|
||||
tempF.root = newRoot
|
||||
// Make new Fs which is the parent
|
||||
err = tempF.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// No root so return old f
|
||||
return f, nil
|
||||
}
|
||||
_, err := tempF.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// File doesn't exist so return old f
|
||||
return f, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
f.features.Fill(ctx, &tempF)
|
||||
f.dirCache = tempF.dirCache
|
||||
f.root = tempF.root
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, folder := range listing.Folders {
|
||||
folderName := f.opt.Enc.ToStandardName(folder.Name)
|
||||
fullPath := getRemote(dir, folderName)
|
||||
modTime := time.Unix(int64(folder.CreatedAt), 0)
|
||||
d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
|
||||
entries = append(entries, d)
|
||||
f.dirCache.Put(fullPath, folder.ID)
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
fileName := f.opt.Enc.ToStandardName(file.Name)
|
||||
remote := getRemote(dir, fileName)
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}
|
||||
entries = append(entries, o)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
if f.opt.Enc.ToStandardName(file.Name) == leaf {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
// Put uploads a new file
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
existingObj, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return existingObj, existingObj.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// put uploads src as a new file
|
||||
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
remote := src.Remote()
|
||||
|
||||
_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: fileFromUpload(resp, modTime),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fileFromUpload builds the FileItem for the object an upload just created.
|
||||
// The single-shot upload response omits updated_at, so the modification time
|
||||
// is taken from what was sent (modTime) rather than defaulting to the create
|
||||
// time - it is the time the server stored via the source-mtime header. The
|
||||
// SHA-256 comes straight from the response (empty for a multipart upload).
|
||||
func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
|
||||
updatedAt := resp.File.CreatedAt
|
||||
if !modTime.IsZero() {
|
||||
updatedAt = float64(modTime.Unix())
|
||||
}
|
||||
return api.FileItem{
|
||||
ID: resp.File.ID,
|
||||
Name: resp.File.Name,
|
||||
SizeBytes: resp.File.SizeBytes,
|
||||
MimeType: resp.File.MimeType,
|
||||
Extension: resp.File.Extension,
|
||||
Region: resp.File.Region,
|
||||
CreatedAt: resp.File.CreatedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
ContentHash: resp.File.ContentHash,
|
||||
}
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
return err
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The server checks emptiness itself (?if_empty=1) and answers
|
||||
// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
|
||||
err = f.removeFolder(ctx, directoryID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move src to this remote using server-side move operations
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't move - not same remote type")
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
|
||||
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find destination directory
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
needsMove := srcDirID != dstDirID
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
|
||||
// Moving and renaming in the same request (rather than a moveFile call
|
||||
// followed by a separate renameFile call) matters when both are needed:
|
||||
// a two-step move-then-rename can transiently place the file's CURRENT
|
||||
// name into a destination folder that already holds an unrelated file
|
||||
// with that name, even though the FINAL name is free - the server only
|
||||
// allows one active name per folder, so that intermediate state 409s.
|
||||
if needsMove || needsRename {
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Return new object. file.Name must reflect the destination leaf, not
|
||||
// the source's - a caller that reuses this object as the src of a
|
||||
// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
|
||||
// recomputes srcLeaf from file.Name; a stale name there makes that
|
||||
// second move see srcLeaf == dstLeaf and skip the rename entirely,
|
||||
// silently leaving the object under the wrong name.
|
||||
newFile := srcObj.file
|
||||
newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: newFile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
|
||||
srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srcLeaf := srcRemote
|
||||
if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
|
||||
srcLeaf = srcRemote[idx+1:]
|
||||
}
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
|
||||
// Move folder to new parent only if parent changed. The new name goes
|
||||
// in the same request, as for files in Move: a move followed by a
|
||||
// separate rename can collide with an unrelated folder sharing the
|
||||
// CURRENT name in the destination even though the final name is free.
|
||||
if srcDirectoryID != dstDirectoryID {
|
||||
renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move directory: %w", err)
|
||||
}
|
||||
if renamed {
|
||||
needsRename = false
|
||||
}
|
||||
}
|
||||
|
||||
if needsRename {
|
||||
err = f.renameFolder(ctx, srcID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side copy operations
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The copy API creates a new row named after the source file when
|
||||
// copying across folders, which collides with one already at the
|
||||
// destination. Copy under a temporary name, then delete the old
|
||||
// destination, then move the copy onto the name - so a copy that fails
|
||||
// leaves the destination as it was. Deleting it up front lost it
|
||||
// (prior art: backend/box).
|
||||
existingObj, err := f.NewObject(ctx, remote)
|
||||
switch err {
|
||||
case nil:
|
||||
tempRemote := remote + "-rclone-copy-" + random.String(8)
|
||||
fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
|
||||
tempObj, err := f.Copy(ctx, src, tempRemote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs.Debugf(tempRemote, "moving to real name %v", remote)
|
||||
if err := existingObj.Remove(ctx); err != nil {
|
||||
return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
|
||||
}
|
||||
return f.Move(ctx, tempObj, remote)
|
||||
case fs.ErrorObjectNotFound:
|
||||
// nothing at the destination, so copy straight onto the name
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dstName := f.opt.Enc.FromStandardName(dstLeaf)
|
||||
|
||||
resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
|
||||
// Servers that predate the name field ignore it and name the copy after
|
||||
// the source, or "Copy of <original>" in the same folder. Rename to the
|
||||
// requested destination name if it differs.
|
||||
if resp.Name != dstName {
|
||||
err = f.renameFile(ctx, resp.FileID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't rename copied file: %w", err)
|
||||
}
|
||||
resp.Name = dstName
|
||||
}
|
||||
|
||||
// A server-side copy is byte-identical to the source and the API
|
||||
// preserves the source's modification time and content hash onto the
|
||||
// copy, so the returned object must report the SOURCE's modtime and hash,
|
||||
// not the copy's wall-clock time. Reading "now" here made the object
|
||||
// rclone holds in memory disagree with the one read back from the remote
|
||||
// (its Fingerprint check compares the two exactly).
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: api.FileItem{
|
||||
ID: resp.FileID,
|
||||
Name: resp.Name,
|
||||
SizeBytes: srcObj.file.SizeBytes,
|
||||
MimeType: srcObj.file.MimeType,
|
||||
Extension: srcObj.file.Extension,
|
||||
Region: srcObj.file.Region,
|
||||
CreatedAt: srcObj.file.CreatedAt,
|
||||
UpdatedAt: srcObj.file.UpdatedAt,
|
||||
ContentHash: srcObj.file.ContentHash,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Purge deletes all the files and the container
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
|
||||
err = f.removeFolder(ctx, directoryID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
obj := o.(*Object)
|
||||
|
||||
resp, err := f.createShareLink(ctx, obj.file.ID, expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return resp.Link.URL, nil
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing as an
|
||||
// optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
info, err := f.getWorkspaceInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage := &fs.Usage{
|
||||
Used: fs.NewUsageValue(info.Storage.Used),
|
||||
Total: fs.NewUsageValue(info.Storage.Total),
|
||||
Free: fs.NewUsageValue(info.Storage.Free),
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Object methods
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// String returns a description of the Object
|
||||
func (o *Object) String() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// ModTime returns the modification date of the file
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
if o.file.UpdatedAt > 0 {
|
||||
return time.Unix(int64(o.file.UpdatedAt), 0)
|
||||
}
|
||||
return time.Unix(int64(o.file.CreatedAt), 0)
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return o.file.SizeBytes
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.SHA256 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// May be "" - a multipart-uploaded object carries no server-side hash,
|
||||
// which rclone treats as "hash unavailable for this object" (as it does
|
||||
// for S3 multipart objects).
|
||||
return o.file.ContentHash, nil
|
||||
}
|
||||
|
||||
// Storable says whether this object can be stored
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
return fs.ErrorCantSetModTime
|
||||
}
|
||||
|
||||
// Open opens the file for read
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
fs.FixRangeOption(options, o.file.SizeBytes)
|
||||
return o.fs.downloadFile(ctx, o.file.ID, options)
|
||||
}
|
||||
|
||||
// Update replaces the file content
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
// Find the directory ID for this file
|
||||
_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload as new version of existing file
|
||||
fileID := o.file.ID
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update object metadata
|
||||
o.file = fileFromUpload(resp, modTime)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove removes this object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.deleteFile(ctx, o.file.ID)
|
||||
}
|
||||
|
||||
// MimeType of an Object if known, "" otherwise
|
||||
func (o *Object) MimeType(ctx context.Context) string {
|
||||
return o.file.MimeType
|
||||
}
|
||||
|
||||
// ID returns the ID of the Object if known, or "" if not
|
||||
func (o *Object) ID() string {
|
||||
return o.file.ID
|
||||
}
|
||||
|
||||
func getRemote(dir, fileName string) string {
|
||||
if dir == "" {
|
||||
return fileName
|
||||
}
|
||||
return dir + "/" + fileName
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ dircache.DirCacher = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
)
|
||||
@@ -1,880 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// sourceMTimeHeader is the request header the upload endpoints read to set a
|
||||
// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
|
||||
const sourceMTimeHeader = "x-dosya-source-mtime"
|
||||
|
||||
// modTimeHeaders returns the ExtraHeaders that carry a source modification time
|
||||
// on an upload, or nil when there is nothing plausible to send. The server
|
||||
// clamps implausible values itself, but a zero time is never worth a header.
|
||||
func modTimeHeaders(modTime time.Time) map[string]string {
|
||||
if modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
|
||||
}
|
||||
|
||||
// workspaceHintHeader names the workspace a by-id request is about. The API
|
||||
// maps an id to its workspace through an index the workspace fills about a
|
||||
// second after a write, so an id this client was handed moments ago can still
|
||||
// miss; the hint lets the server look for it in the named workspace instead.
|
||||
// It only locates: every route still checks membership against the workspace
|
||||
// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
|
||||
const workspaceHintHeader = "X-Dosya-Workspace"
|
||||
|
||||
// byIDHeaders returns the ExtraHeaders every /api/files/:id and
|
||||
// /api/folders/:id call carries, or nil when no workspace is configured.
|
||||
func (f *Fs) byIDHeaders() map[string]string {
|
||||
if f.opt.WorkspaceID == "" {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
|
||||
}
|
||||
|
||||
// retryErrorCodes is a slice of error codes that we will retry
|
||||
var retryErrorCodes = []int{
|
||||
429, // Too Many Requests
|
||||
500, // Internal Server Error
|
||||
502, // Bad Gateway
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
|
||||
if retry {
|
||||
if d, ok := retryAfter(resp); ok {
|
||||
// The API answers 503 + Retry-After while a workspace is being
|
||||
// moved between storage homes, and Cloudflare sends it with 429.
|
||||
// The pacer's own backoff tops out at maxSleep, far short of that.
|
||||
fs.Debugf(nil, "Server asked to retry after %v", d)
|
||||
err = pacer.RetryAfterError(err, d)
|
||||
}
|
||||
}
|
||||
return retry, err
|
||||
}
|
||||
|
||||
// maxRetryAfter caps how long a single Retry-After is honoured for
|
||||
const maxRetryAfter = 5 * time.Minute
|
||||
|
||||
// retryAfter reads a 429 or 503 response's Retry-After header, which the API
|
||||
// sends as whole seconds
|
||||
func retryAfter(resp *http.Response) (time.Duration, bool) {
|
||||
if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
|
||||
return 0, false
|
||||
}
|
||||
seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
|
||||
if err != nil || seconds <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
|
||||
}
|
||||
|
||||
// errorHandler decodes a non-2xx response into an *api.Error so callers can
|
||||
// act on the machine-readable error_code rather than the message text.
|
||||
func errorHandler(resp *http.Response) error {
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading error out of body: %w", err)
|
||||
}
|
||||
apiErr := &api.Error{
|
||||
StatusCode: resp.StatusCode,
|
||||
Status: resp.Status,
|
||||
}
|
||||
if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
|
||||
// Not the API's JSON shape (e.g. an edge response that never
|
||||
// reached the API) - keep the body text so it isn't lost.
|
||||
apiErr.ErrorCode = ""
|
||||
apiErr.Message = strings.TrimSpace(string(body))
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
||||
// The workspace caps the number of in-flight uploads per user. The API
|
||||
// refuses an upload/init past that cap with a 400 carrying
|
||||
// error_code "concurrent_upload_limit", and documents it as an error to
|
||||
// wait on rather than give up on: slots free as uploads finish, and the
|
||||
// server marks sessions idle for 10 minutes as failed, so waiting up to
|
||||
// that long guarantees progress unless the workspace really is saturated.
|
||||
// A 400 is deliberately outside the pacer's retry codes, so the wait
|
||||
// lives in initUpload instead of shouldRetry.
|
||||
var (
|
||||
concurrentUploadRetryDelay = 5 * time.Second
|
||||
concurrentUploadWaitMax = 10 * time.Minute
|
||||
)
|
||||
|
||||
// abortUploadTimeout bounds the abort of a session the client is giving up on,
|
||||
// which runs outside the transfer's context so a cancelled transfer still
|
||||
// sends it
|
||||
const abortUploadTimeout = 10 * time.Second
|
||||
|
||||
// isConcurrentUploadLimit reports whether err is the API refusing an
|
||||
// upload because the workspace's concurrent upload limit is reached
|
||||
func isConcurrentUploadLimit(err error) bool {
|
||||
var apiErr *api.Error
|
||||
return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
|
||||
}
|
||||
|
||||
// listPageSize is the largest page GET /api/files will serve
|
||||
const listPageSize = 500
|
||||
|
||||
// listFilesAndFolders lists files and folders in a directory
|
||||
//
|
||||
// The API pages files (at most listPageSize per page) but returns every
|
||||
// folder on each page, so the files from all pages are gathered and the
|
||||
// folders are taken from the first page only. Pages are ordered by name,
|
||||
// which the API tie-breaks on id, so offsets are stable between requests.
|
||||
func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
|
||||
var listing *api.ListResponse
|
||||
for page := 1; ; page++ {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files",
|
||||
Parameters: map[string][]string{
|
||||
"workspace_id": {f.opt.WorkspaceID},
|
||||
"per_page": {strconv.Itoa(listPageSize)},
|
||||
"page": {strconv.Itoa(page)},
|
||||
"sort": {"name"},
|
||||
"dir": {"asc"},
|
||||
},
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
opts.Parameters["folder_id"] = []string{folderID}
|
||||
}
|
||||
|
||||
var result api.ListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list files: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if listing == nil {
|
||||
listing = &result
|
||||
} else {
|
||||
listing.Files = append(listing.Files, result.Files...)
|
||||
}
|
||||
if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
|
||||
return listing, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// createFolder creates a folder
|
||||
func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
|
||||
request := api.CreateFolderRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
Name: name,
|
||||
}
|
||||
if parentID != "" && parentID != rootID {
|
||||
request.ParentID = &parentID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/folders",
|
||||
}
|
||||
|
||||
var result api.CreateFolderResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// removeFolder deletes a folder
|
||||
//
|
||||
// ?permanent=1 trashes and purges a live folder in one request on today's
|
||||
// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
|
||||
// HTTP 409 "Folder is not empty" instead of purging a folder that still
|
||||
// holds a live file or a live child folder; that 409 is turned into
|
||||
// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
|
||||
// error, and it is never retried (409 is not one of retryErrorCodes). A big
|
||||
// subtree is purged in bounded passes (202, complete=false), so keep
|
||||
// calling until the purge is complete. An older server that ignores
|
||||
// permanent=1 only trashes on the first call (answering permanent=false),
|
||||
// so the loop's second pass sends the plain DELETE that purges it.
|
||||
func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
|
||||
parameters := map[string][]string{"permanent": {"1"}}
|
||||
if ifEmpty {
|
||||
parameters["if_empty"] = []string{"1"}
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/folders/" + folderID,
|
||||
Parameters: parameters,
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
for pass := 0; ; pass++ {
|
||||
var result api.DeleteFolderResponse
|
||||
var statusCode int
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
if resp != nil {
|
||||
statusCode = resp.StatusCode
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if ifEmpty && statusCode == http.StatusConflict {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if pass == 0 {
|
||||
return fmt.Errorf("couldn't remove folder: %w", err)
|
||||
}
|
||||
return fmt.Errorf("couldn't permanently remove folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if result.Permanent && result.Complete {
|
||||
return nil
|
||||
}
|
||||
if pass >= maxFolderPurgePasses {
|
||||
return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
|
||||
const maxFolderPurgePasses = 1000
|
||||
|
||||
// renameFolder renames a folder
|
||||
func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
|
||||
request := api.RenameFolderRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFolder moves a folder to a new parent
|
||||
func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
|
||||
request := api.MoveFolderRequest{Name: newName}
|
||||
if newParentID != "" && newParentID != rootID {
|
||||
request.ParentID = &newParentID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.MoveFolderResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("couldn't move folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return false, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
// Servers that predate the name field ignore it and answer without a
|
||||
// name, leaving the caller to rename separately
|
||||
return newName != "" && result.Name == newName, nil
|
||||
}
|
||||
|
||||
// deleteFile deletes a file
|
||||
//
|
||||
// ?permanent=1 trashes and purges it in one request on today's API. An
|
||||
// older server that ignores the flag only soft-deletes it, answering
|
||||
// permanent=false, so a second DELETE on the now soft-deleted file purges
|
||||
// it from the database and R2, as it always has.
|
||||
func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/files/" + fileID,
|
||||
Parameters: map[string][]string{"permanent": {"1"}},
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.DeleteFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't delete file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
|
||||
// Answered into its own value so a field the second response omits
|
||||
// can't be read off the first.
|
||||
if !result.Permanent {
|
||||
var permanent api.DeleteFileResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't permanently delete file: %w", err)
|
||||
}
|
||||
if !permanent.OK {
|
||||
return fmt.Errorf("API error: %s", permanent.Error)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renameFile renames a file
|
||||
func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
|
||||
request := api.RenameFileRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.RenameFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFile moves a file to a new folder and, if newName is non-empty,
|
||||
// renames it in the same request
|
||||
func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
|
||||
request := api.MoveFileRequest{}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
if newName != "" {
|
||||
request.Name = &newName
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies a file to a folder
|
||||
func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
|
||||
request := api.CopyFileRequest{Name: name}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/copy",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.CopyFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// downloadFile downloads a file by its ID
|
||||
//
|
||||
// The download endpoint answers 302 with a presigned R2 URL in Location, so
|
||||
// this is two calls. Both go through the rest client, which is built on
|
||||
// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
|
||||
// --user-agent, --bind, --dump and the proxy settings apply to downloads as
|
||||
// they do to every other call, and errorHandler and shouldRetry decide the
|
||||
// errors and the retries uniformly.
|
||||
func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
|
||||
// NoRedirect stops the client following the 302 with the API's
|
||||
// Authorization header attached. A 302 is not a 2xx, so Call reports it
|
||||
// as an error (having read and closed the body); the Location header
|
||||
// is what actually matters, and finding it clears the error, as in
|
||||
// backend/linkbox.
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files/" + fileID + "/download",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
NoRedirect: true,
|
||||
}
|
||||
var downloadURL string
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
if resp != nil {
|
||||
if location := resp.Header.Get("Location"); location != "" {
|
||||
downloadURL, err = location, nil
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get download URL: %w", err)
|
||||
}
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("no download URL in redirect response")
|
||||
}
|
||||
|
||||
// Fetch the object from the presigned URL. It carries its own
|
||||
// credentials and R2 refuses a request that also has an Authorization
|
||||
// header, so the client-level bearer token is removed by setting the
|
||||
// header empty: the rest client skips a header with an empty value.
|
||||
opts = rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: downloadURL,
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{"Authorization": ""},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't download file: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// uploadSmallFile uploads a file smaller than 10MB in a single request
|
||||
func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
Body: in,
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
// The body is a stream which can't be rewound, so a low level retry
|
||||
// would resend whatever is left of it - nothing - and the server
|
||||
// would store that as an empty file. Don't retry here; rclone's high
|
||||
// level retries redo the whole transfer instead.
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadPart uploads a single part of a multipart upload
|
||||
func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
|
||||
size := int64(len(part))
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
var result api.UploadPartResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
// A fresh reader per attempt so a retry resends the whole part
|
||||
opts.Body = bytes.NewReader(part)
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// completeUpload finalizes a multipart upload
|
||||
func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/" + sessionID + "/complete",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't complete upload: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// initUpload initializes a new upload session
|
||||
func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
|
||||
request := api.UploadInitRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
FileName: fileName,
|
||||
FileSize: fileSize,
|
||||
MimeType: mimeType,
|
||||
FileID: fileID,
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/init",
|
||||
}
|
||||
|
||||
var result api.UploadInitResponse
|
||||
deadline := time.Now().Add(concurrentUploadWaitMax)
|
||||
for {
|
||||
result = api.UploadInitResponse{}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", err)
|
||||
}
|
||||
fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
|
||||
case <-time.After(concurrentUploadRetryDelay):
|
||||
}
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadParts reads the source part by part and uploads the parts.
|
||||
//
|
||||
// Part 1 goes on its own: the server creates the R2 multipart upload (and a
|
||||
// new file's id and key) when it receives the first part, so parts sent
|
||||
// before that has finished would each start an upload of their own. The
|
||||
// rest go out opt.UploadConcurrency at a time. A part is only read from the
|
||||
// source once a buffer is free, so at most that many parts are held in
|
||||
// memory, and a short source is still found by the reader before the part
|
||||
// it could not fill is sent.
|
||||
func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
|
||||
readPart := func(buf []byte, partNum int) ([]byte, error) {
|
||||
currentPartSize := partSize
|
||||
if partNum == totalParts {
|
||||
currentPartSize = size - partSize*int64(totalParts-1)
|
||||
}
|
||||
part := buf[:currentPartSize]
|
||||
if _, err := io.ReadFull(in, part); err != nil {
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
|
||||
}
|
||||
return part, nil
|
||||
}
|
||||
|
||||
concurrency := max(f.opt.UploadConcurrency, 1)
|
||||
buffers := make(chan []byte, concurrency)
|
||||
buffers <- make([]byte, partSize)
|
||||
allocated := 1
|
||||
|
||||
first := <-buffers
|
||||
part, err := readPart(first, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
buffers <- first
|
||||
|
||||
// A failed read has to take the parts already in flight down with it,
|
||||
// or they upload to completion while the error waits on Wait.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var readErr error
|
||||
for partNum := 2; partNum <= totalParts; partNum++ {
|
||||
// Take a free buffer, allocating one while under the limit.
|
||||
var buf []byte
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
default:
|
||||
if allocated < concurrency {
|
||||
buf = make([]byte, partSize)
|
||||
allocated++
|
||||
} else {
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
case <-gCtx.Done():
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf == nil {
|
||||
break // a part failed; its error comes back from Wait
|
||||
}
|
||||
part, err := readPart(buf, partNum)
|
||||
if err != nil {
|
||||
readErr = err
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
defer func() { buffers <- buf }()
|
||||
_, err := f.uploadPart(gCtx, part, sessionID, partNum)
|
||||
return err
|
||||
})
|
||||
}
|
||||
err = g.Wait()
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// abortUpload retires an upload session the client is giving up on.
|
||||
//
|
||||
// Best effort: the server reclaims an abandoned session itself, so a failure
|
||||
// here - including an older server with no such route - costs the slot the
|
||||
// wait it would have cost anyway and is not worth failing an upload over.
|
||||
// Sent on a context of its own so a cancelled transfer still gives its slot
|
||||
// back rather than leaving it to the sweep.
|
||||
func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
|
||||
defer cancel()
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
NoResponse: true,
|
||||
}
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadFile handles the full upload flow (small or multipart)
|
||||
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
|
||||
// Both refusals live here so a new file and a new version of an existing
|
||||
// one answer the same way: upload/init needs a byte count up front, and
|
||||
// the server stores no zero-length object.
|
||||
if size < 0 {
|
||||
return nil, errors.New("can't upload files with unknown size")
|
||||
}
|
||||
if size == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
leaf := f.opt.Enc.FromStandardName(path.Base(remote))
|
||||
|
||||
initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A session abandoned here holds one of the workspace's upload slots
|
||||
// until the server's sweep retires it ten minutes later, and rclone's
|
||||
// high-level retry opens a fresh session rather than resuming this one -
|
||||
// so without this, a few failed attempts use up every slot.
|
||||
defer func() {
|
||||
if err != nil {
|
||||
f.abortUpload(ctx, initResp.SessionID)
|
||||
}
|
||||
}()
|
||||
|
||||
// Count what the source actually supplies, so a source which ends
|
||||
// early is reported as an error rather than stored as a truncated
|
||||
// file.
|
||||
counter := readers.NewCountingReader(in)
|
||||
|
||||
// Small file upload (single PUT, no resumable)
|
||||
if initResp.Resumable == nil {
|
||||
resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := int64(counter.BytesRead()); got != size {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Multipart upload. Each part is read fully into memory before it is
|
||||
// sent, so a short source is caught before the part goes out and a
|
||||
// part which has to be retried is resent with all of its bytes.
|
||||
partSize := initResp.Resumable.PartSize
|
||||
totalParts := initResp.Resumable.TotalParts
|
||||
// Both bounds matter. partSize*(totalParts-1) >= size means a part would
|
||||
// have nothing to carry; size > partSize*totalParts means the last part
|
||||
// is bigger than partSize, and readPart would slice past its buffer and
|
||||
// panic.
|
||||
if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
|
||||
return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
|
||||
}
|
||||
if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.completeUpload(ctx, initResp.SessionID, modTime)
|
||||
}
|
||||
|
||||
// createShareLink creates a public share link for a file
|
||||
func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
|
||||
request := api.ShareLinkRequest{}
|
||||
if expire < fs.DurationOff {
|
||||
days := int(time.Duration(expire).Hours() / 24)
|
||||
if days < 1 {
|
||||
days = 1
|
||||
}
|
||||
request.ExpiresInDays = &days
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/share",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.ShareLinkResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create share link: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// getWorkspaceInfo fetches workspace storage info
|
||||
// listWorkspaces lists the workspaces this credential can reach. An API key
|
||||
// pinned to a workspace sees exactly one: its own.
|
||||
func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces",
|
||||
}
|
||||
|
||||
var result api.WorkspaceListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list workspaces: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return result.Workspaces, nil
|
||||
}
|
||||
|
||||
func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces/" + f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
var result api.WorkspaceInfoResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get workspace info: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
header string
|
||||
want time.Duration
|
||||
ok bool
|
||||
}{
|
||||
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
|
||||
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
|
||||
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
|
||||
{"missing header", http.StatusServiceUnavailable, "", 0, false},
|
||||
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
|
||||
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
|
||||
if tc.header != "" {
|
||||
resp.Header.Set("Retry-After", tc.header)
|
||||
}
|
||||
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
|
||||
assert.True(t, retry)
|
||||
d, ok := pacer.IsRetryAfter(err)
|
||||
assert.Equal(t, tc.ok, ok)
|
||||
if tc.ok {
|
||||
assert.Equal(t, tc.want, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
|
||||
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
|
||||
resp.Header.Set("Retry-After", "30")
|
||||
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
|
||||
assert.False(t, retry)
|
||||
_, ok := pacer.IsRetryAfter(err)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestRemoveFolderPurgesPermanently(t *testing.T) {
|
||||
// a big subtree purges in bounded passes: 202 complete=false repeats
|
||||
// the same request until complete=true
|
||||
answers := []struct {
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
|
||||
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
|
||||
}
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "DELETE", r.Method)
|
||||
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
|
||||
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
|
||||
a := answers[calls]
|
||||
calls++
|
||||
return jsonResp(a.status, a.body), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
|
||||
assert.Equal(t, len(answers), calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "locked file")
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", true)
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRmdirMakesOneRequest(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
require.NoError(t, f.Rmdir(context.Background(), "sub"))
|
||||
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
|
||||
}
|
||||
|
||||
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
err := f.Rmdir(context.Background(), "sub")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
}
|
||||
|
||||
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
permanent bool
|
||||
wantCalls int
|
||||
}{
|
||||
{"today's API purges in one call", true, 1},
|
||||
{"an older server ignores the flag and still gets a second DELETE", false, 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
|
||||
assert.Equal(t, tc.wantCalls, calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// decodeBody reads a JSON request body into a map
|
||||
func decodeBody(t *testing.T, r *http.Request) map[string]any {
|
||||
b, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
var m map[string]any
|
||||
require.NoError(t, json.Unmarshal(b, &m))
|
||||
return m
|
||||
}
|
||||
|
||||
func TestMoveFolderSendsName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
newName string
|
||||
respBody string
|
||||
wantRenamed bool
|
||||
}{
|
||||
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
|
||||
{"older server ignores name", "dst", `{"ok":true}`, false},
|
||||
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_parent", body["parent_id"])
|
||||
if tc.newName == "" {
|
||||
assert.NotContains(t, body, "name")
|
||||
} else {
|
||||
assert.Equal(t, tc.newName, body["name"])
|
||||
}
|
||||
return jsonResp(http.StatusOK, tc.respBody), nil
|
||||
}))
|
||||
|
||||
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantRenamed, renamed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyFileSendsName(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_dst", body["folder_id"])
|
||||
assert.Equal(t, "final.txt", body["name"])
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
|
||||
}))
|
||||
|
||||
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_2", resp.FileID)
|
||||
assert.Equal(t, "final.txt", resp.Name)
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
|
||||
f := &Fs{}
|
||||
assert.Equal(t, time.Second, f.Precision())
|
||||
assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
|
||||
// Exactly SHA-256, nothing else.
|
||||
assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
|
||||
}
|
||||
|
||||
func TestObjectHashReturnsContentHash(t *testing.T) {
|
||||
o := &Object{file: api.FileItem{ContentHash: "abc123"}}
|
||||
|
||||
got, err := o.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "abc123", got)
|
||||
|
||||
// A multipart object with no server-side hash yields "" and no error,
|
||||
// exactly as an S3 multipart object does.
|
||||
empty := &Object{file: api.FileItem{ContentHash: ""}}
|
||||
got, err = empty.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", got)
|
||||
|
||||
// Any other algorithm is unsupported.
|
||||
_, err = o.Hash(context.Background(), hash.MD5)
|
||||
assert.ErrorIs(t, err, hash.ErrUnsupported)
|
||||
}
|
||||
|
||||
func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
|
||||
assert.True(t, mt.Equal(o.ModTime(context.Background())))
|
||||
}
|
||||
|
||||
func TestModTimeHeaders(t *testing.T) {
|
||||
assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
|
||||
h := modTimeHeaders(mt)
|
||||
require.NotNil(t, h)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
|
||||
}
|
||||
|
||||
func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
|
||||
resp := &api.UploadCompleteResponse{}
|
||||
resp.File.ID = "fil_1"
|
||||
resp.File.Name = "x.txt"
|
||||
resp.File.SizeBytes = 10
|
||||
resp.File.CreatedAt = 1_000
|
||||
resp.File.ContentHash = "deadbeef"
|
||||
|
||||
mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
|
||||
item := fileFromUpload(resp, mt)
|
||||
assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
|
||||
assert.Equal(t, "deadbeef", item.ContentHash)
|
||||
assert.Equal(t, float64(1_000), item.CreatedAt)
|
||||
|
||||
// A zero modtime falls back to the create time.
|
||||
item = fileFromUpload(resp, time.Time{})
|
||||
assert.Equal(t, float64(1_000), item.UpdatedAt)
|
||||
}
|
||||
|
||||
// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
|
||||
// fake transport and asserts the source-mtime header actually rides on the PUT.
|
||||
func TestUploadSendsSourceMTimeHeader(t *testing.T) {
|
||||
var putMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
|
||||
}
|
||||
switch {
|
||||
case r.Method == "POST" && r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
|
||||
putMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "h", resp.File.ContentHash)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
|
||||
// the completion request for a multipart upload (the server stores the modtime
|
||||
// there, not on the individual parts).
|
||||
func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
|
||||
var completeMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body)
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
|
||||
case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
case r.URL.Path == "/api/upload/upl_1/complete":
|
||||
completeMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
|
||||
// server-side Copy returns: it must report the SOURCE's modtime and hash (the
|
||||
// copy is byte-identical and the API preserves both), not the copy's wall
|
||||
// clock. Reading "now" here made rclone's in-memory object disagree with the
|
||||
// one read back from the remote and failed FsCopy's fingerprint check.
|
||||
func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case r.Method == "GET" && r.URL.Path == "/api/files":
|
||||
// NewObject's existence probe on the destination: nothing there.
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
|
||||
case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
|
||||
mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
|
||||
ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
|
||||
ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
|
||||
CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
|
||||
}}
|
||||
|
||||
dst, err := f.Copy(context.Background(), src, "f-copy.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
|
||||
assert.Equal(t, int64(100), dst.Size())
|
||||
got, err := dst.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "srchash", got, "copy keeps the source hash")
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
|
||||
|
||||
func TestErrorHandlerDecodesAPIError(t *testing.T) {
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Status: "400 Bad Request",
|
||||
Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
|
||||
assert.Contains(t, err.Error(), "uploads in progress")
|
||||
assert.Contains(t, err.Error(), "concurrent_upload_limit")
|
||||
}
|
||||
|
||||
func TestErrorHandlerNonJSONBody(t *testing.T) {
|
||||
// e.g. an edge (Cloudflare) response that never reached the API
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Status: "429 Too Many Requests",
|
||||
Body: io.NopCloser(strings.NewReader("error code: 1015\n")),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
|
||||
assert.Equal(t, "", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "429")
|
||||
assert.Contains(t, err.Error(), "error code: 1015")
|
||||
}
|
||||
|
||||
func TestShouldRetryAPIErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// The concurrent upload limit is handled by initUpload's slot wait,
|
||||
// not by the pacer's low level retries.
|
||||
limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
|
||||
retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
|
||||
assert.False(t, retry)
|
||||
assert.Equal(t, limitErr, err)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
|
||||
assert.True(t, retry)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
|
||||
assert.False(t, retry)
|
||||
}
|
||||
|
||||
// swapWaitTimings shortens the slot wait timings for a test and returns a
|
||||
// function which restores them
|
||||
func swapWaitTimings(delay, max time.Duration) func() {
|
||||
oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
|
||||
concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
|
||||
return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
|
||||
}
|
||||
|
||||
// newTestFs makes an Fs which talks to srv
|
||||
func newTestFs(srv *httptest.Server) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// limitServer answers the first n init calls with the concurrent upload
|
||||
// limit error and the rest with success
|
||||
func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/upload/init", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if calls.Add(1) <= n {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(concurrentLimitBody))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
}
|
||||
|
||||
func TestInitUploadWaitsForFreeSlot(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 2, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, resp.OK)
|
||||
assert.Equal(t, int32(3), calls.Load())
|
||||
}
|
||||
|
||||
func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "couldn't init upload")
|
||||
assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
|
||||
}
|
||||
|
||||
func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
|
||||
defer swapWaitTimings(time.Minute, time.Hour)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeListServer serves GET /api/files the way the API does: files paged at
|
||||
// per_page, every folder repeated on each page
|
||||
func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/files", r.URL.Path)
|
||||
q := r.URL.Query()
|
||||
assert.Equal(t, "name", q.Get("sort"))
|
||||
assert.Equal(t, "asc", q.Get("dir"))
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
perPage, _ := strconv.Atoi(q.Get("per_page"))
|
||||
require.Positive(t, page)
|
||||
require.Positive(t, perPage)
|
||||
*pages = append(*pages, page)
|
||||
|
||||
var files []string
|
||||
for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
|
||||
files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
|
||||
}
|
||||
totalPages := max(1, (totalFiles+perPage-1)/perPage)
|
||||
body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
|
||||
`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
|
||||
strings.Join(files, ","), page, perPage, totalFiles, totalPages)
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersFollowsPages(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1, 2, 3}, pages)
|
||||
require.Len(t, listing.Files, 2*listPageSize+7)
|
||||
assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
|
||||
// folders repeat on every page and must not be duplicated
|
||||
assert.Len(t, listing.Folders, 1)
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersSinglePage(t *testing.T) {
|
||||
for _, total := range []int{0, 1, listPageSize} {
|
||||
t.Run(strconv.Itoa(total), func(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, total, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), rootID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1}, pages)
|
||||
assert.Len(t, listing.Files, total)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// parallelUploadServer answers the multipart endpoints without holding a
|
||||
// lock across a request, so parts really do overlap, and records what the
|
||||
// ordering rules below are about: how many parts were in flight at once, and
|
||||
// whether any part started before part 1 had finished.
|
||||
//
|
||||
// Part 1 matters because the API creates the R2 multipart upload (and, for a
|
||||
// new file, its id and key) on the first part it sees. Parts that race ahead
|
||||
// of part 1 would each create their own upload, and only one of those would
|
||||
// be completed.
|
||||
type parallelUploadServer struct {
|
||||
t *testing.T
|
||||
resumable string
|
||||
partDelay time.Duration
|
||||
failPart int // answer this part with a 400, 0 for none
|
||||
mu sync.Mutex
|
||||
inFlight int
|
||||
maxFlight int
|
||||
part1Done bool
|
||||
earlyParts []int // parts that started before part 1 finished
|
||||
parts map[int]string
|
||||
completes int
|
||||
aborts int
|
||||
}
|
||||
|
||||
func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
|
||||
s.mu.Lock()
|
||||
if n != 1 && !s.part1Done {
|
||||
s.earlyParts = append(s.earlyParts, n)
|
||||
}
|
||||
s.inFlight++
|
||||
s.maxFlight = max(s.maxFlight, s.inFlight)
|
||||
s.mu.Unlock()
|
||||
|
||||
time.Sleep(s.partDelay)
|
||||
|
||||
s.mu.Lock()
|
||||
s.inFlight--
|
||||
if n == 1 {
|
||||
s.part1Done = true
|
||||
}
|
||||
s.parts[n] = string(body)
|
||||
fail := n == s.failPart
|
||||
s.mu.Unlock()
|
||||
|
||||
if fail {
|
||||
return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.mu.Lock()
|
||||
s.completes++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.mu.Lock()
|
||||
s.aborts++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
|
||||
return ¶llelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
|
||||
}
|
||||
|
||||
const tenParts = `{"part_size":2,"total_parts":10}`
|
||||
|
||||
func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
content := "aabbccddeeffgghhiijj"
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
|
||||
assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
for i := 1; i <= 10; i++ {
|
||||
assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 1
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, srv.maxFlight)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
srv.failPart = 6
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "part 6")
|
||||
assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
|
||||
// 20 bytes declared, 15 supplied: the short read is found by the
|
||||
// reader, whichever parts are already in flight, and nothing completes.
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Equal(t, 0, srv.completes)
|
||||
assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Test Dosya filesystem interface
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestDosya:",
|
||||
})
|
||||
}
|
||||
@@ -1,247 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// rtFunc adapts a function into an http.RoundTripper so the upload path
|
||||
// can be driven without a network. Unlike a real server it always reads
|
||||
// the whole request body and answers, which is exactly the misbehaviour
|
||||
// (accepting a short body as a complete upload) these tests guard against.
|
||||
type rtFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func jsonResp(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{"Content-Type": {"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
ContentLength: int64(len(body)),
|
||||
}
|
||||
}
|
||||
|
||||
func newFakeFs(rt http.RoundTripper) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// fakeUploadServer records what an upload sends and lets a test inject
|
||||
// failures per request
|
||||
type fakeUploadServer struct {
|
||||
t *testing.T
|
||||
mu sync.Mutex
|
||||
resumable string // JSON for the init "resumable" field, "" for a single PUT
|
||||
puts [][]byte // bodies of PUT /api/upload/<id>
|
||||
parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
|
||||
completes int
|
||||
aborts int // DELETE /api/upload/<id>
|
||||
failFirst map[string]bool // path -> answer the first request with a 502
|
||||
seenBefore map[string]bool
|
||||
}
|
||||
|
||||
func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
|
||||
return &fakeUploadServer{
|
||||
t: t,
|
||||
resumable: resumable,
|
||||
parts: map[int][][]byte{},
|
||||
failFirst: map[string]bool{},
|
||||
seenBefore: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
// Bodies are recorded for every attempt, including the ones answered
|
||||
// with a 502, so tests can see exactly what each attempt carried.
|
||||
fail := s.failFirst[path] && !s.seenBefore[path]
|
||||
s.seenBefore[path] = true
|
||||
bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
if s.resumable != "" {
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && path == "/api/upload/upl_1":
|
||||
s.puts = append(s.puts, body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
s.parts[n] = append(s.parts[n], body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.completes++
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.aborts++
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestUploadSmallSendsWholeBody(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := strings.Repeat("a", 100)
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
require.Len(t, srv.puts, 1)
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
func TestUploadSmallRejectsShortSource(t *testing.T) {
|
||||
// The source declares 100 bytes but only supplies 60. A server which
|
||||
// accepts that as a complete upload must not turn into a success.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
|
||||
}
|
||||
|
||||
func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
|
||||
// A retried PUT would resend an already-consumed reader, i.e. an
|
||||
// empty body, which the server would store as an empty file. The
|
||||
// single PUT therefore must not be retried at the low level.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
srv.failFirst["/api/upload/upl_1"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
content := strings.Repeat("a", 100)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
const threeParts = `{"part_size":4,"total_parts":3}`
|
||||
|
||||
func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := "0123456789"
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
require.Len(t, srv.parts, 3)
|
||||
assert.Equal(t, "0123", string(srv.parts[1][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
}
|
||||
|
||||
func TestUploadMultipartRejectsShortSource(t *testing.T) {
|
||||
// 10 bytes declared, 8 supplied: the last part must never be sent
|
||||
// and the upload must not be completed.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
|
||||
assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
|
||||
assert.Equal(t, 0, srv.completes, "upload must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
|
||||
// A part which fails transiently must be resent with all its bytes,
|
||||
// not with whatever is left of a consumed reader.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
srv.failFirst["/api/upload/upl_1/part/2"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][1]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
// An upload that fails after init has to hand its session back: the session
|
||||
// holds one of the workspace's upload slots for ten minutes otherwise, and
|
||||
// rclone's high-level retry opens a new one rather than resuming it.
|
||||
func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
resumable string
|
||||
content string
|
||||
size int64
|
||||
}{
|
||||
{"small file short source", "", "0123", 10},
|
||||
{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, tc.resumable)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
|
||||
"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, srv.aborts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A successful upload must not abort anything
|
||||
func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
|
||||
"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, srv.aborts)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Every by-id call names the workspace, so the API can find an id it minted
|
||||
// moments ago before its file_home index row lands.
|
||||
func TestByIDCallsSendWorkspaceHint(t *testing.T) {
|
||||
seen := map[string]string{}
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
|
||||
switch r.URL.Path {
|
||||
case "/api/folders/fld_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
case "/api/files/fil_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
|
||||
case "/api/folders/fld_1/move":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
|
||||
default:
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
}))
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.removeFolder(ctx, "fld_1", false))
|
||||
require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
|
||||
_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.deleteFile(ctx, "fil_1"))
|
||||
require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
|
||||
require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
|
||||
_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
_, err = f.createShareLink(ctx, "fil_1", 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
for call, header := range seen {
|
||||
assert.Equal(t, "ws_test", header, call)
|
||||
}
|
||||
assert.Len(t, seen, 8)
|
||||
}
|
||||
|
||||
// The download's API request carries the hint; the presigned R2 fetch must not.
|
||||
func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
|
||||
var apiHint, blobHint string
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/files/fil_1/download" {
|
||||
apiHint = r.Header.Get("X-Dosya-Workspace")
|
||||
http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
|
||||
return
|
||||
}
|
||||
blobHint = r.Header.Get("X-Dosya-Workspace")
|
||||
_, _ = w.Write([]byte("bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
f.opt.APIURL = srv.URL
|
||||
|
||||
rc, err := f.downloadFile(context.Background(), "fil_1", nil)
|
||||
require.NoError(t, err)
|
||||
_ = rc.Close()
|
||||
|
||||
assert.Equal(t, "ws_test", apiHint)
|
||||
assert.Equal(t, "", blobHint)
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeWorkspaceServer answers GET /api/workspaces with the given body
|
||||
func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/workspaces", r.URL.Path)
|
||||
*calls++
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.NoError(t, f.findWorkspace(context.Background()))
|
||||
assert.Equal(t, "ws_only", f.opt.WorkspaceID)
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceListsTheChoices(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
|
||||
`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
err := f.findWorkspace(context.Background())
|
||||
require.Error(t, err)
|
||||
// the ids have to be in the message: they are the only place the user
|
||||
// can read them, which is the whole point of listing them
|
||||
assert.Contains(t, err.Error(), "ws_one (One)")
|
||||
assert.Contains(t, err.Error(), "ws_two (Two)")
|
||||
assert.Empty(t, f.opt.WorkspaceID)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceWithNone(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
|
||||
}
|
||||
+17
-33
@@ -692,27 +692,13 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
|
||||
// deleteObject removes an object by ID
|
||||
func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
if f.opt.HardDelete {
|
||||
// Deleting forever doesn't invalidate the server's cache of
|
||||
// the parent folder listing, so the entry keeps being listed
|
||||
// for about a second. Moving to the trash first does.
|
||||
err := f.deleteEntry(ctx, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return f.deleteEntry(ctx, id, f.opt.HardDelete)
|
||||
}
|
||||
|
||||
// deleteEntry moves the entry with ID to the trash or deletes it forever
|
||||
func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) error {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/file-entries/delete",
|
||||
}
|
||||
request := api.DeleteRequest{
|
||||
EntryIDs: []string{id},
|
||||
DeleteForever: deleteForever,
|
||||
DeleteForever: f.opt.HardDelete,
|
||||
}
|
||||
var result api.DeleteResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -1054,28 +1040,17 @@ func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err
|
||||
}
|
||||
|
||||
// copy and rename a file or folder to directoryID with leaf
|
||||
//
|
||||
// If existing is set it is removed once the copy has succeeded.
|
||||
func (f *Fs) copyTo(ctx context.Context, srcID, dstLeaf, dstDirectoryID string, existing fs.Object) (info *api.Item, err error) {
|
||||
func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
|
||||
// Can have duplicates so don't have to be careful here
|
||||
|
||||
// Copy to dstDirectoryID first
|
||||
info, err = f.copy(ctx, srcID, dstDirectoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The server names the copy "name (1)" if the destination
|
||||
// directory already has an entry called "name" and refuses to
|
||||
// rename it to "name" until that entry is removed.
|
||||
if existing != nil {
|
||||
fs.Debugf(existing, "Server side copy: removing existing object after successful copy")
|
||||
err = existing.Remove(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Rename if required
|
||||
if info.Name != f.opt.Enc.FromStandardName(dstLeaf) {
|
||||
if srcLeaf != dstLeaf {
|
||||
info, err = f.rename(ctx, info.ID.String(), dstLeaf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1099,6 +1074,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
srcLeaf := path.Base(srcObj.remote)
|
||||
|
||||
srcPath := srcObj.fs.rootSlash() + srcObj.remote
|
||||
dstPath := f.rootSlash() + remote
|
||||
if srcPath == dstPath {
|
||||
@@ -1107,8 +1084,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
|
||||
// Find existing object
|
||||
existingObj, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
existingObj = nil
|
||||
if err == nil {
|
||||
defer func() {
|
||||
// Don't remove existing object if returning an error
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
|
||||
err = existingObj.Remove(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Create temporary object
|
||||
@@ -1118,7 +1102,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
}
|
||||
|
||||
// Copy the object
|
||||
info, err := f.copyTo(ctx, srcObj.id, dstLeaf, dstDirectoryID, existingObj)
|
||||
info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -8,14 +8,10 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
stdsync "sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -29,113 +25,13 @@ import (
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
service, err := drive.NewService(
|
||||
context.Background(),
|
||||
option.WithHTTPClient(server.Client()),
|
||||
option.WithEndpoint(server.URL+"/"),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return &Fs{
|
||||
svc: service,
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
permissionsMu: new(stdsync.Mutex),
|
||||
permissions: make(map[string]*drive.Permission),
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionConcurrent(t *testing.T) {
|
||||
const requestCount = 2
|
||||
arrived := make(chan string, requestCount)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce stdsync.Once
|
||||
releaseRequests := func() {
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
}
|
||||
defer releaseRequests()
|
||||
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
arrived <- permissionID
|
||||
<-release
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
type permissionResult struct {
|
||||
permission *drive.Permission
|
||||
inherited bool
|
||||
err error
|
||||
}
|
||||
results := make(chan permissionResult, requestCount)
|
||||
permissionIDs := []string{"permission-1", "permission-2"}
|
||||
for _, permissionID := range permissionIDs {
|
||||
go func() {
|
||||
permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
|
||||
results <- permissionResult{permission: permission, inherited: inherited, err: err}
|
||||
}()
|
||||
}
|
||||
|
||||
arrivedIDs := make(map[string]bool, requestCount)
|
||||
timer := time.NewTimer(time.Second)
|
||||
defer timer.Stop()
|
||||
for len(arrivedIDs) < requestCount {
|
||||
select {
|
||||
case permissionID := <-arrived:
|
||||
arrivedIDs[permissionID] = true
|
||||
case <-timer.C:
|
||||
releaseRequests()
|
||||
t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
|
||||
}
|
||||
}
|
||||
releaseRequests()
|
||||
|
||||
assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
|
||||
returnedIDs := make(map[string]bool, requestCount)
|
||||
for range requestCount {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
require.NotNil(t, result.permission)
|
||||
assert.False(t, result.inherited)
|
||||
returnedIDs[result.permission.Id] = true
|
||||
}
|
||||
assert.Equal(t, arrivedIDs, returnedIDs)
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionCacheHit(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
|
||||
assert.Same(t, first, second)
|
||||
assert.Equal(t, int32(1), requests.Load())
|
||||
}
|
||||
|
||||
func TestDriveScopes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
|
||||
@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
|
||||
|
||||
// getPermission returns permissions for the fileID and permissionID passed in
|
||||
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
|
||||
f.permissionsMu.Lock()
|
||||
defer f.permissionsMu.Unlock()
|
||||
if useCache {
|
||||
f.permissionsMu.Lock()
|
||||
perm = f.permissions[permissionID]
|
||||
f.permissionsMu.Unlock()
|
||||
if perm != nil {
|
||||
return perm, false, nil
|
||||
}
|
||||
@@ -131,9 +131,7 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
|
||||
cleanPermission(perm)
|
||||
|
||||
// cache the permission
|
||||
f.permissionsMu.Lock()
|
||||
f.permissions[permissionID] = perm
|
||||
f.permissionsMu.Unlock()
|
||||
|
||||
return perm, inherited, err
|
||||
}
|
||||
|
||||
+18
-30
@@ -11,6 +11,7 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -21,7 +22,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
@@ -112,13 +112,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
|
||||
}
|
||||
|
||||
// Make an http.Request for the range passed in
|
||||
//
|
||||
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
|
||||
// to an io.Closer and close it after each attempt — the chunk buffer is
|
||||
// pool-backed, so an early close would free its pages and a retry would then
|
||||
// read a dead buffer. The upload loop owns the buffer's lifetime.
|
||||
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
|
||||
req.ContentLength = reqSize
|
||||
totalSize := "*"
|
||||
if rx.ContentLength >= 0 {
|
||||
@@ -173,40 +168,37 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
start := int64(0)
|
||||
var StatusCode int
|
||||
var err error
|
||||
chunkSize := int64(rx.f.opt.ChunkSize)
|
||||
buf := make([]byte, int(rx.f.opt.ChunkSize))
|
||||
for finished := false; !finished; {
|
||||
reqSize := chunkSize
|
||||
var reqSize int64
|
||||
var chunk io.ReadSeeker
|
||||
if rx.ContentLength >= 0 {
|
||||
// If size known use repeatable reader for smoother bwlimit
|
||||
if start >= rx.ContentLength {
|
||||
break
|
||||
}
|
||||
reqSize = min(rx.ContentLength-start, chunkSize)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool so reads are
|
||||
// repeatable for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, rx.Media, reqSize)
|
||||
if rx.ContentLength < 0 {
|
||||
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
|
||||
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
|
||||
} else {
|
||||
// If size unknown read into buffer
|
||||
var n int
|
||||
n, err = readers.ReadFill(rx.Media, buf)
|
||||
if err == io.EOF {
|
||||
// Send the last chunk with the correct ContentLength
|
||||
// otherwise Google doesn't know we've finished
|
||||
rx.ContentLength = start + n
|
||||
rx.ContentLength = start + int64(n)
|
||||
finished = true
|
||||
err = nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqSize = n
|
||||
}
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
reqSize = int64(n)
|
||||
chunk = bytes.NewReader(buf[:reqSize])
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
err = rx.f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
|
||||
again, err := rx.f.shouldRetry(ctx, err)
|
||||
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
|
||||
again = false
|
||||
@@ -214,13 +206,9 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
}
|
||||
return again, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
|
||||
start += reqSize
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
|
||||
// retried successfully. The chunk buffer is pool-backed and implements
|
||||
// io.Closer, so if it reaches the http transport unwrapped the transport
|
||||
// closes it after the failed attempt, returning its pages to the pool, and
|
||||
// the retry then reads a freed buffer.
|
||||
func TestResumableUploadRetry(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("resumable"), 512)
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
if attempts.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
client: server.Client(),
|
||||
}
|
||||
f.opt.ChunkSize = fs.SizeSuffix(len(content))
|
||||
rx := &resumableUpload{
|
||||
f: f,
|
||||
remote: "remote",
|
||||
URI: server.URL,
|
||||
Media: bytes.NewReader(content),
|
||||
MediaType: "application/octet-stream",
|
||||
ContentLength: int64(len(content)),
|
||||
}
|
||||
info, err := rx.Upload(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, info)
|
||||
assert.Equal(t, "fake-id", info.Id)
|
||||
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
|
||||
}
|
||||
+18
-28
@@ -51,7 +51,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -634,8 +633,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil // our root it empty so we probably want to list shared folders
|
||||
}
|
||||
|
||||
dir := path.Dir(f.root)
|
||||
if dir == "." {
|
||||
dir = f.root
|
||||
}
|
||||
|
||||
// root is not empty so we have find the right shared folder if it exists
|
||||
id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
|
||||
id, err := f.findSharedFolder(ctx, dir)
|
||||
if err != nil {
|
||||
// if we didn't find the specified shared folder we have to bail out here
|
||||
return nil, err
|
||||
@@ -943,21 +947,13 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// sharedFolderName returns the shared folder name in root, which is its first
|
||||
// path component. root must be the trimmed root as produced by setRoot (no
|
||||
// leading slash).
|
||||
func sharedFolderName(root string) string {
|
||||
name, _, _ := strings.Cut(root, "/")
|
||||
return name
|
||||
}
|
||||
|
||||
// findSharedFolder find the id for a given shared folder name
|
||||
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
|
||||
// so our only option is to iterate over all shared folders
|
||||
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
|
||||
errFoundFile := errors.New("found file")
|
||||
err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
|
||||
if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
|
||||
if entry.(*fs.Dir).Remote() == name {
|
||||
id = entry.(*fs.Dir).ID()
|
||||
return errFoundFile
|
||||
}
|
||||
@@ -1039,7 +1035,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
|
||||
errFoundFile := errors.New("found file")
|
||||
err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
|
||||
if strings.EqualFold(entry.(*Object).remote, name) {
|
||||
if entry.(*Object).remote == name {
|
||||
o = entry.(*Object)
|
||||
return errFoundFile
|
||||
}
|
||||
@@ -2083,6 +2079,11 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
|
||||
// write chunks
|
||||
in := readers.NewCountingReader(in0)
|
||||
bufSize := chunkSize
|
||||
if size >= 0 && size < bufSize {
|
||||
bufSize = size
|
||||
}
|
||||
buf := make([]byte, int(bufSize))
|
||||
cursor := files.UploadSessionCursor{
|
||||
SessionId: res.SessionId,
|
||||
Offset: 0,
|
||||
@@ -2102,21 +2103,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
}
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
|
||||
skip := int64(0)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// seek to the start in case this is a retry
|
||||
if _, err = rw.Seek(skip, io.SeekStart); err != nil {
|
||||
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, chunk)
|
||||
// after session is started, we retry everything
|
||||
if err != nil {
|
||||
// Check for incorrect offset error and retry with new offset
|
||||
@@ -2128,10 +2122,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
|
||||
if skip < 0 {
|
||||
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
|
||||
} else if skip == n {
|
||||
} else if skip == chunkSize {
|
||||
fs.Debugf(o, "%s: chunk received OK - continuing", what)
|
||||
return false, nil
|
||||
} else if skip > n {
|
||||
} else if skip > chunkSize {
|
||||
// This error should never happen
|
||||
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
|
||||
}
|
||||
@@ -2148,13 +2142,9 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
return err != nil, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if size >= 0 {
|
||||
// Check for sources which truncate early
|
||||
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
|
||||
|
||||
@@ -188,21 +188,6 @@ func TestInternalCheckPathLength(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalSharedFolderName(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
root string
|
||||
want string
|
||||
}{
|
||||
{root: "", want: ""},
|
||||
{root: "SharedFolder", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
|
||||
} {
|
||||
assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaperExportRemote(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
info := &files.FileMetadata{
|
||||
@@ -490,48 +475,3 @@ func TestFindSharedFileResolvesDecodedName(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, encodedName, o.remote)
|
||||
}
|
||||
|
||||
// sharedFoldersHandler serves a single shared folder from list_folders.
|
||||
func sharedFoldersHandler(name, id string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.URL.Path, "list_folders") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
resp := sharing.ListFoldersResult{
|
||||
Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
}
|
||||
|
||||
// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
|
||||
// lookup must match regardless of the case of the requested name.
|
||||
func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
|
||||
f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))
|
||||
|
||||
for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
|
||||
id, err := f.findSharedFolder(context.Background(), name)
|
||||
require.NoError(t, err, name)
|
||||
assert.Equal(t, "folder-id", id, name)
|
||||
}
|
||||
|
||||
_, err := f.findSharedFolder(context.Background(), "no-such-folder")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
}
|
||||
|
||||
// The Dropbox backend advertises CaseInsensitive: true, so a received file
|
||||
// lookup must match regardless of the case of the requested name.
|
||||
func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))
|
||||
|
||||
for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
|
||||
o, err := f.findSharedFile(context.Background(), name)
|
||||
require.NoError(t, err, name)
|
||||
assert.Equal(t, "TestFile.txt", o.remote, name)
|
||||
}
|
||||
|
||||
_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
@@ -16,16 +16,13 @@ import (
|
||||
|
||||
// multipartInit starts a new multipart upload and returns server details.
|
||||
func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
|
||||
encodedFolderPath := f.fromStandardPath(folderPath)
|
||||
encodedFilename := f.fromStandardPath(filename)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/multipart/init",
|
||||
Parameters: url.Values{
|
||||
"key": {f.opt.Key},
|
||||
"filename": {encodedFilename},
|
||||
"folder_path": {encodedFolderPath},
|
||||
"filename": {filename},
|
||||
"folder_path": {folderPath},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -10,11 +11,9 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
rclonemultipart "github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -48,32 +47,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
server := initResp.Result.Server
|
||||
objectPath := initResp.Result.ObjectPath
|
||||
|
||||
chunkSize := int64(f.opt.ChunkSize)
|
||||
if chunkSize <= 0 {
|
||||
return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)
|
||||
}
|
||||
for partNo := 1; ; partNo++ {
|
||||
// Buffer the part in memory from the global pool so it can be
|
||||
// re-sent on retry
|
||||
rw := rclonemultipart.NewRW()
|
||||
n, err := io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("read failed: %w", err)
|
||||
}
|
||||
chunkSize := int(f.opt.ChunkSize)
|
||||
buf := make([]byte, 0, chunkSize)
|
||||
tmp := make([]byte, 1024*1024)
|
||||
partNo := 1
|
||||
|
||||
for {
|
||||
n, errRead := in.Read(tmp)
|
||||
if n > 0 {
|
||||
uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)
|
||||
if uploadErr != nil {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)
|
||||
buf = append(buf, tmp[:n]...)
|
||||
|
||||
// If buffer reached chunkSize, upload a full part
|
||||
if len(buf) >= chunkSize {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
partNo++
|
||||
buf = buf[:0]
|
||||
}
|
||||
}
|
||||
if closeErr := rw.Close(); closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if err == io.EOF {
|
||||
|
||||
if errRead == io.EOF {
|
||||
break
|
||||
}
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read failed: %w", errRead)
|
||||
}
|
||||
}
|
||||
|
||||
if len(buf) > 0 {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
|
||||
@@ -84,40 +91,31 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadPart sends a single multipart chunk of size bytes to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: server,
|
||||
Parameters: url.Values{
|
||||
"partNumber": {strconv.Itoa(partNo)},
|
||||
"uploadId": {uploadID},
|
||||
},
|
||||
Body: r,
|
||||
ContentLength: &size,
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-RC-Upload-Id": uploadID,
|
||||
"X-RC-Part-No": strconv.Itoa(partNo),
|
||||
"X-Sess-ID": sessID,
|
||||
"X-Object-Path": objectPath,
|
||||
},
|
||||
NoResponse: true,
|
||||
// uploadPart sends a single multipart chunk to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
|
||||
url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
// rewind the pooled buffer so each attempt sends the whole chunk
|
||||
_, err := r.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)
|
||||
}
|
||||
return shouldRetry(err), err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
req.Header.Set("X-RC-Upload-Id", uploadID)
|
||||
req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
|
||||
req.Header.Set("X-Sess-ID", sessID)
|
||||
req.Header.Set("X-Object-Path", objectPath)
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("uploadPart failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFile uploads a file to FileLu
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
@@ -12,8 +12,5 @@ func TestIntegration(t *testing.T) {
|
||||
RemoteName: "TestFileLu:",
|
||||
NilObject: nil,
|
||||
SkipInvalidUTF8: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: 1,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"github.com/Files-com/files-sdk-go/v3/file"
|
||||
file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
|
||||
"github.com/Files-com/files-sdk-go/v3/folder"
|
||||
files_sdk_lib "github.com/Files-com/files-sdk-go/v3/lib"
|
||||
"github.com/Files-com/files-sdk-go/v3/session"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
@@ -177,13 +176,6 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Errors from the upload storage servers are of this type
|
||||
var httpErr files_sdk_lib.ResponseError
|
||||
if errors.As(err, &httpErr) && slices.Contains(retryErrorCodes, httpErr.StatusCode) {
|
||||
fs.Debugf(nil, "Retrying HTTP error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
|
||||
@@ -353,7 +345,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
|
||||
for it.Next() {
|
||||
item := new(it.File())
|
||||
item := ptr(it.File())
|
||||
remote := f.opt.Enc.ToStandardPath(item.DisplayName)
|
||||
remote = path.Join(dir, remote)
|
||||
if remote == dir {
|
||||
@@ -424,7 +416,7 @@ func (f *Fs) mkdir(ctx context.Context, path string) error {
|
||||
|
||||
params := files_sdk.FolderCreateParams{
|
||||
Path: path,
|
||||
MkdirParents: new(true),
|
||||
MkdirParents: ptr(true),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -466,7 +458,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
|
||||
params := files_sdk.FileDeleteParams{
|
||||
Path: path,
|
||||
Recursive: new(!check),
|
||||
Recursive: ptr(!check),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -537,7 +529,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.
|
||||
params := files_sdk.FileCopyParams{
|
||||
Path: srcPath,
|
||||
Destination: dstPath,
|
||||
Overwrite: new(true),
|
||||
Overwrite: ptr(true),
|
||||
}
|
||||
|
||||
var action files_sdk.FileAction
|
||||
@@ -680,7 +672,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
Paths: []string{f.absPath(remote)},
|
||||
}
|
||||
if expire < fs.DurationOff {
|
||||
params.ExpiresAt = new(time.Now().Add(time.Duration(expire)))
|
||||
params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
|
||||
}
|
||||
|
||||
var bundle files_sdk.Bundle
|
||||
@@ -842,6 +834,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
func isFolderNotEmpty(err error) bool {
|
||||
var re files_sdk.ResponseError
|
||||
ok := errors.As(err, &re)
|
||||
@@ -861,7 +858,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
file.UploadWithProvidedMtime(src.ModTime(ctx)),
|
||||
}
|
||||
|
||||
err := o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
err := o.fs.fileClient.Upload(uploadOpts...)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -43,7 +43,7 @@ func TestCommandEncodingNeutralisesCRLF(t *testing.T) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
|
||||
@@ -194,7 +194,8 @@ var retryErrorCodes = []int{
|
||||
|
||||
// Return true if the api error has the status given
|
||||
func isAPIErr(err error, status string) bool {
|
||||
if apiErr, ok := errors.AsType[api.Error](err); ok {
|
||||
var apiErr api.Error
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.Status == status
|
||||
}
|
||||
return false
|
||||
|
||||
+61
-82
@@ -10,6 +10,7 @@ package hidrive
|
||||
// to be resolved prior to execution with resolvePath(...).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -23,9 +24,8 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/ranges"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/sync/semaphore"
|
||||
@@ -454,17 +454,17 @@ func (f *Fs) deleteObject(ctx context.Context, path string) error {
|
||||
}
|
||||
|
||||
// createFile creates a file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// This guarantees that existing files will not be overwritten.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
// This returns ErrorFileExists if a file already exists at the specified path.
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
if onExist == AutoNameOnExist {
|
||||
@@ -479,14 +479,12 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -512,16 +510,16 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
|
||||
// overwriteFile updates the content of the file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the file does not exist it will be created.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
|
||||
@@ -533,14 +531,12 @@ func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, m
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -638,18 +634,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
}
|
||||
|
||||
// Read a chunk of data.
|
||||
var (
|
||||
chunkReader *pool.RW
|
||||
bytesRead int64
|
||||
)
|
||||
chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)
|
||||
if bytesRead < int64(chunkSize) {
|
||||
chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
|
||||
if bytesRead < chunkSize {
|
||||
startMoreTransfers = false
|
||||
}
|
||||
if readErr != nil || bytesRead <= 0 {
|
||||
if chunkReader != nil {
|
||||
_ = chunkReader.Close()
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
@@ -659,12 +648,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// After this upload is done,
|
||||
// signal that another transfer can be started.
|
||||
defer transferSemaphore.Release(1)
|
||||
defer func() { _ = chunkReader.Close() }()
|
||||
uploadErr := f.patchFile(gCtx, path, chunkReader, chunkOffset, zeroTime)
|
||||
uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
|
||||
if uploadErr == nil {
|
||||
// Remember successfully written chunks.
|
||||
okChunksMu.Lock()
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: bytesRead})
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
|
||||
okChunksMu.Unlock()
|
||||
fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
|
||||
} else {
|
||||
@@ -708,10 +696,10 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// starting at the given offset.
|
||||
//
|
||||
// Replaces the file contents starting from the given byte offset
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the offset is beyond the file end, the file is extended up to the offset.
|
||||
// The maximum size of the update is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
@@ -723,7 +711,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// usually expand the holes into regions of 0-bytes.
|
||||
//
|
||||
// This returns fs.ErrorObjectNotFound if the object is not found.
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offset uint64, modTime time.Time) error {
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetPath(path)
|
||||
parameters.Set("offset", strconv.FormatUint(offset, 10))
|
||||
@@ -735,15 +723,13 @@ func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offse
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
}
|
||||
|
||||
var resp *http.Response
|
||||
@@ -853,48 +839,41 @@ func createHiDriveScopes(role string, access string) []string {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
// unwrapAccounting splits any transfer accounting off reader,
|
||||
// returning the raw stream and the accounting (nil if there is none),
|
||||
// so the stream can be buffered and accounted when the buffer is uploaded.
|
||||
func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Account) {
|
||||
// Any kind of accounter re-wraps into the one type UnWrapAccounting
|
||||
// knows how to take the *Account out of.
|
||||
unwrapped, wrap := accounting.UnWrap(reader)
|
||||
_, acc = accounting.UnWrapAccounting(wrap(unwrapped))
|
||||
return unwrapped, acc
|
||||
}
|
||||
|
||||
// readerForChunk reads up to length bytes from reader into a buffer
|
||||
// from the global memory pool and returns it
|
||||
// along with the number of bytes that have been read from reader.
|
||||
// Reaching the end of reader early is not an error.
|
||||
//
|
||||
// If accountUpload is set, any accounting on reader is applied
|
||||
// when the buffer is read instead, so the upload is what gets accounted.
|
||||
//
|
||||
// This must not be set for a buffer which creates or replaces a file
|
||||
// in a single request. Accounting can fail part way through the upload,
|
||||
// for example when --max-transfer is reached, and HiDrive keeps
|
||||
// what it has received of an aborted request as the file.
|
||||
//
|
||||
// The caller must Close the buffer when done with it.
|
||||
func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {
|
||||
chunkReader = multipart.NewRW()
|
||||
if accountUpload {
|
||||
var acc *accounting.Account
|
||||
reader, acc = unwrapAccounting(reader)
|
||||
if acc != nil {
|
||||
chunkReader.SetAccounting(acc.AccountRead)
|
||||
}
|
||||
// cachedReader returns a version of the reader that caches its contents and
|
||||
// can therefore be reset using Seek.
|
||||
func cachedReader(reader io.Reader) io.ReadSeeker {
|
||||
bytesReader, ok := reader.(*bytes.Reader)
|
||||
if ok {
|
||||
return bytesReader
|
||||
}
|
||||
|
||||
bytesRead, err = io.CopyN(chunkReader, reader, length)
|
||||
if err == io.EOF {
|
||||
repeatableReader, ok := reader.(*readers.RepeatableReader)
|
||||
if ok {
|
||||
return repeatableReader
|
||||
}
|
||||
|
||||
return readers.NewRepeatableReader(reader)
|
||||
}
|
||||
|
||||
// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
|
||||
// Returns a new io.Reader (chunkReader) for that chunk
|
||||
// and the number of bytes that have been read from reader.
|
||||
func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
|
||||
// Unwrap any accounting from the input if present.
|
||||
reader, wrap := accounting.UnWrap(reader)
|
||||
|
||||
// Read a chunk of data.
|
||||
buffer := make([]byte, length)
|
||||
bytesRead, err = io.ReadFull(reader, buffer)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
_ = chunkReader.Close()
|
||||
return nil, bytesRead, err
|
||||
}
|
||||
return chunkReader, bytesRead, nil
|
||||
// Truncate unused capacity.
|
||||
buffer = buffer[:bytesRead]
|
||||
|
||||
// Use wrap to put any accounting back for chunkReader.
|
||||
return wrap(bytes.NewReader(buffer)), bytesRead, nil
|
||||
}
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -515,14 +514,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
// (i.e. everything up to the cutoff) in the first request,
|
||||
// avoids files being created on upload failure for small files.
|
||||
// (As opposed to creating an empty file and then uploading the content.)
|
||||
//
|
||||
// Only buffer as much as the source declares it has,
|
||||
// so small files do not cost the whole cutoff in memory.
|
||||
prefixSize := int64(f.opt.UploadCutoff)
|
||||
if size := src.Size(); size >= 0 && size < prefixSize {
|
||||
prefixSize = size
|
||||
}
|
||||
cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)
|
||||
tmpReader, bytesRead, err := readerForChunk(in, int(f.opt.UploadCutoff))
|
||||
cutoffReader := cachedReader(tmpReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -547,7 +540,6 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
return false, createErr
|
||||
})
|
||||
_ = cutoffReader.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -564,7 +556,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
// If there is more left to write, o.Update needs to skip ahead.
|
||||
// Use a fs.SeekOption with the current offset to do this.
|
||||
options = append(options, &fs.SeekOption{Offset: bytesRead})
|
||||
options = append(options, &fs.SeekOption{Offset: int64(bytesRead)})
|
||||
err = o.Update(ctx, in, src, options...)
|
||||
|
||||
if err == nil {
|
||||
@@ -979,12 +971,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Metadata should be updated even if the upload fails.
|
||||
info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
|
||||
} else {
|
||||
var content *pool.RW
|
||||
content, _, err = readerForChunk(in, src.Size(), false)
|
||||
if err == nil {
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
|
||||
_ = content.Close()
|
||||
}
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, cachedReader(in), modTime)
|
||||
metaErr = err
|
||||
}
|
||||
|
||||
|
||||
+31
-11
@@ -283,7 +283,7 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
u, err := rest.URLJoinRoot(base, f.root)
|
||||
u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -526,20 +526,40 @@ func addHeaders(req *http.Request, opt *Options) {
|
||||
// rest.ErrHTTPSDowngrade and strips the configured headers when the
|
||||
// redirect chain has left the originally requested host at any point.
|
||||
func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
|
||||
headers := make([]string, 0, len(opt.Headers)/2)
|
||||
for i := 0; i < len(opt.Headers); i += 2 {
|
||||
headers = append(headers, opt.Headers[i])
|
||||
}
|
||||
stripHeaders := rest.StripHeadersOnCrossHostRedirectFn(headers...)
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
err := stripHeaders(req, via)
|
||||
if errors.Is(err, rest.ErrHTTPSDowngrade) {
|
||||
err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
|
||||
if err := rest.RefuseHTTPSDowngradeRedirectFn(req, via); err != nil {
|
||||
if errors.Is(err, rest.ErrHTTPSDowngrade) {
|
||||
err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return err
|
||||
if redirectLeavesHost(req, via) {
|
||||
for i := 0; i < len(opt.Headers); i += 2 {
|
||||
req.Header.Del(opt.Headers[i])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// redirectLeavesHost reports whether any hop in the redirect chain
|
||||
// via plus the pending request req is to a different host from the
|
||||
// original request via[0].
|
||||
//
|
||||
// net/http copies the headers afresh from the original request for
|
||||
// every hop, so once the chain has visited another host the headers
|
||||
// must be stripped from every subsequent hop, even one back to the
|
||||
// original host, as the other host chose the URL.
|
||||
func redirectLeavesHost(req *http.Request, via []*http.Request) bool {
|
||||
origin := via[0].URL
|
||||
for _, hop := range via {
|
||||
if !rest.SameHost(hop.URL, origin) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return !rest.SameHost(req.URL, origin)
|
||||
}
|
||||
|
||||
// Adds the configured headers to the request if any
|
||||
func (f *Fs) addHeaders(req *http.Request) {
|
||||
addHeaders(req, &f.opt)
|
||||
@@ -573,7 +593,7 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
|
||||
return nil, fmt.Errorf("failed to readDir: %w", err)
|
||||
}
|
||||
|
||||
contentType, _, _ := strings.Cut(res.Header.Get("Content-Type"), ";")
|
||||
contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
|
||||
switch contentType {
|
||||
case "text/html":
|
||||
names, err = parse(u, res.Body)
|
||||
|
||||
@@ -760,34 +760,3 @@ func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
|
||||
assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRootCannotChangeHost(t *testing.T) {
|
||||
// A root starting with "//" is a network-path reference under the
|
||||
// URL standard. It must not be able to move the remote (and the
|
||||
// configured headers) to a host other than the configured one.
|
||||
var requests int
|
||||
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests++
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer other.Close()
|
||||
|
||||
m := prepareServer(t)
|
||||
otherHost := strings.TrimPrefix(other.URL, "http://")
|
||||
for _, root := range []string{
|
||||
"//" + otherHost + "/",
|
||||
"//" + otherHost + "/one%.txt",
|
||||
"//" + otherHost,
|
||||
"//user:pass@" + otherHost + "/",
|
||||
} {
|
||||
f, err := NewFs(context.Background(), remoteName, root, m)
|
||||
assert.ErrorContains(t, err, "must not change the scheme, host or user", root)
|
||||
assert.Nil(t, f, root)
|
||||
}
|
||||
assert.Equal(t, 0, requests, "request sent to the other host")
|
||||
|
||||
// A same-origin absolute path must still work
|
||||
f, err := NewFs(context.Background(), remoteName, "/", m)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, m["url"]+"/", f.String())
|
||||
}
|
||||
@@ -37,44 +37,44 @@ type FileList struct {
|
||||
|
||||
// File represents a file or folder in Huawei Drive
|
||||
type File struct {
|
||||
ID string `json:"id"`
|
||||
FileName string `json:"fileName"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
EditedTime time.Time `json:"editedTime"`
|
||||
EditedByMeTime time.Time `json:"editedByMeTime"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Owners []User `json:"owners"`
|
||||
LastEditor User `json:"lastEditor"`
|
||||
Permissions []Permission `json:"permissions"`
|
||||
PermissionIDs []string `json:"permissionIds"`
|
||||
Capabilities FileCapabilities `json:"capabilities"`
|
||||
OwnedByMe bool `json:"ownedByMe"`
|
||||
EditedByMe bool `json:"editedByMe"`
|
||||
ViewedByMe bool `json:"viewedByMe"`
|
||||
HasShared bool `json:"hasShared"`
|
||||
Recycled bool `json:"recycled"`
|
||||
DirectlyRecycled bool `json:"directlyRecycled"`
|
||||
Favorite bool `json:"favorite"`
|
||||
ExistThumbnail bool `json:"existThumbnail"`
|
||||
ThumbnailVersion int64 `json:"thumbnailVersion"`
|
||||
IconDownloadLink string `json:"iconDownloadLink,omitempty"`
|
||||
ContentDownloadLink string `json:"contentDownloadLink,omitempty"`
|
||||
ContentVersion string `json:"contentVersion,omitempty"`
|
||||
LastHistoryVersionID string `json:"lastHistoryVersionId,omitempty"`
|
||||
OccupiedSpace int64 `json:"occupiedSpace,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
WritersHasSharePermission bool `json:"writersHasSharePermission"`
|
||||
WriterHasCopyPermission bool `json:"writerHasCopyPermission"`
|
||||
Containers []string `json:"containers"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
ContentExtras *ContentExtras `json:"contentExtras,omitempty"`
|
||||
ID string `json:"id"`
|
||||
FileName string `json:"fileName"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
EditedTime time.Time `json:"editedTime"`
|
||||
EditedByMeTime time.Time `json:"editedByMeTime,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Owners []User `json:"owners"`
|
||||
LastEditor User `json:"lastEditor"`
|
||||
Permissions []Permission `json:"permissions"`
|
||||
PermissionIDs []string `json:"permissionIds"`
|
||||
Capabilities FileCapabilities `json:"capabilities"`
|
||||
OwnedByMe bool `json:"ownedByMe"`
|
||||
EditedByMe bool `json:"editedByMe"`
|
||||
ViewedByMe bool `json:"viewedByMe"`
|
||||
HasShared bool `json:"hasShared"`
|
||||
Recycled bool `json:"recycled"`
|
||||
DirectlyRecycled bool `json:"directlyRecycled"`
|
||||
Favorite bool `json:"favorite"`
|
||||
ExistThumbnail bool `json:"existThumbnail"`
|
||||
ThumbnailVersion int64 `json:"thumbnailVersion"`
|
||||
IconDownloadLink string `json:"iconDownloadLink,omitempty"`
|
||||
ContentDownloadLink string `json:"contentDownloadLink,omitempty"`
|
||||
ContentVersion string `json:"contentVersion,omitempty"`
|
||||
LastHistoryVersionID string `json:"lastHistoryVersionId,omitempty"`
|
||||
OccupiedSpace int64 `json:"occupiedSpace,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
WritersHasSharePermission bool `json:"writersHasSharePermission"`
|
||||
WriterHasCopyPermission bool `json:"writerHasCopyPermission"`
|
||||
Containers []string `json:"containers"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
ContentExtras *ContentExtras `json:"contentExtras,omitempty"`
|
||||
}
|
||||
|
||||
// IsDir returns true if the file is a directory
|
||||
@@ -125,41 +125,41 @@ type Thumbnail struct {
|
||||
|
||||
// CreateFolderRequest represents request to create a folder
|
||||
type CreateFolderRequest struct {
|
||||
FileName string `json:"fileName"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Favorite bool `json:"favorite,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
AutoRename int `json:"autoRename,omitempty"`
|
||||
FileName string `json:"fileName"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Favorite bool `json:"favorite,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AutoRename int `json:"autoRename,omitempty"`
|
||||
}
|
||||
|
||||
// UpdateFileRequest represents request to update a file
|
||||
type UpdateFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Favorite *bool `json:"favorite,omitempty"`
|
||||
Recycled *bool `json:"recycled,omitempty"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
WriterHasCopyPermission *bool `json:"writerHasCopyPermission,omitempty"`
|
||||
WritersHasSharePermission *bool `json:"writersHasSharePermission,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
AddParentFolder []string `json:"addParentFolder,omitempty"`
|
||||
RemoveParentFolder []string `json:"removeParentFolder,omitempty"`
|
||||
CreatedTime *time.Time `json:"createdTime,omitempty"`
|
||||
EditedTime *time.Time `json:"editedTime,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Favorite *bool `json:"favorite,omitempty"`
|
||||
Recycled *bool `json:"recycled,omitempty"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
WriterHasCopyPermission *bool `json:"writerHasCopyPermission,omitempty"`
|
||||
WritersHasSharePermission *bool `json:"writersHasSharePermission,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AddParentFolder []string `json:"addParentFolder,omitempty"`
|
||||
RemoveParentFolder []string `json:"removeParentFolder,omitempty"`
|
||||
CreatedTime *time.Time `json:"createdTime,omitempty"`
|
||||
EditedTime *time.Time `json:"editedTime,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest represents request to copy a file
|
||||
type CopyFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
}
|
||||
|
||||
// ResumeUploadInitResponse represents response to resume upload init
|
||||
@@ -224,6 +224,11 @@ var GlobalDomains = map[string]bool{
|
||||
"drive.cloud.hicloud.com": true,
|
||||
}
|
||||
|
||||
// BoolPtr returns a pointer to a bool value (helper for UpdateFileRequest)
|
||||
func BoolPtr(b bool) *bool {
|
||||
return &b
|
||||
}
|
||||
|
||||
// StartCursor represents response to Changes.getStartCursor endpoint
|
||||
type StartCursor struct {
|
||||
Category string `json:"category"` // "drive#startCursor"
|
||||
|
||||
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -1032,7 +1031,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
fs.Debugf(f, "Copy: failed to get metadata options: %v", err)
|
||||
}
|
||||
if len(mergedMeta) > 0 {
|
||||
copyReq.Properties = make(map[string]any)
|
||||
copyReq.Properties = make(map[string]interface{})
|
||||
for key, value := range mergedMeta {
|
||||
switch key {
|
||||
case "content-type", "sha256", "btime", "mtime", "utime",
|
||||
@@ -1722,7 +1721,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
}
|
||||
|
||||
// Create metadata
|
||||
metadata := map[string]any{
|
||||
metadata := map[string]interface{}{
|
||||
"fileName": leaf,
|
||||
"mimeType": mimeType,
|
||||
}
|
||||
@@ -1742,10 +1741,9 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
metadata["autoRename"] = 3
|
||||
}
|
||||
|
||||
// Build the multipart body in pooled memory so it can be re-sent on retry
|
||||
rw := pool.NewRW(pool.Global())
|
||||
defer fs.CheckClose(rw, &err)
|
||||
writer := multipart.NewWriter(rw)
|
||||
// Create multipart form
|
||||
var buf bytes.Buffer
|
||||
writer := multipart.NewWriter(&buf)
|
||||
|
||||
// Add metadata part - use exact format from Huawei docs
|
||||
metadataWriter, err := writer.CreatePart(textproto.MIMEHeader{
|
||||
@@ -1791,17 +1789,15 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
return fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
bodySize := rw.Size()
|
||||
bodyBytes := buf.Bytes()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Parameters: url.Values{
|
||||
"uploadType": []string{"multipart"},
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
ContentLength: &bodySize,
|
||||
RootURL: o.fs.uploadURL,
|
||||
Body: rw,
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
RootURL: o.fs.uploadURL,
|
||||
}
|
||||
|
||||
if o.id != "" {
|
||||
@@ -1815,10 +1811,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
|
||||
var info *api.File
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = bytes.NewReader(bodyBytes)
|
||||
resp, err := srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -1879,7 +1872,7 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
|
||||
// Prepare metadata for resumable upload initialization
|
||||
metadata := map[string]any{
|
||||
metadata := map[string]interface{}{
|
||||
"fileName": leaf,
|
||||
}
|
||||
|
||||
@@ -1912,26 +1905,25 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
|
||||
// Now upload the content in chunks
|
||||
// Note 256k is the minimum chunk size and 64M the maximum
|
||||
// single upload size according to the Huawei Drive API
|
||||
chunkSize := min(max(int64(o.fs.opt.ChunkSize), 256*1024), 64*1024*1024)
|
||||
chunkSize := int64(o.fs.opt.ChunkSize)
|
||||
if chunkSize < 256*1024 {
|
||||
chunkSize = 256 * 1024 // Minimum chunk size according to Huawei Drive API
|
||||
}
|
||||
if chunkSize > 64*1024*1024 {
|
||||
chunkSize = 64 * 1024 * 1024 // Maximum single upload size
|
||||
}
|
||||
|
||||
buf := make([]byte, chunkSize)
|
||||
var offset int64
|
||||
|
||||
for offset < size {
|
||||
// Buffer the chunk in pooled memory so it can be re-sent on retry
|
||||
rw := pool.NewRW(pool.Global())
|
||||
n, err := io.CopyN(rw, in, min(size-offset, chunkSize))
|
||||
if err != nil {
|
||||
// io.EOF here means the source ran out before the declared size
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
_ = rw.Close()
|
||||
n, err := io.ReadFull(in, buf)
|
||||
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||
return fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)
|
||||
}
|
||||
|
||||
end := offset + n - 1
|
||||
chunk := buf[:n]
|
||||
end := offset + int64(n) - 1
|
||||
|
||||
// Upload this chunk using the original client with OAuth authentication
|
||||
chunkOpts := rest.Opts{
|
||||
@@ -1939,20 +1931,17 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
RootURL: location,
|
||||
Path: "", // Empty path since RootURL contains the full URL
|
||||
ExtraHeaders: map[string]string{
|
||||
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
|
||||
"Content-Type": mimeType,
|
||||
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
|
||||
"Content-Length": strconv.Itoa(n),
|
||||
"Content-Type": mimeType,
|
||||
},
|
||||
ContentLength: &n,
|
||||
Body: rw,
|
||||
}
|
||||
|
||||
var info *api.File
|
||||
var chunkResp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
chunkOpts.Body = bytes.NewReader(chunk)
|
||||
var err error
|
||||
chunkResp, err = srv.CallJSON(ctx, &chunkOpts, nil, &info)
|
||||
// 308 means continue uploading (incomplete)
|
||||
if chunkResp != nil && chunkResp.StatusCode == 308 {
|
||||
@@ -1966,15 +1955,11 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
return shouldRetry(ctx, chunkResp, err)
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
|
||||
offset += n
|
||||
offset += int64(n)
|
||||
|
||||
// If we got file info back, we're done
|
||||
if info != nil && info.ID != "" {
|
||||
@@ -2105,8 +2090,8 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
|
||||
|
||||
// Prepare the update request payload
|
||||
updateReq := api.UpdateFileRequest{
|
||||
Properties: make(map[string]any),
|
||||
AppSettings: make(map[string]any),
|
||||
Properties: make(map[string]interface{}),
|
||||
AppSettings: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Process metadata and separate into properties and app settings
|
||||
@@ -2117,7 +2102,7 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
|
||||
updateReq.Description = value
|
||||
case "favorite":
|
||||
if favorite, err := strconv.ParseBool(value); err == nil {
|
||||
updateReq.Favorite = new(favorite)
|
||||
updateReq.Favorite = api.BoolPtr(favorite)
|
||||
}
|
||||
case "content-type":
|
||||
// Allow setting/overriding MIME type
|
||||
|
||||
@@ -19,49 +19,15 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
)
|
||||
|
||||
// Limits on the chunk size accepted by the resumable upload API
|
||||
const (
|
||||
minChunkSize = 256 * fs.Kibi
|
||||
maxChunkSize = 64 * fs.Mebi
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestHuaweiDrive:",
|
||||
NilObject: (*Object)(nil),
|
||||
SkipBadWindowsCharacters: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: minChunkSize,
|
||||
MaxChunkSize: maxChunkSize,
|
||||
CeilChunkSize: fstests.NextPowerOfTwo,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
|
||||
// TestNewFs tests the NewFs constructor
|
||||
func TestNewFs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@@ -1618,7 +1617,9 @@ func (ps *PhotosService) GetLibraryAlbumCounts(ctx context.Context) (map[string]
|
||||
if err := ps.requestForArea(ctx, area, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to get library album counts: %w", err)
|
||||
}
|
||||
maps.Copy(counts, response.toCounts(order))
|
||||
for k, v := range response.toCounts(order) {
|
||||
counts[k] = v
|
||||
}
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
@@ -2031,7 +2032,7 @@ func (album *Album) fetchPhotosParallel(ctx context.Context, totalPhotos int64)
|
||||
sem := make(chan struct{}, workers)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := range numPartitions {
|
||||
for i := 0; i < numPartitions; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
|
||||
@@ -1191,7 +1191,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for range 100 {
|
||||
for i := 0; i < 100; i++ {
|
||||
lib.deltaMu.Lock()
|
||||
lib.pendingDelta = &deltaPayload{
|
||||
records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},
|
||||
@@ -1201,7 +1201,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
|
||||
lib.applyPendingDelta(context.Background())
|
||||
}
|
||||
}()
|
||||
for range 100 {
|
||||
for i := 0; i < 100; i++ {
|
||||
ps.FlushCaches()
|
||||
// Re-add the library since FlushCaches clears it
|
||||
ps.mu.Lock()
|
||||
|
||||
@@ -718,7 +718,7 @@ func (s *Session) GetAuthState(ctx context.Context) (*AuthStateResponse, error)
|
||||
Path: "",
|
||||
ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
|
||||
RootURL: authEndpoint,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
// Use srv.Call directly to capture the raw response body for debugging
|
||||
resp, err := s.srv.Call(ctx, &opts)
|
||||
@@ -828,7 +828,7 @@ func (s *Session) TrustSession(ctx context.Context) error {
|
||||
ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
|
||||
RootURL: authEndpoint,
|
||||
NoResponse: true,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
|
||||
_, err := s.Request(ctx, opts, nil, nil)
|
||||
@@ -846,7 +846,7 @@ func (s *Session) ValidateSession(ctx context.Context) error {
|
||||
Path: "/validate",
|
||||
ExtraHeaders: s.GetHeaders(map[string]string{}),
|
||||
RootURL: setupEndpoint,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
_, err := s.Request(ctx, opts, nil, &s.AccountInfo)
|
||||
if err != nil {
|
||||
@@ -904,6 +904,8 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
|
||||
return headers
|
||||
}
|
||||
|
||||
func int64Ptr(v int64) *int64 { return &v }
|
||||
|
||||
// NewSession creates a new Session instance with default values
|
||||
func NewSession() *Session {
|
||||
session := &Session{
|
||||
|
||||
@@ -92,7 +92,7 @@ func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, fou
|
||||
item, resp, err = f.service.GetItemByPath(ctx, path.Join(f.root, dir))
|
||||
return shouldRetry(ctx, resp, err)
|
||||
}); err != nil {
|
||||
if item == nil && resp != nil && resp.StatusCode == 404 {
|
||||
if item == nil && resp.StatusCode == 404 {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, err
|
||||
|
||||
@@ -376,7 +376,8 @@ func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
if libraryName, ok := strings.CutPrefix(pathID, "lib:"); ok {
|
||||
if strings.HasPrefix(pathID, "lib:") {
|
||||
libraryName := strings.TrimPrefix(pathID, "lib:")
|
||||
libraries, err := photosService.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
@@ -735,7 +736,10 @@ func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallb
|
||||
}
|
||||
close(jobCh)
|
||||
|
||||
workers := min(len(jobs), fs.GetConfig(ctx).Checkers)
|
||||
workers := fs.GetConfig(ctx).Checkers
|
||||
if len(jobs) < workers {
|
||||
workers = len(jobs)
|
||||
}
|
||||
errs := make(chan error, workers)
|
||||
|
||||
for range workers {
|
||||
|
||||
@@ -477,7 +477,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
|
||||
FolderPath: f.EncodePath(path.Join(f.root, dir)),
|
||||
})
|
||||
|
||||
if res != nil && res.StatusCode == http.StatusNotFound {
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return false, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
@@ -502,7 +502,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
|
||||
FolderPath: f.EncodePath(remote),
|
||||
})
|
||||
|
||||
if res != nil && res.StatusCode == http.StatusNotFound {
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
return false, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
|
||||
+117
-241
@@ -54,7 +54,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
switch httpErr.StatusCode() {
|
||||
case 401:
|
||||
if !f.authFailed {
|
||||
@@ -474,10 +475,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range childFolders {
|
||||
if !recent.stale(e.UUID, id, e.PlainName) {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if len(childFolders) > 0 {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
|
||||
var childFiles []folders.File
|
||||
@@ -489,10 +488,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range childFiles {
|
||||
if !recent.stale(e.UUID, id, joinNameExt(e.PlainName, e.Type)) {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if len(childFiles) > 0 {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
|
||||
// Delete the directory
|
||||
@@ -506,7 +503,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recent.deleted(id)
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
@@ -525,9 +521,6 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, e
|
||||
return "", false, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if recent.stale(e.UUID, pathID, e.PlainName) {
|
||||
continue
|
||||
}
|
||||
if f.opt.Encoding.ToStandardName(e.PlainName) == leaf {
|
||||
return e.UUID, true, nil
|
||||
}
|
||||
@@ -564,166 +557,46 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error)
|
||||
return resp.UUID, nil
|
||||
}
|
||||
|
||||
// splitNameExt splits a basename into the (plainName, type) pair that Internxt
|
||||
// stores a file under.
|
||||
func splitNameExt(baseName string) (name, ext string) {
|
||||
ext = path.Ext(baseName)
|
||||
if ext == baseName {
|
||||
return baseName, ""
|
||||
// preUploadCheck checks if a file exists in the given directory
|
||||
// Returns the file metadata if it exists, nil if not
|
||||
func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
|
||||
// Parse name and extension from the leaf
|
||||
baseName := f.opt.Encoding.FromStandardName(leaf)
|
||||
name := strings.TrimSuffix(baseName, path.Ext(baseName))
|
||||
ext := strings.TrimPrefix(path.Ext(baseName), ".")
|
||||
|
||||
checkResult, err := files.CheckFilesExistence(ctx, f.cfg, directoryID, []files.FileExistenceCheck{
|
||||
{
|
||||
PlainName: name,
|
||||
Type: ext,
|
||||
OriginalFile: struct{}{},
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
// If existence check fails, assume file doesn't exist to allow upload to proceed
|
||||
return nil, nil
|
||||
}
|
||||
return strings.TrimSuffix(baseName, ext), strings.TrimPrefix(ext, ".")
|
||||
}
|
||||
|
||||
// joinNameExt reassembles the basename a stored (plainName, type) pair
|
||||
// displays as, the way List builds its entries.
|
||||
func joinNameExt(name, ext string) string {
|
||||
if ext == "" {
|
||||
return name
|
||||
}
|
||||
return name + "." + ext
|
||||
}
|
||||
if len(checkResult.Files) > 0 && checkResult.Files[0].FileExists() {
|
||||
result := checkResult.Files[0]
|
||||
if result.Type != ext {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// legacyNameExt splits a basename at the final dot, the way this backend used
|
||||
// to before it adopted the convention the other clients share. It differs from
|
||||
// splitNameExt only for a name whose sole dot leads it: ".bashrc" splits into
|
||||
// an empty name of type "bashrc".
|
||||
func legacyNameExt(baseName string) (name, ext string) {
|
||||
return strings.TrimSuffix(baseName, path.Ext(baseName)), strings.TrimPrefix(path.Ext(baseName), ".")
|
||||
}
|
||||
existingUUID := result.UUID
|
||||
if existingUUID != "" {
|
||||
fileMeta, err := files.GetFileMeta(ctx, f.cfg, existingUUID)
|
||||
if err == nil && fileMeta != nil {
|
||||
return convertFileMetaToFile(fileMeta), nil
|
||||
}
|
||||
|
||||
// existenceCheck builds a lookup criterion for one (plainName, type) spelling.
|
||||
func existenceCheck(name, ext string) files.FileExistenceCheck {
|
||||
return files.FileExistenceCheck{
|
||||
PlainName: name,
|
||||
Type: ext,
|
||||
OriginalFile: struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
// staleWindow is how long the API may carry on returning an item from
|
||||
// where it was before this process moved or deleted it.
|
||||
const staleWindow = time.Minute
|
||||
|
||||
// location is where this process last put an item. An empty parentUUID
|
||||
// means the item was deleted.
|
||||
type location struct {
|
||||
parentUUID string
|
||||
name string // encoded name, including any extension
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
// recentChanges holds the files and folders this process has moved or
|
||||
// deleted recently, keyed by UUID.
|
||||
//
|
||||
// The API serves lookups and listings from read replicas which lag behind
|
||||
// writes, so an item can be returned from its old location for a while after
|
||||
// it was moved or deleted. Acting on such a stale entry can be destructive,
|
||||
// eg overwriting it would delete the file from where it was moved to.
|
||||
type recentChanges struct {
|
||||
mu sync.Mutex
|
||||
items map[string]location
|
||||
lastPrune time.Time
|
||||
}
|
||||
|
||||
// recent is shared by all the Fs in the process as UUIDs are global.
|
||||
var recent = recentChanges{items: map[string]location{}}
|
||||
|
||||
// set records the new location of uuid, pruning expired entries.
|
||||
func (r *recentChanges) set(uuid string, loc location) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
now := time.Now()
|
||||
if now.Sub(r.lastPrune) > staleWindow {
|
||||
for k, v := range r.items {
|
||||
if now.After(v.expires) {
|
||||
delete(r.items, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
r.lastPrune = now
|
||||
}
|
||||
loc.expires = now.Add(staleWindow)
|
||||
r.items[uuid] = loc
|
||||
}
|
||||
|
||||
// moved records that uuid was moved to name in parentUUID.
|
||||
func (r *recentChanges) moved(uuid, parentUUID, name string) {
|
||||
r.set(uuid, location{parentUUID: parentUUID, name: name})
|
||||
}
|
||||
|
||||
// deleted records that uuid was deleted.
|
||||
func (r *recentChanges) deleted(uuid string) {
|
||||
r.set(uuid, location{})
|
||||
}
|
||||
|
||||
// stale reports whether an API result saying uuid is called name in
|
||||
// parentUUID is out of date because of a change made by this process.
|
||||
func (r *recentChanges) stale(uuid, parentUUID, name string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
loc, ok := r.items[uuid]
|
||||
if !ok || time.Now().After(loc.expires) {
|
||||
return false
|
||||
}
|
||||
return loc.parentUUID != parentUUID || loc.name != name
|
||||
}
|
||||
|
||||
// findFile looks up a single file by name within directoryID. Returns
|
||||
// (nil, nil) when the file does not exist; surfaces transport/API errors.
|
||||
func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
|
||||
encodedName := f.opt.Encoding.FromStandardName(leaf)
|
||||
name, ext := splitNameExt(encodedName)
|
||||
checks := []files.FileExistenceCheck{existenceCheck(name, ext)}
|
||||
if encodedName != name {
|
||||
checks = append(checks, existenceCheck(encodedName, ""))
|
||||
}
|
||||
if legacyName, legacyExt := legacyNameExt(encodedName); legacyExt != "" && legacyName != name {
|
||||
checks = append(checks, existenceCheck(legacyName, legacyExt))
|
||||
}
|
||||
|
||||
var checkResult *files.CheckFilesExistenceResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, checks)
|
||||
if isNotFoundError(err) {
|
||||
return true, err
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var uuid string
|
||||
for _, result := range checkResult.Files {
|
||||
if !result.FileExists() || result.UUID == "" {
|
||||
continue
|
||||
}
|
||||
if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {
|
||||
fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)
|
||||
continue
|
||||
}
|
||||
if joinNameExt(result.PlainName, result.Type) == encodedName {
|
||||
uuid = result.UUID
|
||||
break
|
||||
}
|
||||
}
|
||||
if uuid == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var fileMeta *files.FileMeta
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
fileMeta, err = files.GetFileMeta(ctx, f.cfg, uuid)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fileMeta == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return convertFileMetaToFile(fileMeta), nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// convertFileMetaToFile converts files.FileMeta to folders.File
|
||||
@@ -761,9 +634,6 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range foldersList {
|
||||
if recent.stale(e.UUID, dirID, e.PlainName) {
|
||||
continue
|
||||
}
|
||||
remote := path.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
|
||||
out = append(out, fs.NewDir(remote, e.ModificationTime))
|
||||
}
|
||||
@@ -781,9 +651,6 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
if len(e.Type) > 0 {
|
||||
remote += "." + e.Type
|
||||
}
|
||||
if recent.stale(e.UUID, dirID, remote) {
|
||||
continue
|
||||
}
|
||||
remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))
|
||||
out = append(out, newObjectWithFile(f, remote, &e))
|
||||
}
|
||||
@@ -808,7 +675,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
return nil, err
|
||||
}
|
||||
|
||||
existingFile, err := f.findFile(ctx, leaf, directoryID)
|
||||
// Check if file already exists
|
||||
existingFile, err := f.preUploadCheck(ctx, leaf, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -856,28 +724,44 @@ func (f *Fs) Remove(ctx context.Context, remote string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recent.deleted(dirID)
|
||||
f.dirCache.FlushDir(remote)
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewObject creates a new object by looking up a single file's metadata.
|
||||
// NewObject creates a new object
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
parentDir := path.Dir(remote)
|
||||
|
||||
if parentDir == "." {
|
||||
parentDir = ""
|
||||
}
|
||||
file, err := f.findFile(ctx, leaf, directoryID)
|
||||
|
||||
dirID, err := f.dirCache.FindDir(ctx, parentDir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if file == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return newObjectWithFile(f, remote, file), nil
|
||||
|
||||
var files []folders.File
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
files, err = folders.ListAllFiles(ctx, f.cfg, dirID)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetName := path.Base(remote)
|
||||
for _, e := range files {
|
||||
name := e.PlainName
|
||||
if len(e.Type) > 0 {
|
||||
name += "." + e.Type
|
||||
}
|
||||
decodedName := f.opt.Encoding.ToStandardName(name)
|
||||
if decodedName == targetName {
|
||||
return newObjectWithFile(f, remote, &e), nil
|
||||
}
|
||||
}
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
// newObjectWithFile returns a new object by file info
|
||||
@@ -981,23 +865,19 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
newName, newType := splitNameExt(f.opt.Encoding.FromStandardName(leaf))
|
||||
// Parse name and extension from the leaf
|
||||
baseName := f.opt.Encoding.FromStandardName(leaf)
|
||||
newName := strings.TrimSuffix(baseName, path.Ext(baseName))
|
||||
newType := strings.TrimPrefix(path.Ext(baseName), ".")
|
||||
|
||||
// Move the file server-side
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err := files.MoveFile(ctx, f.cfg, srcObj.uuid, directoryID, newName, newType)
|
||||
// The move is checked against lagging read replicas so it can
|
||||
// fail with 404 if the destination directory was just created
|
||||
// or 409 if a file at the destination was just deleted.
|
||||
if err != nil && isStaleReadError(err, true) {
|
||||
return true, err
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))
|
||||
|
||||
dstObj := &Object{
|
||||
f: f,
|
||||
@@ -1033,17 +913,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
encodedLeaf := f.opt.Encoding.FromStandardName(dstLeaf)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
err := folders.MoveFolder(ctx, f.cfg, srcID, dstDirectoryID, encodedLeaf)
|
||||
// Moving a large folder can outlast the gateway timeout (520/502)
|
||||
// yet still complete, in which case the retry reports this.
|
||||
if err != nil && isConflictError(err) && strings.Contains(err.Error(), "already moved to that location") {
|
||||
return false, nil
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recent.moved(srcID, dstDirectoryID, encodedLeaf)
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
@@ -1091,7 +965,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
remote := o.remote
|
||||
|
||||
origName, origType := splitNameExt(path.Base(remote))
|
||||
origBaseName := path.Base(remote)
|
||||
origName := strings.TrimSuffix(origBaseName, path.Ext(origBaseName))
|
||||
origType := strings.TrimPrefix(path.Ext(origBaseName), ".")
|
||||
|
||||
// Create directory if it doesn't exist
|
||||
_, dirID, err := o.f.dirCache.FindPath(ctx, remote, true)
|
||||
@@ -1108,7 +984,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Step 1: If file exists, rename to backup (preserves old file during upload)
|
||||
if oldUUID != "" {
|
||||
name, ext := splitNameExt(path.Base(remote))
|
||||
// Generate unique backup name
|
||||
baseName := path.Base(remote)
|
||||
name := strings.TrimSuffix(baseName, path.Ext(baseName))
|
||||
ext := strings.TrimPrefix(path.Ext(baseName), ".")
|
||||
|
||||
backupSuffix := fmt.Sprintf(".rclone-backup-%s", random.String(8))
|
||||
backupName = o.f.opt.Encoding.FromStandardName(name + backupSuffix)
|
||||
backupType = ext
|
||||
@@ -1181,11 +1061,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return o.f.tooLargeError(remote, tooLarge)
|
||||
}
|
||||
|
||||
if err != nil && isStaleReadError(err, backupUUID != "") {
|
||||
o.restoreBackupFile(ctx, backupUUID, origName, origType)
|
||||
return fserrors.RetryError(err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
|
||||
}
|
||||
@@ -1208,7 +1083,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
err := o.f.pacer.Call(func() (bool, error) {
|
||||
err := files.DeleteFile(ctx, o.f.cfg, backupUUID)
|
||||
if err != nil {
|
||||
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
// Treat 404 (Not Found) and 204 (No Content) as success
|
||||
switch httpErr.StatusCode() {
|
||||
case 404, 204:
|
||||
@@ -1223,7 +1099,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
backupName, backupType, backupUUID, err)
|
||||
// Don't fail the upload just because backup deletion failed
|
||||
} else {
|
||||
recent.deleted(backupUUID)
|
||||
fs.Debugf(o.f, "Successfully deleted backup file")
|
||||
}
|
||||
}
|
||||
@@ -1243,19 +1118,6 @@ func isTimeoutError(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// isNotFoundError reports whether err is a 404 from the API.
|
||||
func isNotFoundError(err error) bool {
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
return errors.As(err, &httpErr) && httpErr.StatusCode() == 404
|
||||
}
|
||||
|
||||
// isStaleReadError reports whether err is the API contradicting a write this
|
||||
// backend has already made, which it does because lookups are served by read
|
||||
// replicas that lag behind the primary.
|
||||
func isStaleReadError(err error, renamed bool) bool {
|
||||
return isNotFoundError(err) || (renamed && isConflictError(err))
|
||||
}
|
||||
|
||||
// isConflictError checks if an error indicates a file conflict (409)
|
||||
func isConflictError(err error) bool {
|
||||
errMsg := err.Error()
|
||||
@@ -1274,7 +1136,8 @@ func isEmptyFileLimitError(err error) bool {
|
||||
// fileTooLargeError extracts the SDK's FileTooLargeError from a wrapped error
|
||||
// chain, returning it (or nil) so callers can branch on the size limit.
|
||||
func fileTooLargeError(err error) *sdkerrors.FileTooLargeError {
|
||||
if tooLarge, ok := errors.AsType[*sdkerrors.FileTooLargeError](err); ok {
|
||||
var tooLarge *sdkerrors.FileTooLargeError
|
||||
if errors.As(err, &tooLarge) {
|
||||
return tooLarge
|
||||
}
|
||||
return nil
|
||||
@@ -1296,22 +1159,40 @@ func (o *Object) recoverFromTimeoutConflict(ctx context.Context, uploadErr error
|
||||
}
|
||||
|
||||
baseName := path.Base(remote)
|
||||
encodedName := o.f.opt.Encoding.FromStandardName(baseName)
|
||||
|
||||
existingFile, err := o.f.findFile(ctx, baseName, dirID)
|
||||
if err != nil || existingFile == nil {
|
||||
var meta *buckets.CreateMetaResponse
|
||||
checkErr := o.f.pacer.Call(func() (bool, error) {
|
||||
existingFile, err := o.f.preUploadCheck(ctx, encodedName, dirID)
|
||||
if err != nil {
|
||||
return o.f.shouldRetry(ctx, err)
|
||||
}
|
||||
if existingFile != nil {
|
||||
name := strings.TrimSuffix(baseName, path.Ext(baseName))
|
||||
ext := strings.TrimPrefix(path.Ext(baseName), ".")
|
||||
|
||||
meta = &buckets.CreateMetaResponse{
|
||||
UUID: existingFile.UUID,
|
||||
FileID: existingFile.FileID,
|
||||
Name: name,
|
||||
PlainName: name,
|
||||
Type: ext,
|
||||
Size: existingFile.Size,
|
||||
}
|
||||
o.id = existingFile.FileID
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if checkErr != nil {
|
||||
return nil, uploadErr
|
||||
}
|
||||
|
||||
name, ext := splitNameExt(baseName)
|
||||
o.id = existingFile.FileID
|
||||
return &buckets.CreateMetaResponse{
|
||||
UUID: existingFile.UUID,
|
||||
FileID: existingFile.FileID,
|
||||
Name: name,
|
||||
PlainName: name,
|
||||
Type: ext,
|
||||
Size: existingFile.Size,
|
||||
}, nil
|
||||
if meta != nil {
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
return nil, uploadErr
|
||||
}
|
||||
|
||||
// restoreBackupFile restores a backup file after upload failure
|
||||
@@ -1329,13 +1210,8 @@ func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, or
|
||||
|
||||
// Remove deletes a file
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
err := o.f.pacer.Call(func() (bool, error) {
|
||||
return o.f.pacer.Call(func() (bool, error) {
|
||||
err := files.DeleteFile(ctx, o.f.cfg, o.uuid)
|
||||
return o.f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recent.deleted(o.uuid)
|
||||
return nil
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
package internxt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/internxt/rclone-adapter/config"
|
||||
"github.com/internxt/rclone-adapter/endpoints"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// storedFile is a file as the service holds it: a (plainName, type) pair
|
||||
// split however the client that uploaded it chose to split it.
|
||||
type storedFile struct {
|
||||
uuid string
|
||||
plainName string
|
||||
fileType string
|
||||
}
|
||||
|
||||
// wireType renders a stored type the way the service does: a file with no
|
||||
// extension comes back as JSON null, not an empty string.
|
||||
func wireType(fileType string) any {
|
||||
if fileType == "" {
|
||||
return nil
|
||||
}
|
||||
return fileType
|
||||
}
|
||||
|
||||
// existenceServer answers the existence check and file meta endpoints from
|
||||
// files, matching plainName exactly and only constraining type when the
|
||||
// criterion supplies one, as the service does.
|
||||
func existenceServer(t *testing.T, stored []storedFile) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if strings.HasSuffix(r.URL.Path, "/meta") {
|
||||
uuid := strings.Split(strings.TrimPrefix(r.URL.Path, "/drive/files/"), "/")[0]
|
||||
for _, f := range stored {
|
||||
if f.uuid == uuid {
|
||||
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
|
||||
"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
|
||||
}))
|
||||
return
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Files []struct {
|
||||
PlainName string `json:"plainName"`
|
||||
Type string `json:"type"`
|
||||
} `json:"files"`
|
||||
}
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
|
||||
var existent []map[string]any
|
||||
for _, f := range stored {
|
||||
for _, c := range req.Files {
|
||||
if c.PlainName == f.plainName && (c.Type == "" || c.Type == f.fileType) {
|
||||
existent = append(existent, map[string]any{
|
||||
"exists": true, "status": "EXISTS",
|
||||
"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"existentFiles": existent}))
|
||||
}))
|
||||
}
|
||||
|
||||
func testFs(t *testing.T, server *httptest.Server) *Fs {
|
||||
t.Helper()
|
||||
f := &Fs{
|
||||
opt: Options{Encoding: encoder.EncodeInvalidUtf8},
|
||||
cfg: &config.Config{
|
||||
HTTPClient: server.Client(),
|
||||
Endpoints: endpoints.NewConfig(server.URL),
|
||||
},
|
||||
}
|
||||
f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
|
||||
return f
|
||||
}
|
||||
|
||||
func TestSplitJoinNameExt(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
baseName string
|
||||
name string
|
||||
ext string
|
||||
}{
|
||||
{"foo.txt", "foo", "txt"},
|
||||
{"foo.tar.gz", "foo.tar", "gz"},
|
||||
{"README", "README", ""},
|
||||
{".bashrc", ".bashrc", ""},
|
||||
{"..hidden", ".", "hidden"},
|
||||
{"", "", ""},
|
||||
} {
|
||||
name, ext := splitNameExt(test.baseName)
|
||||
assert.Equal(t, test.name, name, test.baseName)
|
||||
assert.Equal(t, test.ext, ext, test.baseName)
|
||||
assert.Equal(t, test.baseName, joinNameExt(name, ext), test.baseName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFile(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
what string
|
||||
stored []storedFile
|
||||
leaf string
|
||||
wantUUID string
|
||||
}{{
|
||||
what: "split as rclone splits it",
|
||||
stored: []storedFile{{"uuid-1", "foo.tar", "gz"}},
|
||||
leaf: "foo.tar.gz",
|
||||
wantUUID: "uuid-1",
|
||||
}, {
|
||||
what: "whole name stored as plainName by another client",
|
||||
stored: []storedFile{{"uuid-2", "foo.tar.gz", ""}},
|
||||
leaf: "foo.tar.gz",
|
||||
wantUUID: "uuid-2",
|
||||
}, {
|
||||
what: "extensionless name is not confused with an extended one",
|
||||
stored: []storedFile{{"uuid-3", "README", "md"}, {"uuid-4", "README", ""}},
|
||||
leaf: "README",
|
||||
wantUUID: "uuid-4",
|
||||
}, {
|
||||
what: "dotfile stored the way the web client splits it",
|
||||
stored: []storedFile{{"uuid-6", ".bashrc", ""}},
|
||||
leaf: ".bashrc",
|
||||
wantUUID: "uuid-6",
|
||||
}, {
|
||||
what: "dotfile stored the way older rclone versions split it",
|
||||
stored: []storedFile{{"uuid-7", "", "bashrc"}},
|
||||
leaf: ".bashrc",
|
||||
wantUUID: "uuid-7",
|
||||
}, {
|
||||
what: "trailing dot kept in plainName by a rename",
|
||||
stored: []storedFile{{"uuid-8", "trailing.", ""}},
|
||||
leaf: "trailing.",
|
||||
wantUUID: "uuid-8",
|
||||
}, {
|
||||
what: "missing file",
|
||||
stored: []storedFile{{"uuid-5", "other", "txt"}},
|
||||
leaf: "foo.txt",
|
||||
}} {
|
||||
t.Run(test.what, func(t *testing.T) {
|
||||
server := existenceServer(t, test.stored)
|
||||
defer server.Close()
|
||||
file, err := testFs(t, server).findFile(context.Background(), test.leaf, "dir-uuid")
|
||||
require.NoError(t, err)
|
||||
if test.wantUUID == "" {
|
||||
assert.Nil(t, file)
|
||||
return
|
||||
}
|
||||
require.NotNil(t, file)
|
||||
assert.Equal(t, test.wantUUID, file.UUID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFileIgnoresStale(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
server := existenceServer(t, []storedFile{{"uuid-moved", "foo", "txt"}, {"uuid-deleted", "bar", "txt"}, {"uuid-renamed", "baz", "txt"}})
|
||||
defer server.Close()
|
||||
f := testFs(t, server)
|
||||
|
||||
recent.moved("uuid-moved", "other-dir-uuid", "foo.txt.bak")
|
||||
recent.deleted("uuid-deleted")
|
||||
recent.moved("uuid-renamed", "dir-uuid", "baz.txt")
|
||||
|
||||
file, err := f.findFile(ctx, "foo.txt", "dir-uuid")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, file, "moved away")
|
||||
|
||||
file, err = f.findFile(ctx, "bar.txt", "dir-uuid")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, file, "deleted")
|
||||
|
||||
file, err = f.findFile(ctx, "baz.txt", "dir-uuid")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, file, "moved here")
|
||||
assert.Equal(t, "uuid-renamed", file.UUID)
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/internxt/rclone-adapter/buckets"
|
||||
@@ -262,7 +263,9 @@ func (w *internxtChunkWriter) Close(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Create file metadata in Internxt Drive
|
||||
name, ext := splitNameExt(w.f.opt.Encoding.FromStandardName(path.Base(w.remote)))
|
||||
baseName := w.f.opt.Encoding.FromStandardName(path.Base(w.remote))
|
||||
name := strings.TrimSuffix(baseName, path.Ext(baseName))
|
||||
ext := strings.TrimPrefix(path.Ext(baseName), ".")
|
||||
|
||||
var meta *buckets.CreateMetaResponse
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
|
||||
@@ -34,7 +34,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
@@ -1925,9 +1924,8 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
|
||||
// nothing to clean up by default
|
||||
cleanup = func() {}
|
||||
|
||||
// don't cache small files on disk to reduce wear of the disk, but
|
||||
// spool unknown sized streams there as they can't be bounded in memory
|
||||
if size > threshold || size < 0 {
|
||||
// don't cache small files on disk to reduce wear of the disk
|
||||
if size > threshold {
|
||||
var tempFile *os.File
|
||||
|
||||
// create the cache file
|
||||
@@ -1957,15 +1955,15 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
|
||||
// replace the already read source with a reader of our cached file
|
||||
out = tempFile
|
||||
} else {
|
||||
// that's a small file, just read it into memory from the global pool
|
||||
rw := multipart.NewRW()
|
||||
cleanup = func() {
|
||||
_ = rw.Close()
|
||||
}
|
||||
if _, err = io.Copy(rw, teeReader); err != nil {
|
||||
// that's a small file, just read it into memory
|
||||
var inData []byte
|
||||
inData, err = io.ReadAll(teeReader)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
out = rw
|
||||
|
||||
// set the reader to our read memory block
|
||||
out = bytes.NewReader(inData)
|
||||
}
|
||||
return hex.EncodeToString(md5Hasher.Sum(nil)), out, cleanup, nil
|
||||
}
|
||||
|
||||
@@ -5,14 +5,12 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -30,16 +28,11 @@ func TestReadMD5(t *testing.T) {
|
||||
wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
for _, threshold := range []int64{512, 1024, 10 * 1024, 20 * 1024} {
|
||||
t.Run(fmt.Sprintf("%d", threshold), func(t *testing.T) {
|
||||
inUse := pool.Global().InUse()
|
||||
in := readers.NewPatternReader(size)
|
||||
gotMD5, out, cleanup, err := readMD5(in, size, threshold)
|
||||
defer cleanup()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
if size > threshold {
|
||||
assert.IsType(t, (*os.File)(nil), out, "big files should be spooled to disk")
|
||||
} else {
|
||||
assert.IsType(t, (*pool.RW)(nil), out, "small files should be buffered in the pool")
|
||||
}
|
||||
|
||||
// check md5hash of out
|
||||
hasher := md5.New()
|
||||
@@ -48,36 +41,10 @@ func TestReadMD5(t *testing.T) {
|
||||
assert.Equal(t, n, size)
|
||||
outMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
assert.Equal(t, wantMD5, outMD5)
|
||||
|
||||
// check cleanup returns the buffer to the pool
|
||||
cleanup()
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Unknown sized streams can't be bounded in memory so must be spooled to disk
|
||||
t.Run("UnknownSize", func(t *testing.T) {
|
||||
const size = 1024
|
||||
inUse := pool.Global().InUse()
|
||||
hasher := md5.New()
|
||||
_, err := io.Copy(hasher, readers.NewPatternReader(size))
|
||||
require.NoError(t, err)
|
||||
wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
|
||||
gotMD5, out, cleanup, err := readMD5(readers.NewPatternReader(size), -1, 10*size)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
assert.IsType(t, (*os.File)(nil), out)
|
||||
hasher.Reset()
|
||||
n, err := io.Copy(hasher, out)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(size), n)
|
||||
assert.Equal(t, wantMD5, fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
cleanup()
|
||||
assert.Equal(t, inUse, pool.Global().InUse())
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestMetadata(t *testing.T) {
|
||||
|
||||
@@ -11,10 +11,10 @@ package linkbox
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -35,9 +35,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -48,8 +46,7 @@ const (
|
||||
pacerBurst = 1
|
||||
linkboxAPIURL = "https://www.linkbox.to/api/open/"
|
||||
linkboxWebAPIURL = "https://www.linkbox.to/api/"
|
||||
rootID = "0" // ID of root directory
|
||||
prefixSize = 10 * 1024 * 1024 // the API wants the MD5 of the first 10 MiB of a file
|
||||
rootID = "0" // ID of root directory
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -670,21 +667,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
// readPrefix reads up to prefixSize bytes of in into a buffer from the
|
||||
// global pool returning it along with the hex MD5 of what was read.
|
||||
//
|
||||
// The caller must Close the returned buffer when done with it.
|
||||
func readPrefix(in io.Reader) (rw *pool.RW, md5sum string, err error) {
|
||||
rw = multipart.NewRW()
|
||||
hasher := md5.New()
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hasher), prefixSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, "", err
|
||||
}
|
||||
return rw, hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
@@ -719,13 +701,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
prefix, prefixMD5, err := readPrefix(in)
|
||||
first10m := io.LimitReader(in, 10_485_760)
|
||||
first10mBytes, err := io.ReadAll(first10m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Update err in reading file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = prefix.Close()
|
||||
}()
|
||||
|
||||
// get upload authorization (step 1)
|
||||
opts := &rest.Opts{
|
||||
@@ -735,7 +715,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
Parameters: url.Values{
|
||||
"token": {o.fs.opt.Token},
|
||||
"fileMd5ofPre10m": {prefixMD5},
|
||||
"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
|
||||
"fileSize": {itoa64(size)},
|
||||
},
|
||||
}
|
||||
@@ -777,7 +757,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return fmt.Errorf("head upload URL: %w", err)
|
||||
}
|
||||
|
||||
file := io.MultiReader(prefix, in)
|
||||
file := io.MultiReader(bytes.NewReader(first10mBytes), in)
|
||||
|
||||
opts.Method = "PUT"
|
||||
opts.Body = file
|
||||
@@ -817,7 +797,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
Parameters: url.Values{
|
||||
"token": {o.fs.opt.Token},
|
||||
"fileMd5ofPre10m": {prefixMD5},
|
||||
"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
|
||||
"fileSize": {itoa64(size)},
|
||||
"pid": {dirID},
|
||||
"diyName": {leaf},
|
||||
@@ -1039,7 +1019,8 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
|
||||
// intermittently returns an HTML challenge page with a 200
|
||||
// status instead of JSON. This surfaces as a JSON syntax error,
|
||||
// so retry it to let the pacer back off until the block lifts.
|
||||
if _, ok := errors.AsType[*json.SyntaxError](err); ok {
|
||||
var syntaxErr *json.SyntaxError
|
||||
if errors.As(err, &syntaxErr) {
|
||||
return true, err
|
||||
}
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package linkbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestReadPrefix(t *testing.T) {
|
||||
for _, size := range []int64{1, 1024, prefixSize - 1, prefixSize, prefixSize + 1, 3 * prefixSize} {
|
||||
t.Run(fmt.Sprintf("%d", size), func(t *testing.T) {
|
||||
inUse := pool.Global().InUse()
|
||||
content := bytes.Repeat([]byte("x"), int(size))
|
||||
for i := range content {
|
||||
content[i] = byte(i)
|
||||
}
|
||||
wantPrefix := content[:min(size, prefixSize)]
|
||||
wantMD5 := fmt.Sprintf("%x", md5.Sum(wantPrefix))
|
||||
|
||||
in := bytes.NewReader(content)
|
||||
rw, gotMD5, err := readPrefix(in)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
assert.Equal(t, int64(len(wantPrefix)), rw.Size())
|
||||
|
||||
// The prefix followed by the rest of in must reproduce the content
|
||||
got, err := io.ReadAll(io.MultiReader(rw, in))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, got)
|
||||
|
||||
require.NoError(t, rw.Close())
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
|
||||
})
|
||||
}
|
||||
}
|
||||
+38
-127
@@ -12,7 +12,6 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -37,10 +36,6 @@ import (
|
||||
const (
|
||||
devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
|
||||
useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
|
||||
// maxLinkTargetSize is the largest symlink target accepted when
|
||||
// translating a .rclonelink object, comfortably above any OS path
|
||||
// limit (Windows allows 32767 UTF-16 units).
|
||||
maxLinkTargetSize = 128 * 1024
|
||||
)
|
||||
|
||||
// timeType allows the user to choose what exactly ModTime() returns
|
||||
@@ -316,10 +311,10 @@ set this flag.`,
|
||||
Name: "fatal_if_no_space",
|
||||
Help: `Make out-of-space errors fatal during transfers.
|
||||
|
||||
When enabled, an out of space error while writing, creating a directory,
|
||||
or creating a file returns a fatal error so that rclone aborts rather
|
||||
than retrying the operation. Useful for backup scripts that should halt
|
||||
loudly on a full disk rather than spin retrying.`,
|
||||
When enabled, an ENOSPC error during a write returns a fatal error so
|
||||
that rclone aborts rather than retrying the operation. Useful for
|
||||
backup scripts that should halt loudly on a full disk rather than spin
|
||||
retrying.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
@@ -443,9 +438,6 @@ var (
|
||||
errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
|
||||
errLinksNeedsSuffix = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
|
||||
errPathEscapes = errors.New("file name is not a path within the local root - check the encoding")
|
||||
errSymlinkLoop = errors.New("loop detected: points to a parent directory")
|
||||
errLinkTargetTooLong = errors.New("symlink target is too long to be a path")
|
||||
errLinkNameInvalid = errors.New("translated link needs a file name before the \"" + fs.LinkSuffix + "\" suffix")
|
||||
)
|
||||
|
||||
// NewFs constructs an Fs from the path
|
||||
@@ -573,26 +565,30 @@ func (f *Fs) caseInsensitive() bool {
|
||||
return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
|
||||
}
|
||||
|
||||
// translateLink checks whether the remote is a translated link
|
||||
// and returns a new path, removing the suffix as needed,
|
||||
// It also returns whether this is a translated link at all
|
||||
//
|
||||
// for regular files, localPath is returned unchanged
|
||||
func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
|
||||
isTranslatedLink = strings.HasSuffix(remote, fs.LinkSuffix)
|
||||
newLocalPath = strings.TrimSuffix(localPath, fs.LinkSuffix)
|
||||
return newLocalPath, isTranslatedLink
|
||||
}
|
||||
|
||||
// newObject makes a half completed Object
|
||||
func (f *Fs) newObject(remote string) (*Object, error) {
|
||||
name, translatedLink := remote, false
|
||||
if f.opt.TranslateSymlinks {
|
||||
name, translatedLink = strings.CutSuffix(remote, fs.LinkSuffix)
|
||||
}
|
||||
if translatedLink {
|
||||
// Check this makes a valid native file name
|
||||
native := f.nativePath(name)
|
||||
switch native[strings.LastIndex(native, string(filepath.Separator))+1:] {
|
||||
case "", ".", "..":
|
||||
return nil, fserrors.NoRetryError(fmt.Errorf("%q: %w", remote, errLinkNameInvalid))
|
||||
}
|
||||
}
|
||||
// Check the name after any suffix removal
|
||||
localPath, err := f.localPath(name)
|
||||
translatedLink := false
|
||||
localPath, err := f.localPath(remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if f.opt.TranslateSymlinks {
|
||||
// Possibly receive a new name for localPath
|
||||
localPath, translatedLink = translateLink(remote, localPath)
|
||||
}
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
@@ -641,23 +637,15 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
}
|
||||
|
||||
// Create new directory object from the info passed in
|
||||
//
|
||||
// Directories are never translated links, so the name is used as is
|
||||
// whatever suffix it has.
|
||||
func (f *Fs) newDirectory(dir string, fi os.FileInfo) (*Directory, error) {
|
||||
localPath, err := f.localPath(dir)
|
||||
o, err := f.newObject(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &Directory{
|
||||
Object: Object{
|
||||
fs: f,
|
||||
remote: dir,
|
||||
path: localPath,
|
||||
},
|
||||
}
|
||||
d.setMetadata(fi)
|
||||
return d, nil
|
||||
o.setMetadata(fi)
|
||||
return &Directory{
|
||||
Object: *o,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
@@ -680,7 +668,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
if err != nil {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
var parents []os.FileInfo // fsDirPath and its parents, read on first use
|
||||
|
||||
fd, err := os.Open(fsDirPath)
|
||||
if err != nil {
|
||||
@@ -761,28 +748,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
if f.opt.FollowSymlinks && (mode&symlinkFlag) != 0 {
|
||||
localPath := filepath.Join(fsDirPath, name)
|
||||
fi, err = os.Stat(localPath)
|
||||
// A directory symlink pointing to a parent would be followed
|
||||
// until the OS returns ELOOP, which can take a very long time
|
||||
if err == nil && fi.IsDir() {
|
||||
if parents == nil {
|
||||
parents = statParents(fsDirPath)
|
||||
}
|
||||
if slices.ContainsFunc(parents, func(parent os.FileInfo) bool { return os.SameFile(fi, parent) }) {
|
||||
// Quietly skip loops the directory filters exclude as
|
||||
// the layer above wouldn't recurse into them
|
||||
if useFilter {
|
||||
if include, dirErr := filter.IncludeDirectory(ctx, f)(newRemote); dirErr == nil && !include {
|
||||
continue
|
||||
}
|
||||
}
|
||||
err = errSymlinkLoop
|
||||
}
|
||||
}
|
||||
// Quietly skip errors on excluded files and directories
|
||||
if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
|
||||
continue
|
||||
}
|
||||
if os.IsNotExist(err) || isCircularSymlinkError(err) || errors.Is(err, errSymlinkLoop) {
|
||||
if os.IsNotExist(err) || isCircularSymlinkError(err) {
|
||||
// Skip bad symlinks and circular symlinks
|
||||
err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
|
||||
fs.Errorf(newRemote, "Listing error: %v", err)
|
||||
@@ -815,12 +785,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
continue
|
||||
}
|
||||
fso, err := f.newObjectWithInfo(newRemote, fi)
|
||||
if errors.Is(err, errLinkNameInvalid) {
|
||||
// Skip a badly named ".rclonelink" file
|
||||
fs.Errorf(newRemote, "Listing error: %v", err)
|
||||
_ = accounting.Stats(ctx).Error(err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -833,21 +797,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// statParents returns the info for dirPath and each of its parents up
|
||||
// to the root of the file system, skipping any which can't be read.
|
||||
func statParents(dirPath string) (parents []os.FileInfo) {
|
||||
for {
|
||||
if fi, err := os.Stat(dirPath); err == nil {
|
||||
parents = append(parents, fi)
|
||||
}
|
||||
parent := filepath.Dir(dirPath)
|
||||
if parent == dirPath {
|
||||
return parents
|
||||
}
|
||||
dirPath = parent
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) cleanRemote(dir, filename string) (remote string) {
|
||||
if f.opt.UTFNorm {
|
||||
filename = norm.NFC.String(filename)
|
||||
@@ -865,19 +814,14 @@ func (f *Fs) cleanRemote(dir, filename string) (remote string) {
|
||||
return
|
||||
}
|
||||
|
||||
// nativePath returns name in the OS encoding with OS path separators,
|
||||
// relative to f.root, without cleaning it.
|
||||
func (f *Fs) nativePath(name string) string {
|
||||
return filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
|
||||
}
|
||||
|
||||
// localPath returns the OS path for the object called name, which is
|
||||
// always underneath f.root.
|
||||
//
|
||||
// It returns errPathEscapes if name resolves outside f.root which can
|
||||
// happen depending on the encoding.
|
||||
func (f *Fs) localPath(name string) (string, error) {
|
||||
localPath := filepath.Join(f.root, f.nativePath(name))
|
||||
native := filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
|
||||
localPath := filepath.Join(f.root, native)
|
||||
rel, err := filepath.Rel(f.root, localPath)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return "", fserrors.NoRetryError(fmt.Errorf("%q: %w", name, errPathEscapes))
|
||||
@@ -1690,17 +1634,12 @@ func isDiskFullError(err error) bool {
|
||||
return errors.Is(err, file.ErrDiskFull) || fserrors.IsErrNoSpace(err)
|
||||
}
|
||||
|
||||
func wrapFatalIfNoSpace(err error, enabled bool) error {
|
||||
if err != nil && enabled && isDiskFullError(err) {
|
||||
return fserrors.FatalError(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Update the object from in with modTime and size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
defer func() {
|
||||
err = wrapFatalIfNoSpace(err, o.fs.opt.FatalIfNoSpace)
|
||||
if err != nil && o.fs.opt.FatalIfNoSpace && isDiskFullError(err) {
|
||||
err = fserrors.FatalError(err)
|
||||
}
|
||||
}()
|
||||
var out io.WriteCloser
|
||||
var hasher *hash.MultiHasher
|
||||
@@ -1757,28 +1696,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
out = f
|
||||
} else {
|
||||
// The target is buffered in memory, so stop reading just past
|
||||
// the longest acceptable one rather than trust the source
|
||||
in = io.LimitReader(in, maxLinkTargetSize+1)
|
||||
out = nopWriterCloser{&symlinkData}
|
||||
}
|
||||
|
||||
// Calculate the hash of the object as we write it
|
||||
var w io.Writer = out
|
||||
// Calculate the hash of the object we are reading as we go along
|
||||
if hasher != nil {
|
||||
w = io.MultiWriter(w, hasher)
|
||||
in = io.TeeReader(in, hasher)
|
||||
}
|
||||
|
||||
_, err = io.Copy(w, in)
|
||||
_, err = io.Copy(out, in)
|
||||
closeErr := out.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
|
||||
if o.translatedLink {
|
||||
if err == nil && symlinkData.Len() > maxLinkTargetSize {
|
||||
err = fserrors.NoRetryError(errLinkTargetTooLong)
|
||||
}
|
||||
if err == nil {
|
||||
// Use the contents of the copied object to create a symlink,
|
||||
// without following or creating it through a planted symlink
|
||||
@@ -1829,30 +1761,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
var sparseWarning sync.Once
|
||||
|
||||
type fatalIfNoSpaceWriterAt struct {
|
||||
fs.WriterAtCloser
|
||||
enabled bool
|
||||
}
|
||||
|
||||
func (w *fatalIfNoSpaceWriterAt) WriteAt(p []byte, off int64) (n int, err error) {
|
||||
n, err = w.WriterAtCloser.WriteAt(p, off)
|
||||
return n, wrapFatalIfNoSpace(err, w.enabled)
|
||||
}
|
||||
|
||||
func (w *fatalIfNoSpaceWriterAt) Close() error {
|
||||
return wrapFatalIfNoSpace(w.WriterAtCloser.Close(), w.enabled)
|
||||
}
|
||||
|
||||
// OpenWriterAt opens with a handle for random access writes
|
||||
//
|
||||
// Pass in the remote desired and the size if known.
|
||||
//
|
||||
// It truncates any existing object
|
||||
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (writer fs.WriterAtCloser, err error) {
|
||||
defer func() {
|
||||
err = wrapFatalIfNoSpace(err, f.opt.FatalIfNoSpace)
|
||||
}()
|
||||
|
||||
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
|
||||
// Temporary Object under construction
|
||||
o, err := f.newObject(remote)
|
||||
if err != nil {
|
||||
@@ -1890,10 +1804,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (write
|
||||
}
|
||||
}
|
||||
|
||||
return &fatalIfNoSpaceWriterAt{
|
||||
WriterAtCloser: out,
|
||||
enabled: f.opt.FatalIfNoSpace,
|
||||
}, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// setMetadata sets the file info from the os.FileInfo passed in
|
||||
|
||||
@@ -60,7 +60,7 @@ func TestIsDiskFullError(t *testing.T) {
|
||||
// FatalIfNoSpace setting, returning the error from Update.
|
||||
func updateWithReader(t *testing.T, fatalIfNoSpace bool, readerErr error) error {
|
||||
t.Helper()
|
||||
r := fstest.NewRunIndividual(t)
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
f.opt.FatalIfNoSpace = fatalIfNoSpace
|
||||
|
||||
@@ -96,87 +96,3 @@ func TestUpdateFatalIfNoSpaceOnButNotDiskFull(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
assert.False(t, fserrors.IsFatalError(err), "non-disk-full errors must not be fatal regardless of option")
|
||||
}
|
||||
|
||||
type diskFullWriterAtCloser struct {
|
||||
writeErr error
|
||||
closeErr error
|
||||
n int
|
||||
data []byte
|
||||
offset int64
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (w *diskFullWriterAtCloser) WriteAt(p []byte, off int64) (int, error) {
|
||||
w.data = append([]byte(nil), p...)
|
||||
w.offset = off
|
||||
return w.n, w.writeErr
|
||||
}
|
||||
|
||||
func (w *diskFullWriterAtCloser) Close() error {
|
||||
w.closed = true
|
||||
return w.closeErr
|
||||
}
|
||||
|
||||
func testOpenWriterAtError(t *testing.T, injected error, diskFull bool) {
|
||||
t.Helper()
|
||||
for _, enabled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("enabled=%v", enabled), func(t *testing.T) {
|
||||
r := fstest.NewRunIndividual(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
f.opt.FatalIfNoSpace = enabled
|
||||
f.opt.NoPreAllocate = true
|
||||
writer, err := f.OpenWriterAt(context.Background(), "test.txt", 0)
|
||||
require.NoError(t, err)
|
||||
wrapped, ok := writer.(*fatalIfNoSpaceWriterAt)
|
||||
require.True(t, ok)
|
||||
require.NoError(t, wrapped.WriterAtCloser.Close())
|
||||
wantN := 2
|
||||
if injected == nil {
|
||||
wantN = 4
|
||||
}
|
||||
underlying := &diskFullWriterAtCloser{writeErr: injected, closeErr: injected, n: wantN}
|
||||
wrapped.WriterAtCloser = underlying
|
||||
n, err := writer.WriteAt([]byte("data"), 17)
|
||||
assert.Equal(t, wantN, n)
|
||||
assert.Equal(t, []byte("data"), underlying.data)
|
||||
assert.Equal(t, int64(17), underlying.offset)
|
||||
assert.ErrorIs(t, err, injected)
|
||||
assert.Equal(t, enabled && diskFull, fserrors.IsFatalError(err))
|
||||
err = writer.Close()
|
||||
assert.True(t, underlying.closed)
|
||||
assert.ErrorIs(t, err, injected)
|
||||
assert.Equal(t, enabled && diskFull, fserrors.IsFatalError(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenWriterAtFatalIfNoSpace(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
diskFull bool
|
||||
}{
|
||||
{"success", nil, false},
|
||||
{"unrelated", syscall.EPERM, false},
|
||||
{"ENOSPC", syscall.ENOSPC, true},
|
||||
{"ErrDiskFull", file.ErrDiskFull, true},
|
||||
{"wrapped", fmt.Errorf("write: %w", syscall.ENOSPC), true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) { testOpenWriterAtError(t, test.err, test.diskFull) })
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenWriterAtSetupError(t *testing.T) {
|
||||
for _, enabled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("enabled=%v", enabled), func(t *testing.T) {
|
||||
r := fstest.NewRunIndividual(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
f.opt.FatalIfNoSpace = enabled
|
||||
require.NoError(t, os.WriteFile(filepath.Join(r.LocalName, "parent"), nil, 0600))
|
||||
writer, err := f.OpenWriterAt(context.Background(), "parent/child", 0)
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, writer)
|
||||
assert.False(t, fserrors.IsFatalError(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
package local
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func TestUpdateFatalIfNoSpaceWindows(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"ERROR_DISK_FULL", windows.ERROR_DISK_FULL},
|
||||
{"openat PathError", &os.PathError{Op: "openat", Path: "test.txt", Err: windows.ERROR_DISK_FULL}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Run("off", func(t *testing.T) {
|
||||
err := updateWithReader(t, false, test.err)
|
||||
require.Error(t, err)
|
||||
assert.False(t, fserrors.IsFatalError(err))
|
||||
})
|
||||
t.Run("on", func(t *testing.T) {
|
||||
err := updateWithReader(t, true, test.err)
|
||||
require.Error(t, err)
|
||||
assert.True(t, fserrors.IsFatalError(err))
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenWriterAtFatalIfNoSpaceWindows(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
diskFull bool
|
||||
}{
|
||||
{"ERROR_DISK_FULL", windows.ERROR_DISK_FULL, true},
|
||||
{"ERROR_HANDLE_DISK_FULL", windows.ERROR_HANDLE_DISK_FULL, true},
|
||||
{"PathError", &os.PathError{Op: "write", Path: "test.txt", Err: windows.ERROR_DISK_FULL}, true},
|
||||
{"ERROR_ACCESS_DENIED", windows.ERROR_ACCESS_DENIED, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) { testOpenWriterAtError(t, test.err, test.diskFull) })
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package local
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
@@ -24,7 +23,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/file"
|
||||
@@ -99,19 +97,7 @@ func TestVerifyCopy(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
// skipIfNoSymlinks skips the test if this process can't create symlinks.
|
||||
//
|
||||
// Windows grants the privilege only to an elevated process or one running with
|
||||
// Developer Mode enabled, so an ordinary user gets ERROR_PRIVILEGE_NOT_HELD.
|
||||
func skipIfNoSymlinks(t *testing.T) {
|
||||
t.Helper()
|
||||
if err := os.Symlink("target", filepath.Join(t.TempDir(), "link")); err != nil {
|
||||
t.Skipf("Skipping as symlinks are unavailable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSymlink(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
@@ -272,29 +258,6 @@ func linksMode(f *Fs) {
|
||||
f.lstat = os.Lstat
|
||||
}
|
||||
|
||||
// TestSymlinkTargetTooLong checks that a .rclonelink object whose body is
|
||||
// far bigger than any path is refused without being buffered in memory,
|
||||
// and that nothing is created at the destination.
|
||||
func TestSymlinkTargetTooLong(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
// A source which never ends, so the read must be bounded
|
||||
src := object.NewStaticObjectInfo("big"+fs.LinkSuffix, fstest.Time("2001-02-03T04:05:10Z"), -1, true, nil, nil)
|
||||
in := readers.NewCountingReader(readers.NewPatternReader(1 << 40))
|
||||
_, err := f.Put(ctx, in, src)
|
||||
require.ErrorIs(t, err, errLinkTargetTooLong)
|
||||
assert.True(t, fserrors.IsNoRetryError(err))
|
||||
assert.LessOrEqual(t, in.BytesRead(), uint64(maxLinkTargetSize+1), "read more of the body than needed")
|
||||
|
||||
_, err = os.Lstat(filepath.Join(f.root, "big"))
|
||||
assert.True(t, os.IsNotExist(err), "nothing should be created for a refused symlink")
|
||||
}
|
||||
|
||||
// TestSymlinkEscapeWriteThroughBlocked mirrors the GHSA-cf44-9pgv-m4xc PoC: a
|
||||
// malicious --links source serves "pwn.rclonelink" whose body is a path outside
|
||||
// the destination, plus a sibling "pwn/authkeys" that sorts after it and would
|
||||
@@ -302,7 +265,6 @@ func TestSymlinkTargetTooLong(t *testing.T) {
|
||||
// faithful backup of the source) but must refuse to write through it, so
|
||||
// nothing lands outside the destination (CWE-59).
|
||||
func TestSymlinkEscapeWriteThroughBlocked(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// A directory outside the destination the attacker wants to write into
|
||||
@@ -338,7 +300,6 @@ func TestSymlinkEscapeWriteThroughBlocked(t *testing.T) {
|
||||
// re-validated against the root, so the write-through is refused and nothing
|
||||
// escapes.
|
||||
func TestSymlinkEscapeNestedBlocked(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
|
||||
evil := t.TempDir()
|
||||
@@ -390,7 +351,6 @@ func TestSymlinkEscapeConcurrent(t *testing.T) {
|
||||
// use: an in-tree symlink to a sibling directory can still be created and
|
||||
// written through, since that write stays inside the destination.
|
||||
func TestSymlinkInTreeWriteThroughWorks(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
@@ -646,154 +606,6 @@ func TestLocalPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestLinkSuffixEscapeBlocked checks that in --links mode a name which
|
||||
// passes the root check with its ".rclonelink" suffix can't escape the
|
||||
// destination once the suffix is removed: "...rclonelink" must not
|
||||
// become "..", the parent of the root.
|
||||
func TestLinkSuffixEscapeBlocked(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
outer := t.TempDir()
|
||||
newLinksFs := func(dir, encoding string) *Fs {
|
||||
fRaw, err := NewFs(ctx, "local", filepath.Join(outer, dir), configmap.Simple{"encoding": encoding})
|
||||
require.NoError(t, err)
|
||||
f := fRaw.(*Fs)
|
||||
linksMode(f)
|
||||
require.NoError(t, f.Mkdir(ctx, ""))
|
||||
return f
|
||||
}
|
||||
f := newLinksFs("dst", encoder.OS.String())
|
||||
s := newLinksFs("slash", "Slash")
|
||||
|
||||
// Stamp the parent of the destinations so a write to it shows up.
|
||||
before := fstest.Time("2011-12-25T12:59:59Z")
|
||||
require.NoError(t, os.Chtimes(outer, before, before))
|
||||
parentUnchanged := func() {
|
||||
t.Helper()
|
||||
fi, err := os.Lstat(outer)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, fi.ModTime().Equal(before), "the parent of the destination was modified")
|
||||
}
|
||||
|
||||
// A directory called "...rclonelink" is created under its own name
|
||||
// and its metadata goes there, not to the parent of the root which
|
||||
// trimming the suffix off its path would address.
|
||||
remote := ".." + fs.LinkSuffix
|
||||
metadata := fs.Metadata{"mtime": "2001-02-03T04:05:06Z"}
|
||||
_, err := f.MkdirMetadata(ctx, remote, metadata)
|
||||
require.NoError(t, err)
|
||||
parentUnchanged()
|
||||
|
||||
// A file called "...rclonelink" is a link called "..", which the
|
||||
// default encoding stores as fullwidth dots like any other name.
|
||||
require.NoError(t, putFile(ctx, f, remote, "target"))
|
||||
fi, err := os.Lstat(filepath.Join(f.root, ".."))
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, fi.Mode()&os.ModeSymlink)
|
||||
parentUnchanged()
|
||||
|
||||
// Ordinary translated links still resolve to the file without the suffix.
|
||||
o, err := f.newObject("sub/link" + fs.LinkSuffix)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, o.translatedLink)
|
||||
assert.Equal(t, filepath.Join(f.root, "sub", "link"), o.path)
|
||||
|
||||
// A name which merely contains the suffix is a regular file.
|
||||
o, err = f.newObject("sub/link" + fs.LinkSuffix + ".txt")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, o.translatedLink)
|
||||
assert.Equal(t, filepath.Join(f.root, "sub", "link"+fs.LinkSuffix+".txt"), o.path)
|
||||
|
||||
// An encoding without Dot leaves ".." as a real ".." once the suffix
|
||||
// is gone, so the name is refused instead.
|
||||
require.ErrorIs(t, putFile(ctx, s, remote, "PWNED"), errLinkNameInvalid)
|
||||
_, err = s.MkdirMetadata(ctx, remote, metadata)
|
||||
require.NoError(t, err)
|
||||
parentUnchanged()
|
||||
|
||||
// As is any other name which doesn't leave a file name once the
|
||||
// suffix is removed and the encoding applied.
|
||||
for _, name := range []string{
|
||||
remote,
|
||||
"sub/.." + fs.LinkSuffix,
|
||||
"sub/." + fs.LinkSuffix,
|
||||
"sub/.." + fs.LinkSuffix,
|
||||
"sub/sub2/.." + fs.LinkSuffix,
|
||||
"sub/." + fs.LinkSuffix,
|
||||
"sub/" + fs.LinkSuffix,
|
||||
fs.LinkSuffix,
|
||||
} {
|
||||
_, err = s.newObject(name)
|
||||
assert.ErrorIs(t, err, errLinkNameInvalid, name)
|
||||
assert.True(t, fserrors.IsNoRetryError(err), name)
|
||||
}
|
||||
|
||||
// The root check still applies to the name of the link.
|
||||
_, err = s.newObject("../link" + fs.LinkSuffix)
|
||||
require.ErrorIs(t, err, errPathEscapes)
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
// A backslash is a path separator on Windows, so an encoding
|
||||
// without BackSlash can hide ".." behind one.
|
||||
w := newLinksFs("win", "Slash,Dot")
|
||||
_, err = w.newObject(`sub\..` + fs.LinkSuffix)
|
||||
assert.ErrorIs(t, err, errLinkNameInvalid)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLinkSuffixListSkipsInvalidName checks that with --links a regular
|
||||
// file whose name is just the suffix is reported as an error and skipped
|
||||
// rather than failing the listing of its whole directory.
|
||||
func TestLinkSuffixListSkipsInvalidName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
when := fstest.Time("2001-02-03T04:05:06Z")
|
||||
r.WriteFile("sub/"+fs.LinkSuffix, "not a link", when)
|
||||
r.WriteFile("sub/file.txt", "hello", when)
|
||||
|
||||
accounting.Stats(ctx).ResetErrors()
|
||||
entries, err := f.List(ctx, "sub")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "sub/file.txt", entries[0].Remote())
|
||||
assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors())
|
||||
accounting.Stats(ctx).ResetErrors()
|
||||
}
|
||||
|
||||
// TestLinkSuffixDirectory checks that in --links mode a directory whose
|
||||
// name ends in ".rclonelink" is an ordinary directory. Only files are
|
||||
// translated into links, so the suffix must stay in its path rather
|
||||
// than making it address a sibling.
|
||||
func TestLinkSuffixDirectory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
dir := "sub" + fs.LinkSuffix
|
||||
want := filepath.Join(f.root, dir)
|
||||
mtime := fstest.Time("2001-02-03T04:05:06Z")
|
||||
|
||||
d, err := f.MkdirMetadata(ctx, dir, fs.Metadata{"mtime": mtime.Format(time.RFC3339Nano)})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, d.(*Directory).path)
|
||||
fi, err := os.Lstat(want)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, fi.IsDir())
|
||||
assert.True(t, fi.ModTime().Equal(mtime), "metadata was not written to the directory")
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
got, ok := entries[0].(*Directory)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, dir, got.Remote())
|
||||
assert.Equal(t, want, got.path)
|
||||
assert.False(t, got.translatedLink)
|
||||
}
|
||||
|
||||
func TestHashWithTypeNone(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
@@ -841,32 +653,6 @@ func TestHashOnUpdate(t *testing.T) {
|
||||
assert.Equal(t, "45685e95985e20822fb2538a522a5ccf", md5)
|
||||
}
|
||||
|
||||
// Test the hash cached by Update matches a HashesOption hint passed by the caller
|
||||
func TestHashOnUpdateWithHashOption(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
const filePath = "file.txt"
|
||||
when := time.Now()
|
||||
r.WriteFile(filePath, "x", when)
|
||||
f := r.Flocal.(*Fs)
|
||||
|
||||
o, err := f.NewObject(ctx, filePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := bytes.NewBufferString("content")
|
||||
src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)
|
||||
options := []fs.OpenOption{&fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}}
|
||||
require.NoError(t, o.Update(ctx, b, src, options...))
|
||||
|
||||
gotContent, err := os.ReadFile(filepath.Join(f.root, filePath))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "content", string(gotContent))
|
||||
|
||||
md5, err := o.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)
|
||||
}
|
||||
|
||||
// Test hashes on deleting an object
|
||||
func TestHashOnDelete(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
@@ -897,7 +683,6 @@ func TestHashOnDelete(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMetadata(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
const filePath = "metafile.txt"
|
||||
@@ -1174,7 +959,6 @@ func TestFilter(t *testing.T) {
|
||||
}
|
||||
|
||||
func testFilterSymlink(t *testing.T, copyLinks bool) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
@@ -1279,7 +1063,6 @@ func TestFilterSymlinkLinks(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCopySymlink(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
@@ -1349,121 +1132,3 @@ func TestCopySymlink(t *testing.T) {
|
||||
want = fstest.NewItem("dst2/file.txt", "hello world", when)
|
||||
fstest.CompareItems(t, []fs.DirEntry{dst}, []fstest.Item{want}, nil, f.precision, "")
|
||||
}
|
||||
|
||||
// TestCopyLinksLoop checks that -L/--copy-links follows a symlink to a
|
||||
// sibling directory but skips symlinks which point to the directory
|
||||
// being listed or one of its parents, as these would loop forever.
|
||||
func TestCopyLinksLoop(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
dir := t.TempDir()
|
||||
|
||||
// folder-A/link-to-B and folder-B/link-to-A point at each other
|
||||
// and folder-A/link-to-parent points at dir
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-A"), 0700))
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-B"), 0700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-A", "file-A"), []byte("A"), 0600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-B", "file-B"), []byte("B"), 0600))
|
||||
require.NoError(t, os.Symlink(filepath.Join("..", "folder-B"), filepath.Join(dir, "folder-A", "link-to-B")))
|
||||
require.NoError(t, os.Symlink(filepath.Join("..", "folder-A"), filepath.Join(dir, "folder-B", "link-to-A")))
|
||||
require.NoError(t, os.Symlink("..", filepath.Join(dir, "folder-A", "link-to-parent")))
|
||||
|
||||
wantRoot := []string{
|
||||
"folder-A",
|
||||
"folder-A/file-A",
|
||||
"folder-A/link-to-B",
|
||||
"folder-A/link-to-B/file-B",
|
||||
"folder-B",
|
||||
"folder-B/file-B",
|
||||
"folder-B/link-to-A",
|
||||
"folder-B/link-to-A/file-A",
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
root string
|
||||
exclude string
|
||||
want []string
|
||||
wantErrors int64
|
||||
}{{
|
||||
name: "Root",
|
||||
root: dir,
|
||||
want: wantRoot,
|
||||
// folder-A/link-to-parent, folder-A/link-to-B/link-to-A,
|
||||
// folder-B/link-to-A/link-to-B, folder-B/link-to-A/link-to-parent
|
||||
wantErrors: 4,
|
||||
}, {
|
||||
// link-to-parent points above the root of the Fs
|
||||
name: "SubDir",
|
||||
root: filepath.Join(dir, "folder-A"),
|
||||
want: []string{
|
||||
"file-A",
|
||||
"link-to-B",
|
||||
"link-to-B/file-B",
|
||||
},
|
||||
// link-to-parent, link-to-B/link-to-A
|
||||
wantErrors: 2,
|
||||
}, {
|
||||
// Excluded loops aren't reported
|
||||
name: "Excluded",
|
||||
root: dir,
|
||||
exclude: "link-to-parent/**",
|
||||
want: wantRoot,
|
||||
// folder-A/link-to-B/link-to-A, folder-B/link-to-A/link-to-B
|
||||
wantErrors: 2,
|
||||
}} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if test.exclude != "" {
|
||||
var fi *filter.Filter
|
||||
ctx, fi = filter.AddConfig(ctx)
|
||||
require.NoError(t, fi.AddRule("- "+test.exclude))
|
||||
}
|
||||
f, err := NewFs(ctx, "local", test.root, configmap.Simple{
|
||||
"copy_links": "true",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
accounting.Stats(ctx).ResetErrors()
|
||||
defer accounting.Stats(ctx).ResetErrors()
|
||||
|
||||
var got []string
|
||||
err = walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error {
|
||||
for _, entry := range entries {
|
||||
got = append(got, entry.Remote())
|
||||
}
|
||||
// Stop a runaway listing rather than waiting for ELOOP
|
||||
if len(got) > 100 {
|
||||
return errors.New("too many entries - symlink loop not detected")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, test.want, got)
|
||||
assert.Equal(t, test.wantErrors, accounting.Stats(ctx).GetErrors(), "global errors found")
|
||||
})
|
||||
}
|
||||
|
||||
// A single List reports and skips the link to a parent, but still
|
||||
// returns the link to a sibling
|
||||
t.Run("List", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f, err := NewFs(ctx, "local", dir, configmap.Simple{
|
||||
"copy_links": "true",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
accounting.Stats(ctx).ResetErrors()
|
||||
defer accounting.Stats(ctx).ResetErrors()
|
||||
|
||||
entries, err := f.List(ctx, "folder-A")
|
||||
require.NoError(t, err)
|
||||
var got []string
|
||||
for _, entry := range entries {
|
||||
got = append(got, entry.Remote())
|
||||
}
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, []string{"folder-A/file-A", "folder-A/link-to-B"}, got)
|
||||
assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
|
||||
})
|
||||
}
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -1615,6 +1614,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
|
||||
var (
|
||||
fileBuf []byte
|
||||
fileHash []byte
|
||||
newHash []byte
|
||||
slowHash bool
|
||||
@@ -1660,19 +1660,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Attempt to put by calculating hash in memory
|
||||
if trySpeedup && size <= int64(o.fs.opt.SpeedupMaxMem) {
|
||||
rw := multipart.NewRW()
|
||||
defer func() {
|
||||
_ = rw.Close()
|
||||
}()
|
||||
hasher := mrhash.New()
|
||||
if _, err = io.Copy(rw, io.TeeReader(in, hasher)); err != nil {
|
||||
fileBuf, err = io.ReadAll(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileHash = hasher.Sum(nil)
|
||||
fileHash = mrhash.Sum(fileBuf)
|
||||
if o.putByHash(ctx, fileHash, src, "memory") {
|
||||
return nil
|
||||
}
|
||||
wrapIn = rw
|
||||
wrapIn = bytes.NewReader(fileBuf)
|
||||
trySpeedup = false // speedup failed, force upload
|
||||
}
|
||||
|
||||
@@ -1706,8 +1702,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Upload object data
|
||||
if size <= mrhash.Size {
|
||||
// Optimize upload: skip extra request if data fits in the hash buffer.
|
||||
var fileBuf []byte
|
||||
fileBuf, err = io.ReadAll(wrapIn)
|
||||
if fileBuf == nil {
|
||||
fileBuf, err = io.ReadAll(wrapIn)
|
||||
}
|
||||
if fileHash == nil && err == nil {
|
||||
fileHash = mrhash.Sum(fileBuf)
|
||||
}
|
||||
@@ -1872,12 +1869,6 @@ func (o *Object) upload(ctx context.Context, in io.Reader, size int64, options .
|
||||
strHash string
|
||||
)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// Rewind seekable bodies so a retry resends the whole file
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
if _, err := seeker.Seek(0, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
res, err = o.fs.srv.Call(ctx, &opts)
|
||||
if err == nil {
|
||||
strHash, err = readBodyWord(res)
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
package mailru
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/mailru/mrhash"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
// TestUploadRetry checks that an upload from a seekable pooled buffer
|
||||
// whose first attempt fails with a retriable error is retried with the
|
||||
// full body rather than the drained reader.
|
||||
func TestUploadRetry(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("mailru"), 512)
|
||||
wantHash := mrhash.Sum(content)
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
if attempts.Add(1) == 1 {
|
||||
// Drop the connection so the client sees a retriable error
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
require.NoError(t, err)
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
assert.Equal(t, content, body, "retried upload should re-send the full body")
|
||||
_, _ = w.Write([]byte(hex.EncodeToString(wantHash) + "\n"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
srv: rest.NewClient(server.Client()),
|
||||
source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "token"}),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
|
||||
shardURL: server.URL,
|
||||
shardExpiry: time.Now().Add(time.Hour),
|
||||
}
|
||||
o := &Object{fs: f, remote: "remote"}
|
||||
|
||||
rw := multipart.NewRW()
|
||||
defer func() { _ = rw.Close() }()
|
||||
_, err := rw.Write(content)
|
||||
require.NoError(t, err)
|
||||
|
||||
gotHash, err := o.upload(ctx, rw, int64(len(content)))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantHash, gotHash)
|
||||
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
|
||||
}
|
||||
@@ -236,9 +236,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't parse URL %q: %w", opt.Endpoint, err)
|
||||
}
|
||||
u, err := rest.URLJoinRoot(base, root)
|
||||
u, err := rest.URLJoin(base, rest.URLPathEscape(root))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("couldn't join URL %q and %q: %w", base.String(), root, err)
|
||||
}
|
||||
client := fshttp.NewClient(ctx)
|
||||
|
||||
@@ -573,12 +573,10 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
fs.Errorf(nil, "Unable to parse URL %q: %v", f.endpointURL, err)
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
// A bad resume start must fail the listing rather than
|
||||
// return fs.ErrorDirNotFound which sync treats as an empty
|
||||
// directory
|
||||
resumeURL, err := rest.URLJoinRoot(u, resumeStart)
|
||||
resumeURL, err := rest.URLJoin(u, rest.URLPathEscape(resumeStart))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to join URL %q for resumeStart %q: %w", f.endpointURL, resumeStart, err)
|
||||
fs.Errorf(nil, "Unable to join URL %q for resumeStart %s: %v", f.endpointURL, resumeStart, err)
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
URL = resumeURL.String()
|
||||
}
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
package netstorage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestListResumeCannotChangeHost(t *testing.T) {
|
||||
// The resume start returned by a list request comes from the
|
||||
// server. It must not be able to move the signed continuation
|
||||
// request to another host.
|
||||
var requests int
|
||||
other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests++
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}))
|
||||
defer other.Close()
|
||||
otherHost := strings.TrimPrefix(other.URL, "http://")
|
||||
|
||||
var listRequests int
|
||||
configured := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
action := r.Header.Get("X-Akamai-ACS-Action")
|
||||
switch {
|
||||
case strings.Contains(action, "action=stat"):
|
||||
_, _ = w.Write([]byte(`<stat directory="/123456/dir"><file type="dir" name="dir" mtime="0"/></stat>`))
|
||||
case strings.Contains(action, "action=list"):
|
||||
listRequests++
|
||||
_, _ = w.Write([]byte(`<list><file type="file" name="123456/dir/a.txt" size="1" mtime="0"/><resume start="//` + otherHost + `/123456/dir/b.txt"/></list>`))
|
||||
default:
|
||||
http.Error(w, "unexpected action "+action, http.StatusBadRequest)
|
||||
}
|
||||
}))
|
||||
defer configured.Close()
|
||||
|
||||
m := configmap.Simple{
|
||||
"type": "netstorage",
|
||||
"protocol": "http",
|
||||
"host": strings.TrimPrefix(configured.URL, "http://") + "/123456/",
|
||||
"account": "account",
|
||||
"secret": obscure.MustObscure("secret"),
|
||||
}
|
||||
f, err := NewFs(context.Background(), "TestNetStorage", "dir", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
// The listing must fail loudly rather than with
|
||||
// fs.ErrorDirNotFound which sync treats as an empty directory
|
||||
err = f.(*Fs).ListR(context.Background(), "", func(entries fs.DirEntries) error { return nil })
|
||||
assert.ErrorContains(t, err, "must not change the scheme, host or user")
|
||||
assert.NotErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
assert.Equal(t, 1, listRequests, "list request sent to the configured host")
|
||||
assert.Equal(t, 0, requests, "request sent to the other host")
|
||||
}
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
const (
|
||||
dirMimeType = "inode/directory"
|
||||
timeFormatIn = time.RFC3339
|
||||
timeFormatOut = "2006-01-02T15:04:05.999Z" // keeps mS if present, though OneDrive only stores S
|
||||
timeFormatOut = "2006-01-02T15:04:05.999Z" // mS for OneDrive Personal, otherwise only S
|
||||
)
|
||||
|
||||
// system metadata keys which this backend owns
|
||||
@@ -32,17 +32,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||
ReadOnly: true,
|
||||
},
|
||||
"mtime": {
|
||||
Help: "Time of last modification with S accuracy.",
|
||||
Help: "Time of last modification with S accuracy (mS for OneDrive Personal).",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05Z",
|
||||
},
|
||||
"btime": {
|
||||
Help: "Time of file birth (creation) with S accuracy.",
|
||||
Help: "Time of file birth (creation) with S accuracy (mS for OneDrive Personal).",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05Z",
|
||||
},
|
||||
"utime": {
|
||||
Help: "Time of upload with S accuracy.",
|
||||
Help: "Time of upload with S accuracy (mS for OneDrive Personal).",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05Z",
|
||||
ReadOnly: true,
|
||||
@@ -113,7 +113,7 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||
ReadOnly: true,
|
||||
},
|
||||
"shared-time": {
|
||||
Help: "Time when the item was shared, with S accuracy.",
|
||||
Help: "Time when the item was shared, with S accuracy (mS for OneDrive Personal).",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05Z",
|
||||
ReadOnly: true,
|
||||
|
||||
+92
-169
@@ -37,9 +37,9 @@ import (
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -56,16 +56,7 @@ const (
|
||||
driveTypeSharepoint = "documentLibrary"
|
||||
defaultChunkSize = 10 * fs.Mebi
|
||||
chunkSizeMultiple = 320 * fs.Kibi
|
||||
defaultTenantAPIVersion = "v2.0"
|
||||
// maxSinglePartSize is the size at which Graph stops accepting an upload in a
|
||||
// single request (PUT /items/{id}/content). Microsoft documents this as
|
||||
// "250 MB", see
|
||||
// https://learn.microsoft.com/en-us/graph/api/driveitem-put-content
|
||||
// Measured against SharePoint Online the figure is binary and exclusive: a
|
||||
// body of 262143999 bytes is accepted, one of 262144000 is not. That matches
|
||||
// upload_cutoff being an exclusive threshold in Update below, so a cutoff of
|
||||
// exactly 250Mi sends everything the server would refuse to multipart.
|
||||
maxSinglePartSize = 250 * fs.Mebi
|
||||
maxSinglePartSize = 4 * fs.Mebi
|
||||
|
||||
regionGlobal = "global"
|
||||
regionUS = "us"
|
||||
@@ -167,7 +158,7 @@ See: https://github.com/rclone/rclone/issues/1716
|
||||
Name: "tenant_url",
|
||||
Help: `The tenant URL for non-admin OneDrive access.
|
||||
|
||||
Set this to your SharePoint tenant URL to use the SharePoint API
|
||||
Set this to your SharePoint tenant URL to use the SharePoint v2.0 API
|
||||
endpoint instead of the standard Microsoft Graph API. This allows
|
||||
accessing business OneDrive without admin consent.
|
||||
|
||||
@@ -178,16 +169,6 @@ for "driveAccessToken" in the network requests. Look for the
|
||||
Example: https://your-tenant.sharepoint.com/_api`,
|
||||
Default: "",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "tenant_api_version",
|
||||
Help: `The SharePoint API version to use with tenant_url.
|
||||
|
||||
Set this to the SharePoint API version matching the browser-extracted
|
||||
access token. For example, use v2.1 with a driveAccessTokenV21 token.
|
||||
|
||||
This only applies when tenant_url is set.`,
|
||||
Default: defaultTenantAPIVersion,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
|
||||
@@ -294,13 +275,16 @@ cases, rclone will fall back to normal copy (which will be slightly slower).`,
|
||||
Default: false,
|
||||
Help: `Remove all versions on modifying operations.
|
||||
|
||||
Onedrive creates versions when rclone uploads new files
|
||||
Onedrive for business creates versions when rclone uploads new files
|
||||
overwriting an existing one and when it sets the modification time.
|
||||
|
||||
These versions take up space out of the quota.
|
||||
|
||||
This flag checks for versions after file upload and setting
|
||||
modification time and removes all but the last version.
|
||||
|
||||
**NB** Onedrive personal can't currently delete versions so don't use
|
||||
this flag there.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -310,7 +294,8 @@ modification time and removes all but the last version.
|
||||
Normally files will get sent to the recycle bin on deletion. Setting
|
||||
this flag causes them to be permanently deleted. Use with care.
|
||||
|
||||
This works with OneDrive for Business, SharePoint document libraries, and OneDrive personal accounts, including free accounts.
|
||||
OneDrive personal accounts do not support the permanentDelete API,
|
||||
it only applies to OneDrive for Business and SharePoint document libraries.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
@@ -339,18 +324,14 @@ This works with OneDrive for Business, SharePoint document libraries, and OneDri
|
||||
Help: "Creates a read-write link to the item.",
|
||||
}, {
|
||||
Value: "embed",
|
||||
Help: "Creates an embeddable link to the item.\nOnly available in OneDrive personal.",
|
||||
Help: "Creates an embeddable link to the item.",
|
||||
}},
|
||||
}, {
|
||||
Name: "link_password",
|
||||
Default: "",
|
||||
Help: `Set the password for links created by the link command.
|
||||
|
||||
At the time of writing this works with OneDrive for Business and
|
||||
OneDrive personal paid accounts.
|
||||
|
||||
OneDrive personal free accounts can't set a password or an expiry time
|
||||
(with --expire) on links.
|
||||
At the time of writing this only works with OneDrive personal paid accounts.
|
||||
`,
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
@@ -438,12 +419,21 @@ Setting this flag speeds up these things greatly:
|
||||
rclone size onedrive:
|
||||
rclone rc vfs/refresh recursive=true
|
||||
|
||||
Rclone asks for the delta listing of the directory being listed. If
|
||||
the drive only supports delta listings at the root of the drive (as
|
||||
some older OneDrive for Business and SharePoint drives do) then
|
||||
rclone lists from the root and discards all the data that is not
|
||||
under the directory you asked for. So it will be correct but may not
|
||||
be very efficient.
|
||||
**However** the delta listing API **only** works at the root of the
|
||||
drive. If you use it not at the root then it recurses from the root
|
||||
and discards all the data that is not under the directory you asked
|
||||
for. So it will be correct but may not be very efficient.
|
||||
|
||||
This is why this flag is not set as the default.
|
||||
|
||||
As a rule of thumb if nearly all of your data is under rclone's root
|
||||
directory (the |root/directory| in |onedrive:root/directory|) then
|
||||
using this flag will be a big performance win. If your data is
|
||||
mostly not under the root then using this flag will be a big
|
||||
performance loss.
|
||||
|
||||
It is recommended if you are mounting your onedrive at the root
|
||||
(or near the root when using crypt) and using rclone |rc vfs/refresh|.
|
||||
`, "|", "`"),
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -512,20 +502,12 @@ func getRegionURL(m configmap.Mapper) (region, graphURL string) {
|
||||
// Check if tenant_url is provided for non-admin mode
|
||||
tenantURL, _ := m.Get("tenant_url")
|
||||
if tenantURL != "" {
|
||||
tenantAPIVersion, _ := m.Get("tenant_api_version")
|
||||
graphURL = tenantAPIEndpoint(tenantURL, tenantAPIVersion)
|
||||
graphURL = tenantURL + "/v2.0"
|
||||
}
|
||||
|
||||
return region, graphURL
|
||||
}
|
||||
|
||||
func tenantAPIEndpoint(tenantURL, tenantAPIVersion string) string {
|
||||
if tenantAPIVersion == "" {
|
||||
tenantAPIVersion = defaultTenantAPIVersion
|
||||
}
|
||||
return strings.TrimRight(tenantURL, "/") + "/" + strings.TrimLeft(tenantAPIVersion, "/")
|
||||
}
|
||||
|
||||
// Config for chooseDrive
|
||||
type chooseDriveOpt struct {
|
||||
opts rest.Opts
|
||||
@@ -819,7 +801,6 @@ type Options struct {
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
TenantURL string `config:"tenant_url"`
|
||||
TenantAPIVersion string `config:"tenant_api_version"`
|
||||
DriveID string `config:"drive_id"`
|
||||
DriveType string `config:"drive_type"`
|
||||
RootFolderID string `config:"root_folder_id"`
|
||||
@@ -1127,7 +1108,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID
|
||||
|
||||
if opt.TenantURL != "" {
|
||||
rootURL = tenantAPIEndpoint(opt.TenantURL, opt.TenantAPIVersion) + "/drives/" + opt.DriveID
|
||||
rootURL = opt.TenantURL + "/v2.0" + "/drives/" + opt.DriveID
|
||||
}
|
||||
|
||||
oauthConfig, err := makeOauthConfig(ctx, opt)
|
||||
@@ -1414,8 +1395,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, dir string, info *api.Item) (en
|
||||
id := info.GetID()
|
||||
f.dirCache.Put(remote, id)
|
||||
d := f.newDir(id, remote)
|
||||
// OneDrive reports the total size of a folder's contents
|
||||
d.size = info.GetSize()
|
||||
d.items = folder.ChildCount
|
||||
f.setSystemMetadata(info, d.meta, remote, dirMimeType)
|
||||
entry = d
|
||||
@@ -1504,6 +1483,10 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
return err
|
||||
}
|
||||
|
||||
// ListR only works at the root of a onedrive, not on a folder
|
||||
// So we have to filter things outside of the root which is
|
||||
// inefficient.
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
|
||||
// list a folder conventionally - used for shared folders
|
||||
@@ -1579,26 +1562,17 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
return nil
|
||||
}
|
||||
|
||||
listDelta := func(opts rest.Opts) error {
|
||||
opts.Parameters = url.Values{
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/root/delta",
|
||||
Parameters: map[string][]string{
|
||||
// "token": {token},
|
||||
"$top": {fmt.Sprintf("%d", f.opt.ListChunk)},
|
||||
}
|
||||
var result api.DeltaResponse
|
||||
return f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
|
||||
},
|
||||
}
|
||||
|
||||
err = listDelta(f.newOptsCall(directoryID, "GET", "/delta"))
|
||||
// Some drives only support delta listings at the root of the
|
||||
// drive, in which case list the whole drive and filter out the
|
||||
// items outside dir.
|
||||
var apiErr *api.Error
|
||||
if err != nil && len(seen) == 0 && errors.As(err, &apiErr) {
|
||||
fs.Debugf(f, "Delta listing of directory failed, listing from the root of the drive instead: %v", err)
|
||||
err = listDelta(rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/root/delta",
|
||||
})
|
||||
}
|
||||
var result api.DeltaResponse
|
||||
err = f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2774,26 +2748,12 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
|
||||
position := int64(0)
|
||||
for remaining > 0 {
|
||||
n := min(remaining, int64(o.fs.opt.ChunkSize))
|
||||
// Buffer the chunk in memory from the global pool so it can be
|
||||
// re-sent (or partly re-sent after a 416) on retry
|
||||
rw := multipart.NewRW()
|
||||
_, err = io.CopyN(rw, in, n)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
if err == io.EOF {
|
||||
err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, position, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
seg := readers.NewRepeatableReader(io.LimitReader(in, n))
|
||||
fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
|
||||
info, err = o.uploadFragment(ctx, uploadURL, position, size, rw, n, options...)
|
||||
closeErr := rw.Close()
|
||||
info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
remaining -= n
|
||||
position += n
|
||||
}
|
||||
@@ -2812,13 +2772,13 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
|
||||
return info, o.setMetaData(info)
|
||||
}
|
||||
|
||||
// Update the content of a remote file smaller than maxSinglePartSize in one single request
|
||||
// Update the content of a remote file within 4 MiB size in one single request
|
||||
// (currently only used when size is exactly 0)
|
||||
// This function will set modtime and metadata after uploading, which will create a new version for the remote file
|
||||
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
|
||||
size := src.Size()
|
||||
if size < 0 || size >= int64(maxSinglePartSize) {
|
||||
return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and < %v", maxSinglePartSize)
|
||||
if size < 0 || size > int64(maxSinglePartSize) {
|
||||
return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)
|
||||
}
|
||||
|
||||
fs.Debugf(o, "Starting singlepart upload")
|
||||
@@ -2937,7 +2897,7 @@ func (o *Object) ID() string {
|
||||
func (f *Fs) parseNormalizedID(ID string) (string, string, string) {
|
||||
var rootURL string
|
||||
if f.opt.TenantURL != "" {
|
||||
rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
|
||||
rootURL = f.opt.TenantURL + "/v2.0/drives"
|
||||
} else {
|
||||
rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
|
||||
}
|
||||
@@ -3105,15 +3065,9 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}
|
||||
case <-tickerC:
|
||||
fs.Debugf(f, "Checking for changes on remote")
|
||||
var newDeltaToken string
|
||||
newDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
|
||||
nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
|
||||
if err != nil {
|
||||
// Keep the token we already have: replacing it with the
|
||||
// empty token from a failed poll would reset the delta
|
||||
// listing to the start and lose every change in between.
|
||||
fs.Infof(f, "Change notify listener failure: %s", err)
|
||||
} else {
|
||||
nextDeltaToken = newDeltaToken
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3121,7 +3075,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, "latest", "")
|
||||
delta, err := f.changeNotifyNextChange(ctx, "latest")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -3133,37 +3087,18 @@ func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken str
|
||||
return
|
||||
}
|
||||
|
||||
// changeNotifyNextChange fetches one page of the drive delta listing.
|
||||
//
|
||||
// Pass the delta token to resume from in token, or the @odata.nextLink URL
|
||||
// returned by the previous page in nextLink. Exactly one of the two should
|
||||
// be set.
|
||||
func (f *Fs) changeNotifyNextChange(ctx context.Context, token, nextLink string) (delta api.DeltaResponse, err error) {
|
||||
opts := f.buildDriveDeltaOpts(token, nextLink)
|
||||
func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta api.DeltaResponse, err error) {
|
||||
opts := f.buildDriveDeltaOpts(token)
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var resp *http.Response
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// buildDriveDeltaOpts builds the request for one page of the drive delta
|
||||
// listing. nextLink, when set, is the complete @odata.nextLink URL from the
|
||||
// previous page and is used as-is, overriding the drive root.
|
||||
func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
|
||||
if nextLink != "" {
|
||||
return rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: nextLink,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
|
||||
var rootURL string
|
||||
if f.opt.TenantURL != "" {
|
||||
rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
|
||||
rootURL = f.opt.TenantURL + "/v2.0/drives"
|
||||
} else {
|
||||
rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
|
||||
}
|
||||
@@ -3176,58 +3111,46 @@ func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
|
||||
}
|
||||
}
|
||||
|
||||
// changeNotifyRunner polls for changes and calls notifyFunc for each one.
|
||||
//
|
||||
// It walks every page of the change set and returns the token to resume from
|
||||
// next time. The returned token is only valid when err is nil: the caller
|
||||
// must keep the token it passed in on error, otherwise a failed poll would
|
||||
// reset the delta listing to the start and lose the changes in between.
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (string, error) {
|
||||
token, nextLink := deltaToken, ""
|
||||
for {
|
||||
delta, err := f.changeNotifyNextChange(ctx, token, nextLink)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, item := range delta.Value {
|
||||
isDriveRootFolder := item.GetParentReference().ID == ""
|
||||
if isDriveRootFolder {
|
||||
continue
|
||||
}
|
||||
|
||||
fullPath, err := getItemFullPath(&item)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get item full path: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if fullPath == f.root {
|
||||
continue
|
||||
}
|
||||
|
||||
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
|
||||
if !insideRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
if item.GetFile() != nil {
|
||||
notifyFunc(relName, fs.EntryObject)
|
||||
} else if item.GetFolder() != nil {
|
||||
notifyFunc(relName, fs.EntryDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
if delta.NextLink == "" {
|
||||
// end of the change set - the deltaLink carries the resume token
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return parsedURL.Query().Get("token"), nil
|
||||
}
|
||||
nextLink = delta.NextLink
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, deltaToken)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
nextDeltaToken = parsedURL.Query().Get("token")
|
||||
|
||||
for _, item := range delta.Value {
|
||||
isDriveRootFolder := item.GetParentReference().ID == ""
|
||||
if isDriveRootFolder {
|
||||
continue
|
||||
}
|
||||
|
||||
fullPath, err := getItemFullPath(&item)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get item full path: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if fullPath == f.root {
|
||||
continue
|
||||
}
|
||||
|
||||
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
|
||||
if !insideRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
if item.GetFile() != nil {
|
||||
notifyFunc(relName, fs.EntryObject)
|
||||
} else if item.GetFolder() != nil {
|
||||
notifyFunc(relName, fs.EntryDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func getItemFullPath(item *api.Item) (fullPath string, err error) {
|
||||
|
||||
@@ -4,11 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -18,10 +15,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -48,40 +42,6 @@ func TestMain(m *testing.M) {
|
||||
fstest.TestMain(m)
|
||||
}
|
||||
|
||||
func TestTenantAPIEndpoint(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
tenantURL string
|
||||
tenantAPIVersion string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "default",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.0",
|
||||
}, {
|
||||
name: "explicit v2.1",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api",
|
||||
tenantAPIVersion: "v2.1",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.1",
|
||||
}, {
|
||||
name: "future version",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api",
|
||||
tenantAPIVersion: "v2.2",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.2",
|
||||
}, {
|
||||
name: "trailing slash",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api/",
|
||||
tenantAPIVersion: "/v2.1",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.1",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, tenantAPIEndpoint(tc.tenantURL, tc.tenantAPIVersion))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sharingRefused caches whether the remote refuses sharing invitations.
|
||||
var sharingRefused *bool
|
||||
|
||||
@@ -556,30 +516,6 @@ func (f *Fs) resetTestDefaults(r *fstest.Run) {
|
||||
r.Finalise()
|
||||
}
|
||||
|
||||
// TestCheckUploadCutoff checks the bounds of the --onedrive-upload-cutoff
|
||||
// validation, in particular that the documented API maximum is accepted.
|
||||
func TestCheckUploadCutoff(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in fs.SizeSuffix
|
||||
wantErr bool
|
||||
}{
|
||||
{in: 0, wantErr: false},
|
||||
{in: 4 * fs.Mebi, wantErr: false},
|
||||
// maxSinglePartSize is a legal cutoff even though the server refuses a
|
||||
// body of that size, because the cutoff is an exclusive threshold: at
|
||||
// this setting a file of exactly maxSinglePartSize goes multipart.
|
||||
{in: maxSinglePartSize, wantErr: false},
|
||||
{in: maxSinglePartSize + 1, wantErr: true},
|
||||
} {
|
||||
err := checkUploadCutoff(test.in)
|
||||
if test.wantErr {
|
||||
assert.Error(t, err, test.in.String())
|
||||
} else {
|
||||
assert.NoError(t, err, test.in.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// InternalTest dispatches all internal tests
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
newTestF := func() (*Fs, *fstest.Run) {
|
||||
@@ -614,159 +550,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// newChangeNotifyTestFs returns an Fs whose delta listing is served by
|
||||
// handler on a local test server, plus the server itself. The tenant URL
|
||||
// is pointed at the server so no request leaves the test.
|
||||
func newChangeNotifyTestFs(t *testing.T, handler http.Handler) (*Fs, *httptest.Server) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
ci: fs.GetConfig(ctx),
|
||||
root: "",
|
||||
driveID: "testdrive",
|
||||
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
|
||||
}
|
||||
f.opt.TenantURL = server.URL
|
||||
f.opt.TenantAPIVersion = defaultTenantAPIVersion
|
||||
return f, server
|
||||
}
|
||||
|
||||
// deltaItem builds a delta entry for a file named name inside a folder.
|
||||
func deltaItem(name, folder string) api.Item {
|
||||
return api.Item{
|
||||
ID: name,
|
||||
Name: name,
|
||||
File: &api.FileFacet{},
|
||||
ParentReference: &api.ItemReference{
|
||||
ID: "parentid",
|
||||
Path: "driveid:/" + folder,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestChangeNotifyRetriesAndKeepsToken covers a transient failure of the
|
||||
// delta poll: the request must go through the pacer so it is retried, and
|
||||
// a poll which fails outright must not hand back an empty token, since the
|
||||
// caller stores whatever it returns. An empty token would reset the delta
|
||||
// listing to the start and silently lose every change in between.
|
||||
func TestChangeNotifyRetriesAndKeepsToken(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var attempts atomic.Int32
|
||||
var tokens []string
|
||||
fail := true
|
||||
f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts.Add(1)
|
||||
tokens = append(tokens, r.URL.Query().Get("token"))
|
||||
if fail {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
|
||||
"code": "serviceNotAvailable",
|
||||
"message": "Service unavailable",
|
||||
}})
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
|
||||
Value: []api.Item{deltaItem("file1.txt", "folder")},
|
||||
DeltaLink: "https://example.com/delta?token=newtoken",
|
||||
})
|
||||
}))
|
||||
|
||||
var notified []string
|
||||
notifyFunc := func(path string, entryType fs.EntryType) {
|
||||
notified = append(notified, path)
|
||||
}
|
||||
|
||||
// The poll fails, but is retried by the pacer first
|
||||
token, err := f.changeNotifyRunner(ctx, notifyFunc, "tok1")
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, token, "a failed poll must not return a token")
|
||||
assert.Equal(t, fs.GetConfig(ctx).LowLevelRetries, int(attempts.Load()), "the failed poll should have been retried by the pacer")
|
||||
assert.Equal(t, slices.Repeat([]string{"tok1"}, int(attempts.Load())), tokens, "every retry should resume from the same token")
|
||||
assert.Empty(t, notified)
|
||||
|
||||
// Keeping the previous token (as ChangeNotify does) resumes correctly
|
||||
fail = false
|
||||
attempts.Store(0)
|
||||
tokens = nil
|
||||
token, err = f.changeNotifyRunner(ctx, notifyFunc, "tok1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "newtoken", token)
|
||||
assert.Equal(t, []string{"tok1"}, tokens, "the second poll should resume from the retained token")
|
||||
assert.Equal(t, []string{"folder/file1.txt"}, notified)
|
||||
}
|
||||
|
||||
// TestChangeNotifyFollowsNextLink covers a change set which spans more than
|
||||
// one page. Graph returns @odata.nextLink and no @odata.deltaLink until the
|
||||
// last page, so a runner which stops after the first page both loses the
|
||||
// remaining changes and ends up with an empty resume token.
|
||||
func TestChangeNotifyFollowsNextLink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var urls []string
|
||||
var server *httptest.Server
|
||||
f, server := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
urls = append(urls, r.URL.RequestURI())
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch len(urls) {
|
||||
case 1:
|
||||
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
|
||||
Value: []api.Item{deltaItem("page1.txt", "folder")},
|
||||
NextLink: server.URL + "/nextpage?$skiptoken=abc",
|
||||
})
|
||||
case 2:
|
||||
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
|
||||
Value: []api.Item{deltaItem("page2.txt", "folder")},
|
||||
DeltaLink: "https://example.com/delta?token=lasttoken",
|
||||
})
|
||||
default:
|
||||
t.Errorf("unexpected extra request to %s", r.URL)
|
||||
}
|
||||
}))
|
||||
|
||||
var notified []string
|
||||
token, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
|
||||
notified = append(notified, path)
|
||||
}, "tok1")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "lasttoken", token, "the resume token should come from the last page")
|
||||
assert.Equal(t, []string{"folder/page1.txt", "folder/page2.txt"}, notified, "changes from every page should be notified")
|
||||
require.Len(t, urls, 2, "the nextLink page should have been fetched")
|
||||
assert.Contains(t, urls[0], "/testdrive/root/delta")
|
||||
assert.Contains(t, urls[0], "token=tok1")
|
||||
assert.Equal(t, "/nextpage?$skiptoken=abc", urls[1], "the nextLink should be used as-is, without the delta token")
|
||||
}
|
||||
|
||||
// TestListDirectorySize checks that a listed directory carries the size
|
||||
// OneDrive reports for it, the total size of its contents, instead of -1.
|
||||
func TestListDirectorySize(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/items/rootid/children", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(api.ListChildrenResponse{
|
||||
Value: []api.Item{{
|
||||
ID: "dirid",
|
||||
Name: "photos",
|
||||
Size: 2646204760,
|
||||
Folder: &api.FolderFacet{ChildCount: 3},
|
||||
}},
|
||||
})
|
||||
}))
|
||||
f.dirCache = dircache.New("", "rootid", f)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
dir, ok := entries[0].(fs.Directory)
|
||||
require.True(t, ok, "entry should be a directory")
|
||||
assert.Equal(t, "photos", dir.Remote())
|
||||
assert.Equal(t, int64(2646204760), dir.Size())
|
||||
assert.Equal(t, int64(3), dir.Items())
|
||||
}
|
||||
Loaded 100 of 485 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user