Compare commits
285
Commits
test
..
v1.75-stable
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be8d6044ea | ||
|
|
bd07f481e2 | ||
|
|
56059c47c8 | ||
|
|
35ab21109b | ||
|
|
fc7a57b447 | ||
|
|
028f2b4568 | ||
|
|
62e770eb54 | ||
|
|
652bbb0908 | ||
|
|
4df376e2db | ||
|
|
68e43e5b88 | ||
|
|
1dcc246e03 | ||
|
|
4cc69c2a62 | ||
|
|
8d97a50c4a | ||
|
|
7c2405c035 | ||
|
|
10335538d1 | ||
|
|
a7b1599204 | ||
|
|
518f43dee9 | ||
|
|
43d3374465 | ||
|
|
10875eec70 | ||
|
|
40eaa114c1 | ||
|
|
a34b606abf | ||
|
|
37aa0cb629 | ||
|
|
a7f0259e55 | ||
|
|
323b0b4559 | ||
|
|
26175553a7 | ||
|
|
4a944dff0f | ||
|
|
e9b9b48a68 | ||
|
|
3a14789eb4 | ||
|
|
47027efd12 | ||
|
|
2f43b99abe | ||
|
|
133ced103a | ||
|
|
317c885857 | ||
|
|
a34f1f153a | ||
|
|
e7d4d425bc | ||
|
|
01a9d67e06 | ||
|
|
aece53963a | ||
|
|
291016ea75 | ||
|
|
7e57142de1 | ||
|
|
722c14d806 | ||
|
|
89b8a59e8b | ||
|
|
1b500594d3 | ||
|
|
9b495f2717 | ||
|
|
f6290108df | ||
|
|
ed9f1d836e | ||
|
|
59ea591bcb | ||
|
|
2cc9088374 | ||
|
|
d8dbeb9de4 | ||
|
|
9cf1af177a | ||
|
|
bb42b076af | ||
|
|
0992024791 | ||
|
|
48ac01c2b8 | ||
|
|
b928e8ddd0 | ||
|
|
9ac7855da9 | ||
|
|
5d20bd2832 | ||
|
|
2f84013eec | ||
|
|
675f9c3684 | ||
|
|
b8a697888c | ||
|
|
dddf808b47 | ||
|
|
451346f01c | ||
|
|
6fadaa46aa | ||
|
|
563919b0c0 | ||
|
|
1ecdeb4466 | ||
|
|
b85a681721 | ||
|
|
61b8a90db8 | ||
|
|
419c5f9844 | ||
|
|
50d2430b4a | ||
|
|
cd6941ed93 | ||
|
|
8290a478c8 | ||
|
|
d89f30a136 | ||
|
|
f8899ca982 | ||
|
|
d77c1bf5c6 | ||
|
|
19bf14aea4 | ||
|
|
353167fd38 | ||
|
|
09e1967c73 | ||
|
|
1d1eb94c34 | ||
|
|
fd3b9bd383 | ||
|
|
6676f2673e | ||
|
|
f6af51420f | ||
|
|
a755455609 | ||
|
|
0e6ed13131 | ||
|
|
96d2633836 | ||
|
|
f625ac98a6 | ||
|
|
19068b6608 | ||
|
|
ee40507d38 | ||
|
|
b4cb0426a3 | ||
|
|
5977137b3b | ||
|
|
512f6193a2 | ||
|
|
46cf167c22 | ||
|
|
2d24951658 | ||
|
|
ad9ad335f4 | ||
|
|
7d3e148212 | ||
|
|
2ff47e8274 | ||
|
|
f3bec5b1aa | ||
|
|
d418a5a882 | ||
|
|
db96e30c5e | ||
|
|
111fda1287 | ||
|
|
10d999e3ee | ||
|
|
6a7dcf12c3 | ||
|
|
fecd1d578a | ||
|
|
af92ea943d | ||
|
|
b46bb7647b | ||
|
|
3f6b38c9d9 | ||
|
|
a1fba2e8c1 | ||
|
|
dca3902fef | ||
|
|
06387f878a | ||
|
|
13ae25b561 | ||
|
|
5ea1851d52 | ||
|
|
2b65691945 | ||
|
|
5de8a01717 | ||
|
|
4d50f6d878 | ||
|
|
34ad19a1eb | ||
|
|
5bc26e57b4 | ||
|
|
8d15b2bb75 | ||
|
|
c062ec1b12 | ||
|
|
7b647c729b | ||
|
|
4347404119 | ||
|
|
366b188a5a | ||
|
|
0dd95c2840 | ||
|
|
efba10c065 | ||
|
|
cc53a4cdac | ||
|
|
89c6cb0911 | ||
|
|
a5f9653807 | ||
|
|
620ce60d00 | ||
|
|
fa75e2b178 | ||
|
|
b5055efecf | ||
|
|
ab8a00e159 | ||
|
|
70700de356 | ||
|
|
277d3ea146 | ||
|
|
fc65a1b9cb | ||
|
|
b1677718cd | ||
|
|
4dde74364b | ||
|
|
bb7b312b19 | ||
|
|
43162d20e0 | ||
|
|
ba3a6c17ce | ||
|
|
7c1a79e071 | ||
|
|
0d582bfffe | ||
|
|
459501873b | ||
|
|
f1590144a4 | ||
|
|
9d25ce63a3 | ||
|
|
18c39dc283 | ||
|
|
79ccf3d26b | ||
|
|
d659d10afd | ||
|
|
b8431d4f50 | ||
|
|
6c566ca014 | ||
|
|
cd7cef72db | ||
|
|
33ee8c0ef8 | ||
|
|
bb0633ee43 | ||
|
|
347738843f | ||
|
|
4d547c4b7b | ||
|
|
ab95b42328 | ||
|
|
409ba3b1f4 | ||
|
|
274342326c | ||
|
|
97eee1fecf | ||
|
|
98274d294a | ||
|
|
34aeb94efa | ||
|
|
de09fecf1c | ||
|
|
b7b89b0708 | ||
|
|
4079b6f493 | ||
|
|
bc482e79ff | ||
|
|
7ef04886b9 | ||
|
|
b70ebaba7b | ||
|
|
b03a9f8ea3 | ||
|
|
8a2bed2f71 | ||
|
|
df2a2d1127 | ||
|
|
2e18447255 | ||
|
|
ef6bae24c5 | ||
|
|
687d264b68 | ||
|
|
4158f63d2f | ||
|
|
c47832ad34 | ||
|
|
60e96416df | ||
|
|
c841c3bba3 | ||
|
|
79fbc0842f | ||
|
|
22859b7e69 | ||
|
|
925fb4fb21 | ||
|
|
96f298bdec | ||
|
|
60fb55dc6a | ||
|
|
181ef190ce | ||
|
|
ad8cd41c49 | ||
|
|
5dae3adbf5 | ||
|
|
6507e13d5a | ||
|
|
e4913c9e91 | ||
|
|
28bf49d66f | ||
|
|
17b0c03338 | ||
|
|
a7ab39d3d1 | ||
|
|
7a594b09c6 | ||
|
|
f376c64784 | ||
|
|
afeb26ae74 | ||
|
|
c6af0b57c2 | ||
|
|
90595f34f2 | ||
|
|
b3cf0444a2 | ||
|
|
11ff6e49bb | ||
|
|
bcef98db3f | ||
|
|
f18aacf2c7 | ||
|
|
ab1f458013 | ||
|
|
7c1dfd99f3 | ||
|
|
57842c5ee4 | ||
|
|
739403963a | ||
|
|
1bec2133c5 | ||
|
|
64de81e6a0 | ||
|
|
0b187f3d7a | ||
|
|
0e5100203c | ||
|
|
ba3d34b9f8 | ||
|
|
77e6390f5b | ||
|
|
f190b34d95 | ||
|
|
49e7016d12 | ||
|
|
6240cbb694 | ||
|
|
66bc465d1a | ||
|
|
b576cbdf40 | ||
|
|
66dba8239f | ||
|
|
92ef010fd2 | ||
|
|
feb0664b64 | ||
|
|
61427e9729 | ||
|
|
3d1fd51f61 | ||
|
|
6c44400bf7 | ||
|
|
041b766428 | ||
|
|
3c505b1e99 | ||
|
|
6317faccc0 | ||
|
|
5a490a31c5 | ||
|
|
b8470e4cac | ||
|
|
e7ae39f42d | ||
|
|
43f107e049 | ||
|
|
748eaf9b28 | ||
|
|
11dd394670 | ||
|
|
21483522a6 | ||
|
|
f92262d48c | ||
|
|
ad87bf6c93 | ||
|
|
f1c06a047e | ||
|
|
d6fb148d9b | ||
|
|
930de88f69 | ||
|
|
fad0359e6b | ||
|
|
c14e507609 | ||
|
|
0720a194cf | ||
|
|
194d22ce71 | ||
|
|
7995d87cd5 | ||
|
|
2d261879dd | ||
|
|
2e0718167f | ||
|
|
9718ed3fc2 | ||
|
|
645e1ea5be | ||
|
|
40dab7ecdd | ||
|
|
43a2a92950 | ||
|
|
62f8f944e2 | ||
|
|
841ff2d418 | ||
|
|
33ab81ce6d | ||
|
|
632ff74375 | ||
|
|
39b926a2ce | ||
|
|
31f4c78c5e | ||
|
|
fcb317e226 | ||
|
|
82a4a63b92 | ||
|
|
d407018995 | ||
|
|
b5f0ea9b7b | ||
|
|
d166eaf635 | ||
|
|
62a1df7271 | ||
|
|
e171f996d6 | ||
|
|
2bf9e0d20e | ||
|
|
afb1daa8fa | ||
|
|
5775f0430b | ||
|
|
d0f00a22d5 | ||
|
|
ad7e0c64af | ||
|
|
674d7a8a70 | ||
|
|
1274301521 | ||
|
|
840755543f | ||
|
|
2472718b7c | ||
|
|
a4f1c1558e | ||
|
|
76d5c49ae6 | ||
|
|
9fec073ab7 | ||
|
|
a9c4548d92 | ||
|
|
d038b8b1c9 | ||
|
|
3ceea42329 | ||
|
|
6beb4a5bd2 | ||
|
|
7d70921bf5 | ||
|
|
76c6992260 | ||
|
|
f1297d6298 | ||
|
|
9a112f40c1 | ||
|
|
a3dcf7f0ad | ||
|
|
3084177049 | ||
|
|
e4b778b5e3 | ||
|
|
ee363b1826 | ||
|
|
e7f9c7bc10 | ||
|
|
10eb7b5b10 | ||
|
|
e0faeeb5d6 | ||
|
|
dcf15b5f9c | ||
|
|
c93cc1e1ae | ||
|
|
21c98ff5ff | ||
|
|
5fbc677ed4 | ||
|
|
00347ecf36 |
No files matched your search
@@ -2,9 +2,6 @@
|
||||
go.mod text eol=lf
|
||||
go.sum text eol=lf
|
||||
|
||||
# Embedded templates must render the same on every OS
|
||||
lib/http/templates/** text eol=lf
|
||||
|
||||
# Ignore generated files in GitHub language statistics and diffs
|
||||
/MANUAL.* linguist-generated=true
|
||||
/rclone.1 linguist-generated=true
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.9'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +48,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.9'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.9'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +64,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.9'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.9'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,7 +81,7 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.9'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
@@ -284,7 +284,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
go-version: '~1.26.9'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
@@ -399,7 +399,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
go-version: '~1.26.9'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
@@ -20,8 +20,6 @@ The core parts of rclone under `fs` and `vfs` need to work with all backends and
|
||||
|
||||
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
|
||||
|
||||
Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
|
||||
|
||||
## Build and Test Commands
|
||||
|
||||
```bash
|
||||
|
||||
+1
-4
@@ -35,9 +35,6 @@ RUN echo "**** Print Version Binary ****" && \
|
||||
# Begin final image
|
||||
FROM alpine:latest
|
||||
|
||||
ARG RCLONE_UID=1009
|
||||
ARG RCLONE_GID=1009
|
||||
|
||||
RUN echo "**** Install Dependencies ****" && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
@@ -49,7 +46,7 @@ RUN echo "**** Install Dependencies ****" && \
|
||||
|
||||
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
|
||||
|
||||
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
|
||||
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
|
||||
|
||||
ENTRYPOINT [ "rclone" ]
|
||||
|
||||
|
||||
Generated
+3164
-1639
File diff suppressed because it is too large.
Load diff
Generated
+1640
-319
File diff suppressed because it is too large.
Load diff
@@ -38,9 +38,6 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
|
||||
BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
|
||||
BETA_UPLOAD_ROOT := beta.rclone.org:
|
||||
BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
|
||||
# Update the beta site's directory listings after an upload - add
|
||||
# --changed for the uploaded directories
|
||||
BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
# Pass in GOTAGS=xyz on the make command line to set build tags
|
||||
ifdef GOTAGS
|
||||
BUILDTAGS=-tags "$(GOTAGS)"
|
||||
@@ -237,7 +234,6 @@ check_sign:
|
||||
upload:
|
||||
rclone -P copy build/ downloads.rclone.org:/$(TAG)
|
||||
rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
|
||||
rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
|
||||
upload_github:
|
||||
./bin/upload-github $(TAG)
|
||||
@@ -270,7 +266,6 @@ ci_upload:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
|
||||
endif
|
||||
./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
|
||||
@echo Beta release ready at $(BETA_URL)/testbuilds
|
||||
|
||||
ci_beta:
|
||||
@@ -280,7 +275,6 @@ ci_beta:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
|
||||
endif
|
||||
rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
|
||||
@echo Beta release ready at $(BETA_URL)
|
||||
|
||||
# Fetch the binary builds from GitHub actions
|
||||
@@ -302,26 +296,21 @@ retag:
|
||||
@echo "Version is $(VERSION)"
|
||||
git tag -f -s -m "Version $(VERSION)" $(VERSION)
|
||||
|
||||
# Start development of the next version after a release: the next
|
||||
# minor version on master or the next patch version on a stable branch.
|
||||
startdev:
|
||||
@branch=$$(git rev-parse --abbrev-ref HEAD); \
|
||||
case "$$branch" in \
|
||||
master) next="$(NEXT_VERSION)" ;; \
|
||||
v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
|
||||
*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
|
||||
echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
|
||||
fi; \
|
||||
echo "Version is $(VERSION)"; \
|
||||
echo "Next version is $$next"; \
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
|
||||
echo -n "$$next" > docs/layouts/partials/version.html; \
|
||||
echo "$$next" > VERSION; \
|
||||
git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next version is $(NEXT_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
startstable: startdev
|
||||
startstable:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_PATCH_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
winzip:
|
||||
zip -9 rclone-$(TAG).zip rclone.exe
|
||||
|
||||
@@ -39,7 +39,6 @@ directories to and from different cloud storage providers.
|
||||
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
|
||||
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
|
||||
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
|
||||
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
|
||||
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
@@ -111,7 +110,6 @@ directories to and from different cloud storage providers.
|
||||
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
|
||||
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
|
||||
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
|
||||
- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
|
||||
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
|
||||
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
|
||||
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
|
||||
|
||||
+18
-69
@@ -6,37 +6,27 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
- [gh the github cli](https://github.com/cli/cli) for uploading packages
|
||||
- pandoc for making the html and man pages
|
||||
- hugo for building the website
|
||||
- golangci-lint and docker for `make check` (markdownlint runs in docker)
|
||||
- a gpg key for signing the tag and the checksums
|
||||
- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
|
||||
`www.rclone.org:` and `test-rclone-org:`
|
||||
- the sponsor logos - see [Sponsor logos](#sponsor-logos)
|
||||
|
||||
## Making a release
|
||||
|
||||
- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
|
||||
- git checkout master # see below for stable branch
|
||||
- git pull # IMPORTANT
|
||||
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
|
||||
- git status - make sure everything is checked in
|
||||
- Check GitHub actions build for master is Green
|
||||
- Check the integration tests at <https://integration.rclone.org>
|
||||
- cat VERSION # check this is the version being released
|
||||
- make tag # signed tag on HEAD + new changelog entry + docs
|
||||
- edit docs/content/changelog.md # entries already in point releases are
|
||||
removed automatically
|
||||
- make test # see integration test server or run locally
|
||||
- make tag
|
||||
- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
|
||||
releases
|
||||
- make tidy
|
||||
- make doc # again to pick up the changelog edits
|
||||
- git status - to check for new command docs in docs/content/commands - git add them
|
||||
- make doc
|
||||
- git status - to check for new man pages - git add them
|
||||
- git commit -a -v -m "Version v1.XX.0"
|
||||
- make check # if this fails, fix, git commit --amend and run it again
|
||||
- make retag # moves the tag onto the Version commit
|
||||
- make check
|
||||
- make retag
|
||||
- git push origin # without --follow-tags so it doesn't push the tag if it fails
|
||||
- git push --follow-tags origin
|
||||
- \# Wait for the GitHub builds to complete then...
|
||||
- \# Don't commit anything until after make upload_github as these steps
|
||||
use the tag on HEAD to find the version
|
||||
- make fetch_binaries
|
||||
- make tarball
|
||||
- make vendorball
|
||||
@@ -45,17 +35,9 @@ This file describes how to make the various kinds of releases
|
||||
- make upload
|
||||
- make upload_test_website # check sponsors are correct/unchanged
|
||||
- make upload_website
|
||||
- make upload_github # this triggers the docker plugin and winget workflows
|
||||
- Check the docker image, docker plugin and winget GitHub actions succeeded
|
||||
- make startdev # works out the next version for master or a stable branch
|
||||
- git push
|
||||
- Disable the release freeze rule
|
||||
- Release any security advisories
|
||||
- request a CVE for each advisory if not done already
|
||||
- set the patched version on each advisory and publish it
|
||||
- when the CVEs are assigned replace `CVE-PENDING` in the changelog on
|
||||
master and the stable branch then `make upload_website`
|
||||
- \# announce with forum post, twitter post, and post to announcements list
|
||||
- make upload_github
|
||||
- make startdev # make startstable for stable branch
|
||||
- \# announce with forum post, twitter post, and post to annoucements list
|
||||
|
||||
## Update dependencies
|
||||
|
||||
@@ -208,53 +190,20 @@ First make the release branch. If this is a second point release then
|
||||
this will be done already.
|
||||
|
||||
- git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
|
||||
- make startdev # sets the version to ${BASE_TAG}.1
|
||||
- git push -u origin ${BASE_TAG}-stable
|
||||
- make startstable
|
||||
|
||||
Now
|
||||
|
||||
- Enable release freeze rule
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- make fetch-gui-and-commit # update the GUI on master first
|
||||
- git push
|
||||
- git co ${BASE_TAG}-stable
|
||||
- git pull
|
||||
- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
|
||||
- `git cherry-pick -x` any fixes, including the GUI update if there was one
|
||||
- for security releases consider updating the Go patch version in
|
||||
`.github/workflows/build.yml` and dependencies with CVE fixes
|
||||
- fixes may depend on commits which weren't picked so push to CI
|
||||
and check it is green before tagging
|
||||
- Do the steps in "Making a release" above, except
|
||||
- stay on ${BASE_TAG}-stable - don't checkout master or run
|
||||
make fetch-gui-and-commit
|
||||
- check the GitHub actions build for ${BASE_TAG}-stable not master
|
||||
- run the integration tests on ${BASE_TAG}-stable as described below
|
||||
- the release freeze rule is already enabled - leave it until the end
|
||||
- `git cherry-pick -x` any fixes
|
||||
- make startstable
|
||||
- Do the steps as above
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- `#` cherry pick the changes to the changelog - check the diff to make sure it
|
||||
is correct
|
||||
- git checkout ${BASE_TAG}-stable docs/content/changelog.md
|
||||
- git diff --cached # check only the ${NEW_TAG} entry has changed
|
||||
- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
|
||||
- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
|
||||
- git push
|
||||
- Disable release freeze rule
|
||||
|
||||
Point releases are only made for the latest release. Once v1.(XX+1).0
|
||||
is out the ${BASE_TAG}-stable branch is abandoned.
|
||||
|
||||
### Running the integration tests on the stable branch
|
||||
|
||||
The integration tests at <https://integration.rclone.org> test master.
|
||||
To test the stable branch push it to the testing server and run the
|
||||
tests there in `screen` as they take a while.
|
||||
|
||||
```console
|
||||
git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
|
||||
ssh rclone@rclone-testing
|
||||
screen
|
||||
cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
|
||||
```
|
||||
|
||||
## Sponsor logos
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/compress"
|
||||
_ "github.com/rclone/rclone/backend/crypt"
|
||||
_ "github.com/rclone/rclone/backend/doi"
|
||||
_ "github.com/rclone/rclone/backend/dosya"
|
||||
_ "github.com/rclone/rclone/backend/drime"
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/dropbox"
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -23,7 +22,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -417,103 +415,3 @@ func TestIsDirectChild(t *testing.T) {
|
||||
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
|
||||
}
|
||||
}
|
||||
|
||||
// A symlink in a squashfs image is skipped unless -l/--links is in use,
|
||||
// as it is on the other backends. With the flag it is exposed under its
|
||||
// name with the link suffix and reads back as its target.
|
||||
//
|
||||
// Note that this uses mksquashfs as an external binary.
|
||||
func TestArchiveSquashfsSymlink(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "mksquashfs")
|
||||
ctx := context.Background()
|
||||
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "input")
|
||||
require.NoError(t, os.Mkdir(input, 0777))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
|
||||
require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
|
||||
// a regular file which happens to carry the link suffix must stay
|
||||
// readable under the flag, not be taken for a symlink
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
|
||||
image := filepath.Join(dir, "test.sqfs")
|
||||
run(t, "mksquashfs", input, image)
|
||||
|
||||
remotes := func(t *testing.T, f fs.Fs) []string {
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
names = append(names, entry.Remote())
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
read := func(t *testing.T, f fs.Fs, remote string) string {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
require.NoError(t, err)
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
contents, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, o.Size(), int64(len(contents)))
|
||||
return string(contents)
|
||||
}
|
||||
|
||||
f, err := fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
|
||||
assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
|
||||
}
|
||||
|
||||
// A hostile archive must not be able to write outside the destination.
|
||||
// A symlink pointing above the root with a member inside it is the
|
||||
// classic way to try it, and the refusal comes from the local backend,
|
||||
// so pin it from end to end here rather than trusting it stays.
|
||||
func TestArchiveZipSymlinkEscape(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
hdr := &zip.FileHeader{Name: "a"}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err := zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("../outside"))
|
||||
require.NoError(t, err)
|
||||
w, err = zw.Create("a/pwned.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("pwned"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
zipPath := filepath.Join(dir, "evil.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
dst := filepath.Join(dir, "dst")
|
||||
require.NoError(t, os.Mkdir(dst, 0777))
|
||||
|
||||
fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
|
||||
require.NoError(t, err)
|
||||
fdst, err := fs.NewFs(ctx, dst)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
|
||||
|
||||
_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
|
||||
assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
|
||||
}
|
||||
@@ -43,7 +43,6 @@ type Fs struct {
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
file string // remote of the single file root points at, if any
|
||||
links bool // read symlinks as .rclonelink files
|
||||
}
|
||||
|
||||
// recoverParsePanic converts a panic raised while parsing a squashfs image
|
||||
@@ -100,7 +99,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
|
||||
c: c,
|
||||
remote: remote,
|
||||
root: strings.Trim(root, "/"),
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -222,30 +220,6 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
|
||||
}
|
||||
}
|
||||
|
||||
// Convert a symlink entry into an Object holding its target
|
||||
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
|
||||
size: int64(len(target)),
|
||||
modTime: item.ModTime(),
|
||||
isLink: true,
|
||||
link: target,
|
||||
}
|
||||
}
|
||||
|
||||
// readlink returns the target of the symlink the info describes.
|
||||
//
|
||||
// A symlink may carry an empty target, so the caller decides what is a
|
||||
// symlink from the mode rather than from the target being set.
|
||||
func readlink(info os.FileInfo) (string, error) {
|
||||
st, ok := info.Sys().(*squashfs.StatT)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
|
||||
}
|
||||
return st.LinkTarget, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
@@ -275,7 +249,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, item := range items {
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
// Skip entry names that aren't safe
|
||||
@@ -298,20 +271,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = f.objectFromFileInfo(nativeDir, info)
|
||||
} else if item.Type()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry = f.objectFromLink(nativeDir, info, target)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
@@ -327,9 +286,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
@@ -351,17 +307,7 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
// Name the entry as List does, so that the two agree on
|
||||
// which names exist
|
||||
isLink := fi.Type()&os.ModeSymlink != 0
|
||||
if isLink && !f.links {
|
||||
continue
|
||||
}
|
||||
name := fi.Name()
|
||||
if isLink {
|
||||
name += fs.LinkSuffix
|
||||
}
|
||||
if name == leaf {
|
||||
if fi.Name() == leaf {
|
||||
if fi.IsDir() {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
@@ -369,14 +315,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
|
||||
}
|
||||
if isLink {
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o = f.objectFromLink(dir, info, target)
|
||||
break
|
||||
}
|
||||
o = f.objectFromFileInfo(dir, info)
|
||||
break
|
||||
}
|
||||
@@ -455,8 +393,6 @@ type Object struct {
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
isLink bool // whether this is a symlink
|
||||
link string // target of the symlink
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
@@ -528,17 +464,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
}
|
||||
}
|
||||
|
||||
if o.isLink {
|
||||
// Clamp offset into range to avoid panic
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset > int64(len(o.link)) {
|
||||
offset = int64(len(o.link))
|
||||
}
|
||||
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
|
||||
}
|
||||
|
||||
remote, err := o.fs.toNative(o.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -43,7 +43,6 @@ type Fs struct {
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
links bool // read symlinks as .rclonelink files
|
||||
dt dirtree.DirTree // read from zipfile
|
||||
}
|
||||
|
||||
@@ -68,7 +67,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -140,7 +138,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
}
|
||||
dt := dirtree.New()
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, file := range zr.File {
|
||||
// Skip entries whose name escapes the archive's own namespace
|
||||
remote, err := sanitize.Path(file.Name)
|
||||
@@ -155,15 +152,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !isDir && file.Mode()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
// The member holds the link target, so it reads back
|
||||
// like a .rclonelink file
|
||||
remote += fs.LinkSuffix
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root, requiring a path
|
||||
@@ -203,9 +191,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -37,26 +36,6 @@ func writeZip(t *testing.T, dir, name string, names ...string) string {
|
||||
return name
|
||||
}
|
||||
|
||||
// writeZipSymlink builds a zip holding file and a symlink entry link
|
||||
// pointing at target, and returns its leaf name.
|
||||
func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create(file)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data for " + file))
|
||||
require.NoError(t, err)
|
||||
hdr := &zip.FileHeader{Name: link}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err = zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(target))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// allRemotes returns every Object remote in the mounted archive's dirtree.
|
||||
func allRemotes(t *testing.T, f fs.Fs) []string {
|
||||
zf, ok := f.(*Fs)
|
||||
@@ -191,37 +170,3 @@ func TestListCacheNotAliased(t *testing.T) {
|
||||
}
|
||||
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A symlink stored in the archive is skipped unless -l/--links is in
|
||||
// use, as it is on the other backends. With the flag it is exposed
|
||||
// under its name with the link suffix and reads back as its target.
|
||||
func TestReadZipSymlink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
|
||||
|
||||
o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len("file.txt")), o.Size())
|
||||
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
target, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, "file.txt", string(target))
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Parallel listing support for azureblob, built on the Apache Arrow
|
||||
// listing format.
|
||||
@@ -62,7 +62,7 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
|
||||
|
||||
// Schedule every shard; SetLimit bounds concurrency, so more shards than
|
||||
// workers gives free load balancing as workers pick up the next shard.
|
||||
var total atomic.Int64
|
||||
var total int64
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(f.opt.ListParallelism)
|
||||
for _, sh := range shards {
|
||||
@@ -85,12 +85,12 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
|
||||
shardOpts.EndBefore = nil
|
||||
}
|
||||
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
|
||||
total.Add(int64(n))
|
||||
atomic.AddInt64(&total, int64(n))
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := g.Wait()
|
||||
found := int(total.Load())
|
||||
found := int(atomic.LoadInt64(&total))
|
||||
if isEndBeforeUnsupported(err) {
|
||||
// The account can't honour the shards' endBefore bounds, so parallel
|
||||
// listing can't be used on it.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
+38
-171
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -365,22 +364,7 @@ creates an empty object ending with "/", to persist the folder.
|
||||
|
||||
This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
|
||||
the Microsoft standard.
|
||||
|
||||
This is not needed on storage accounts with a hierarchical namespace
|
||||
(see the hns option) as these always support empty directories.
|
||||
`,
|
||||
}, {
|
||||
Name: "hns",
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
Help: `Set if the storage account has a hierarchical namespace.
|
||||
|
||||
Storage accounts with hierarchical namespace enabled (Azure Data Lake
|
||||
Storage Gen2) have real directories which can be empty.
|
||||
|
||||
If this is unset then rclone will read it from the storage account
|
||||
and save it in the config file.
|
||||
`,
|
||||
}, {
|
||||
Name: "no_check_container",
|
||||
Help: `If set, don't attempt to check the container exists or create it.
|
||||
@@ -448,7 +432,6 @@ type Options struct {
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
PublicAccess string `config:"public_access"`
|
||||
DirectoryMarkers bool `config:"directory_markers"`
|
||||
HNS fs.Tristate `config:"hns"`
|
||||
NoCheckContainer bool `config:"no_check_container"`
|
||||
NoHeadObject bool `config:"no_head_object"`
|
||||
DeleteSnapshots string `config:"delete_snapshots"`
|
||||
@@ -472,7 +455,6 @@ type Fs struct {
|
||||
rootContainer string // container part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
isLimited bool // if limited to one container
|
||||
hns bool // if the storage account has a hierarchical namespace
|
||||
cache *bucket.Cache // cache for container creation status
|
||||
pacer *fs.Pacer // To pace and retry the API calls
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
@@ -539,7 +521,7 @@ func parsePath(path string) (root string) {
|
||||
// relative to f.root
|
||||
func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
|
||||
containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
|
||||
if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
|
||||
if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
|
||||
containerPath = containerPath[:len(containerPath)-1]
|
||||
}
|
||||
return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
|
||||
@@ -587,7 +569,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
// General errors from:
|
||||
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
|
||||
// Blob specific errors from:
|
||||
@@ -706,6 +689,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ServerSideAcrossConfigs: true,
|
||||
DoubleSlash: true,
|
||||
}).Fill(ctx, f)
|
||||
if opt.DirectoryMarkers {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
|
||||
DefaultBaseURL: storageDefaultBaseURL,
|
||||
@@ -738,28 +725,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.isLimited = true
|
||||
}
|
||||
|
||||
if opt.HNS.Valid {
|
||||
f.hns = opt.HNS.Value
|
||||
} else if !f.anonymous {
|
||||
containerName := f.rootContainer
|
||||
if containerName == "" {
|
||||
containerName = res.Container
|
||||
}
|
||||
f.hns, err = f.readHNS(ctx, containerName)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
|
||||
} else {
|
||||
m.Set("hns", strconv.FormatBool(f.hns))
|
||||
}
|
||||
}
|
||||
if f.hns {
|
||||
fs.Debugf(f, "Storage account has a hierarchical namespace")
|
||||
}
|
||||
if f.dirMarkers() {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
if f.rootContainer != "" && f.rootDirectory != "" {
|
||||
// Check to see if the (container,directory) is actually an existing file
|
||||
oldRoot := f.root
|
||||
@@ -780,39 +745,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// readHNS reads whether the storage account has a hierarchical namespace
|
||||
//
|
||||
// If containerName is set then the container client is used which works
|
||||
// with credentials limited to that container.
|
||||
func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
|
||||
var isHNS *bool
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
if containerName != "" {
|
||||
var resp container.GetAccountInfoResponse
|
||||
resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
} else {
|
||||
var resp service.GetAccountInfoResponse
|
||||
resp, err = f.svc.GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if isHNS == nil {
|
||||
return false, errors.New("not returned by server")
|
||||
}
|
||||
return *isHNS, nil
|
||||
}
|
||||
|
||||
// dirMarkers returns true if directories are persisted with directory markers
|
||||
func (f *Fs) dirMarkers() bool {
|
||||
return f.opt.DirectoryMarkers || f.hns
|
||||
}
|
||||
|
||||
// return the container client for the container passed in
|
||||
func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
|
||||
f.cntSVCcacheMu.Lock()
|
||||
@@ -850,12 +782,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
o, err := f.newObjectWithInfo(ctx, remote, nil)
|
||||
if err == fs.ErrorNotAFile {
|
||||
// The blob is a directory
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, err
|
||||
return f.newObjectWithInfo(ctx, remote, nil)
|
||||
}
|
||||
|
||||
// getBlobSVC creates a blob client
|
||||
@@ -924,15 +851,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
|
||||
lowerKey := strings.ToLower(k)
|
||||
switch lowerKey {
|
||||
case "cache-control":
|
||||
headers.BlobCacheControl = new(v)
|
||||
headers.BlobCacheControl = pString(v)
|
||||
case "content-disposition":
|
||||
headers.BlobContentDisposition = new(v)
|
||||
headers.BlobContentDisposition = pString(v)
|
||||
case "content-encoding":
|
||||
headers.BlobContentEncoding = new(v)
|
||||
headers.BlobContentEncoding = pString(v)
|
||||
case "content-language":
|
||||
headers.BlobContentLanguage = new(v)
|
||||
headers.BlobContentLanguage = pString(v)
|
||||
case "content-type":
|
||||
headers.BlobContentType = new(v)
|
||||
headers.BlobContentType = pString(v)
|
||||
case "x-ms-tags":
|
||||
parsed, perr := parseXMsTags(v)
|
||||
if perr != nil {
|
||||
@@ -1169,33 +1096,18 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
|
||||
if endsWithSlash || remote == "" {
|
||||
return true
|
||||
}
|
||||
return hasDirMetadata(metadata)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether the metadata marks the blob as a directory
|
||||
func hasDirMetadata(metadata map[string]*string) bool {
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether props are for a directory on a storage account with
|
||||
// a hierarchical namespace
|
||||
//
|
||||
// These ignore a trailing "/" on the name so a file can be returned
|
||||
// when looking for a directory.
|
||||
func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
|
||||
return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
|
||||
}
|
||||
|
||||
// listFn is called from list to handle an object
|
||||
type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
|
||||
|
||||
@@ -1254,18 +1166,15 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f.dirMarkers() && foundItems == 0 && directory != "" {
|
||||
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
|
||||
// Determine whether the directory exists or not by whether it has a marker
|
||||
props, err := f.readMetaData(ctx, containerName, directory)
|
||||
_, err := f.readMetaData(ctx, containerName, directory)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
if f.hns && !isHNSDirectory(props) {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1318,13 +1227,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
fs.Debugf(f, "Nil name received")
|
||||
continue
|
||||
}
|
||||
// Storage accounts with a hierarchical namespace return
|
||||
// the file "dir" when listing "dir/"
|
||||
if !strings.HasPrefix(*file.Name, directory) {
|
||||
fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
|
||||
foundItems--
|
||||
continue
|
||||
}
|
||||
remote := f.opt.Enc.ToStandardPath(*file.Name)
|
||||
if !strings.HasPrefix(remote, prefix) {
|
||||
fs.Debugf(f, "Odd name received %q", remote)
|
||||
@@ -1586,7 +1488,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
|
||||
func (f *Fs) listContainersToFn(fn listContainerFn) error {
|
||||
max := int32(f.opt.ListChunkSize)
|
||||
pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
|
||||
Include: service.ListContainersInclude{Metadata: true},
|
||||
Include: service.ListContainersInclude{Metadata: true, Deleted: true},
|
||||
MaxResults: &max,
|
||||
})
|
||||
ctx := context.Background()
|
||||
@@ -1636,7 +1538,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
|
||||
// Create directory marker file and parents
|
||||
func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
|
||||
if !f.dirMarkers() || container == "" {
|
||||
if !f.opt.DirectoryMarkers || container == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1693,11 +1595,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
|
||||
// mkdirParent creates the parent bucket/directory if it doesn't exist
|
||||
func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
|
||||
if f.hns {
|
||||
// Parent directories are created by the upload
|
||||
container, _ := f.split(remote)
|
||||
return f.makeContainer(ctx, container)
|
||||
}
|
||||
remote, _ = strings.CutSuffix(remote, "/")
|
||||
dir := path.Dir(remote)
|
||||
if dir == "/" || dir == "." {
|
||||
@@ -1803,9 +1700,6 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
|
||||
// Returns an error if it isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
container, directory := f.split(dir)
|
||||
if f.hns && container != "" && directory != "" {
|
||||
return f.rmdirHNS(ctx, container, directory)
|
||||
}
|
||||
// Remove directory marker file
|
||||
if f.opt.DirectoryMarkers && container != "" && directory != "" {
|
||||
o := &Object{
|
||||
@@ -1828,39 +1722,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return f.deleteContainer(ctx, container)
|
||||
}
|
||||
|
||||
// rmdirHNS removes the directory on a storage account with a
|
||||
// hierarchical namespace
|
||||
//
|
||||
// The directory must be deleted without a trailing "/" and the server
|
||||
// refuses to delete it if it isn't empty.
|
||||
func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
|
||||
directory, _ = strings.CutSuffix(directory, "/")
|
||||
// Check it is a directory as deleting a file would succeed
|
||||
props, err := f.readMetaData(ctx, container, directory)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isHNSDirectory(props) {
|
||||
return fs.ErrorIsFile
|
||||
}
|
||||
blb := f.getBlobSVC(container, directory)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err := blb.Delete(ctx, nil)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if storageErr.StatusCode == http.StatusNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Precision of the remote
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Nanosecond
|
||||
@@ -2699,6 +2560,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return downloadResponse.Body, nil
|
||||
}
|
||||
|
||||
// Converts a string into a pointer to a string
|
||||
func pString(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
|
||||
type readSeekCloser struct {
|
||||
io.Reader
|
||||
@@ -2853,7 +2719,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
// isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
|
||||
// returning true if it is found
|
||||
func isInvalidBlockOrBlob(err error) bool {
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
|
||||
}
|
||||
return false
|
||||
@@ -3225,7 +3092,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
|
||||
// Start with default content-type based on source
|
||||
ui.httpHeaders = blob.HTTPHeaders{
|
||||
BlobContentType: new(fs.MimeType(ctx, src)),
|
||||
BlobContentType: pString(fs.MimeType(ctx, src)),
|
||||
}
|
||||
|
||||
// Apply mapped metadata/headers/tags if requested
|
||||
@@ -3271,15 +3138,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
o.tags[parts[0]] = parts[1]
|
||||
}
|
||||
case "cache-control":
|
||||
ui.httpHeaders.BlobCacheControl = new(value)
|
||||
ui.httpHeaders.BlobCacheControl = pString(value)
|
||||
case "content-disposition":
|
||||
ui.httpHeaders.BlobContentDisposition = new(value)
|
||||
ui.httpHeaders.BlobContentDisposition = pString(value)
|
||||
case "content-encoding":
|
||||
ui.httpHeaders.BlobContentEncoding = new(value)
|
||||
ui.httpHeaders.BlobContentEncoding = pString(value)
|
||||
case "content-language":
|
||||
ui.httpHeaders.BlobContentLanguage = new(value)
|
||||
ui.httpHeaders.BlobContentLanguage = pString(value)
|
||||
case "content-type":
|
||||
ui.httpHeaders.BlobContentType = new(value)
|
||||
ui.httpHeaders.BlobContentType = pString(value)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
@@ -325,14 +325,6 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
|
||||
}
|
||||
|
||||
// helper to read blob tags for an object
|
||||
// skipIfHNS skips the test if the storage account has a hierarchical
|
||||
// namespace as these don't support blob tags
|
||||
func skipIfHNS(t *testing.T, f *Fs) {
|
||||
if f.hns {
|
||||
t.Skip("blob tags not supported with hierarchical namespace")
|
||||
}
|
||||
}
|
||||
|
||||
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
||||
ao := o.(*Object)
|
||||
blb := ao.getBlobSVC()
|
||||
@@ -418,7 +410,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart upload
|
||||
t.Run("PutSinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -434,7 +425,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart upload
|
||||
t.Run("PutMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize + 2048))
|
||||
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -622,7 +612,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart copy
|
||||
t.Run("CopySinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
@@ -648,7 +637,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart copy
|
||||
t.Run("CopyMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
||||
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || solaris || js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -650,6 +650,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
|
||||
@@ -813,8 +818,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
|
||||
}
|
||||
opt := file.RenameOptions{
|
||||
IgnoreReadOnly: new(true),
|
||||
ReplaceIfExists: new(true),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
ReplaceIfExists: ptr(true),
|
||||
}
|
||||
dstAbsPath := f.absPath(remote)
|
||||
fc := srcObj.fileClient()
|
||||
@@ -859,8 +864,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
opt := directory.RenameOptions{
|
||||
IgnoreReadOnly: new(false),
|
||||
ReplaceIfExists: new(false),
|
||||
IgnoreReadOnly: ptr(false),
|
||||
ReplaceIfExists: ptr(false),
|
||||
}
|
||||
dstAbsPath := dstFs.absPath(dstRemote)
|
||||
dirClient := srcFs.dirClient(srcRemote)
|
||||
@@ -899,8 +904,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
ChangeTime: file.SourceCopyFileChangeTime{},
|
||||
CreationTime: file.SourceCopyFileCreationTime{},
|
||||
LastWriteTime: file.SourceCopyFileLastWriteTime{},
|
||||
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: new(true),
|
||||
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
},
|
||||
}
|
||||
srcURL := srcObj.fileClient().URL()
|
||||
|
||||
+15
-23
@@ -3,6 +3,7 @@
|
||||
package box
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
@@ -19,7 +20,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
|
||||
}
|
||||
|
||||
// uploadPart uploads a part in an upload session
|
||||
//
|
||||
// chunk must contain exactly the part's data and is read from the start
|
||||
// on each attempt.
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
hasher := sha1.New()
|
||||
chunkSize, err := io.Copy(hasher, chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
chunkSize := int64(len(chunk))
|
||||
sha1sum := sha1.Sum(chunk)
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/files/upload_sessions/" + SessionID,
|
||||
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
|
||||
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Digest": sha1Digest(hasher.Sum(nil)),
|
||||
"Digest": sha1Digest(sha1sum[:]),
|
||||
},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = chunk.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = wrap(chunk)
|
||||
opts.Body = wrap(bytes.NewReader(chunk))
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -223,25 +213,27 @@ outer:
|
||||
|
||||
reqSize := min(remaining, chunkSize)
|
||||
|
||||
// Read the chunk into memory from the global pool, hashing
|
||||
// it for the whole file hash (must be done sequentially)
|
||||
rw := multipart.NewRW().Reserve(reqSize)
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
|
||||
// Make a block of memory
|
||||
buf := make([]byte, reqSize)
|
||||
|
||||
// Read the chunk
|
||||
_, err = io.ReadFull(in, buf)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
err = fmt.Errorf("multipart upload failed to read source: %w", err)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Make the global hash (must be done sequentially)
|
||||
_, _ = hash.Write(buf)
|
||||
|
||||
// Transfer the chunk
|
||||
wg.Add(1)
|
||||
o.fs.uploadToken.Get()
|
||||
go func(part int, position int64) {
|
||||
defer wg.Done()
|
||||
defer o.fs.uploadToken.Put()
|
||||
defer func() { _ = rw.Close() }()
|
||||
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
|
||||
select {
|
||||
|
||||
@@ -43,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
|
||||
@@ -472,8 +472,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
if updateOptions, ok := option.(*api.UpdateOptions); ok {
|
||||
if updateOptions.PublicID != "" {
|
||||
updateObject = true
|
||||
params.Overwrite = new(true)
|
||||
params.Invalidate = new(true)
|
||||
params.Overwrite = SDKApi.Bool(true)
|
||||
params.Invalidate = SDKApi.Bool(true)
|
||||
params.PublicID = updateOptions.PublicID
|
||||
params.ResourceType = updateOptions.ResourceType
|
||||
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -564,19 +563,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
|
||||
|
||||
// cache small files in memory and do normal upload
|
||||
rw := multipart.NewRW()
|
||||
defer fs.CheckClose(rw, &err)
|
||||
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
|
||||
if err == io.EOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, rw, src, options...)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
buf := make([]byte, f.opt.RAMCacheLimit)
|
||||
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
|
||||
}
|
||||
|
||||
// Need to include what we already read
|
||||
in = &ReadCloserWrapper{
|
||||
Reader: io.MultiReader(rw, in),
|
||||
Reader: io.MultiReader(bytes.NewReader(buf), in),
|
||||
Closer: in,
|
||||
}
|
||||
|
||||
@@ -1157,15 +1152,6 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return objErr
|
||||
}
|
||||
|
||||
// countingDiscard is an io.Writer which discards its input and counts
|
||||
// the bytes written
|
||||
type countingDiscard int64
|
||||
|
||||
func (c *countingDiscard) Write(p []byte) (int, error) {
|
||||
*c += countingDiscard(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
|
||||
type ReadCloserWrapper struct {
|
||||
io.Reader
|
||||
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type gzipModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
|
||||
// PutStream. It reads each body twice when it is seekable to model a
|
||||
// backend retrying an upload.
|
||||
type putRecorder struct {
|
||||
fs.Fs
|
||||
t *testing.T
|
||||
stream bool // expose PutStream from the wrapped Fs
|
||||
size int64
|
||||
seekable bool
|
||||
body []byte
|
||||
streamed bool
|
||||
}
|
||||
|
||||
func (r *putRecorder) Features() *fs.Features {
|
||||
features := *r.Fs.Features()
|
||||
features.PutStream = nil
|
||||
if r.stream {
|
||||
features.PutStream = r.PutStream
|
||||
}
|
||||
return &features
|
||||
}
|
||||
|
||||
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
|
||||
r.size = src.Size()
|
||||
body, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
r.seekable = true
|
||||
_, err = seeker.Seek(0, io.SeekStart)
|
||||
require.NoError(r.t, err)
|
||||
again, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
assert.Equal(r.t, body, again, "retried body should be identical")
|
||||
}
|
||||
r.body = body
|
||||
return body
|
||||
}
|
||||
|
||||
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
body := r.record(in, src)
|
||||
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
|
||||
}
|
||||
|
||||
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
r.streamed = true
|
||||
return r.Put(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// TestRcat checks the three upload paths rcat takes: small files buffered
|
||||
// in memory and uploaded with a known size and a seekable body, large
|
||||
// files streamed when the wrapped remote supports it, and large files
|
||||
// spooled to a temporary file when it doesn't. Pool pages taken for the
|
||||
// buffer must be returned on every path.
|
||||
func TestRcat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const limit = 1024
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
stream bool
|
||||
wantSize int64
|
||||
}{
|
||||
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
|
||||
{name: "Empty", size: 0, stream: true, wantSize: 0},
|
||||
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
|
||||
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
|
||||
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
|
||||
require.NoError(t, err)
|
||||
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
|
||||
f := &Fs{Fs: rec}
|
||||
f.opt.RAMCacheLimit = limit
|
||||
content := []byte(random.String(test.size))
|
||||
inUse := pool.Global().InUse()
|
||||
|
||||
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
|
||||
|
||||
assert.Equal(t, test.wantSize, rec.size)
|
||||
assert.Equal(t, content, rec.body)
|
||||
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
|
||||
if test.size < limit {
|
||||
assert.True(t, rec.seekable, "small file body should be seekable")
|
||||
}
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, content, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsCompressible checks the compressibility heuristic on data at
|
||||
// both ends of the scale for every compression mode which has one.
|
||||
func TestIsCompressible(t *testing.T) {
|
||||
compressible := bytes.Repeat([]byte("compress me "), 4096)
|
||||
incompressible := make([]byte, len(compressible))
|
||||
_, err := io.ReadFull(rand.Reader, incompressible)
|
||||
require.NoError(t, err)
|
||||
for name, handler := range map[string]compressionModeHandler{
|
||||
"gzip": &gzipModeHandler{},
|
||||
"zstd": &zstdModeHandler{},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type zstdModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
// Package api has type definitions for dosya.dev
|
||||
package api
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Response is the base response wrapper
|
||||
type Response struct {
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Error is an error response from the API.
|
||||
//
|
||||
// StatusCode and Status are filled in from the HTTP response. The API
|
||||
// sets ErrorCode on the errors a client is expected to act on rather
|
||||
// than just report, e.g. "concurrent_upload_limit".
|
||||
type Error struct {
|
||||
StatusCode int `json:"-"`
|
||||
Status string `json:"-"`
|
||||
OK bool `json:"ok"`
|
||||
Message string `json:"error"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
|
||||
}
|
||||
|
||||
// Error satisfies the error interface
|
||||
func (e *Error) Error() string {
|
||||
out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
|
||||
if e.Message != "" {
|
||||
out += ": " + e.Message
|
||||
}
|
||||
if e.ErrorCode != "" {
|
||||
out += " (" + e.ErrorCode + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileItem represents a file from the dosya API
|
||||
type FileItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
UpdatedAt float64 `json:"updated_at"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
|
||||
// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
|
||||
// are never re-read to hash the whole object - the same limitation S3 has).
|
||||
ContentHash string `json:"content_hash"`
|
||||
}
|
||||
|
||||
// FolderItem represents a folder from the dosya API
|
||||
type FolderItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
FileCount int `json:"file_count"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
}
|
||||
|
||||
// ListResponse is the response from GET /api/files
|
||||
type ListResponse struct {
|
||||
Response
|
||||
Files []FileItem `json:"files"`
|
||||
Folders []FolderItem `json:"folders"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
// Pagination describes which page of files a ListResponse holds
|
||||
type Pagination struct {
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest is the request body for POST /api/folders
|
||||
type CreateFolderRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse is the response from POST /api/folders
|
||||
type CreateFolderResponse struct {
|
||||
Response
|
||||
Folder struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Workspace string `json:"workspace_id"`
|
||||
} `json:"folder"`
|
||||
}
|
||||
|
||||
// DeleteFolderResponse is the response from DELETE /api/folders/:id
|
||||
type DeleteFolderResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
Complete bool `json:"complete"`
|
||||
Remaining int `json:"remaining"`
|
||||
FilesAffected int `json:"files_affected"`
|
||||
FoldersRemoved int `json:"folders_removed"`
|
||||
}
|
||||
|
||||
// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
|
||||
type RenameFolderRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFolderRequest is the request body for PUT /api/folders/:id/move
|
||||
type MoveFolderRequest struct {
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// MoveFolderResponse is the response from PUT /api/folders/:id/move
|
||||
type MoveFolderResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// UploadInitRequest is the request body for POST /api/upload/init
|
||||
type UploadInitRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Region string `json:"region,omitempty"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
FileID *string `json:"file_id,omitempty"`
|
||||
}
|
||||
|
||||
// UploadInitResponse is the response from POST /api/upload/init
|
||||
type UploadInitResponse struct {
|
||||
Response
|
||||
SessionID string `json:"session_id"`
|
||||
UploadURL string `json:"upload_url"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Resumable *struct {
|
||||
PartSize int64 `json:"part_size"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
PartUploadURL string `json:"part_upload_url"`
|
||||
CompleteURL string `json:"complete_url"`
|
||||
StatusURL string `json:"status_url"`
|
||||
} `json:"resumable"`
|
||||
}
|
||||
|
||||
// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
|
||||
type UploadCompleteResponse struct {
|
||||
Response
|
||||
File struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Version int `json:"version"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
|
||||
// "" for a multipart upload, which the server does not hash.
|
||||
ContentHash string `json:"content_hash"`
|
||||
} `json:"file"`
|
||||
}
|
||||
|
||||
// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
|
||||
type UploadPartResponse struct {
|
||||
Response
|
||||
PartNumber int `json:"part_number"`
|
||||
ETag string `json:"etag"`
|
||||
BytesUploaded int64 `json:"bytes_uploaded"`
|
||||
PartsCompleted int `json:"parts_completed"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
}
|
||||
|
||||
// DeleteFileResponse is the response from DELETE /api/files/:id
|
||||
type DeleteFileResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
}
|
||||
|
||||
// RenameFileRequest is the request body for PUT /api/files/:id/rename
|
||||
type RenameFileRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// RenameFileResponse is the response from PUT /api/files/:id/rename
|
||||
type RenameFileResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFileRequest is the request body for PUT /api/files/:id/move
|
||||
type MoveFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest is the request body for POST /api/files/:id/copy
|
||||
type CopyFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileResponse is the response from POST /api/files/:id/copy
|
||||
type CopyFileResponse struct {
|
||||
Response
|
||||
FileID string `json:"file_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// ShareLinkRequest is the request body for POST /api/files/:id/share
|
||||
type ShareLinkRequest struct {
|
||||
ExpiresInDays *int `json:"expires_in_days,omitempty"`
|
||||
}
|
||||
|
||||
// ShareLinkResponse is the response from POST /api/files/:id/share
|
||||
type ShareLinkResponse struct {
|
||||
Response
|
||||
Link struct {
|
||||
ID string `json:"id"`
|
||||
Token string `json:"token"`
|
||||
URL string `json:"url"`
|
||||
ExpiresAt *int64 `json:"expires_at"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
} `json:"link"`
|
||||
}
|
||||
|
||||
// Workspace is one entry of the workspace list
|
||||
type Workspace struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// WorkspaceListResponse is the response from GET /api/workspaces
|
||||
type WorkspaceListResponse struct {
|
||||
Response
|
||||
Workspaces []Workspace `json:"workspaces"`
|
||||
}
|
||||
|
||||
// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
|
||||
type WorkspaceInfoResponse struct {
|
||||
Response
|
||||
Storage struct {
|
||||
Used int64 `json:"used"`
|
||||
Total int64 `json:"total"`
|
||||
Free int64 `json:"free"`
|
||||
} `json:"storage"`
|
||||
}
|
||||
@@ -1,772 +0,0 @@
|
||||
// Package dosya provides an interface to the dosya.dev cloud storage system.
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
rootID = "root"
|
||||
defaultBaseURL = "https://api.dosya.dev"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2
|
||||
attackConstant = 0
|
||||
)
|
||||
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "dosya",
|
||||
Description: "dosya.dev",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
|
||||
Name: "api_key",
|
||||
Sensitive: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "workspace_id",
|
||||
Help: `Your workspace ID.
|
||||
|
||||
Leave this blank to use the one workspace the API key can reach, which
|
||||
is what a key restricted to a single workspace does. If the key can
|
||||
reach several, rclone will refuse and list their IDs for you to choose
|
||||
from.`,
|
||||
}, {
|
||||
Help: "API base URL.",
|
||||
Name: "api_url",
|
||||
Default: defaultBaseURL,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
Help: `Concurrency for multipart uploads.
|
||||
|
||||
This is the number of parts of the same file that are uploaded
|
||||
concurrently. The first part is always uploaded on its own, because
|
||||
the server creates the upload when it receives it.
|
||||
|
||||
Each part in flight is held in memory, so memory use grows with
|
||||
this value times the part size the server chooses.
|
||||
|
||||
If you are uploading large files over high-speed links and these
|
||||
uploads do not fully utilize your bandwidth, then increasing this
|
||||
may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeSingleQuote |
|
||||
encoder.EncodeDoubleQuote |
|
||||
encoder.EncodeLtGt |
|
||||
encoder.EncodeCtl |
|
||||
encoder.EncodeLeftSpace |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
APIKey string `config:"api_key"`
|
||||
WorkspaceID string `config:"workspace_id"`
|
||||
APIURL string `config:"api_url"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dosya.dev filesystem
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
features *fs.Features
|
||||
rest *rest.Client
|
||||
pacer *fs.Pacer
|
||||
dirCache *dircache.DirCache
|
||||
}
|
||||
|
||||
// Object represents a dosya.dev file
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
file api.FileItem
|
||||
}
|
||||
|
||||
// findWorkspace fills in the workspace ID when the config leaves it blank.
|
||||
//
|
||||
// The ID is not shown anywhere in the dosya.dev web interface, so requiring
|
||||
// it meant asking people to find it in a developer console. A key restricted
|
||||
// to one workspace can only mean that one, and an account with a single
|
||||
// workspace has only one possible answer; anything else genuinely has to be
|
||||
// chosen, so list the choices in the error.
|
||||
func (f *Fs) findWorkspace(ctx context.Context) error {
|
||||
workspaces, err := f.listWorkspaces(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch len(workspaces) {
|
||||
case 0:
|
||||
return errors.New("no workspaces found for this API key")
|
||||
case 1:
|
||||
f.opt.WorkspaceID = workspaces[0].ID
|
||||
fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
|
||||
return nil
|
||||
}
|
||||
var choices strings.Builder
|
||||
for _, w := range workspaces {
|
||||
_, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
|
||||
}
|
||||
return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
|
||||
len(workspaces), choices.String())
|
||||
}
|
||||
|
||||
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
||||
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
|
||||
listing, err := f.listFilesAndFolders(ctx, pathID)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, folder := range listing.Folders {
|
||||
if f.opt.Enc.ToStandardName(folder.Name) == leaf {
|
||||
return folder.ID, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// CreateDir makes a directory with pathID as parent and name leaf
|
||||
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
|
||||
name := f.opt.Enc.FromStandardName(leaf)
|
||||
resp, err := f.createFolder(ctx, name, pathID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.Folder.ID, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("dosya.dev root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
// Modification times are stored and returned to the second (the API
|
||||
// keeps them as Unix seconds, sent on upload via the source-mtime header).
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash types of the filesystem
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
// The server computes and stores the SHA-256 of a single-shot upload's
|
||||
// bytes; multipart uploads have none (see Object.Hash).
|
||||
return hash.NewHashSet(hash.SHA256)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(config, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
|
||||
}
|
||||
|
||||
// No DuplicateFiles: an upload that names no existing file but whose
|
||||
// (workspace, folder, name) is already taken becomes a new VERSION of
|
||||
// that file rather than a second row, so a name is unique within a
|
||||
// folder and the same upload twice returns the same file id.
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
ReadMimeType: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
|
||||
f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
|
||||
|
||||
if f.opt.WorkspaceID == "" {
|
||||
err = f.findWorkspace(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, rootID, f)
|
||||
|
||||
// Find the current root
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// Assume it is a file
|
||||
newRoot, remote := dircache.SplitPath(root)
|
||||
tempF := *f
|
||||
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
|
||||
tempF.root = newRoot
|
||||
// Make new Fs which is the parent
|
||||
err = tempF.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// No root so return old f
|
||||
return f, nil
|
||||
}
|
||||
_, err := tempF.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// File doesn't exist so return old f
|
||||
return f, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
f.features.Fill(ctx, &tempF)
|
||||
f.dirCache = tempF.dirCache
|
||||
f.root = tempF.root
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, folder := range listing.Folders {
|
||||
folderName := f.opt.Enc.ToStandardName(folder.Name)
|
||||
fullPath := getRemote(dir, folderName)
|
||||
modTime := time.Unix(int64(folder.CreatedAt), 0)
|
||||
d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
|
||||
entries = append(entries, d)
|
||||
f.dirCache.Put(fullPath, folder.ID)
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
fileName := f.opt.Enc.ToStandardName(file.Name)
|
||||
remote := getRemote(dir, fileName)
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}
|
||||
entries = append(entries, o)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
if f.opt.Enc.ToStandardName(file.Name) == leaf {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
// Put uploads a new file
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
existingObj, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return existingObj, existingObj.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// put uploads src as a new file
|
||||
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
remote := src.Remote()
|
||||
|
||||
_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: fileFromUpload(resp, modTime),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fileFromUpload builds the FileItem for the object an upload just created.
|
||||
// The single-shot upload response omits updated_at, so the modification time
|
||||
// is taken from what was sent (modTime) rather than defaulting to the create
|
||||
// time - it is the time the server stored via the source-mtime header. The
|
||||
// SHA-256 comes straight from the response (empty for a multipart upload).
|
||||
func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
|
||||
updatedAt := resp.File.CreatedAt
|
||||
if !modTime.IsZero() {
|
||||
updatedAt = float64(modTime.Unix())
|
||||
}
|
||||
return api.FileItem{
|
||||
ID: resp.File.ID,
|
||||
Name: resp.File.Name,
|
||||
SizeBytes: resp.File.SizeBytes,
|
||||
MimeType: resp.File.MimeType,
|
||||
Extension: resp.File.Extension,
|
||||
Region: resp.File.Region,
|
||||
CreatedAt: resp.File.CreatedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
ContentHash: resp.File.ContentHash,
|
||||
}
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
return err
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The server checks emptiness itself (?if_empty=1) and answers
|
||||
// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
|
||||
err = f.removeFolder(ctx, directoryID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move src to this remote using server-side move operations
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't move - not same remote type")
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
|
||||
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find destination directory
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
needsMove := srcDirID != dstDirID
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
|
||||
// Moving and renaming in the same request (rather than a moveFile call
|
||||
// followed by a separate renameFile call) matters when both are needed:
|
||||
// a two-step move-then-rename can transiently place the file's CURRENT
|
||||
// name into a destination folder that already holds an unrelated file
|
||||
// with that name, even though the FINAL name is free - the server only
|
||||
// allows one active name per folder, so that intermediate state 409s.
|
||||
if needsMove || needsRename {
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Return new object. file.Name must reflect the destination leaf, not
|
||||
// the source's - a caller that reuses this object as the src of a
|
||||
// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
|
||||
// recomputes srcLeaf from file.Name; a stale name there makes that
|
||||
// second move see srcLeaf == dstLeaf and skip the rename entirely,
|
||||
// silently leaving the object under the wrong name.
|
||||
newFile := srcObj.file
|
||||
newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: newFile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
|
||||
srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srcLeaf := srcRemote
|
||||
if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
|
||||
srcLeaf = srcRemote[idx+1:]
|
||||
}
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
|
||||
// Move folder to new parent only if parent changed. The new name goes
|
||||
// in the same request, as for files in Move: a move followed by a
|
||||
// separate rename can collide with an unrelated folder sharing the
|
||||
// CURRENT name in the destination even though the final name is free.
|
||||
if srcDirectoryID != dstDirectoryID {
|
||||
renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move directory: %w", err)
|
||||
}
|
||||
if renamed {
|
||||
needsRename = false
|
||||
}
|
||||
}
|
||||
|
||||
if needsRename {
|
||||
err = f.renameFolder(ctx, srcID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side copy operations
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The copy API creates a new row named after the source file when
|
||||
// copying across folders, which collides with one already at the
|
||||
// destination. Copy under a temporary name, then delete the old
|
||||
// destination, then move the copy onto the name - so a copy that fails
|
||||
// leaves the destination as it was. Deleting it up front lost it
|
||||
// (prior art: backend/box).
|
||||
existingObj, err := f.NewObject(ctx, remote)
|
||||
switch err {
|
||||
case nil:
|
||||
tempRemote := remote + "-rclone-copy-" + random.String(8)
|
||||
fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
|
||||
tempObj, err := f.Copy(ctx, src, tempRemote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs.Debugf(tempRemote, "moving to real name %v", remote)
|
||||
if err := existingObj.Remove(ctx); err != nil {
|
||||
return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
|
||||
}
|
||||
return f.Move(ctx, tempObj, remote)
|
||||
case fs.ErrorObjectNotFound:
|
||||
// nothing at the destination, so copy straight onto the name
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dstName := f.opt.Enc.FromStandardName(dstLeaf)
|
||||
|
||||
resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
|
||||
// Servers that predate the name field ignore it and name the copy after
|
||||
// the source, or "Copy of <original>" in the same folder. Rename to the
|
||||
// requested destination name if it differs.
|
||||
if resp.Name != dstName {
|
||||
err = f.renameFile(ctx, resp.FileID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't rename copied file: %w", err)
|
||||
}
|
||||
resp.Name = dstName
|
||||
}
|
||||
|
||||
// A server-side copy is byte-identical to the source and the API
|
||||
// preserves the source's modification time and content hash onto the
|
||||
// copy, so the returned object must report the SOURCE's modtime and hash,
|
||||
// not the copy's wall-clock time. Reading "now" here made the object
|
||||
// rclone holds in memory disagree with the one read back from the remote
|
||||
// (its Fingerprint check compares the two exactly).
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: api.FileItem{
|
||||
ID: resp.FileID,
|
||||
Name: resp.Name,
|
||||
SizeBytes: srcObj.file.SizeBytes,
|
||||
MimeType: srcObj.file.MimeType,
|
||||
Extension: srcObj.file.Extension,
|
||||
Region: srcObj.file.Region,
|
||||
CreatedAt: srcObj.file.CreatedAt,
|
||||
UpdatedAt: srcObj.file.UpdatedAt,
|
||||
ContentHash: srcObj.file.ContentHash,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Purge deletes all the files and the container
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
|
||||
err = f.removeFolder(ctx, directoryID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
obj := o.(*Object)
|
||||
|
||||
resp, err := f.createShareLink(ctx, obj.file.ID, expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return resp.Link.URL, nil
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing as an
|
||||
// optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
info, err := f.getWorkspaceInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage := &fs.Usage{
|
||||
Used: fs.NewUsageValue(info.Storage.Used),
|
||||
Total: fs.NewUsageValue(info.Storage.Total),
|
||||
Free: fs.NewUsageValue(info.Storage.Free),
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Object methods
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// String returns a description of the Object
|
||||
func (o *Object) String() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// ModTime returns the modification date of the file
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
if o.file.UpdatedAt > 0 {
|
||||
return time.Unix(int64(o.file.UpdatedAt), 0)
|
||||
}
|
||||
return time.Unix(int64(o.file.CreatedAt), 0)
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return o.file.SizeBytes
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.SHA256 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// May be "" - a multipart-uploaded object carries no server-side hash,
|
||||
// which rclone treats as "hash unavailable for this object" (as it does
|
||||
// for S3 multipart objects).
|
||||
return o.file.ContentHash, nil
|
||||
}
|
||||
|
||||
// Storable says whether this object can be stored
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
return fs.ErrorCantSetModTime
|
||||
}
|
||||
|
||||
// Open opens the file for read
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
fs.FixRangeOption(options, o.file.SizeBytes)
|
||||
return o.fs.downloadFile(ctx, o.file.ID, options)
|
||||
}
|
||||
|
||||
// Update replaces the file content
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
// Find the directory ID for this file
|
||||
_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload as new version of existing file
|
||||
fileID := o.file.ID
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update object metadata
|
||||
o.file = fileFromUpload(resp, modTime)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove removes this object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.deleteFile(ctx, o.file.ID)
|
||||
}
|
||||
|
||||
// MimeType of an Object if known, "" otherwise
|
||||
func (o *Object) MimeType(ctx context.Context) string {
|
||||
return o.file.MimeType
|
||||
}
|
||||
|
||||
// ID returns the ID of the Object if known, or "" if not
|
||||
func (o *Object) ID() string {
|
||||
return o.file.ID
|
||||
}
|
||||
|
||||
func getRemote(dir, fileName string) string {
|
||||
if dir == "" {
|
||||
return fileName
|
||||
}
|
||||
return dir + "/" + fileName
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ dircache.DirCacher = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
)
|
||||
@@ -1,880 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// sourceMTimeHeader is the request header the upload endpoints read to set a
|
||||
// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
|
||||
const sourceMTimeHeader = "x-dosya-source-mtime"
|
||||
|
||||
// modTimeHeaders returns the ExtraHeaders that carry a source modification time
|
||||
// on an upload, or nil when there is nothing plausible to send. The server
|
||||
// clamps implausible values itself, but a zero time is never worth a header.
|
||||
func modTimeHeaders(modTime time.Time) map[string]string {
|
||||
if modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
|
||||
}
|
||||
|
||||
// workspaceHintHeader names the workspace a by-id request is about. The API
|
||||
// maps an id to its workspace through an index the workspace fills about a
|
||||
// second after a write, so an id this client was handed moments ago can still
|
||||
// miss; the hint lets the server look for it in the named workspace instead.
|
||||
// It only locates: every route still checks membership against the workspace
|
||||
// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
|
||||
const workspaceHintHeader = "X-Dosya-Workspace"
|
||||
|
||||
// byIDHeaders returns the ExtraHeaders every /api/files/:id and
|
||||
// /api/folders/:id call carries, or nil when no workspace is configured.
|
||||
func (f *Fs) byIDHeaders() map[string]string {
|
||||
if f.opt.WorkspaceID == "" {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
|
||||
}
|
||||
|
||||
// retryErrorCodes is a slice of error codes that we will retry
|
||||
var retryErrorCodes = []int{
|
||||
429, // Too Many Requests
|
||||
500, // Internal Server Error
|
||||
502, // Bad Gateway
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
|
||||
if retry {
|
||||
if d, ok := retryAfter(resp); ok {
|
||||
// The API answers 503 + Retry-After while a workspace is being
|
||||
// moved between storage homes, and Cloudflare sends it with 429.
|
||||
// The pacer's own backoff tops out at maxSleep, far short of that.
|
||||
fs.Debugf(nil, "Server asked to retry after %v", d)
|
||||
err = pacer.RetryAfterError(err, d)
|
||||
}
|
||||
}
|
||||
return retry, err
|
||||
}
|
||||
|
||||
// maxRetryAfter caps how long a single Retry-After is honoured for
|
||||
const maxRetryAfter = 5 * time.Minute
|
||||
|
||||
// retryAfter reads a 429 or 503 response's Retry-After header, which the API
|
||||
// sends as whole seconds
|
||||
func retryAfter(resp *http.Response) (time.Duration, bool) {
|
||||
if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
|
||||
return 0, false
|
||||
}
|
||||
seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
|
||||
if err != nil || seconds <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
|
||||
}
|
||||
|
||||
// errorHandler decodes a non-2xx response into an *api.Error so callers can
|
||||
// act on the machine-readable error_code rather than the message text.
|
||||
func errorHandler(resp *http.Response) error {
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading error out of body: %w", err)
|
||||
}
|
||||
apiErr := &api.Error{
|
||||
StatusCode: resp.StatusCode,
|
||||
Status: resp.Status,
|
||||
}
|
||||
if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
|
||||
// Not the API's JSON shape (e.g. an edge response that never
|
||||
// reached the API) - keep the body text so it isn't lost.
|
||||
apiErr.ErrorCode = ""
|
||||
apiErr.Message = strings.TrimSpace(string(body))
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
||||
// The workspace caps the number of in-flight uploads per user. The API
|
||||
// refuses an upload/init past that cap with a 400 carrying
|
||||
// error_code "concurrent_upload_limit", and documents it as an error to
|
||||
// wait on rather than give up on: slots free as uploads finish, and the
|
||||
// server marks sessions idle for 10 minutes as failed, so waiting up to
|
||||
// that long guarantees progress unless the workspace really is saturated.
|
||||
// A 400 is deliberately outside the pacer's retry codes, so the wait
|
||||
// lives in initUpload instead of shouldRetry.
|
||||
var (
|
||||
concurrentUploadRetryDelay = 5 * time.Second
|
||||
concurrentUploadWaitMax = 10 * time.Minute
|
||||
)
|
||||
|
||||
// abortUploadTimeout bounds the abort of a session the client is giving up on,
|
||||
// which runs outside the transfer's context so a cancelled transfer still
|
||||
// sends it
|
||||
const abortUploadTimeout = 10 * time.Second
|
||||
|
||||
// isConcurrentUploadLimit reports whether err is the API refusing an
|
||||
// upload because the workspace's concurrent upload limit is reached
|
||||
func isConcurrentUploadLimit(err error) bool {
|
||||
var apiErr *api.Error
|
||||
return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
|
||||
}
|
||||
|
||||
// listPageSize is the largest page GET /api/files will serve
|
||||
const listPageSize = 500
|
||||
|
||||
// listFilesAndFolders lists files and folders in a directory
|
||||
//
|
||||
// The API pages files (at most listPageSize per page) but returns every
|
||||
// folder on each page, so the files from all pages are gathered and the
|
||||
// folders are taken from the first page only. Pages are ordered by name,
|
||||
// which the API tie-breaks on id, so offsets are stable between requests.
|
||||
func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
|
||||
var listing *api.ListResponse
|
||||
for page := 1; ; page++ {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files",
|
||||
Parameters: map[string][]string{
|
||||
"workspace_id": {f.opt.WorkspaceID},
|
||||
"per_page": {strconv.Itoa(listPageSize)},
|
||||
"page": {strconv.Itoa(page)},
|
||||
"sort": {"name"},
|
||||
"dir": {"asc"},
|
||||
},
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
opts.Parameters["folder_id"] = []string{folderID}
|
||||
}
|
||||
|
||||
var result api.ListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list files: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if listing == nil {
|
||||
listing = &result
|
||||
} else {
|
||||
listing.Files = append(listing.Files, result.Files...)
|
||||
}
|
||||
if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
|
||||
return listing, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// createFolder creates a folder
|
||||
func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
|
||||
request := api.CreateFolderRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
Name: name,
|
||||
}
|
||||
if parentID != "" && parentID != rootID {
|
||||
request.ParentID = &parentID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/folders",
|
||||
}
|
||||
|
||||
var result api.CreateFolderResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// removeFolder deletes a folder
|
||||
//
|
||||
// ?permanent=1 trashes and purges a live folder in one request on today's
|
||||
// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
|
||||
// HTTP 409 "Folder is not empty" instead of purging a folder that still
|
||||
// holds a live file or a live child folder; that 409 is turned into
|
||||
// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
|
||||
// error, and it is never retried (409 is not one of retryErrorCodes). A big
|
||||
// subtree is purged in bounded passes (202, complete=false), so keep
|
||||
// calling until the purge is complete. An older server that ignores
|
||||
// permanent=1 only trashes on the first call (answering permanent=false),
|
||||
// so the loop's second pass sends the plain DELETE that purges it.
|
||||
func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
|
||||
parameters := map[string][]string{"permanent": {"1"}}
|
||||
if ifEmpty {
|
||||
parameters["if_empty"] = []string{"1"}
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/folders/" + folderID,
|
||||
Parameters: parameters,
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
for pass := 0; ; pass++ {
|
||||
var result api.DeleteFolderResponse
|
||||
var statusCode int
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
if resp != nil {
|
||||
statusCode = resp.StatusCode
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if ifEmpty && statusCode == http.StatusConflict {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if pass == 0 {
|
||||
return fmt.Errorf("couldn't remove folder: %w", err)
|
||||
}
|
||||
return fmt.Errorf("couldn't permanently remove folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if result.Permanent && result.Complete {
|
||||
return nil
|
||||
}
|
||||
if pass >= maxFolderPurgePasses {
|
||||
return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
|
||||
const maxFolderPurgePasses = 1000
|
||||
|
||||
// renameFolder renames a folder
|
||||
func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
|
||||
request := api.RenameFolderRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFolder moves a folder to a new parent
|
||||
func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
|
||||
request := api.MoveFolderRequest{Name: newName}
|
||||
if newParentID != "" && newParentID != rootID {
|
||||
request.ParentID = &newParentID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.MoveFolderResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("couldn't move folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return false, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
// Servers that predate the name field ignore it and answer without a
|
||||
// name, leaving the caller to rename separately
|
||||
return newName != "" && result.Name == newName, nil
|
||||
}
|
||||
|
||||
// deleteFile deletes a file
|
||||
//
|
||||
// ?permanent=1 trashes and purges it in one request on today's API. An
|
||||
// older server that ignores the flag only soft-deletes it, answering
|
||||
// permanent=false, so a second DELETE on the now soft-deleted file purges
|
||||
// it from the database and R2, as it always has.
|
||||
func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/files/" + fileID,
|
||||
Parameters: map[string][]string{"permanent": {"1"}},
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.DeleteFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't delete file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
|
||||
// Answered into its own value so a field the second response omits
|
||||
// can't be read off the first.
|
||||
if !result.Permanent {
|
||||
var permanent api.DeleteFileResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't permanently delete file: %w", err)
|
||||
}
|
||||
if !permanent.OK {
|
||||
return fmt.Errorf("API error: %s", permanent.Error)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renameFile renames a file
|
||||
func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
|
||||
request := api.RenameFileRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.RenameFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFile moves a file to a new folder and, if newName is non-empty,
|
||||
// renames it in the same request
|
||||
func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
|
||||
request := api.MoveFileRequest{}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
if newName != "" {
|
||||
request.Name = &newName
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies a file to a folder
|
||||
func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
|
||||
request := api.CopyFileRequest{Name: name}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/copy",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.CopyFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// downloadFile downloads a file by its ID
|
||||
//
|
||||
// The download endpoint answers 302 with a presigned R2 URL in Location, so
|
||||
// this is two calls. Both go through the rest client, which is built on
|
||||
// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
|
||||
// --user-agent, --bind, --dump and the proxy settings apply to downloads as
|
||||
// they do to every other call, and errorHandler and shouldRetry decide the
|
||||
// errors and the retries uniformly.
|
||||
func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
|
||||
// NoRedirect stops the client following the 302 with the API's
|
||||
// Authorization header attached. A 302 is not a 2xx, so Call reports it
|
||||
// as an error (having read and closed the body); the Location header
|
||||
// is what actually matters, and finding it clears the error, as in
|
||||
// backend/linkbox.
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files/" + fileID + "/download",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
NoRedirect: true,
|
||||
}
|
||||
var downloadURL string
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
if resp != nil {
|
||||
if location := resp.Header.Get("Location"); location != "" {
|
||||
downloadURL, err = location, nil
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get download URL: %w", err)
|
||||
}
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("no download URL in redirect response")
|
||||
}
|
||||
|
||||
// Fetch the object from the presigned URL. It carries its own
|
||||
// credentials and R2 refuses a request that also has an Authorization
|
||||
// header, so the client-level bearer token is removed by setting the
|
||||
// header empty: the rest client skips a header with an empty value.
|
||||
opts = rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: downloadURL,
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{"Authorization": ""},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't download file: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// uploadSmallFile uploads a file smaller than 10MB in a single request
|
||||
func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
Body: in,
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
// The body is a stream which can't be rewound, so a low level retry
|
||||
// would resend whatever is left of it - nothing - and the server
|
||||
// would store that as an empty file. Don't retry here; rclone's high
|
||||
// level retries redo the whole transfer instead.
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadPart uploads a single part of a multipart upload
|
||||
func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
|
||||
size := int64(len(part))
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
var result api.UploadPartResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
// A fresh reader per attempt so a retry resends the whole part
|
||||
opts.Body = bytes.NewReader(part)
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// completeUpload finalizes a multipart upload
|
||||
func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/" + sessionID + "/complete",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't complete upload: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// initUpload initializes a new upload session
|
||||
func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
|
||||
request := api.UploadInitRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
FileName: fileName,
|
||||
FileSize: fileSize,
|
||||
MimeType: mimeType,
|
||||
FileID: fileID,
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/init",
|
||||
}
|
||||
|
||||
var result api.UploadInitResponse
|
||||
deadline := time.Now().Add(concurrentUploadWaitMax)
|
||||
for {
|
||||
result = api.UploadInitResponse{}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", err)
|
||||
}
|
||||
fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
|
||||
case <-time.After(concurrentUploadRetryDelay):
|
||||
}
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadParts reads the source part by part and uploads the parts.
|
||||
//
|
||||
// Part 1 goes on its own: the server creates the R2 multipart upload (and a
|
||||
// new file's id and key) when it receives the first part, so parts sent
|
||||
// before that has finished would each start an upload of their own. The
|
||||
// rest go out opt.UploadConcurrency at a time. A part is only read from the
|
||||
// source once a buffer is free, so at most that many parts are held in
|
||||
// memory, and a short source is still found by the reader before the part
|
||||
// it could not fill is sent.
|
||||
func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
|
||||
readPart := func(buf []byte, partNum int) ([]byte, error) {
|
||||
currentPartSize := partSize
|
||||
if partNum == totalParts {
|
||||
currentPartSize = size - partSize*int64(totalParts-1)
|
||||
}
|
||||
part := buf[:currentPartSize]
|
||||
if _, err := io.ReadFull(in, part); err != nil {
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
|
||||
}
|
||||
return part, nil
|
||||
}
|
||||
|
||||
concurrency := max(f.opt.UploadConcurrency, 1)
|
||||
buffers := make(chan []byte, concurrency)
|
||||
buffers <- make([]byte, partSize)
|
||||
allocated := 1
|
||||
|
||||
first := <-buffers
|
||||
part, err := readPart(first, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
buffers <- first
|
||||
|
||||
// A failed read has to take the parts already in flight down with it,
|
||||
// or they upload to completion while the error waits on Wait.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var readErr error
|
||||
for partNum := 2; partNum <= totalParts; partNum++ {
|
||||
// Take a free buffer, allocating one while under the limit.
|
||||
var buf []byte
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
default:
|
||||
if allocated < concurrency {
|
||||
buf = make([]byte, partSize)
|
||||
allocated++
|
||||
} else {
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
case <-gCtx.Done():
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf == nil {
|
||||
break // a part failed; its error comes back from Wait
|
||||
}
|
||||
part, err := readPart(buf, partNum)
|
||||
if err != nil {
|
||||
readErr = err
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
defer func() { buffers <- buf }()
|
||||
_, err := f.uploadPart(gCtx, part, sessionID, partNum)
|
||||
return err
|
||||
})
|
||||
}
|
||||
err = g.Wait()
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// abortUpload retires an upload session the client is giving up on.
|
||||
//
|
||||
// Best effort: the server reclaims an abandoned session itself, so a failure
|
||||
// here - including an older server with no such route - costs the slot the
|
||||
// wait it would have cost anyway and is not worth failing an upload over.
|
||||
// Sent on a context of its own so a cancelled transfer still gives its slot
|
||||
// back rather than leaving it to the sweep.
|
||||
func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
|
||||
defer cancel()
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
NoResponse: true,
|
||||
}
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadFile handles the full upload flow (small or multipart)
|
||||
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
|
||||
// Both refusals live here so a new file and a new version of an existing
|
||||
// one answer the same way: upload/init needs a byte count up front, and
|
||||
// the server stores no zero-length object.
|
||||
if size < 0 {
|
||||
return nil, errors.New("can't upload files with unknown size")
|
||||
}
|
||||
if size == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
leaf := f.opt.Enc.FromStandardName(path.Base(remote))
|
||||
|
||||
initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A session abandoned here holds one of the workspace's upload slots
|
||||
// until the server's sweep retires it ten minutes later, and rclone's
|
||||
// high-level retry opens a fresh session rather than resuming this one -
|
||||
// so without this, a few failed attempts use up every slot.
|
||||
defer func() {
|
||||
if err != nil {
|
||||
f.abortUpload(ctx, initResp.SessionID)
|
||||
}
|
||||
}()
|
||||
|
||||
// Count what the source actually supplies, so a source which ends
|
||||
// early is reported as an error rather than stored as a truncated
|
||||
// file.
|
||||
counter := readers.NewCountingReader(in)
|
||||
|
||||
// Small file upload (single PUT, no resumable)
|
||||
if initResp.Resumable == nil {
|
||||
resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := int64(counter.BytesRead()); got != size {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Multipart upload. Each part is read fully into memory before it is
|
||||
// sent, so a short source is caught before the part goes out and a
|
||||
// part which has to be retried is resent with all of its bytes.
|
||||
partSize := initResp.Resumable.PartSize
|
||||
totalParts := initResp.Resumable.TotalParts
|
||||
// Both bounds matter. partSize*(totalParts-1) >= size means a part would
|
||||
// have nothing to carry; size > partSize*totalParts means the last part
|
||||
// is bigger than partSize, and readPart would slice past its buffer and
|
||||
// panic.
|
||||
if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
|
||||
return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
|
||||
}
|
||||
if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.completeUpload(ctx, initResp.SessionID, modTime)
|
||||
}
|
||||
|
||||
// createShareLink creates a public share link for a file
|
||||
func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
|
||||
request := api.ShareLinkRequest{}
|
||||
if expire < fs.DurationOff {
|
||||
days := int(time.Duration(expire).Hours() / 24)
|
||||
if days < 1 {
|
||||
days = 1
|
||||
}
|
||||
request.ExpiresInDays = &days
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/share",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.ShareLinkResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create share link: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// getWorkspaceInfo fetches workspace storage info
|
||||
// listWorkspaces lists the workspaces this credential can reach. An API key
|
||||
// pinned to a workspace sees exactly one: its own.
|
||||
func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces",
|
||||
}
|
||||
|
||||
var result api.WorkspaceListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list workspaces: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return result.Workspaces, nil
|
||||
}
|
||||
|
||||
func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces/" + f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
var result api.WorkspaceInfoResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get workspace info: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
header string
|
||||
want time.Duration
|
||||
ok bool
|
||||
}{
|
||||
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
|
||||
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
|
||||
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
|
||||
{"missing header", http.StatusServiceUnavailable, "", 0, false},
|
||||
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
|
||||
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
|
||||
if tc.header != "" {
|
||||
resp.Header.Set("Retry-After", tc.header)
|
||||
}
|
||||
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
|
||||
assert.True(t, retry)
|
||||
d, ok := pacer.IsRetryAfter(err)
|
||||
assert.Equal(t, tc.ok, ok)
|
||||
if tc.ok {
|
||||
assert.Equal(t, tc.want, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
|
||||
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
|
||||
resp.Header.Set("Retry-After", "30")
|
||||
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
|
||||
assert.False(t, retry)
|
||||
_, ok := pacer.IsRetryAfter(err)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestRemoveFolderPurgesPermanently(t *testing.T) {
|
||||
// a big subtree purges in bounded passes: 202 complete=false repeats
|
||||
// the same request until complete=true
|
||||
answers := []struct {
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
|
||||
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
|
||||
}
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "DELETE", r.Method)
|
||||
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
|
||||
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
|
||||
a := answers[calls]
|
||||
calls++
|
||||
return jsonResp(a.status, a.body), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
|
||||
assert.Equal(t, len(answers), calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "locked file")
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", true)
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRmdirMakesOneRequest(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
require.NoError(t, f.Rmdir(context.Background(), "sub"))
|
||||
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
|
||||
}
|
||||
|
||||
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
err := f.Rmdir(context.Background(), "sub")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
}
|
||||
|
||||
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
permanent bool
|
||||
wantCalls int
|
||||
}{
|
||||
{"today's API purges in one call", true, 1},
|
||||
{"an older server ignores the flag and still gets a second DELETE", false, 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
|
||||
assert.Equal(t, tc.wantCalls, calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// decodeBody reads a JSON request body into a map
|
||||
func decodeBody(t *testing.T, r *http.Request) map[string]any {
|
||||
b, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
var m map[string]any
|
||||
require.NoError(t, json.Unmarshal(b, &m))
|
||||
return m
|
||||
}
|
||||
|
||||
func TestMoveFolderSendsName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
newName string
|
||||
respBody string
|
||||
wantRenamed bool
|
||||
}{
|
||||
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
|
||||
{"older server ignores name", "dst", `{"ok":true}`, false},
|
||||
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_parent", body["parent_id"])
|
||||
if tc.newName == "" {
|
||||
assert.NotContains(t, body, "name")
|
||||
} else {
|
||||
assert.Equal(t, tc.newName, body["name"])
|
||||
}
|
||||
return jsonResp(http.StatusOK, tc.respBody), nil
|
||||
}))
|
||||
|
||||
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantRenamed, renamed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyFileSendsName(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_dst", body["folder_id"])
|
||||
assert.Equal(t, "final.txt", body["name"])
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
|
||||
}))
|
||||
|
||||
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_2", resp.FileID)
|
||||
assert.Equal(t, "final.txt", resp.Name)
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
|
||||
f := &Fs{}
|
||||
assert.Equal(t, time.Second, f.Precision())
|
||||
assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
|
||||
// Exactly SHA-256, nothing else.
|
||||
assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
|
||||
}
|
||||
|
||||
func TestObjectHashReturnsContentHash(t *testing.T) {
|
||||
o := &Object{file: api.FileItem{ContentHash: "abc123"}}
|
||||
|
||||
got, err := o.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "abc123", got)
|
||||
|
||||
// A multipart object with no server-side hash yields "" and no error,
|
||||
// exactly as an S3 multipart object does.
|
||||
empty := &Object{file: api.FileItem{ContentHash: ""}}
|
||||
got, err = empty.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", got)
|
||||
|
||||
// Any other algorithm is unsupported.
|
||||
_, err = o.Hash(context.Background(), hash.MD5)
|
||||
assert.ErrorIs(t, err, hash.ErrUnsupported)
|
||||
}
|
||||
|
||||
func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
|
||||
assert.True(t, mt.Equal(o.ModTime(context.Background())))
|
||||
}
|
||||
|
||||
func TestModTimeHeaders(t *testing.T) {
|
||||
assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
|
||||
h := modTimeHeaders(mt)
|
||||
require.NotNil(t, h)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
|
||||
}
|
||||
|
||||
func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
|
||||
resp := &api.UploadCompleteResponse{}
|
||||
resp.File.ID = "fil_1"
|
||||
resp.File.Name = "x.txt"
|
||||
resp.File.SizeBytes = 10
|
||||
resp.File.CreatedAt = 1_000
|
||||
resp.File.ContentHash = "deadbeef"
|
||||
|
||||
mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
|
||||
item := fileFromUpload(resp, mt)
|
||||
assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
|
||||
assert.Equal(t, "deadbeef", item.ContentHash)
|
||||
assert.Equal(t, float64(1_000), item.CreatedAt)
|
||||
|
||||
// A zero modtime falls back to the create time.
|
||||
item = fileFromUpload(resp, time.Time{})
|
||||
assert.Equal(t, float64(1_000), item.UpdatedAt)
|
||||
}
|
||||
|
||||
// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
|
||||
// fake transport and asserts the source-mtime header actually rides on the PUT.
|
||||
func TestUploadSendsSourceMTimeHeader(t *testing.T) {
|
||||
var putMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
|
||||
}
|
||||
switch {
|
||||
case r.Method == "POST" && r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
|
||||
putMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "h", resp.File.ContentHash)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
|
||||
// the completion request for a multipart upload (the server stores the modtime
|
||||
// there, not on the individual parts).
|
||||
func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
|
||||
var completeMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body)
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
|
||||
case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
case r.URL.Path == "/api/upload/upl_1/complete":
|
||||
completeMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
|
||||
// server-side Copy returns: it must report the SOURCE's modtime and hash (the
|
||||
// copy is byte-identical and the API preserves both), not the copy's wall
|
||||
// clock. Reading "now" here made rclone's in-memory object disagree with the
|
||||
// one read back from the remote and failed FsCopy's fingerprint check.
|
||||
func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case r.Method == "GET" && r.URL.Path == "/api/files":
|
||||
// NewObject's existence probe on the destination: nothing there.
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
|
||||
case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
|
||||
mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
|
||||
ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
|
||||
ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
|
||||
CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
|
||||
}}
|
||||
|
||||
dst, err := f.Copy(context.Background(), src, "f-copy.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
|
||||
assert.Equal(t, int64(100), dst.Size())
|
||||
got, err := dst.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "srchash", got, "copy keeps the source hash")
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
|
||||
|
||||
func TestErrorHandlerDecodesAPIError(t *testing.T) {
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Status: "400 Bad Request",
|
||||
Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
|
||||
assert.Contains(t, err.Error(), "uploads in progress")
|
||||
assert.Contains(t, err.Error(), "concurrent_upload_limit")
|
||||
}
|
||||
|
||||
func TestErrorHandlerNonJSONBody(t *testing.T) {
|
||||
// e.g. an edge (Cloudflare) response that never reached the API
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Status: "429 Too Many Requests",
|
||||
Body: io.NopCloser(strings.NewReader("error code: 1015\n")),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
|
||||
assert.Equal(t, "", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "429")
|
||||
assert.Contains(t, err.Error(), "error code: 1015")
|
||||
}
|
||||
|
||||
func TestShouldRetryAPIErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// The concurrent upload limit is handled by initUpload's slot wait,
|
||||
// not by the pacer's low level retries.
|
||||
limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
|
||||
retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
|
||||
assert.False(t, retry)
|
||||
assert.Equal(t, limitErr, err)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
|
||||
assert.True(t, retry)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
|
||||
assert.False(t, retry)
|
||||
}
|
||||
|
||||
// swapWaitTimings shortens the slot wait timings for a test and returns a
|
||||
// function which restores them
|
||||
func swapWaitTimings(delay, max time.Duration) func() {
|
||||
oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
|
||||
concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
|
||||
return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
|
||||
}
|
||||
|
||||
// newTestFs makes an Fs which talks to srv
|
||||
func newTestFs(srv *httptest.Server) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// limitServer answers the first n init calls with the concurrent upload
|
||||
// limit error and the rest with success
|
||||
func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/upload/init", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if calls.Add(1) <= n {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(concurrentLimitBody))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
}
|
||||
|
||||
func TestInitUploadWaitsForFreeSlot(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 2, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, resp.OK)
|
||||
assert.Equal(t, int32(3), calls.Load())
|
||||
}
|
||||
|
||||
func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "couldn't init upload")
|
||||
assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
|
||||
}
|
||||
|
||||
func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
|
||||
defer swapWaitTimings(time.Minute, time.Hour)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeListServer serves GET /api/files the way the API does: files paged at
|
||||
// per_page, every folder repeated on each page
|
||||
func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/files", r.URL.Path)
|
||||
q := r.URL.Query()
|
||||
assert.Equal(t, "name", q.Get("sort"))
|
||||
assert.Equal(t, "asc", q.Get("dir"))
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
perPage, _ := strconv.Atoi(q.Get("per_page"))
|
||||
require.Positive(t, page)
|
||||
require.Positive(t, perPage)
|
||||
*pages = append(*pages, page)
|
||||
|
||||
var files []string
|
||||
for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
|
||||
files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
|
||||
}
|
||||
totalPages := max(1, (totalFiles+perPage-1)/perPage)
|
||||
body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
|
||||
`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
|
||||
strings.Join(files, ","), page, perPage, totalFiles, totalPages)
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersFollowsPages(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1, 2, 3}, pages)
|
||||
require.Len(t, listing.Files, 2*listPageSize+7)
|
||||
assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
|
||||
// folders repeat on every page and must not be duplicated
|
||||
assert.Len(t, listing.Folders, 1)
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersSinglePage(t *testing.T) {
|
||||
for _, total := range []int{0, 1, listPageSize} {
|
||||
t.Run(strconv.Itoa(total), func(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, total, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), rootID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1}, pages)
|
||||
assert.Len(t, listing.Files, total)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// parallelUploadServer answers the multipart endpoints without holding a
|
||||
// lock across a request, so parts really do overlap, and records what the
|
||||
// ordering rules below are about: how many parts were in flight at once, and
|
||||
// whether any part started before part 1 had finished.
|
||||
//
|
||||
// Part 1 matters because the API creates the R2 multipart upload (and, for a
|
||||
// new file, its id and key) on the first part it sees. Parts that race ahead
|
||||
// of part 1 would each create their own upload, and only one of those would
|
||||
// be completed.
|
||||
type parallelUploadServer struct {
|
||||
t *testing.T
|
||||
resumable string
|
||||
partDelay time.Duration
|
||||
failPart int // answer this part with a 400, 0 for none
|
||||
mu sync.Mutex
|
||||
inFlight int
|
||||
maxFlight int
|
||||
part1Done bool
|
||||
earlyParts []int // parts that started before part 1 finished
|
||||
parts map[int]string
|
||||
completes int
|
||||
aborts int
|
||||
}
|
||||
|
||||
func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
|
||||
s.mu.Lock()
|
||||
if n != 1 && !s.part1Done {
|
||||
s.earlyParts = append(s.earlyParts, n)
|
||||
}
|
||||
s.inFlight++
|
||||
s.maxFlight = max(s.maxFlight, s.inFlight)
|
||||
s.mu.Unlock()
|
||||
|
||||
time.Sleep(s.partDelay)
|
||||
|
||||
s.mu.Lock()
|
||||
s.inFlight--
|
||||
if n == 1 {
|
||||
s.part1Done = true
|
||||
}
|
||||
s.parts[n] = string(body)
|
||||
fail := n == s.failPart
|
||||
s.mu.Unlock()
|
||||
|
||||
if fail {
|
||||
return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.mu.Lock()
|
||||
s.completes++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.mu.Lock()
|
||||
s.aborts++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
|
||||
return ¶llelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
|
||||
}
|
||||
|
||||
const tenParts = `{"part_size":2,"total_parts":10}`
|
||||
|
||||
func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
content := "aabbccddeeffgghhiijj"
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
|
||||
assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
for i := 1; i <= 10; i++ {
|
||||
assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 1
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, srv.maxFlight)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
srv.failPart = 6
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "part 6")
|
||||
assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
|
||||
// 20 bytes declared, 15 supplied: the short read is found by the
|
||||
// reader, whichever parts are already in flight, and nothing completes.
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Equal(t, 0, srv.completes)
|
||||
assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Test Dosya filesystem interface
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestDosya:",
|
||||
})
|
||||
}
|
||||
@@ -1,247 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// rtFunc adapts a function into an http.RoundTripper so the upload path
|
||||
// can be driven without a network. Unlike a real server it always reads
|
||||
// the whole request body and answers, which is exactly the misbehaviour
|
||||
// (accepting a short body as a complete upload) these tests guard against.
|
||||
type rtFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func jsonResp(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{"Content-Type": {"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
ContentLength: int64(len(body)),
|
||||
}
|
||||
}
|
||||
|
||||
func newFakeFs(rt http.RoundTripper) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// fakeUploadServer records what an upload sends and lets a test inject
|
||||
// failures per request
|
||||
type fakeUploadServer struct {
|
||||
t *testing.T
|
||||
mu sync.Mutex
|
||||
resumable string // JSON for the init "resumable" field, "" for a single PUT
|
||||
puts [][]byte // bodies of PUT /api/upload/<id>
|
||||
parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
|
||||
completes int
|
||||
aborts int // DELETE /api/upload/<id>
|
||||
failFirst map[string]bool // path -> answer the first request with a 502
|
||||
seenBefore map[string]bool
|
||||
}
|
||||
|
||||
func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
|
||||
return &fakeUploadServer{
|
||||
t: t,
|
||||
resumable: resumable,
|
||||
parts: map[int][][]byte{},
|
||||
failFirst: map[string]bool{},
|
||||
seenBefore: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
// Bodies are recorded for every attempt, including the ones answered
|
||||
// with a 502, so tests can see exactly what each attempt carried.
|
||||
fail := s.failFirst[path] && !s.seenBefore[path]
|
||||
s.seenBefore[path] = true
|
||||
bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
if s.resumable != "" {
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && path == "/api/upload/upl_1":
|
||||
s.puts = append(s.puts, body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
s.parts[n] = append(s.parts[n], body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.completes++
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.aborts++
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestUploadSmallSendsWholeBody(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := strings.Repeat("a", 100)
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
require.Len(t, srv.puts, 1)
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
func TestUploadSmallRejectsShortSource(t *testing.T) {
|
||||
// The source declares 100 bytes but only supplies 60. A server which
|
||||
// accepts that as a complete upload must not turn into a success.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
|
||||
}
|
||||
|
||||
func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
|
||||
// A retried PUT would resend an already-consumed reader, i.e. an
|
||||
// empty body, which the server would store as an empty file. The
|
||||
// single PUT therefore must not be retried at the low level.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
srv.failFirst["/api/upload/upl_1"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
content := strings.Repeat("a", 100)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
const threeParts = `{"part_size":4,"total_parts":3}`
|
||||
|
||||
func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := "0123456789"
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
require.Len(t, srv.parts, 3)
|
||||
assert.Equal(t, "0123", string(srv.parts[1][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
}
|
||||
|
||||
func TestUploadMultipartRejectsShortSource(t *testing.T) {
|
||||
// 10 bytes declared, 8 supplied: the last part must never be sent
|
||||
// and the upload must not be completed.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
|
||||
assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
|
||||
assert.Equal(t, 0, srv.completes, "upload must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
|
||||
// A part which fails transiently must be resent with all its bytes,
|
||||
// not with whatever is left of a consumed reader.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
srv.failFirst["/api/upload/upl_1/part/2"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][1]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
// An upload that fails after init has to hand its session back: the session
|
||||
// holds one of the workspace's upload slots for ten minutes otherwise, and
|
||||
// rclone's high-level retry opens a new one rather than resuming it.
|
||||
func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
resumable string
|
||||
content string
|
||||
size int64
|
||||
}{
|
||||
{"small file short source", "", "0123", 10},
|
||||
{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, tc.resumable)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
|
||||
"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, srv.aborts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A successful upload must not abort anything
|
||||
func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
|
||||
"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, srv.aborts)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Every by-id call names the workspace, so the API can find an id it minted
|
||||
// moments ago before its file_home index row lands.
|
||||
func TestByIDCallsSendWorkspaceHint(t *testing.T) {
|
||||
seen := map[string]string{}
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
|
||||
switch r.URL.Path {
|
||||
case "/api/folders/fld_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
case "/api/files/fil_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
|
||||
case "/api/folders/fld_1/move":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
|
||||
default:
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
}))
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.removeFolder(ctx, "fld_1", false))
|
||||
require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
|
||||
_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.deleteFile(ctx, "fil_1"))
|
||||
require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
|
||||
require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
|
||||
_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
_, err = f.createShareLink(ctx, "fil_1", 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
for call, header := range seen {
|
||||
assert.Equal(t, "ws_test", header, call)
|
||||
}
|
||||
assert.Len(t, seen, 8)
|
||||
}
|
||||
|
||||
// The download's API request carries the hint; the presigned R2 fetch must not.
|
||||
func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
|
||||
var apiHint, blobHint string
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/files/fil_1/download" {
|
||||
apiHint = r.Header.Get("X-Dosya-Workspace")
|
||||
http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
|
||||
return
|
||||
}
|
||||
blobHint = r.Header.Get("X-Dosya-Workspace")
|
||||
_, _ = w.Write([]byte("bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
f.opt.APIURL = srv.URL
|
||||
|
||||
rc, err := f.downloadFile(context.Background(), "fil_1", nil)
|
||||
require.NoError(t, err)
|
||||
_ = rc.Close()
|
||||
|
||||
assert.Equal(t, "ws_test", apiHint)
|
||||
assert.Equal(t, "", blobHint)
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeWorkspaceServer answers GET /api/workspaces with the given body
|
||||
func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/workspaces", r.URL.Path)
|
||||
*calls++
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.NoError(t, f.findWorkspace(context.Background()))
|
||||
assert.Equal(t, "ws_only", f.opt.WorkspaceID)
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceListsTheChoices(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
|
||||
`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
err := f.findWorkspace(context.Background())
|
||||
require.Error(t, err)
|
||||
// the ids have to be in the message: they are the only place the user
|
||||
// can read them, which is the whole point of listing them
|
||||
assert.Contains(t, err.Error(), "ws_one (One)")
|
||||
assert.Contains(t, err.Error(), "ws_two (Two)")
|
||||
assert.Empty(t, f.opt.WorkspaceID)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceWithNone(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
|
||||
}
|
||||
@@ -8,14 +8,10 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
stdsync "sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -29,113 +25,13 @@ import (
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
service, err := drive.NewService(
|
||||
context.Background(),
|
||||
option.WithHTTPClient(server.Client()),
|
||||
option.WithEndpoint(server.URL+"/"),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return &Fs{
|
||||
svc: service,
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
permissionsMu: new(stdsync.Mutex),
|
||||
permissions: make(map[string]*drive.Permission),
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionConcurrent(t *testing.T) {
|
||||
const requestCount = 2
|
||||
arrived := make(chan string, requestCount)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce stdsync.Once
|
||||
releaseRequests := func() {
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
}
|
||||
defer releaseRequests()
|
||||
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
arrived <- permissionID
|
||||
<-release
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
type permissionResult struct {
|
||||
permission *drive.Permission
|
||||
inherited bool
|
||||
err error
|
||||
}
|
||||
results := make(chan permissionResult, requestCount)
|
||||
permissionIDs := []string{"permission-1", "permission-2"}
|
||||
for _, permissionID := range permissionIDs {
|
||||
go func() {
|
||||
permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
|
||||
results <- permissionResult{permission: permission, inherited: inherited, err: err}
|
||||
}()
|
||||
}
|
||||
|
||||
arrivedIDs := make(map[string]bool, requestCount)
|
||||
timer := time.NewTimer(time.Second)
|
||||
defer timer.Stop()
|
||||
for len(arrivedIDs) < requestCount {
|
||||
select {
|
||||
case permissionID := <-arrived:
|
||||
arrivedIDs[permissionID] = true
|
||||
case <-timer.C:
|
||||
releaseRequests()
|
||||
t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
|
||||
}
|
||||
}
|
||||
releaseRequests()
|
||||
|
||||
assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
|
||||
returnedIDs := make(map[string]bool, requestCount)
|
||||
for range requestCount {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
require.NotNil(t, result.permission)
|
||||
assert.False(t, result.inherited)
|
||||
returnedIDs[result.permission.Id] = true
|
||||
}
|
||||
assert.Equal(t, arrivedIDs, returnedIDs)
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionCacheHit(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
|
||||
assert.Same(t, first, second)
|
||||
assert.Equal(t, int32(1), requests.Load())
|
||||
}
|
||||
|
||||
func TestDriveScopes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
|
||||
@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
|
||||
|
||||
// getPermission returns permissions for the fileID and permissionID passed in
|
||||
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
|
||||
f.permissionsMu.Lock()
|
||||
defer f.permissionsMu.Unlock()
|
||||
if useCache {
|
||||
f.permissionsMu.Lock()
|
||||
perm = f.permissions[permissionID]
|
||||
f.permissionsMu.Unlock()
|
||||
if perm != nil {
|
||||
return perm, false, nil
|
||||
}
|
||||
@@ -131,9 +131,7 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
|
||||
cleanPermission(perm)
|
||||
|
||||
// cache the permission
|
||||
f.permissionsMu.Lock()
|
||||
f.permissions[permissionID] = perm
|
||||
f.permissionsMu.Unlock()
|
||||
|
||||
return perm, inherited, err
|
||||
}
|
||||
|
||||
+18
-30
@@ -11,6 +11,7 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -21,7 +22,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
@@ -112,13 +112,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
|
||||
}
|
||||
|
||||
// Make an http.Request for the range passed in
|
||||
//
|
||||
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
|
||||
// to an io.Closer and close it after each attempt — the chunk buffer is
|
||||
// pool-backed, so an early close would free its pages and a retry would then
|
||||
// read a dead buffer. The upload loop owns the buffer's lifetime.
|
||||
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
|
||||
req.ContentLength = reqSize
|
||||
totalSize := "*"
|
||||
if rx.ContentLength >= 0 {
|
||||
@@ -173,40 +168,37 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
start := int64(0)
|
||||
var StatusCode int
|
||||
var err error
|
||||
chunkSize := int64(rx.f.opt.ChunkSize)
|
||||
buf := make([]byte, int(rx.f.opt.ChunkSize))
|
||||
for finished := false; !finished; {
|
||||
reqSize := chunkSize
|
||||
var reqSize int64
|
||||
var chunk io.ReadSeeker
|
||||
if rx.ContentLength >= 0 {
|
||||
// If size known use repeatable reader for smoother bwlimit
|
||||
if start >= rx.ContentLength {
|
||||
break
|
||||
}
|
||||
reqSize = min(rx.ContentLength-start, chunkSize)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool so reads are
|
||||
// repeatable for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, rx.Media, reqSize)
|
||||
if rx.ContentLength < 0 {
|
||||
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
|
||||
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
|
||||
} else {
|
||||
// If size unknown read into buffer
|
||||
var n int
|
||||
n, err = readers.ReadFill(rx.Media, buf)
|
||||
if err == io.EOF {
|
||||
// Send the last chunk with the correct ContentLength
|
||||
// otherwise Google doesn't know we've finished
|
||||
rx.ContentLength = start + n
|
||||
rx.ContentLength = start + int64(n)
|
||||
finished = true
|
||||
err = nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqSize = n
|
||||
}
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
reqSize = int64(n)
|
||||
chunk = bytes.NewReader(buf[:reqSize])
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
err = rx.f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
|
||||
again, err := rx.f.shouldRetry(ctx, err)
|
||||
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
|
||||
again = false
|
||||
@@ -214,13 +206,9 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
}
|
||||
return again, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
|
||||
start += reqSize
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
|
||||
// retried successfully. The chunk buffer is pool-backed and implements
|
||||
// io.Closer, so if it reaches the http transport unwrapped the transport
|
||||
// closes it after the failed attempt, returning its pages to the pool, and
|
||||
// the retry then reads a freed buffer.
|
||||
func TestResumableUploadRetry(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("resumable"), 512)
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
if attempts.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
client: server.Client(),
|
||||
}
|
||||
f.opt.ChunkSize = fs.SizeSuffix(len(content))
|
||||
rx := &resumableUpload{
|
||||
f: f,
|
||||
remote: "remote",
|
||||
URI: server.URL,
|
||||
Media: bytes.NewReader(content),
|
||||
MediaType: "application/octet-stream",
|
||||
ContentLength: int64(len(content)),
|
||||
}
|
||||
info, err := rx.Upload(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, info)
|
||||
assert.Equal(t, "fake-id", info.Id)
|
||||
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
|
||||
}
|
||||
+10
-17
@@ -51,7 +51,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -2083,6 +2082,11 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
|
||||
// write chunks
|
||||
in := readers.NewCountingReader(in0)
|
||||
bufSize := chunkSize
|
||||
if size >= 0 && size < bufSize {
|
||||
bufSize = size
|
||||
}
|
||||
buf := make([]byte, int(bufSize))
|
||||
cursor := files.UploadSessionCursor{
|
||||
SessionId: res.SessionId,
|
||||
Offset: 0,
|
||||
@@ -2102,21 +2106,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
}
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
|
||||
skip := int64(0)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// seek to the start in case this is a retry
|
||||
if _, err = rw.Seek(skip, io.SeekStart); err != nil {
|
||||
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, chunk)
|
||||
// after session is started, we retry everything
|
||||
if err != nil {
|
||||
// Check for incorrect offset error and retry with new offset
|
||||
@@ -2128,10 +2125,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
|
||||
if skip < 0 {
|
||||
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
|
||||
} else if skip == n {
|
||||
} else if skip == chunkSize {
|
||||
fs.Debugf(o, "%s: chunk received OK - continuing", what)
|
||||
return false, nil
|
||||
} else if skip > n {
|
||||
} else if skip > chunkSize {
|
||||
// This error should never happen
|
||||
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
|
||||
}
|
||||
@@ -2148,13 +2145,9 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
return err != nil, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if size >= 0 {
|
||||
// Check for sources which truncate early
|
||||
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
|
||||
|
||||
@@ -16,16 +16,13 @@ import (
|
||||
|
||||
// multipartInit starts a new multipart upload and returns server details.
|
||||
func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
|
||||
encodedFolderPath := f.fromStandardPath(folderPath)
|
||||
encodedFilename := f.fromStandardPath(filename)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/multipart/init",
|
||||
Parameters: url.Values{
|
||||
"key": {f.opt.Key},
|
||||
"filename": {encodedFilename},
|
||||
"folder_path": {encodedFolderPath},
|
||||
"filename": {filename},
|
||||
"folder_path": {folderPath},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -10,11 +11,9 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
rclonemultipart "github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -48,32 +47,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
server := initResp.Result.Server
|
||||
objectPath := initResp.Result.ObjectPath
|
||||
|
||||
chunkSize := int64(f.opt.ChunkSize)
|
||||
if chunkSize <= 0 {
|
||||
return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)
|
||||
}
|
||||
for partNo := 1; ; partNo++ {
|
||||
// Buffer the part in memory from the global pool so it can be
|
||||
// re-sent on retry
|
||||
rw := rclonemultipart.NewRW()
|
||||
n, err := io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("read failed: %w", err)
|
||||
}
|
||||
chunkSize := int(f.opt.ChunkSize)
|
||||
buf := make([]byte, 0, chunkSize)
|
||||
tmp := make([]byte, 1024*1024)
|
||||
partNo := 1
|
||||
|
||||
for {
|
||||
n, errRead := in.Read(tmp)
|
||||
if n > 0 {
|
||||
uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)
|
||||
if uploadErr != nil {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)
|
||||
buf = append(buf, tmp[:n]...)
|
||||
|
||||
// If buffer reached chunkSize, upload a full part
|
||||
if len(buf) >= chunkSize {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
partNo++
|
||||
buf = buf[:0]
|
||||
}
|
||||
}
|
||||
if closeErr := rw.Close(); closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if err == io.EOF {
|
||||
|
||||
if errRead == io.EOF {
|
||||
break
|
||||
}
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read failed: %w", errRead)
|
||||
}
|
||||
}
|
||||
|
||||
if len(buf) > 0 {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
|
||||
@@ -84,40 +91,31 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadPart sends a single multipart chunk of size bytes to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: server,
|
||||
Parameters: url.Values{
|
||||
"partNumber": {strconv.Itoa(partNo)},
|
||||
"uploadId": {uploadID},
|
||||
},
|
||||
Body: r,
|
||||
ContentLength: &size,
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-RC-Upload-Id": uploadID,
|
||||
"X-RC-Part-No": strconv.Itoa(partNo),
|
||||
"X-Sess-ID": sessID,
|
||||
"X-Object-Path": objectPath,
|
||||
},
|
||||
NoResponse: true,
|
||||
// uploadPart sends a single multipart chunk to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
|
||||
url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
// rewind the pooled buffer so each attempt sends the whole chunk
|
||||
_, err := r.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)
|
||||
}
|
||||
return shouldRetry(err), err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
req.Header.Set("X-RC-Upload-Id", uploadID)
|
||||
req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
|
||||
req.Header.Set("X-Sess-ID", sessID)
|
||||
req.Header.Set("X-Object-Path", objectPath)
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("uploadPart failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFile uploads a file to FileLu
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
@@ -12,8 +12,5 @@ func TestIntegration(t *testing.T) {
|
||||
RemoteName: "TestFileLu:",
|
||||
NilObject: nil,
|
||||
SkipInvalidUTF8: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: 1,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -353,7 +353,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
|
||||
for it.Next() {
|
||||
item := new(it.File())
|
||||
item := ptr(it.File())
|
||||
remote := f.opt.Enc.ToStandardPath(item.DisplayName)
|
||||
remote = path.Join(dir, remote)
|
||||
if remote == dir {
|
||||
@@ -424,7 +424,7 @@ func (f *Fs) mkdir(ctx context.Context, path string) error {
|
||||
|
||||
params := files_sdk.FolderCreateParams{
|
||||
Path: path,
|
||||
MkdirParents: new(true),
|
||||
MkdirParents: ptr(true),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -466,7 +466,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
|
||||
params := files_sdk.FileDeleteParams{
|
||||
Path: path,
|
||||
Recursive: new(!check),
|
||||
Recursive: ptr(!check),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -537,7 +537,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.
|
||||
params := files_sdk.FileCopyParams{
|
||||
Path: srcPath,
|
||||
Destination: dstPath,
|
||||
Overwrite: new(true),
|
||||
Overwrite: ptr(true),
|
||||
}
|
||||
|
||||
var action files_sdk.FileAction
|
||||
@@ -680,7 +680,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
Paths: []string{f.absPath(remote)},
|
||||
}
|
||||
if expire < fs.DurationOff {
|
||||
params.ExpiresAt = new(time.Now().Add(time.Duration(expire)))
|
||||
params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
|
||||
}
|
||||
|
||||
var bundle files_sdk.Bundle
|
||||
@@ -842,6 +842,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
func isFolderNotEmpty(err error) bool {
|
||||
var re files_sdk.ResponseError
|
||||
ok := errors.As(err, &re)
|
||||
|
||||
@@ -43,7 +43,7 @@ func TestCommandEncodingNeutralisesCRLF(t *testing.T) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
|
||||
@@ -194,7 +194,8 @@ var retryErrorCodes = []int{
|
||||
|
||||
// Return true if the api error has the status given
|
||||
func isAPIErr(err error, status string) bool {
|
||||
if apiErr, ok := errors.AsType[api.Error](err); ok {
|
||||
var apiErr api.Error
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.Status == status
|
||||
}
|
||||
return false
|
||||
|
||||
+61
-82
@@ -10,6 +10,7 @@ package hidrive
|
||||
// to be resolved prior to execution with resolvePath(...).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -23,9 +24,8 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/ranges"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/sync/semaphore"
|
||||
@@ -454,17 +454,17 @@ func (f *Fs) deleteObject(ctx context.Context, path string) error {
|
||||
}
|
||||
|
||||
// createFile creates a file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// This guarantees that existing files will not be overwritten.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
// This returns ErrorFileExists if a file already exists at the specified path.
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
if onExist == AutoNameOnExist {
|
||||
@@ -479,14 +479,12 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -512,16 +510,16 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
|
||||
// overwriteFile updates the content of the file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the file does not exist it will be created.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
|
||||
@@ -533,14 +531,12 @@ func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, m
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -638,18 +634,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
}
|
||||
|
||||
// Read a chunk of data.
|
||||
var (
|
||||
chunkReader *pool.RW
|
||||
bytesRead int64
|
||||
)
|
||||
chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)
|
||||
if bytesRead < int64(chunkSize) {
|
||||
chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
|
||||
if bytesRead < chunkSize {
|
||||
startMoreTransfers = false
|
||||
}
|
||||
if readErr != nil || bytesRead <= 0 {
|
||||
if chunkReader != nil {
|
||||
_ = chunkReader.Close()
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
@@ -659,12 +648,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// After this upload is done,
|
||||
// signal that another transfer can be started.
|
||||
defer transferSemaphore.Release(1)
|
||||
defer func() { _ = chunkReader.Close() }()
|
||||
uploadErr := f.patchFile(gCtx, path, chunkReader, chunkOffset, zeroTime)
|
||||
uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
|
||||
if uploadErr == nil {
|
||||
// Remember successfully written chunks.
|
||||
okChunksMu.Lock()
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: bytesRead})
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
|
||||
okChunksMu.Unlock()
|
||||
fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
|
||||
} else {
|
||||
@@ -708,10 +696,10 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// starting at the given offset.
|
||||
//
|
||||
// Replaces the file contents starting from the given byte offset
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the offset is beyond the file end, the file is extended up to the offset.
|
||||
// The maximum size of the update is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
@@ -723,7 +711,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// usually expand the holes into regions of 0-bytes.
|
||||
//
|
||||
// This returns fs.ErrorObjectNotFound if the object is not found.
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offset uint64, modTime time.Time) error {
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetPath(path)
|
||||
parameters.Set("offset", strconv.FormatUint(offset, 10))
|
||||
@@ -735,15 +723,13 @@ func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offse
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
}
|
||||
|
||||
var resp *http.Response
|
||||
@@ -853,48 +839,41 @@ func createHiDriveScopes(role string, access string) []string {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
// unwrapAccounting splits any transfer accounting off reader,
|
||||
// returning the raw stream and the accounting (nil if there is none),
|
||||
// so the stream can be buffered and accounted when the buffer is uploaded.
|
||||
func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Account) {
|
||||
// Any kind of accounter re-wraps into the one type UnWrapAccounting
|
||||
// knows how to take the *Account out of.
|
||||
unwrapped, wrap := accounting.UnWrap(reader)
|
||||
_, acc = accounting.UnWrapAccounting(wrap(unwrapped))
|
||||
return unwrapped, acc
|
||||
}
|
||||
|
||||
// readerForChunk reads up to length bytes from reader into a buffer
|
||||
// from the global memory pool and returns it
|
||||
// along with the number of bytes that have been read from reader.
|
||||
// Reaching the end of reader early is not an error.
|
||||
//
|
||||
// If accountUpload is set, any accounting on reader is applied
|
||||
// when the buffer is read instead, so the upload is what gets accounted.
|
||||
//
|
||||
// This must not be set for a buffer which creates or replaces a file
|
||||
// in a single request. Accounting can fail part way through the upload,
|
||||
// for example when --max-transfer is reached, and HiDrive keeps
|
||||
// what it has received of an aborted request as the file.
|
||||
//
|
||||
// The caller must Close the buffer when done with it.
|
||||
func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {
|
||||
chunkReader = multipart.NewRW()
|
||||
if accountUpload {
|
||||
var acc *accounting.Account
|
||||
reader, acc = unwrapAccounting(reader)
|
||||
if acc != nil {
|
||||
chunkReader.SetAccounting(acc.AccountRead)
|
||||
}
|
||||
// cachedReader returns a version of the reader that caches its contents and
|
||||
// can therefore be reset using Seek.
|
||||
func cachedReader(reader io.Reader) io.ReadSeeker {
|
||||
bytesReader, ok := reader.(*bytes.Reader)
|
||||
if ok {
|
||||
return bytesReader
|
||||
}
|
||||
|
||||
bytesRead, err = io.CopyN(chunkReader, reader, length)
|
||||
if err == io.EOF {
|
||||
repeatableReader, ok := reader.(*readers.RepeatableReader)
|
||||
if ok {
|
||||
return repeatableReader
|
||||
}
|
||||
|
||||
return readers.NewRepeatableReader(reader)
|
||||
}
|
||||
|
||||
// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
|
||||
// Returns a new io.Reader (chunkReader) for that chunk
|
||||
// and the number of bytes that have been read from reader.
|
||||
func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
|
||||
// Unwrap any accounting from the input if present.
|
||||
reader, wrap := accounting.UnWrap(reader)
|
||||
|
||||
// Read a chunk of data.
|
||||
buffer := make([]byte, length)
|
||||
bytesRead, err = io.ReadFull(reader, buffer)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
_ = chunkReader.Close()
|
||||
return nil, bytesRead, err
|
||||
}
|
||||
return chunkReader, bytesRead, nil
|
||||
// Truncate unused capacity.
|
||||
buffer = buffer[:bytesRead]
|
||||
|
||||
// Use wrap to put any accounting back for chunkReader.
|
||||
return wrap(bytes.NewReader(buffer)), bytesRead, nil
|
||||
}
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -515,14 +514,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
// (i.e. everything up to the cutoff) in the first request,
|
||||
// avoids files being created on upload failure for small files.
|
||||
// (As opposed to creating an empty file and then uploading the content.)
|
||||
//
|
||||
// Only buffer as much as the source declares it has,
|
||||
// so small files do not cost the whole cutoff in memory.
|
||||
prefixSize := int64(f.opt.UploadCutoff)
|
||||
if size := src.Size(); size >= 0 && size < prefixSize {
|
||||
prefixSize = size
|
||||
}
|
||||
cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)
|
||||
tmpReader, bytesRead, err := readerForChunk(in, int(f.opt.UploadCutoff))
|
||||
cutoffReader := cachedReader(tmpReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -547,7 +540,6 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
return false, createErr
|
||||
})
|
||||
_ = cutoffReader.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -564,7 +556,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
// If there is more left to write, o.Update needs to skip ahead.
|
||||
// Use a fs.SeekOption with the current offset to do this.
|
||||
options = append(options, &fs.SeekOption{Offset: bytesRead})
|
||||
options = append(options, &fs.SeekOption{Offset: int64(bytesRead)})
|
||||
err = o.Update(ctx, in, src, options...)
|
||||
|
||||
if err == nil {
|
||||
@@ -979,12 +971,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Metadata should be updated even if the upload fails.
|
||||
info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
|
||||
} else {
|
||||
var content *pool.RW
|
||||
content, _, err = readerForChunk(in, src.Size(), false)
|
||||
if err == nil {
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
|
||||
_ = content.Close()
|
||||
}
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, cachedReader(in), modTime)
|
||||
metaErr = err
|
||||
}
|
||||
|
||||
|
||||
@@ -573,7 +573,7 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
|
||||
return nil, fmt.Errorf("failed to readDir: %w", err)
|
||||
}
|
||||
|
||||
contentType, _, _ := strings.Cut(res.Header.Get("Content-Type"), ";")
|
||||
contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
|
||||
switch contentType {
|
||||
case "text/html":
|
||||
names, err = parse(u, res.Body)
|
||||
|
||||
@@ -37,44 +37,44 @@ type FileList struct {
|
||||
|
||||
// File represents a file or folder in Huawei Drive
|
||||
type File struct {
|
||||
ID string `json:"id"`
|
||||
FileName string `json:"fileName"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
EditedTime time.Time `json:"editedTime"`
|
||||
EditedByMeTime time.Time `json:"editedByMeTime"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Owners []User `json:"owners"`
|
||||
LastEditor User `json:"lastEditor"`
|
||||
Permissions []Permission `json:"permissions"`
|
||||
PermissionIDs []string `json:"permissionIds"`
|
||||
Capabilities FileCapabilities `json:"capabilities"`
|
||||
OwnedByMe bool `json:"ownedByMe"`
|
||||
EditedByMe bool `json:"editedByMe"`
|
||||
ViewedByMe bool `json:"viewedByMe"`
|
||||
HasShared bool `json:"hasShared"`
|
||||
Recycled bool `json:"recycled"`
|
||||
DirectlyRecycled bool `json:"directlyRecycled"`
|
||||
Favorite bool `json:"favorite"`
|
||||
ExistThumbnail bool `json:"existThumbnail"`
|
||||
ThumbnailVersion int64 `json:"thumbnailVersion"`
|
||||
IconDownloadLink string `json:"iconDownloadLink,omitempty"`
|
||||
ContentDownloadLink string `json:"contentDownloadLink,omitempty"`
|
||||
ContentVersion string `json:"contentVersion,omitempty"`
|
||||
LastHistoryVersionID string `json:"lastHistoryVersionId,omitempty"`
|
||||
OccupiedSpace int64 `json:"occupiedSpace,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
WritersHasSharePermission bool `json:"writersHasSharePermission"`
|
||||
WriterHasCopyPermission bool `json:"writerHasCopyPermission"`
|
||||
Containers []string `json:"containers"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
ContentExtras *ContentExtras `json:"contentExtras,omitempty"`
|
||||
ID string `json:"id"`
|
||||
FileName string `json:"fileName"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
EditedTime time.Time `json:"editedTime"`
|
||||
EditedByMeTime time.Time `json:"editedByMeTime,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Owners []User `json:"owners"`
|
||||
LastEditor User `json:"lastEditor"`
|
||||
Permissions []Permission `json:"permissions"`
|
||||
PermissionIDs []string `json:"permissionIds"`
|
||||
Capabilities FileCapabilities `json:"capabilities"`
|
||||
OwnedByMe bool `json:"ownedByMe"`
|
||||
EditedByMe bool `json:"editedByMe"`
|
||||
ViewedByMe bool `json:"viewedByMe"`
|
||||
HasShared bool `json:"hasShared"`
|
||||
Recycled bool `json:"recycled"`
|
||||
DirectlyRecycled bool `json:"directlyRecycled"`
|
||||
Favorite bool `json:"favorite"`
|
||||
ExistThumbnail bool `json:"existThumbnail"`
|
||||
ThumbnailVersion int64 `json:"thumbnailVersion"`
|
||||
IconDownloadLink string `json:"iconDownloadLink,omitempty"`
|
||||
ContentDownloadLink string `json:"contentDownloadLink,omitempty"`
|
||||
ContentVersion string `json:"contentVersion,omitempty"`
|
||||
LastHistoryVersionID string `json:"lastHistoryVersionId,omitempty"`
|
||||
OccupiedSpace int64 `json:"occupiedSpace,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
WritersHasSharePermission bool `json:"writersHasSharePermission"`
|
||||
WriterHasCopyPermission bool `json:"writerHasCopyPermission"`
|
||||
Containers []string `json:"containers"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
ContentExtras *ContentExtras `json:"contentExtras,omitempty"`
|
||||
}
|
||||
|
||||
// IsDir returns true if the file is a directory
|
||||
@@ -125,41 +125,41 @@ type Thumbnail struct {
|
||||
|
||||
// CreateFolderRequest represents request to create a folder
|
||||
type CreateFolderRequest struct {
|
||||
FileName string `json:"fileName"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Favorite bool `json:"favorite,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
AutoRename int `json:"autoRename,omitempty"`
|
||||
FileName string `json:"fileName"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Favorite bool `json:"favorite,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AutoRename int `json:"autoRename,omitempty"`
|
||||
}
|
||||
|
||||
// UpdateFileRequest represents request to update a file
|
||||
type UpdateFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Favorite *bool `json:"favorite,omitempty"`
|
||||
Recycled *bool `json:"recycled,omitempty"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
WriterHasCopyPermission *bool `json:"writerHasCopyPermission,omitempty"`
|
||||
WritersHasSharePermission *bool `json:"writersHasSharePermission,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
AddParentFolder []string `json:"addParentFolder,omitempty"`
|
||||
RemoveParentFolder []string `json:"removeParentFolder,omitempty"`
|
||||
CreatedTime *time.Time `json:"createdTime,omitempty"`
|
||||
EditedTime *time.Time `json:"editedTime,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Favorite *bool `json:"favorite,omitempty"`
|
||||
Recycled *bool `json:"recycled,omitempty"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
WriterHasCopyPermission *bool `json:"writerHasCopyPermission,omitempty"`
|
||||
WritersHasSharePermission *bool `json:"writersHasSharePermission,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AddParentFolder []string `json:"addParentFolder,omitempty"`
|
||||
RemoveParentFolder []string `json:"removeParentFolder,omitempty"`
|
||||
CreatedTime *time.Time `json:"createdTime,omitempty"`
|
||||
EditedTime *time.Time `json:"editedTime,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest represents request to copy a file
|
||||
type CopyFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
}
|
||||
|
||||
// ResumeUploadInitResponse represents response to resume upload init
|
||||
@@ -224,6 +224,11 @@ var GlobalDomains = map[string]bool{
|
||||
"drive.cloud.hicloud.com": true,
|
||||
}
|
||||
|
||||
// BoolPtr returns a pointer to a bool value (helper for UpdateFileRequest)
|
||||
func BoolPtr(b bool) *bool {
|
||||
return &b
|
||||
}
|
||||
|
||||
// StartCursor represents response to Changes.getStartCursor endpoint
|
||||
type StartCursor struct {
|
||||
Category string `json:"category"` // "drive#startCursor"
|
||||
|
||||
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -1032,7 +1031,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
fs.Debugf(f, "Copy: failed to get metadata options: %v", err)
|
||||
}
|
||||
if len(mergedMeta) > 0 {
|
||||
copyReq.Properties = make(map[string]any)
|
||||
copyReq.Properties = make(map[string]interface{})
|
||||
for key, value := range mergedMeta {
|
||||
switch key {
|
||||
case "content-type", "sha256", "btime", "mtime", "utime",
|
||||
@@ -1722,7 +1721,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
}
|
||||
|
||||
// Create metadata
|
||||
metadata := map[string]any{
|
||||
metadata := map[string]interface{}{
|
||||
"fileName": leaf,
|
||||
"mimeType": mimeType,
|
||||
}
|
||||
@@ -1742,10 +1741,9 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
metadata["autoRename"] = 3
|
||||
}
|
||||
|
||||
// Build the multipart body in pooled memory so it can be re-sent on retry
|
||||
rw := pool.NewRW(pool.Global())
|
||||
defer fs.CheckClose(rw, &err)
|
||||
writer := multipart.NewWriter(rw)
|
||||
// Create multipart form
|
||||
var buf bytes.Buffer
|
||||
writer := multipart.NewWriter(&buf)
|
||||
|
||||
// Add metadata part - use exact format from Huawei docs
|
||||
metadataWriter, err := writer.CreatePart(textproto.MIMEHeader{
|
||||
@@ -1791,17 +1789,15 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
return fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
bodySize := rw.Size()
|
||||
bodyBytes := buf.Bytes()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Parameters: url.Values{
|
||||
"uploadType": []string{"multipart"},
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
ContentLength: &bodySize,
|
||||
RootURL: o.fs.uploadURL,
|
||||
Body: rw,
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
RootURL: o.fs.uploadURL,
|
||||
}
|
||||
|
||||
if o.id != "" {
|
||||
@@ -1815,10 +1811,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
|
||||
var info *api.File
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = bytes.NewReader(bodyBytes)
|
||||
resp, err := srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -1879,7 +1872,7 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
|
||||
// Prepare metadata for resumable upload initialization
|
||||
metadata := map[string]any{
|
||||
metadata := map[string]interface{}{
|
||||
"fileName": leaf,
|
||||
}
|
||||
|
||||
@@ -1912,26 +1905,25 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
|
||||
// Now upload the content in chunks
|
||||
// Note 256k is the minimum chunk size and 64M the maximum
|
||||
// single upload size according to the Huawei Drive API
|
||||
chunkSize := min(max(int64(o.fs.opt.ChunkSize), 256*1024), 64*1024*1024)
|
||||
chunkSize := int64(o.fs.opt.ChunkSize)
|
||||
if chunkSize < 256*1024 {
|
||||
chunkSize = 256 * 1024 // Minimum chunk size according to Huawei Drive API
|
||||
}
|
||||
if chunkSize > 64*1024*1024 {
|
||||
chunkSize = 64 * 1024 * 1024 // Maximum single upload size
|
||||
}
|
||||
|
||||
buf := make([]byte, chunkSize)
|
||||
var offset int64
|
||||
|
||||
for offset < size {
|
||||
// Buffer the chunk in pooled memory so it can be re-sent on retry
|
||||
rw := pool.NewRW(pool.Global())
|
||||
n, err := io.CopyN(rw, in, min(size-offset, chunkSize))
|
||||
if err != nil {
|
||||
// io.EOF here means the source ran out before the declared size
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
_ = rw.Close()
|
||||
n, err := io.ReadFull(in, buf)
|
||||
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||
return fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)
|
||||
}
|
||||
|
||||
end := offset + n - 1
|
||||
chunk := buf[:n]
|
||||
end := offset + int64(n) - 1
|
||||
|
||||
// Upload this chunk using the original client with OAuth authentication
|
||||
chunkOpts := rest.Opts{
|
||||
@@ -1939,20 +1931,17 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
RootURL: location,
|
||||
Path: "", // Empty path since RootURL contains the full URL
|
||||
ExtraHeaders: map[string]string{
|
||||
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
|
||||
"Content-Type": mimeType,
|
||||
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
|
||||
"Content-Length": strconv.Itoa(n),
|
||||
"Content-Type": mimeType,
|
||||
},
|
||||
ContentLength: &n,
|
||||
Body: rw,
|
||||
}
|
||||
|
||||
var info *api.File
|
||||
var chunkResp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
chunkOpts.Body = bytes.NewReader(chunk)
|
||||
var err error
|
||||
chunkResp, err = srv.CallJSON(ctx, &chunkOpts, nil, &info)
|
||||
// 308 means continue uploading (incomplete)
|
||||
if chunkResp != nil && chunkResp.StatusCode == 308 {
|
||||
@@ -1966,15 +1955,11 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
return shouldRetry(ctx, chunkResp, err)
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
|
||||
offset += n
|
||||
offset += int64(n)
|
||||
|
||||
// If we got file info back, we're done
|
||||
if info != nil && info.ID != "" {
|
||||
@@ -2105,8 +2090,8 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
|
||||
|
||||
// Prepare the update request payload
|
||||
updateReq := api.UpdateFileRequest{
|
||||
Properties: make(map[string]any),
|
||||
AppSettings: make(map[string]any),
|
||||
Properties: make(map[string]interface{}),
|
||||
AppSettings: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Process metadata and separate into properties and app settings
|
||||
@@ -2117,7 +2102,7 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
|
||||
updateReq.Description = value
|
||||
case "favorite":
|
||||
if favorite, err := strconv.ParseBool(value); err == nil {
|
||||
updateReq.Favorite = new(favorite)
|
||||
updateReq.Favorite = api.BoolPtr(favorite)
|
||||
}
|
||||
case "content-type":
|
||||
// Allow setting/overriding MIME type
|
||||
|
||||
@@ -19,49 +19,15 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
)
|
||||
|
||||
// Limits on the chunk size accepted by the resumable upload API
|
||||
const (
|
||||
minChunkSize = 256 * fs.Kibi
|
||||
maxChunkSize = 64 * fs.Mebi
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestHuaweiDrive:",
|
||||
NilObject: (*Object)(nil),
|
||||
SkipBadWindowsCharacters: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: minChunkSize,
|
||||
MaxChunkSize: maxChunkSize,
|
||||
CeilChunkSize: fstests.NextPowerOfTwo,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
|
||||
// TestNewFs tests the NewFs constructor
|
||||
func TestNewFs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@@ -1618,7 +1617,9 @@ func (ps *PhotosService) GetLibraryAlbumCounts(ctx context.Context) (map[string]
|
||||
if err := ps.requestForArea(ctx, area, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to get library album counts: %w", err)
|
||||
}
|
||||
maps.Copy(counts, response.toCounts(order))
|
||||
for k, v := range response.toCounts(order) {
|
||||
counts[k] = v
|
||||
}
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
@@ -2031,7 +2032,7 @@ func (album *Album) fetchPhotosParallel(ctx context.Context, totalPhotos int64)
|
||||
sem := make(chan struct{}, workers)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := range numPartitions {
|
||||
for i := 0; i < numPartitions; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
|
||||
@@ -1191,7 +1191,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for range 100 {
|
||||
for i := 0; i < 100; i++ {
|
||||
lib.deltaMu.Lock()
|
||||
lib.pendingDelta = &deltaPayload{
|
||||
records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},
|
||||
@@ -1201,7 +1201,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
|
||||
lib.applyPendingDelta(context.Background())
|
||||
}
|
||||
}()
|
||||
for range 100 {
|
||||
for i := 0; i < 100; i++ {
|
||||
ps.FlushCaches()
|
||||
// Re-add the library since FlushCaches clears it
|
||||
ps.mu.Lock()
|
||||
|
||||
@@ -718,7 +718,7 @@ func (s *Session) GetAuthState(ctx context.Context) (*AuthStateResponse, error)
|
||||
Path: "",
|
||||
ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
|
||||
RootURL: authEndpoint,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
// Use srv.Call directly to capture the raw response body for debugging
|
||||
resp, err := s.srv.Call(ctx, &opts)
|
||||
@@ -828,7 +828,7 @@ func (s *Session) TrustSession(ctx context.Context) error {
|
||||
ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
|
||||
RootURL: authEndpoint,
|
||||
NoResponse: true,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
|
||||
_, err := s.Request(ctx, opts, nil, nil)
|
||||
@@ -846,7 +846,7 @@ func (s *Session) ValidateSession(ctx context.Context) error {
|
||||
Path: "/validate",
|
||||
ExtraHeaders: s.GetHeaders(map[string]string{}),
|
||||
RootURL: setupEndpoint,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
_, err := s.Request(ctx, opts, nil, &s.AccountInfo)
|
||||
if err != nil {
|
||||
@@ -904,6 +904,8 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
|
||||
return headers
|
||||
}
|
||||
|
||||
func int64Ptr(v int64) *int64 { return &v }
|
||||
|
||||
// NewSession creates a new Session instance with default values
|
||||
func NewSession() *Session {
|
||||
session := &Session{
|
||||
|
||||
@@ -376,7 +376,8 @@ func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
if libraryName, ok := strings.CutPrefix(pathID, "lib:"); ok {
|
||||
if strings.HasPrefix(pathID, "lib:") {
|
||||
libraryName := strings.TrimPrefix(pathID, "lib:")
|
||||
libraries, err := photosService.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
@@ -735,7 +736,10 @@ func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallb
|
||||
}
|
||||
close(jobCh)
|
||||
|
||||
workers := min(len(jobs), fs.GetConfig(ctx).Checkers)
|
||||
workers := fs.GetConfig(ctx).Checkers
|
||||
if len(jobs) < workers {
|
||||
workers = len(jobs)
|
||||
}
|
||||
errs := make(chan error, workers)
|
||||
|
||||
for range workers {
|
||||
|
||||
@@ -54,7 +54,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
switch httpErr.StatusCode() {
|
||||
case 401:
|
||||
if !f.authFailed {
|
||||
@@ -1208,7 +1209,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
err := o.f.pacer.Call(func() (bool, error) {
|
||||
err := files.DeleteFile(ctx, o.f.cfg, backupUUID)
|
||||
if err != nil {
|
||||
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
// Treat 404 (Not Found) and 204 (No Content) as success
|
||||
switch httpErr.StatusCode() {
|
||||
case 404, 204:
|
||||
@@ -1274,7 +1276,8 @@ func isEmptyFileLimitError(err error) bool {
|
||||
// fileTooLargeError extracts the SDK's FileTooLargeError from a wrapped error
|
||||
// chain, returning it (or nil) so callers can branch on the size limit.
|
||||
func fileTooLargeError(err error) *sdkerrors.FileTooLargeError {
|
||||
if tooLarge, ok := errors.AsType[*sdkerrors.FileTooLargeError](err); ok {
|
||||
var tooLarge *sdkerrors.FileTooLargeError
|
||||
if errors.As(err, &tooLarge) {
|
||||
return tooLarge
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -34,7 +34,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
@@ -1925,9 +1924,8 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
|
||||
// nothing to clean up by default
|
||||
cleanup = func() {}
|
||||
|
||||
// don't cache small files on disk to reduce wear of the disk, but
|
||||
// spool unknown sized streams there as they can't be bounded in memory
|
||||
if size > threshold || size < 0 {
|
||||
// don't cache small files on disk to reduce wear of the disk
|
||||
if size > threshold {
|
||||
var tempFile *os.File
|
||||
|
||||
// create the cache file
|
||||
@@ -1957,15 +1955,15 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
|
||||
// replace the already read source with a reader of our cached file
|
||||
out = tempFile
|
||||
} else {
|
||||
// that's a small file, just read it into memory from the global pool
|
||||
rw := multipart.NewRW()
|
||||
cleanup = func() {
|
||||
_ = rw.Close()
|
||||
}
|
||||
if _, err = io.Copy(rw, teeReader); err != nil {
|
||||
// that's a small file, just read it into memory
|
||||
var inData []byte
|
||||
inData, err = io.ReadAll(teeReader)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
out = rw
|
||||
|
||||
// set the reader to our read memory block
|
||||
out = bytes.NewReader(inData)
|
||||
}
|
||||
return hex.EncodeToString(md5Hasher.Sum(nil)), out, cleanup, nil
|
||||
}
|
||||
|
||||
@@ -5,14 +5,12 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -30,16 +28,11 @@ func TestReadMD5(t *testing.T) {
|
||||
wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
for _, threshold := range []int64{512, 1024, 10 * 1024, 20 * 1024} {
|
||||
t.Run(fmt.Sprintf("%d", threshold), func(t *testing.T) {
|
||||
inUse := pool.Global().InUse()
|
||||
in := readers.NewPatternReader(size)
|
||||
gotMD5, out, cleanup, err := readMD5(in, size, threshold)
|
||||
defer cleanup()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
if size > threshold {
|
||||
assert.IsType(t, (*os.File)(nil), out, "big files should be spooled to disk")
|
||||
} else {
|
||||
assert.IsType(t, (*pool.RW)(nil), out, "small files should be buffered in the pool")
|
||||
}
|
||||
|
||||
// check md5hash of out
|
||||
hasher := md5.New()
|
||||
@@ -48,36 +41,10 @@ func TestReadMD5(t *testing.T) {
|
||||
assert.Equal(t, n, size)
|
||||
outMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
assert.Equal(t, wantMD5, outMD5)
|
||||
|
||||
// check cleanup returns the buffer to the pool
|
||||
cleanup()
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Unknown sized streams can't be bounded in memory so must be spooled to disk
|
||||
t.Run("UnknownSize", func(t *testing.T) {
|
||||
const size = 1024
|
||||
inUse := pool.Global().InUse()
|
||||
hasher := md5.New()
|
||||
_, err := io.Copy(hasher, readers.NewPatternReader(size))
|
||||
require.NoError(t, err)
|
||||
wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
|
||||
gotMD5, out, cleanup, err := readMD5(readers.NewPatternReader(size), -1, 10*size)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
assert.IsType(t, (*os.File)(nil), out)
|
||||
hasher.Reset()
|
||||
n, err := io.Copy(hasher, out)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(size), n)
|
||||
assert.Equal(t, wantMD5, fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
cleanup()
|
||||
assert.Equal(t, inUse, pool.Global().InUse())
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestMetadata(t *testing.T) {
|
||||
|
||||
@@ -11,10 +11,10 @@ package linkbox
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -35,9 +35,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -48,8 +46,7 @@ const (
|
||||
pacerBurst = 1
|
||||
linkboxAPIURL = "https://www.linkbox.to/api/open/"
|
||||
linkboxWebAPIURL = "https://www.linkbox.to/api/"
|
||||
rootID = "0" // ID of root directory
|
||||
prefixSize = 10 * 1024 * 1024 // the API wants the MD5 of the first 10 MiB of a file
|
||||
rootID = "0" // ID of root directory
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -670,21 +667,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
// readPrefix reads up to prefixSize bytes of in into a buffer from the
|
||||
// global pool returning it along with the hex MD5 of what was read.
|
||||
//
|
||||
// The caller must Close the returned buffer when done with it.
|
||||
func readPrefix(in io.Reader) (rw *pool.RW, md5sum string, err error) {
|
||||
rw = multipart.NewRW()
|
||||
hasher := md5.New()
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hasher), prefixSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, "", err
|
||||
}
|
||||
return rw, hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
@@ -719,13 +701,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
prefix, prefixMD5, err := readPrefix(in)
|
||||
first10m := io.LimitReader(in, 10_485_760)
|
||||
first10mBytes, err := io.ReadAll(first10m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Update err in reading file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = prefix.Close()
|
||||
}()
|
||||
|
||||
// get upload authorization (step 1)
|
||||
opts := &rest.Opts{
|
||||
@@ -735,7 +715,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
Parameters: url.Values{
|
||||
"token": {o.fs.opt.Token},
|
||||
"fileMd5ofPre10m": {prefixMD5},
|
||||
"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
|
||||
"fileSize": {itoa64(size)},
|
||||
},
|
||||
}
|
||||
@@ -777,7 +757,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return fmt.Errorf("head upload URL: %w", err)
|
||||
}
|
||||
|
||||
file := io.MultiReader(prefix, in)
|
||||
file := io.MultiReader(bytes.NewReader(first10mBytes), in)
|
||||
|
||||
opts.Method = "PUT"
|
||||
opts.Body = file
|
||||
@@ -817,7 +797,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
Parameters: url.Values{
|
||||
"token": {o.fs.opt.Token},
|
||||
"fileMd5ofPre10m": {prefixMD5},
|
||||
"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
|
||||
"fileSize": {itoa64(size)},
|
||||
"pid": {dirID},
|
||||
"diyName": {leaf},
|
||||
@@ -1039,7 +1019,8 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
|
||||
// intermittently returns an HTML challenge page with a 200
|
||||
// status instead of JSON. This surfaces as a JSON syntax error,
|
||||
// so retry it to let the pacer back off until the block lifts.
|
||||
if _, ok := errors.AsType[*json.SyntaxError](err); ok {
|
||||
var syntaxErr *json.SyntaxError
|
||||
if errors.As(err, &syntaxErr) {
|
||||
return true, err
|
||||
}
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package linkbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestReadPrefix(t *testing.T) {
|
||||
for _, size := range []int64{1, 1024, prefixSize - 1, prefixSize, prefixSize + 1, 3 * prefixSize} {
|
||||
t.Run(fmt.Sprintf("%d", size), func(t *testing.T) {
|
||||
inUse := pool.Global().InUse()
|
||||
content := bytes.Repeat([]byte("x"), int(size))
|
||||
for i := range content {
|
||||
content[i] = byte(i)
|
||||
}
|
||||
wantPrefix := content[:min(size, prefixSize)]
|
||||
wantMD5 := fmt.Sprintf("%x", md5.Sum(wantPrefix))
|
||||
|
||||
in := bytes.NewReader(content)
|
||||
rw, gotMD5, err := readPrefix(in)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
assert.Equal(t, int64(len(wantPrefix)), rw.Size())
|
||||
|
||||
// The prefix followed by the rest of in must reproduce the content
|
||||
got, err := io.ReadAll(io.MultiReader(rw, in))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, got)
|
||||
|
||||
require.NoError(t, rw.Close())
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
|
||||
})
|
||||
}
|
||||
}
|
||||
+13
-86
@@ -12,7 +12,6 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -37,10 +36,6 @@ import (
|
||||
const (
|
||||
devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
|
||||
useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
|
||||
// maxLinkTargetSize is the largest symlink target accepted when
|
||||
// translating a .rclonelink object, comfortably above any OS path
|
||||
// limit (Windows allows 32767 UTF-16 units).
|
||||
maxLinkTargetSize = 128 * 1024
|
||||
)
|
||||
|
||||
// timeType allows the user to choose what exactly ModTime() returns
|
||||
@@ -316,10 +311,10 @@ set this flag.`,
|
||||
Name: "fatal_if_no_space",
|
||||
Help: `Make out-of-space errors fatal during transfers.
|
||||
|
||||
When enabled, an out of space error while writing, creating a directory,
|
||||
or creating a file returns a fatal error so that rclone aborts rather
|
||||
than retrying the operation. Useful for backup scripts that should halt
|
||||
loudly on a full disk rather than spin retrying.`,
|
||||
When enabled, an ENOSPC error during a write returns a fatal error so
|
||||
that rclone aborts rather than retrying the operation. Useful for
|
||||
backup scripts that should halt loudly on a full disk rather than spin
|
||||
retrying.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
@@ -443,8 +438,6 @@ var (
|
||||
errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
|
||||
errLinksNeedsSuffix = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
|
||||
errPathEscapes = errors.New("file name is not a path within the local root - check the encoding")
|
||||
errSymlinkLoop = errors.New("loop detected: points to a parent directory")
|
||||
errLinkTargetTooLong = errors.New("symlink target is too long to be a path")
|
||||
errLinkNameInvalid = errors.New("translated link needs a file name before the \"" + fs.LinkSuffix + "\" suffix")
|
||||
)
|
||||
|
||||
@@ -680,7 +673,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
if err != nil {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
var parents []os.FileInfo // fsDirPath and its parents, read on first use
|
||||
|
||||
fd, err := os.Open(fsDirPath)
|
||||
if err != nil {
|
||||
@@ -761,28 +753,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
if f.opt.FollowSymlinks && (mode&symlinkFlag) != 0 {
|
||||
localPath := filepath.Join(fsDirPath, name)
|
||||
fi, err = os.Stat(localPath)
|
||||
// A directory symlink pointing to a parent would be followed
|
||||
// until the OS returns ELOOP, which can take a very long time
|
||||
if err == nil && fi.IsDir() {
|
||||
if parents == nil {
|
||||
parents = statParents(fsDirPath)
|
||||
}
|
||||
if slices.ContainsFunc(parents, func(parent os.FileInfo) bool { return os.SameFile(fi, parent) }) {
|
||||
// Quietly skip loops the directory filters exclude as
|
||||
// the layer above wouldn't recurse into them
|
||||
if useFilter {
|
||||
if include, dirErr := filter.IncludeDirectory(ctx, f)(newRemote); dirErr == nil && !include {
|
||||
continue
|
||||
}
|
||||
}
|
||||
err = errSymlinkLoop
|
||||
}
|
||||
}
|
||||
// Quietly skip errors on excluded files and directories
|
||||
if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
|
||||
continue
|
||||
}
|
||||
if os.IsNotExist(err) || isCircularSymlinkError(err) || errors.Is(err, errSymlinkLoop) {
|
||||
if os.IsNotExist(err) || isCircularSymlinkError(err) {
|
||||
// Skip bad symlinks and circular symlinks
|
||||
err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
|
||||
fs.Errorf(newRemote, "Listing error: %v", err)
|
||||
@@ -833,21 +808,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// statParents returns the info for dirPath and each of its parents up
|
||||
// to the root of the file system, skipping any which can't be read.
|
||||
func statParents(dirPath string) (parents []os.FileInfo) {
|
||||
for {
|
||||
if fi, err := os.Stat(dirPath); err == nil {
|
||||
parents = append(parents, fi)
|
||||
}
|
||||
parent := filepath.Dir(dirPath)
|
||||
if parent == dirPath {
|
||||
return parents
|
||||
}
|
||||
dirPath = parent
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) cleanRemote(dir, filename string) (remote string) {
|
||||
if f.opt.UTFNorm {
|
||||
filename = norm.NFC.String(filename)
|
||||
@@ -1690,17 +1650,12 @@ func isDiskFullError(err error) bool {
|
||||
return errors.Is(err, file.ErrDiskFull) || fserrors.IsErrNoSpace(err)
|
||||
}
|
||||
|
||||
func wrapFatalIfNoSpace(err error, enabled bool) error {
|
||||
if err != nil && enabled && isDiskFullError(err) {
|
||||
return fserrors.FatalError(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Update the object from in with modTime and size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
defer func() {
|
||||
err = wrapFatalIfNoSpace(err, o.fs.opt.FatalIfNoSpace)
|
||||
if err != nil && o.fs.opt.FatalIfNoSpace && isDiskFullError(err) {
|
||||
err = fserrors.FatalError(err)
|
||||
}
|
||||
}()
|
||||
var out io.WriteCloser
|
||||
var hasher *hash.MultiHasher
|
||||
@@ -1757,28 +1712,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
out = f
|
||||
} else {
|
||||
// The target is buffered in memory, so stop reading just past
|
||||
// the longest acceptable one rather than trust the source
|
||||
in = io.LimitReader(in, maxLinkTargetSize+1)
|
||||
out = nopWriterCloser{&symlinkData}
|
||||
}
|
||||
|
||||
// Calculate the hash of the object as we write it
|
||||
var w io.Writer = out
|
||||
// Calculate the hash of the object we are reading as we go along
|
||||
if hasher != nil {
|
||||
w = io.MultiWriter(w, hasher)
|
||||
in = io.TeeReader(in, hasher)
|
||||
}
|
||||
|
||||
_, err = io.Copy(w, in)
|
||||
_, err = io.Copy(out, in)
|
||||
closeErr := out.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
|
||||
if o.translatedLink {
|
||||
if err == nil && symlinkData.Len() > maxLinkTargetSize {
|
||||
err = fserrors.NoRetryError(errLinkTargetTooLong)
|
||||
}
|
||||
if err == nil {
|
||||
// Use the contents of the copied object to create a symlink,
|
||||
// without following or creating it through a planted symlink
|
||||
@@ -1829,30 +1777,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
var sparseWarning sync.Once
|
||||
|
||||
type fatalIfNoSpaceWriterAt struct {
|
||||
fs.WriterAtCloser
|
||||
enabled bool
|
||||
}
|
||||
|
||||
func (w *fatalIfNoSpaceWriterAt) WriteAt(p []byte, off int64) (n int, err error) {
|
||||
n, err = w.WriterAtCloser.WriteAt(p, off)
|
||||
return n, wrapFatalIfNoSpace(err, w.enabled)
|
||||
}
|
||||
|
||||
func (w *fatalIfNoSpaceWriterAt) Close() error {
|
||||
return wrapFatalIfNoSpace(w.WriterAtCloser.Close(), w.enabled)
|
||||
}
|
||||
|
||||
// OpenWriterAt opens with a handle for random access writes
|
||||
//
|
||||
// Pass in the remote desired and the size if known.
|
||||
//
|
||||
// It truncates any existing object
|
||||
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (writer fs.WriterAtCloser, err error) {
|
||||
defer func() {
|
||||
err = wrapFatalIfNoSpace(err, f.opt.FatalIfNoSpace)
|
||||
}()
|
||||
|
||||
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
|
||||
// Temporary Object under construction
|
||||
o, err := f.newObject(remote)
|
||||
if err != nil {
|
||||
@@ -1890,10 +1820,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (write
|
||||
}
|
||||
}
|
||||
|
||||
return &fatalIfNoSpaceWriterAt{
|
||||
WriterAtCloser: out,
|
||||
enabled: f.opt.FatalIfNoSpace,
|
||||
}, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// setMetadata sets the file info from the os.FileInfo passed in
|
||||
|
||||
@@ -60,7 +60,7 @@ func TestIsDiskFullError(t *testing.T) {
|
||||
// FatalIfNoSpace setting, returning the error from Update.
|
||||
func updateWithReader(t *testing.T, fatalIfNoSpace bool, readerErr error) error {
|
||||
t.Helper()
|
||||
r := fstest.NewRunIndividual(t)
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
f.opt.FatalIfNoSpace = fatalIfNoSpace
|
||||
|
||||
@@ -96,87 +96,3 @@ func TestUpdateFatalIfNoSpaceOnButNotDiskFull(t *testing.T) {
|
||||
require.Error(t, err)
|
||||
assert.False(t, fserrors.IsFatalError(err), "non-disk-full errors must not be fatal regardless of option")
|
||||
}
|
||||
|
||||
type diskFullWriterAtCloser struct {
|
||||
writeErr error
|
||||
closeErr error
|
||||
n int
|
||||
data []byte
|
||||
offset int64
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (w *diskFullWriterAtCloser) WriteAt(p []byte, off int64) (int, error) {
|
||||
w.data = append([]byte(nil), p...)
|
||||
w.offset = off
|
||||
return w.n, w.writeErr
|
||||
}
|
||||
|
||||
func (w *diskFullWriterAtCloser) Close() error {
|
||||
w.closed = true
|
||||
return w.closeErr
|
||||
}
|
||||
|
||||
func testOpenWriterAtError(t *testing.T, injected error, diskFull bool) {
|
||||
t.Helper()
|
||||
for _, enabled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("enabled=%v", enabled), func(t *testing.T) {
|
||||
r := fstest.NewRunIndividual(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
f.opt.FatalIfNoSpace = enabled
|
||||
f.opt.NoPreAllocate = true
|
||||
writer, err := f.OpenWriterAt(context.Background(), "test.txt", 0)
|
||||
require.NoError(t, err)
|
||||
wrapped, ok := writer.(*fatalIfNoSpaceWriterAt)
|
||||
require.True(t, ok)
|
||||
require.NoError(t, wrapped.WriterAtCloser.Close())
|
||||
wantN := 2
|
||||
if injected == nil {
|
||||
wantN = 4
|
||||
}
|
||||
underlying := &diskFullWriterAtCloser{writeErr: injected, closeErr: injected, n: wantN}
|
||||
wrapped.WriterAtCloser = underlying
|
||||
n, err := writer.WriteAt([]byte("data"), 17)
|
||||
assert.Equal(t, wantN, n)
|
||||
assert.Equal(t, []byte("data"), underlying.data)
|
||||
assert.Equal(t, int64(17), underlying.offset)
|
||||
assert.ErrorIs(t, err, injected)
|
||||
assert.Equal(t, enabled && diskFull, fserrors.IsFatalError(err))
|
||||
err = writer.Close()
|
||||
assert.True(t, underlying.closed)
|
||||
assert.ErrorIs(t, err, injected)
|
||||
assert.Equal(t, enabled && diskFull, fserrors.IsFatalError(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenWriterAtFatalIfNoSpace(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
diskFull bool
|
||||
}{
|
||||
{"success", nil, false},
|
||||
{"unrelated", syscall.EPERM, false},
|
||||
{"ENOSPC", syscall.ENOSPC, true},
|
||||
{"ErrDiskFull", file.ErrDiskFull, true},
|
||||
{"wrapped", fmt.Errorf("write: %w", syscall.ENOSPC), true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) { testOpenWriterAtError(t, test.err, test.diskFull) })
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenWriterAtSetupError(t *testing.T) {
|
||||
for _, enabled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("enabled=%v", enabled), func(t *testing.T) {
|
||||
r := fstest.NewRunIndividual(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
f.opt.FatalIfNoSpace = enabled
|
||||
require.NoError(t, os.WriteFile(filepath.Join(r.LocalName, "parent"), nil, 0600))
|
||||
writer, err := f.OpenWriterAt(context.Background(), "parent/child", 0)
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, writer)
|
||||
assert.False(t, fserrors.IsFatalError(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -35,18 +35,3 @@ func TestUpdateFatalIfNoSpaceWindows(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenWriterAtFatalIfNoSpaceWindows(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
err error
|
||||
diskFull bool
|
||||
}{
|
||||
{"ERROR_DISK_FULL", windows.ERROR_DISK_FULL, true},
|
||||
{"ERROR_HANDLE_DISK_FULL", windows.ERROR_HANDLE_DISK_FULL, true},
|
||||
{"PathError", &os.PathError{Op: "write", Path: "test.txt", Err: windows.ERROR_DISK_FULL}, true},
|
||||
{"ERROR_ACCESS_DENIED", windows.ERROR_ACCESS_DENIED, false},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) { testOpenWriterAtError(t, test.err, test.diskFull) })
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package local
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
@@ -24,7 +23,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/file"
|
||||
@@ -272,29 +270,6 @@ func linksMode(f *Fs) {
|
||||
f.lstat = os.Lstat
|
||||
}
|
||||
|
||||
// TestSymlinkTargetTooLong checks that a .rclonelink object whose body is
|
||||
// far bigger than any path is refused without being buffered in memory,
|
||||
// and that nothing is created at the destination.
|
||||
func TestSymlinkTargetTooLong(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
ctx := context.Background()
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
// A source which never ends, so the read must be bounded
|
||||
src := object.NewStaticObjectInfo("big"+fs.LinkSuffix, fstest.Time("2001-02-03T04:05:10Z"), -1, true, nil, nil)
|
||||
in := readers.NewCountingReader(readers.NewPatternReader(1 << 40))
|
||||
_, err := f.Put(ctx, in, src)
|
||||
require.ErrorIs(t, err, errLinkTargetTooLong)
|
||||
assert.True(t, fserrors.IsNoRetryError(err))
|
||||
assert.LessOrEqual(t, in.BytesRead(), uint64(maxLinkTargetSize+1), "read more of the body than needed")
|
||||
|
||||
_, err = os.Lstat(filepath.Join(f.root, "big"))
|
||||
assert.True(t, os.IsNotExist(err), "nothing should be created for a refused symlink")
|
||||
}
|
||||
|
||||
// TestSymlinkEscapeWriteThroughBlocked mirrors the GHSA-cf44-9pgv-m4xc PoC: a
|
||||
// malicious --links source serves "pwn.rclonelink" whose body is a path outside
|
||||
// the destination, plus a sibling "pwn/authkeys" that sorts after it and would
|
||||
@@ -841,32 +816,6 @@ func TestHashOnUpdate(t *testing.T) {
|
||||
assert.Equal(t, "45685e95985e20822fb2538a522a5ccf", md5)
|
||||
}
|
||||
|
||||
// Test the hash cached by Update matches a HashesOption hint passed by the caller
|
||||
func TestHashOnUpdateWithHashOption(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
const filePath = "file.txt"
|
||||
when := time.Now()
|
||||
r.WriteFile(filePath, "x", when)
|
||||
f := r.Flocal.(*Fs)
|
||||
|
||||
o, err := f.NewObject(ctx, filePath)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := bytes.NewBufferString("content")
|
||||
src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)
|
||||
options := []fs.OpenOption{&fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}}
|
||||
require.NoError(t, o.Update(ctx, b, src, options...))
|
||||
|
||||
gotContent, err := os.ReadFile(filepath.Join(f.root, filePath))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "content", string(gotContent))
|
||||
|
||||
md5, err := o.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)
|
||||
}
|
||||
|
||||
// Test hashes on deleting an object
|
||||
func TestHashOnDelete(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
@@ -1349,121 +1298,3 @@ func TestCopySymlink(t *testing.T) {
|
||||
want = fstest.NewItem("dst2/file.txt", "hello world", when)
|
||||
fstest.CompareItems(t, []fs.DirEntry{dst}, []fstest.Item{want}, nil, f.precision, "")
|
||||
}
|
||||
|
||||
// TestCopyLinksLoop checks that -L/--copy-links follows a symlink to a
|
||||
// sibling directory but skips symlinks which point to the directory
|
||||
// being listed or one of its parents, as these would loop forever.
|
||||
func TestCopyLinksLoop(t *testing.T) {
|
||||
skipIfNoSymlinks(t)
|
||||
dir := t.TempDir()
|
||||
|
||||
// folder-A/link-to-B and folder-B/link-to-A point at each other
|
||||
// and folder-A/link-to-parent points at dir
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-A"), 0700))
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-B"), 0700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-A", "file-A"), []byte("A"), 0600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-B", "file-B"), []byte("B"), 0600))
|
||||
require.NoError(t, os.Symlink(filepath.Join("..", "folder-B"), filepath.Join(dir, "folder-A", "link-to-B")))
|
||||
require.NoError(t, os.Symlink(filepath.Join("..", "folder-A"), filepath.Join(dir, "folder-B", "link-to-A")))
|
||||
require.NoError(t, os.Symlink("..", filepath.Join(dir, "folder-A", "link-to-parent")))
|
||||
|
||||
wantRoot := []string{
|
||||
"folder-A",
|
||||
"folder-A/file-A",
|
||||
"folder-A/link-to-B",
|
||||
"folder-A/link-to-B/file-B",
|
||||
"folder-B",
|
||||
"folder-B/file-B",
|
||||
"folder-B/link-to-A",
|
||||
"folder-B/link-to-A/file-A",
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
root string
|
||||
exclude string
|
||||
want []string
|
||||
wantErrors int64
|
||||
}{{
|
||||
name: "Root",
|
||||
root: dir,
|
||||
want: wantRoot,
|
||||
// folder-A/link-to-parent, folder-A/link-to-B/link-to-A,
|
||||
// folder-B/link-to-A/link-to-B, folder-B/link-to-A/link-to-parent
|
||||
wantErrors: 4,
|
||||
}, {
|
||||
// link-to-parent points above the root of the Fs
|
||||
name: "SubDir",
|
||||
root: filepath.Join(dir, "folder-A"),
|
||||
want: []string{
|
||||
"file-A",
|
||||
"link-to-B",
|
||||
"link-to-B/file-B",
|
||||
},
|
||||
// link-to-parent, link-to-B/link-to-A
|
||||
wantErrors: 2,
|
||||
}, {
|
||||
// Excluded loops aren't reported
|
||||
name: "Excluded",
|
||||
root: dir,
|
||||
exclude: "link-to-parent/**",
|
||||
want: wantRoot,
|
||||
// folder-A/link-to-B/link-to-A, folder-B/link-to-A/link-to-B
|
||||
wantErrors: 2,
|
||||
}} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if test.exclude != "" {
|
||||
var fi *filter.Filter
|
||||
ctx, fi = filter.AddConfig(ctx)
|
||||
require.NoError(t, fi.AddRule("- "+test.exclude))
|
||||
}
|
||||
f, err := NewFs(ctx, "local", test.root, configmap.Simple{
|
||||
"copy_links": "true",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
accounting.Stats(ctx).ResetErrors()
|
||||
defer accounting.Stats(ctx).ResetErrors()
|
||||
|
||||
var got []string
|
||||
err = walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error {
|
||||
for _, entry := range entries {
|
||||
got = append(got, entry.Remote())
|
||||
}
|
||||
// Stop a runaway listing rather than waiting for ELOOP
|
||||
if len(got) > 100 {
|
||||
return errors.New("too many entries - symlink loop not detected")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, test.want, got)
|
||||
assert.Equal(t, test.wantErrors, accounting.Stats(ctx).GetErrors(), "global errors found")
|
||||
})
|
||||
}
|
||||
|
||||
// A single List reports and skips the link to a parent, but still
|
||||
// returns the link to a sibling
|
||||
t.Run("List", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f, err := NewFs(ctx, "local", dir, configmap.Simple{
|
||||
"copy_links": "true",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
accounting.Stats(ctx).ResetErrors()
|
||||
defer accounting.Stats(ctx).ResetErrors()
|
||||
|
||||
entries, err := f.List(ctx, "folder-A")
|
||||
require.NoError(t, err)
|
||||
var got []string
|
||||
for _, entry := range entries {
|
||||
got = append(got, entry.Remote())
|
||||
}
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, []string{"folder-A/file-A", "folder-A/link-to-B"}, got)
|
||||
assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
|
||||
})
|
||||
}
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -1615,6 +1614,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
|
||||
var (
|
||||
fileBuf []byte
|
||||
fileHash []byte
|
||||
newHash []byte
|
||||
slowHash bool
|
||||
@@ -1660,19 +1660,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Attempt to put by calculating hash in memory
|
||||
if trySpeedup && size <= int64(o.fs.opt.SpeedupMaxMem) {
|
||||
rw := multipart.NewRW()
|
||||
defer func() {
|
||||
_ = rw.Close()
|
||||
}()
|
||||
hasher := mrhash.New()
|
||||
if _, err = io.Copy(rw, io.TeeReader(in, hasher)); err != nil {
|
||||
fileBuf, err = io.ReadAll(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileHash = hasher.Sum(nil)
|
||||
fileHash = mrhash.Sum(fileBuf)
|
||||
if o.putByHash(ctx, fileHash, src, "memory") {
|
||||
return nil
|
||||
}
|
||||
wrapIn = rw
|
||||
wrapIn = bytes.NewReader(fileBuf)
|
||||
trySpeedup = false // speedup failed, force upload
|
||||
}
|
||||
|
||||
@@ -1706,8 +1702,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Upload object data
|
||||
if size <= mrhash.Size {
|
||||
// Optimize upload: skip extra request if data fits in the hash buffer.
|
||||
var fileBuf []byte
|
||||
fileBuf, err = io.ReadAll(wrapIn)
|
||||
if fileBuf == nil {
|
||||
fileBuf, err = io.ReadAll(wrapIn)
|
||||
}
|
||||
if fileHash == nil && err == nil {
|
||||
fileHash = mrhash.Sum(fileBuf)
|
||||
}
|
||||
@@ -1872,12 +1869,6 @@ func (o *Object) upload(ctx context.Context, in io.Reader, size int64, options .
|
||||
strHash string
|
||||
)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// Rewind seekable bodies so a retry resends the whole file
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
if _, err := seeker.Seek(0, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
res, err = o.fs.srv.Call(ctx, &opts)
|
||||
if err == nil {
|
||||
strHash, err = readBodyWord(res)
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
package mailru
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/mailru/mrhash"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
// TestUploadRetry checks that an upload from a seekable pooled buffer
|
||||
// whose first attempt fails with a retriable error is retried with the
|
||||
// full body rather than the drained reader.
|
||||
func TestUploadRetry(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("mailru"), 512)
|
||||
wantHash := mrhash.Sum(content)
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
if attempts.Add(1) == 1 {
|
||||
// Drop the connection so the client sees a retriable error
|
||||
conn, _, err := w.(http.Hijacker).Hijack()
|
||||
require.NoError(t, err)
|
||||
_ = conn.Close()
|
||||
return
|
||||
}
|
||||
assert.Equal(t, content, body, "retried upload should re-send the full body")
|
||||
_, _ = w.Write([]byte(hex.EncodeToString(wantHash) + "\n"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
srv: rest.NewClient(server.Client()),
|
||||
source: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "token"}),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
|
||||
shardURL: server.URL,
|
||||
shardExpiry: time.Now().Add(time.Hour),
|
||||
}
|
||||
o := &Object{fs: f, remote: "remote"}
|
||||
|
||||
rw := multipart.NewRW()
|
||||
defer func() { _ = rw.Close() }()
|
||||
_, err := rw.Write(content)
|
||||
require.NoError(t, err)
|
||||
|
||||
gotHash, err := o.upload(ctx, rw, int64(len(content)))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantHash, gotHash)
|
||||
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
|
||||
}
|
||||
+84
-161
@@ -37,9 +37,9 @@ import (
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -56,16 +56,7 @@ const (
|
||||
driveTypeSharepoint = "documentLibrary"
|
||||
defaultChunkSize = 10 * fs.Mebi
|
||||
chunkSizeMultiple = 320 * fs.Kibi
|
||||
defaultTenantAPIVersion = "v2.0"
|
||||
// maxSinglePartSize is the size at which Graph stops accepting an upload in a
|
||||
// single request (PUT /items/{id}/content). Microsoft documents this as
|
||||
// "250 MB", see
|
||||
// https://learn.microsoft.com/en-us/graph/api/driveitem-put-content
|
||||
// Measured against SharePoint Online the figure is binary and exclusive: a
|
||||
// body of 262143999 bytes is accepted, one of 262144000 is not. That matches
|
||||
// upload_cutoff being an exclusive threshold in Update below, so a cutoff of
|
||||
// exactly 250Mi sends everything the server would refuse to multipart.
|
||||
maxSinglePartSize = 250 * fs.Mebi
|
||||
maxSinglePartSize = 4 * fs.Mebi
|
||||
|
||||
regionGlobal = "global"
|
||||
regionUS = "us"
|
||||
@@ -167,7 +158,7 @@ See: https://github.com/rclone/rclone/issues/1716
|
||||
Name: "tenant_url",
|
||||
Help: `The tenant URL for non-admin OneDrive access.
|
||||
|
||||
Set this to your SharePoint tenant URL to use the SharePoint API
|
||||
Set this to your SharePoint tenant URL to use the SharePoint v2.0 API
|
||||
endpoint instead of the standard Microsoft Graph API. This allows
|
||||
accessing business OneDrive without admin consent.
|
||||
|
||||
@@ -178,16 +169,6 @@ for "driveAccessToken" in the network requests. Look for the
|
||||
Example: https://your-tenant.sharepoint.com/_api`,
|
||||
Default: "",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "tenant_api_version",
|
||||
Help: `The SharePoint API version to use with tenant_url.
|
||||
|
||||
Set this to the SharePoint API version matching the browser-extracted
|
||||
access token. For example, use v2.1 with a driveAccessTokenV21 token.
|
||||
|
||||
This only applies when tenant_url is set.`,
|
||||
Default: defaultTenantAPIVersion,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
|
||||
@@ -438,12 +419,21 @@ Setting this flag speeds up these things greatly:
|
||||
rclone size onedrive:
|
||||
rclone rc vfs/refresh recursive=true
|
||||
|
||||
Rclone asks for the delta listing of the directory being listed. If
|
||||
the drive only supports delta listings at the root of the drive (as
|
||||
some older OneDrive for Business and SharePoint drives do) then
|
||||
rclone lists from the root and discards all the data that is not
|
||||
under the directory you asked for. So it will be correct but may not
|
||||
be very efficient.
|
||||
**However** the delta listing API **only** works at the root of the
|
||||
drive. If you use it not at the root then it recurses from the root
|
||||
and discards all the data that is not under the directory you asked
|
||||
for. So it will be correct but may not be very efficient.
|
||||
|
||||
This is why this flag is not set as the default.
|
||||
|
||||
As a rule of thumb if nearly all of your data is under rclone's root
|
||||
directory (the |root/directory| in |onedrive:root/directory|) then
|
||||
using this flag will be a big performance win. If your data is
|
||||
mostly not under the root then using this flag will be a big
|
||||
performance loss.
|
||||
|
||||
It is recommended if you are mounting your onedrive at the root
|
||||
(or near the root when using crypt) and using rclone |rc vfs/refresh|.
|
||||
`, "|", "`"),
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -512,20 +502,12 @@ func getRegionURL(m configmap.Mapper) (region, graphURL string) {
|
||||
// Check if tenant_url is provided for non-admin mode
|
||||
tenantURL, _ := m.Get("tenant_url")
|
||||
if tenantURL != "" {
|
||||
tenantAPIVersion, _ := m.Get("tenant_api_version")
|
||||
graphURL = tenantAPIEndpoint(tenantURL, tenantAPIVersion)
|
||||
graphURL = tenantURL + "/v2.0"
|
||||
}
|
||||
|
||||
return region, graphURL
|
||||
}
|
||||
|
||||
func tenantAPIEndpoint(tenantURL, tenantAPIVersion string) string {
|
||||
if tenantAPIVersion == "" {
|
||||
tenantAPIVersion = defaultTenantAPIVersion
|
||||
}
|
||||
return strings.TrimRight(tenantURL, "/") + "/" + strings.TrimLeft(tenantAPIVersion, "/")
|
||||
}
|
||||
|
||||
// Config for chooseDrive
|
||||
type chooseDriveOpt struct {
|
||||
opts rest.Opts
|
||||
@@ -819,7 +801,6 @@ type Options struct {
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
TenantURL string `config:"tenant_url"`
|
||||
TenantAPIVersion string `config:"tenant_api_version"`
|
||||
DriveID string `config:"drive_id"`
|
||||
DriveType string `config:"drive_type"`
|
||||
RootFolderID string `config:"root_folder_id"`
|
||||
@@ -1127,7 +1108,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID
|
||||
|
||||
if opt.TenantURL != "" {
|
||||
rootURL = tenantAPIEndpoint(opt.TenantURL, opt.TenantAPIVersion) + "/drives/" + opt.DriveID
|
||||
rootURL = opt.TenantURL + "/v2.0" + "/drives/" + opt.DriveID
|
||||
}
|
||||
|
||||
oauthConfig, err := makeOauthConfig(ctx, opt)
|
||||
@@ -1414,8 +1395,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, dir string, info *api.Item) (en
|
||||
id := info.GetID()
|
||||
f.dirCache.Put(remote, id)
|
||||
d := f.newDir(id, remote)
|
||||
// OneDrive reports the total size of a folder's contents
|
||||
d.size = info.GetSize()
|
||||
d.items = folder.ChildCount
|
||||
f.setSystemMetadata(info, d.meta, remote, dirMimeType)
|
||||
entry = d
|
||||
@@ -1504,6 +1483,10 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
return err
|
||||
}
|
||||
|
||||
// ListR only works at the root of a onedrive, not on a folder
|
||||
// So we have to filter things outside of the root which is
|
||||
// inefficient.
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
|
||||
// list a folder conventionally - used for shared folders
|
||||
@@ -1579,26 +1562,17 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
return nil
|
||||
}
|
||||
|
||||
listDelta := func(opts rest.Opts) error {
|
||||
opts.Parameters = url.Values{
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/root/delta",
|
||||
Parameters: map[string][]string{
|
||||
// "token": {token},
|
||||
"$top": {fmt.Sprintf("%d", f.opt.ListChunk)},
|
||||
}
|
||||
var result api.DeltaResponse
|
||||
return f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
|
||||
},
|
||||
}
|
||||
|
||||
err = listDelta(f.newOptsCall(directoryID, "GET", "/delta"))
|
||||
// Some drives only support delta listings at the root of the
|
||||
// drive, in which case list the whole drive and filter out the
|
||||
// items outside dir.
|
||||
var apiErr *api.Error
|
||||
if err != nil && len(seen) == 0 && errors.As(err, &apiErr) {
|
||||
fs.Debugf(f, "Delta listing of directory failed, listing from the root of the drive instead: %v", err)
|
||||
err = listDelta(rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/root/delta",
|
||||
})
|
||||
}
|
||||
var result api.DeltaResponse
|
||||
err = f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2774,26 +2748,12 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
|
||||
position := int64(0)
|
||||
for remaining > 0 {
|
||||
n := min(remaining, int64(o.fs.opt.ChunkSize))
|
||||
// Buffer the chunk in memory from the global pool so it can be
|
||||
// re-sent (or partly re-sent after a 416) on retry
|
||||
rw := multipart.NewRW()
|
||||
_, err = io.CopyN(rw, in, n)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
if err == io.EOF {
|
||||
err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, position, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
seg := readers.NewRepeatableReader(io.LimitReader(in, n))
|
||||
fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
|
||||
info, err = o.uploadFragment(ctx, uploadURL, position, size, rw, n, options...)
|
||||
closeErr := rw.Close()
|
||||
info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
remaining -= n
|
||||
position += n
|
||||
}
|
||||
@@ -2812,13 +2772,13 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
|
||||
return info, o.setMetaData(info)
|
||||
}
|
||||
|
||||
// Update the content of a remote file smaller than maxSinglePartSize in one single request
|
||||
// Update the content of a remote file within 4 MiB size in one single request
|
||||
// (currently only used when size is exactly 0)
|
||||
// This function will set modtime and metadata after uploading, which will create a new version for the remote file
|
||||
func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
|
||||
size := src.Size()
|
||||
if size < 0 || size >= int64(maxSinglePartSize) {
|
||||
return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and < %v", maxSinglePartSize)
|
||||
if size < 0 || size > int64(maxSinglePartSize) {
|
||||
return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)
|
||||
}
|
||||
|
||||
fs.Debugf(o, "Starting singlepart upload")
|
||||
@@ -2937,7 +2897,7 @@ func (o *Object) ID() string {
|
||||
func (f *Fs) parseNormalizedID(ID string) (string, string, string) {
|
||||
var rootURL string
|
||||
if f.opt.TenantURL != "" {
|
||||
rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
|
||||
rootURL = f.opt.TenantURL + "/v2.0/drives"
|
||||
} else {
|
||||
rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
|
||||
}
|
||||
@@ -3105,15 +3065,9 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}
|
||||
case <-tickerC:
|
||||
fs.Debugf(f, "Checking for changes on remote")
|
||||
var newDeltaToken string
|
||||
newDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
|
||||
nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
|
||||
if err != nil {
|
||||
// Keep the token we already have: replacing it with the
|
||||
// empty token from a failed poll would reset the delta
|
||||
// listing to the start and lose every change in between.
|
||||
fs.Infof(f, "Change notify listener failure: %s", err)
|
||||
} else {
|
||||
nextDeltaToken = newDeltaToken
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3121,7 +3075,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, "latest", "")
|
||||
delta, err := f.changeNotifyNextChange(ctx, "latest")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -3133,37 +3087,18 @@ func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken str
|
||||
return
|
||||
}
|
||||
|
||||
// changeNotifyNextChange fetches one page of the drive delta listing.
|
||||
//
|
||||
// Pass the delta token to resume from in token, or the @odata.nextLink URL
|
||||
// returned by the previous page in nextLink. Exactly one of the two should
|
||||
// be set.
|
||||
func (f *Fs) changeNotifyNextChange(ctx context.Context, token, nextLink string) (delta api.DeltaResponse, err error) {
|
||||
opts := f.buildDriveDeltaOpts(token, nextLink)
|
||||
func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta api.DeltaResponse, err error) {
|
||||
opts := f.buildDriveDeltaOpts(token)
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var resp *http.Response
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// buildDriveDeltaOpts builds the request for one page of the drive delta
|
||||
// listing. nextLink, when set, is the complete @odata.nextLink URL from the
|
||||
// previous page and is used as-is, overriding the drive root.
|
||||
func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
|
||||
if nextLink != "" {
|
||||
return rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: nextLink,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
|
||||
var rootURL string
|
||||
if f.opt.TenantURL != "" {
|
||||
rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
|
||||
rootURL = f.opt.TenantURL + "/v2.0/drives"
|
||||
} else {
|
||||
rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
|
||||
}
|
||||
@@ -3176,58 +3111,46 @@ func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
|
||||
}
|
||||
}
|
||||
|
||||
// changeNotifyRunner polls for changes and calls notifyFunc for each one.
|
||||
//
|
||||
// It walks every page of the change set and returns the token to resume from
|
||||
// next time. The returned token is only valid when err is nil: the caller
|
||||
// must keep the token it passed in on error, otherwise a failed poll would
|
||||
// reset the delta listing to the start and lose the changes in between.
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (string, error) {
|
||||
token, nextLink := deltaToken, ""
|
||||
for {
|
||||
delta, err := f.changeNotifyNextChange(ctx, token, nextLink)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, item := range delta.Value {
|
||||
isDriveRootFolder := item.GetParentReference().ID == ""
|
||||
if isDriveRootFolder {
|
||||
continue
|
||||
}
|
||||
|
||||
fullPath, err := getItemFullPath(&item)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get item full path: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if fullPath == f.root {
|
||||
continue
|
||||
}
|
||||
|
||||
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
|
||||
if !insideRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
if item.GetFile() != nil {
|
||||
notifyFunc(relName, fs.EntryObject)
|
||||
} else if item.GetFolder() != nil {
|
||||
notifyFunc(relName, fs.EntryDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
if delta.NextLink == "" {
|
||||
// end of the change set - the deltaLink carries the resume token
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return parsedURL.Query().Get("token"), nil
|
||||
}
|
||||
nextLink = delta.NextLink
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, deltaToken)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
nextDeltaToken = parsedURL.Query().Get("token")
|
||||
|
||||
for _, item := range delta.Value {
|
||||
isDriveRootFolder := item.GetParentReference().ID == ""
|
||||
if isDriveRootFolder {
|
||||
continue
|
||||
}
|
||||
|
||||
fullPath, err := getItemFullPath(&item)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get item full path: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if fullPath == f.root {
|
||||
continue
|
||||
}
|
||||
|
||||
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
|
||||
if !insideRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
if item.GetFile() != nil {
|
||||
notifyFunc(relName, fs.EntryObject)
|
||||
} else if item.GetFolder() != nil {
|
||||
notifyFunc(relName, fs.EntryDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func getItemFullPath(item *api.Item) (fullPath string, err error) {
|
||||
|
||||
@@ -4,11 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -18,10 +15,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -48,40 +42,6 @@ func TestMain(m *testing.M) {
|
||||
fstest.TestMain(m)
|
||||
}
|
||||
|
||||
func TestTenantAPIEndpoint(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
tenantURL string
|
||||
tenantAPIVersion string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "default",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.0",
|
||||
}, {
|
||||
name: "explicit v2.1",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api",
|
||||
tenantAPIVersion: "v2.1",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.1",
|
||||
}, {
|
||||
name: "future version",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api",
|
||||
tenantAPIVersion: "v2.2",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.2",
|
||||
}, {
|
||||
name: "trailing slash",
|
||||
tenantURL: "https://example-my.sharepoint.com/_api/",
|
||||
tenantAPIVersion: "/v2.1",
|
||||
want: "https://example-my.sharepoint.com/_api/v2.1",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, tenantAPIEndpoint(tc.tenantURL, tc.tenantAPIVersion))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sharingRefused caches whether the remote refuses sharing invitations.
|
||||
var sharingRefused *bool
|
||||
|
||||
@@ -556,30 +516,6 @@ func (f *Fs) resetTestDefaults(r *fstest.Run) {
|
||||
r.Finalise()
|
||||
}
|
||||
|
||||
// TestCheckUploadCutoff checks the bounds of the --onedrive-upload-cutoff
|
||||
// validation, in particular that the documented API maximum is accepted.
|
||||
func TestCheckUploadCutoff(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in fs.SizeSuffix
|
||||
wantErr bool
|
||||
}{
|
||||
{in: 0, wantErr: false},
|
||||
{in: 4 * fs.Mebi, wantErr: false},
|
||||
// maxSinglePartSize is a legal cutoff even though the server refuses a
|
||||
// body of that size, because the cutoff is an exclusive threshold: at
|
||||
// this setting a file of exactly maxSinglePartSize goes multipart.
|
||||
{in: maxSinglePartSize, wantErr: false},
|
||||
{in: maxSinglePartSize + 1, wantErr: true},
|
||||
} {
|
||||
err := checkUploadCutoff(test.in)
|
||||
if test.wantErr {
|
||||
assert.Error(t, err, test.in.String())
|
||||
} else {
|
||||
assert.NoError(t, err, test.in.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// InternalTest dispatches all internal tests
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
newTestF := func() (*Fs, *fstest.Run) {
|
||||
@@ -614,159 +550,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// newChangeNotifyTestFs returns an Fs whose delta listing is served by
|
||||
// handler on a local test server, plus the server itself. The tenant URL
|
||||
// is pointed at the server so no request leaves the test.
|
||||
func newChangeNotifyTestFs(t *testing.T, handler http.Handler) (*Fs, *httptest.Server) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
ci: fs.GetConfig(ctx),
|
||||
root: "",
|
||||
driveID: "testdrive",
|
||||
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
|
||||
}
|
||||
f.opt.TenantURL = server.URL
|
||||
f.opt.TenantAPIVersion = defaultTenantAPIVersion
|
||||
return f, server
|
||||
}
|
||||
|
||||
// deltaItem builds a delta entry for a file named name inside a folder.
|
||||
func deltaItem(name, folder string) api.Item {
|
||||
return api.Item{
|
||||
ID: name,
|
||||
Name: name,
|
||||
File: &api.FileFacet{},
|
||||
ParentReference: &api.ItemReference{
|
||||
ID: "parentid",
|
||||
Path: "driveid:/" + folder,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestChangeNotifyRetriesAndKeepsToken covers a transient failure of the
|
||||
// delta poll: the request must go through the pacer so it is retried, and
|
||||
// a poll which fails outright must not hand back an empty token, since the
|
||||
// caller stores whatever it returns. An empty token would reset the delta
|
||||
// listing to the start and silently lose every change in between.
|
||||
func TestChangeNotifyRetriesAndKeepsToken(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var attempts atomic.Int32
|
||||
var tokens []string
|
||||
fail := true
|
||||
f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts.Add(1)
|
||||
tokens = append(tokens, r.URL.Query().Get("token"))
|
||||
if fail {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
|
||||
"code": "serviceNotAvailable",
|
||||
"message": "Service unavailable",
|
||||
}})
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
|
||||
Value: []api.Item{deltaItem("file1.txt", "folder")},
|
||||
DeltaLink: "https://example.com/delta?token=newtoken",
|
||||
})
|
||||
}))
|
||||
|
||||
var notified []string
|
||||
notifyFunc := func(path string, entryType fs.EntryType) {
|
||||
notified = append(notified, path)
|
||||
}
|
||||
|
||||
// The poll fails, but is retried by the pacer first
|
||||
token, err := f.changeNotifyRunner(ctx, notifyFunc, "tok1")
|
||||
require.Error(t, err)
|
||||
assert.Empty(t, token, "a failed poll must not return a token")
|
||||
assert.Equal(t, fs.GetConfig(ctx).LowLevelRetries, int(attempts.Load()), "the failed poll should have been retried by the pacer")
|
||||
assert.Equal(t, slices.Repeat([]string{"tok1"}, int(attempts.Load())), tokens, "every retry should resume from the same token")
|
||||
assert.Empty(t, notified)
|
||||
|
||||
// Keeping the previous token (as ChangeNotify does) resumes correctly
|
||||
fail = false
|
||||
attempts.Store(0)
|
||||
tokens = nil
|
||||
token, err = f.changeNotifyRunner(ctx, notifyFunc, "tok1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "newtoken", token)
|
||||
assert.Equal(t, []string{"tok1"}, tokens, "the second poll should resume from the retained token")
|
||||
assert.Equal(t, []string{"folder/file1.txt"}, notified)
|
||||
}
|
||||
|
||||
// TestChangeNotifyFollowsNextLink covers a change set which spans more than
|
||||
// one page. Graph returns @odata.nextLink and no @odata.deltaLink until the
|
||||
// last page, so a runner which stops after the first page both loses the
|
||||
// remaining changes and ends up with an empty resume token.
|
||||
func TestChangeNotifyFollowsNextLink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var urls []string
|
||||
var server *httptest.Server
|
||||
f, server := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
urls = append(urls, r.URL.RequestURI())
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
switch len(urls) {
|
||||
case 1:
|
||||
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
|
||||
Value: []api.Item{deltaItem("page1.txt", "folder")},
|
||||
NextLink: server.URL + "/nextpage?$skiptoken=abc",
|
||||
})
|
||||
case 2:
|
||||
_ = json.NewEncoder(w).Encode(api.DeltaResponse{
|
||||
Value: []api.Item{deltaItem("page2.txt", "folder")},
|
||||
DeltaLink: "https://example.com/delta?token=lasttoken",
|
||||
})
|
||||
default:
|
||||
t.Errorf("unexpected extra request to %s", r.URL)
|
||||
}
|
||||
}))
|
||||
|
||||
var notified []string
|
||||
token, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
|
||||
notified = append(notified, path)
|
||||
}, "tok1")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "lasttoken", token, "the resume token should come from the last page")
|
||||
assert.Equal(t, []string{"folder/page1.txt", "folder/page2.txt"}, notified, "changes from every page should be notified")
|
||||
require.Len(t, urls, 2, "the nextLink page should have been fetched")
|
||||
assert.Contains(t, urls[0], "/testdrive/root/delta")
|
||||
assert.Contains(t, urls[0], "token=tok1")
|
||||
assert.Equal(t, "/nextpage?$skiptoken=abc", urls[1], "the nextLink should be used as-is, without the delta token")
|
||||
}
|
||||
|
||||
// TestListDirectorySize checks that a listed directory carries the size
|
||||
// OneDrive reports for it, the total size of its contents, instead of -1.
|
||||
func TestListDirectorySize(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/items/rootid/children", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(api.ListChildrenResponse{
|
||||
Value: []api.Item{{
|
||||
ID: "dirid",
|
||||
Name: "photos",
|
||||
Size: 2646204760,
|
||||
Folder: &api.FolderFacet{ChildCount: 3},
|
||||
}},
|
||||
})
|
||||
}))
|
||||
f.dirCache = dircache.New("", "rootid", f)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
dir, ok := entries[0].(fs.Directory)
|
||||
require.True(t, ok, "entry should be a directory")
|
||||
assert.Equal(t, "photos", dir.Remote())
|
||||
assert.Equal(t, int64(2646204760), dir.Size())
|
||||
assert.Equal(t, int64(3), dir.Items())
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package onedrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fragmentServer is a fake OneDrive upload session which records the
|
||||
// fragments it receives. fail is called for each PUT before the body is
|
||||
// accepted and may write an error response, returning true to reject it.
|
||||
type fragmentServer struct {
|
||||
t *testing.T
|
||||
size int64
|
||||
mu sync.Mutex
|
||||
received []byte // bytes accepted so far
|
||||
fragments int // number of accepted fragments
|
||||
fail func(w http.ResponseWriter, attempt int) bool
|
||||
attempts int
|
||||
}
|
||||
|
||||
func (s *fragmentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
switch {
|
||||
case r.Method == "POST" && r.URL.Path == "/root/createUploadSession":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"uploadUrl":"http://%s/upload","nextExpectedRanges":["0-"]}`, r.Host)
|
||||
case r.Method == "GET" && r.URL.Path == "/upload":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)
|
||||
case r.Method == "PUT" && r.URL.Path == "/upload":
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
s.attempts++
|
||||
if s.fail != nil && s.fail(w, s.attempts) {
|
||||
return
|
||||
}
|
||||
var start, end, total int64
|
||||
_, err = fmt.Sscanf(r.Header.Get("Content-Range"), "bytes %d-%d/%d", &start, &end, &total)
|
||||
require.NoError(s.t, err)
|
||||
assert.Equal(s.t, int64(len(s.received)), start, "fragment sent out of order")
|
||||
assert.Equal(s.t, int64(len(body)), end-start+1, "Content-Range disagrees with body length")
|
||||
assert.Equal(s.t, s.size, total)
|
||||
s.received = append(s.received, body...)
|
||||
s.fragments++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if int64(len(s.received)) == s.size {
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, s.size)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)
|
||||
default:
|
||||
s.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// newUploadTest returns an Object wired to a fragmentServer which
|
||||
// uploads content in chunks of chunkSize.
|
||||
func newUploadTest(t *testing.T, content []byte, chunkSize int) (*Object, *fragmentServer, *httptest.Server) {
|
||||
s := &fragmentServer{t: t, size: int64(len(content))}
|
||||
server := httptest.NewServer(s)
|
||||
t.Cleanup(server.Close)
|
||||
client := rest.NewClient(server.Client()).SetRoot(server.URL)
|
||||
f := &Fs{
|
||||
ci: fs.GetConfig(context.Background()),
|
||||
srv: client,
|
||||
unAuth: client,
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
|
||||
}
|
||||
f.opt.ChunkSize = fs.SizeSuffix(chunkSize)
|
||||
f.opt.UploadCutoff = fs.SizeSuffix(chunkSize)
|
||||
return &Object{fs: f}, s, server
|
||||
}
|
||||
|
||||
// TestUploadMultipartSkip checks that a 416 response makes the upload
|
||||
// read the server's position and re-send only the unreceived tail of
|
||||
// the fragment.
|
||||
func TestUploadMultipartSkip(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("onedrive"), 1024)
|
||||
chunkSize := len(content) / 2
|
||||
skip := chunkSize / 4
|
||||
o, s, _ := newUploadTest(t, content, chunkSize)
|
||||
s.fail = func(w http.ResponseWriter, attempt int) bool {
|
||||
if attempt == 1 {
|
||||
// Pretend the first part of the fragment was received
|
||||
// before the connection broke
|
||||
s.received = append(s.received, content[:skip]...)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "invalidRange", "message": "bad range"}})
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
src := object.NewStaticObjectInfo("remote", time.Now(), int64(len(content)), true, nil, nil)
|
||||
info, err := o.uploadMultipart(context.Background(), bytes.NewReader(content), src)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, info)
|
||||
assert.Equal(t, content, s.received)
|
||||
assert.Equal(t, 2, s.fragments)
|
||||
assert.Equal(t, 3, s.attempts, "expected one 416, a partial re-send and the final fragment")
|
||||
}
|
||||
@@ -23,8 +23,8 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -1047,6 +1047,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// resp.Body.Close()
|
||||
// fs.Debugf(nil, "PostOpen: %#v", openResponse)
|
||||
|
||||
buf := make([]byte, o.fs.opt.ChunkSize)
|
||||
chunkOffset := int64(0)
|
||||
remainingBytes := size
|
||||
chunkCounter := 0
|
||||
@@ -1056,27 +1057,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
remainingBytes -= currentChunkSize
|
||||
fs.Debugf(o, "Uploading chunk %d, size=%d, remain=%d", chunkCounter, currentChunkSize, remainingBytes)
|
||||
|
||||
// Buffer the chunk in memory from the global pool so reads are
|
||||
// repeatable for retries
|
||||
rw := multipart.NewRW()
|
||||
_, err = io.CopyN(rw, in, currentChunkSize)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
if err == io.EOF {
|
||||
err = fmt.Errorf("short read of chunk %d: %w", chunkCounter, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, currentChunkSize)
|
||||
var reply uploadFileChunkReply
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// seek to the start in case this is a retry
|
||||
if _, err = rw.Seek(0, io.SeekStart); err != nil {
|
||||
if _, err = chunk.Seek(0, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/upload/upload_file_chunk.json",
|
||||
Body: rw,
|
||||
Body: chunk,
|
||||
MultipartParams: url.Values{
|
||||
"session_id": []string{o.fs.session.SessionID},
|
||||
"file_id": []string{o.id},
|
||||
@@ -1091,13 +1082,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("failed to create file: %w", closeErr)
|
||||
}
|
||||
if reply.TotalWritten != currentChunkSize {
|
||||
return fmt.Errorf("failed to create file: incomplete write of %d/%d bytes", reply.TotalWritten, currentChunkSize)
|
||||
}
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
package opendrive
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var _ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
@@ -13,8 +13,5 @@ func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestOpenDrive:",
|
||||
NilObject: (*opendrive.Object)(nil),
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: 1,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/oracle/oci-go-sdk/v65/common"
|
||||
"github.com/oracle/oci-go-sdk/v65/objectstorage"
|
||||
)
|
||||
|
||||
@@ -72,57 +73,57 @@ func populateSSECustomerKeys(opt *Options) error {
|
||||
// https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingyourencryptionkeys.htm
|
||||
func useBYOKPutObject(fs *Fs, request *objectstorage.PutObjectRequest) {
|
||||
if fs.opt.SSEKMSKeyID != "" {
|
||||
request.OpcSseKmsKeyId = new(fs.opt.SSEKMSKeyID)
|
||||
request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
|
||||
}
|
||||
if fs.opt.SSECustomerAlgorithm != "" {
|
||||
request.OpcSseCustomerAlgorithm = new(fs.opt.SSECustomerAlgorithm)
|
||||
request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
|
||||
}
|
||||
if fs.opt.SSECustomerKey != "" {
|
||||
request.OpcSseCustomerKey = new(fs.opt.SSECustomerKey)
|
||||
request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
|
||||
}
|
||||
if fs.opt.SSECustomerKeySha256 != "" {
|
||||
request.OpcSseCustomerKeySha256 = new(fs.opt.SSECustomerKeySha256)
|
||||
request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
|
||||
}
|
||||
}
|
||||
|
||||
func useBYOKHeadObject(fs *Fs, request *objectstorage.HeadObjectRequest) {
|
||||
if fs.opt.SSECustomerAlgorithm != "" {
|
||||
request.OpcSseCustomerAlgorithm = new(fs.opt.SSECustomerAlgorithm)
|
||||
request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
|
||||
}
|
||||
if fs.opt.SSECustomerKey != "" {
|
||||
request.OpcSseCustomerKey = new(fs.opt.SSECustomerKey)
|
||||
request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
|
||||
}
|
||||
if fs.opt.SSECustomerKeySha256 != "" {
|
||||
request.OpcSseCustomerKeySha256 = new(fs.opt.SSECustomerKeySha256)
|
||||
request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
|
||||
}
|
||||
}
|
||||
|
||||
func useBYOKGetObject(fs *Fs, request *objectstorage.GetObjectRequest) {
|
||||
if fs.opt.SSECustomerAlgorithm != "" {
|
||||
request.OpcSseCustomerAlgorithm = new(fs.opt.SSECustomerAlgorithm)
|
||||
request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
|
||||
}
|
||||
if fs.opt.SSECustomerKey != "" {
|
||||
request.OpcSseCustomerKey = new(fs.opt.SSECustomerKey)
|
||||
request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
|
||||
}
|
||||
if fs.opt.SSECustomerKeySha256 != "" {
|
||||
request.OpcSseCustomerKeySha256 = new(fs.opt.SSECustomerKeySha256)
|
||||
request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
|
||||
}
|
||||
}
|
||||
|
||||
func useBYOKCopyObject(fs *Fs, request *objectstorage.CopyObjectRequest) {
|
||||
if fs.opt.SSEKMSKeyID != "" {
|
||||
request.OpcSseKmsKeyId = new(fs.opt.SSEKMSKeyID)
|
||||
request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
|
||||
}
|
||||
if fs.opt.SSECustomerAlgorithm != "" {
|
||||
request.OpcSseCustomerAlgorithm = new(fs.opt.SSECustomerAlgorithm)
|
||||
request.OpcSourceSseCustomerAlgorithm = new(fs.opt.SSECustomerAlgorithm)
|
||||
request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
|
||||
request.OpcSourceSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
|
||||
}
|
||||
if fs.opt.SSECustomerKey != "" {
|
||||
request.OpcSseCustomerKey = new(fs.opt.SSECustomerKey)
|
||||
request.OpcSourceSseCustomerKey = new(fs.opt.SSECustomerKey)
|
||||
request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
|
||||
request.OpcSourceSseCustomerKey = common.String(fs.opt.SSECustomerKey)
|
||||
}
|
||||
if fs.opt.SSECustomerKeySha256 != "" {
|
||||
request.OpcSseCustomerKeySha256 = new(fs.opt.SSECustomerKeySha256)
|
||||
request.OpcSourceSseCustomerKeySha256 = new(fs.opt.SSECustomerKeySha256)
|
||||
request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
|
||||
request.OpcSourceSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
@@ -63,7 +62,9 @@ func TestGzipEncoding(t *testing.T) {
|
||||
"namespace": "test",
|
||||
"endpoint": srv.URL,
|
||||
}
|
||||
maps.Copy(m, extraConfig)
|
||||
for k, v := range extraConfig {
|
||||
m[k] = v
|
||||
}
|
||||
fsInfo, err := NewFs(ctx, "TestOOSGzip", "bucket", m)
|
||||
require.NoError(t, err)
|
||||
return fsInfo.(*Fs)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/oracle/oci-go-sdk/v65/common"
|
||||
"github.com/oracle/oci-go-sdk/v65/objectstorage"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
@@ -196,7 +197,22 @@ func (f *Fs) rename(ctx context.Context, remote, newName string) (any, error) {
|
||||
}
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
err = f.renameObject(ctx, bucketName, objectPath, newName)
|
||||
details := objectstorage.RenameObjectDetails{
|
||||
SourceName: common.String(objectPath),
|
||||
NewName: common.String(newName),
|
||||
}
|
||||
request := objectstorage.RenameObjectRequest{
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
RenameObjectDetails: details,
|
||||
OpcClientRequestId: nil,
|
||||
RequestMetadata: common.RequestMetadata{},
|
||||
}
|
||||
var response objectstorage.RenameObjectResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
response, err = f.srv.RenameObject(ctx, request)
|
||||
return shouldRetry(ctx, response.HTTPResponse(), err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -268,8 +284,8 @@ func (f *Fs) listMultipartUploadsObject(ctx context.Context, bucketName, directo
|
||||
|
||||
uploads = []*objectstorage.MultipartUpload{}
|
||||
req := objectstorage.ListMultipartUploadsRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
}
|
||||
|
||||
var response objectstorage.ListMultipartUploadsResponse
|
||||
@@ -306,11 +322,11 @@ func (f *Fs) listMultipartUploadParts(ctx context.Context, bucketName, bucketPat
|
||||
uploadedParts map[int]objectstorage.MultipartUploadPartSummary, err error) {
|
||||
uploadedParts = make(map[int]objectstorage.MultipartUploadPartSummary)
|
||||
req := objectstorage.ListMultipartUploadPartsRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
ObjectName: new(bucketPath),
|
||||
UploadId: new(uploadID),
|
||||
Limit: new(1000),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
ObjectName: common.String(bucketPath),
|
||||
UploadId: common.String(uploadID),
|
||||
Limit: common.Int(1000),
|
||||
}
|
||||
|
||||
var response objectstorage.ListMultipartUploadPartsResponse
|
||||
@@ -335,7 +351,7 @@ func (f *Fs) listMultipartUploadParts(ctx context.Context, bucketName, bucketPat
|
||||
|
||||
func (f *Fs) restore(ctx context.Context, opt map[string]string) (any, error) {
|
||||
req := objectstorage.RestoreObjectsRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
RestoreObjectsDetails: objectstorage.RestoreObjectsDetails{},
|
||||
}
|
||||
if hours := opt["hours"]; hours != "" {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/oracle/oci-go-sdk/v65/common"
|
||||
"github.com/oracle/oci-go-sdk/v65/objectstorage"
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
@@ -40,55 +41,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return f.NewObject(ctx, remote)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Implement Mover is an optional interface for Fs
|
||||
//------------------------------------------------------------
|
||||
|
||||
// Move src to this remote using server-side rename operations.
|
||||
//
|
||||
// This is stored with the remote path given.
|
||||
// It returns the destination Object and a possible error.
|
||||
// Will only be called if src.Fs().Name() == f.Name().
|
||||
// If it isn't possible then return fs.ErrorCantMove.
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
srcBucket, srcPath := srcObj.split()
|
||||
dstBucket, dstPath := f.split(remote)
|
||||
if srcBucket != dstBucket {
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
err := f.renameObject(ctx, srcBucket, srcPath, dstPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.NewObject(ctx, remote)
|
||||
}
|
||||
|
||||
// renameObject renames sourceName to newName in bucketName.
|
||||
func (f *Fs) renameObject(ctx context.Context, bucketName, sourceName, newName string) error {
|
||||
req := objectstorage.RenameObjectRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
RenameObjectDetails: objectstorage.RenameObjectDetails{
|
||||
SourceName: new(sourceName),
|
||||
NewName: new(newName),
|
||||
},
|
||||
}
|
||||
var resp objectstorage.RenameObjectResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.srv.RenameObject(ctx, req)
|
||||
return shouldRetry(ctx, resp.HTTPResponse(), err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copy does a server-side copy from dstObj <- srcObj
|
||||
//
|
||||
// If newInfo is nil then the metadata will be copied otherwise it
|
||||
@@ -109,16 +61,16 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object) (err erro
|
||||
}
|
||||
}
|
||||
copyObjectDetails := objectstorage.CopyObjectDetails{
|
||||
SourceObjectName: new(srcPath),
|
||||
DestinationRegion: new(dstObj.fs.opt.Region),
|
||||
DestinationNamespace: new(dstObj.fs.opt.Namespace),
|
||||
DestinationBucket: new(dstBucket),
|
||||
DestinationObjectName: new(dstPath),
|
||||
SourceObjectName: common.String(srcPath),
|
||||
DestinationRegion: common.String(dstObj.fs.opt.Region),
|
||||
DestinationNamespace: common.String(dstObj.fs.opt.Namespace),
|
||||
DestinationBucket: common.String(dstBucket),
|
||||
DestinationObjectName: common.String(dstPath),
|
||||
DestinationObjectMetadata: metadataWithOpcPrefix(srcObj.meta),
|
||||
}
|
||||
req := objectstorage.CopyObjectRequest{
|
||||
NamespaceName: new(srcObj.fs.opt.Namespace),
|
||||
BucketName: new(srcBucket),
|
||||
NamespaceName: common.String(srcObj.fs.opt.Namespace),
|
||||
BucketName: common.String(srcBucket),
|
||||
CopyObjectDetails: copyObjectDetails,
|
||||
}
|
||||
useBYOKCopyObject(f, &req)
|
||||
|
||||
@@ -161,13 +161,13 @@ func (w *objectChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, rea
|
||||
}
|
||||
}
|
||||
req := objectstorage.UploadPartRequest{
|
||||
NamespaceName: new(w.f.opt.Namespace),
|
||||
NamespaceName: common.String(w.f.opt.Namespace),
|
||||
BucketName: w.bucket,
|
||||
ObjectName: w.key,
|
||||
UploadId: w.uploadID,
|
||||
UploadPartNum: new(ossPartNumber),
|
||||
ContentLength: new(currentChunkSize),
|
||||
ContentMD5: new(md5sum),
|
||||
UploadPartNum: common.Int(ossPartNumber),
|
||||
ContentLength: common.Int64(currentChunkSize),
|
||||
ContentMD5: common.String(md5sum),
|
||||
}
|
||||
w.o.applyPartUploadOptions(w.ui.req, &req)
|
||||
var resp objectstorage.UploadPartResponse
|
||||
@@ -213,7 +213,7 @@ func (w *objectChunkWriter) addCompletedPart(partNum *int, eTag *string) {
|
||||
|
||||
func (w *objectChunkWriter) Close(ctx context.Context) (err error) {
|
||||
req := objectstorage.CommitMultipartUploadRequest{
|
||||
NamespaceName: new(w.f.opt.Namespace),
|
||||
NamespaceName: common.String(w.f.opt.Namespace),
|
||||
BucketName: w.bucket,
|
||||
ObjectName: w.key,
|
||||
UploadId: w.uploadID,
|
||||
@@ -290,9 +290,9 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
bucket, bucketPath := o.split()
|
||||
|
||||
ui.req = &objectstorage.PutObjectRequest{
|
||||
NamespaceName: new(o.fs.opt.Namespace),
|
||||
BucketName: new(bucket),
|
||||
ObjectName: new(bucketPath),
|
||||
NamespaceName: common.String(o.fs.opt.Namespace),
|
||||
BucketName: common.String(bucket),
|
||||
ObjectName: common.String(bucketPath),
|
||||
}
|
||||
|
||||
// Set the mtime in the metadata
|
||||
@@ -305,7 +305,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
ui.req.OpcMeta = make(map[string]string, len(meta)+2)
|
||||
// merge metadata into request and user metadata
|
||||
for k, v := range meta {
|
||||
pv := new(v)
|
||||
pv := common.String(v)
|
||||
k = strings.ToLower(k)
|
||||
switch k {
|
||||
case "cache-control":
|
||||
@@ -366,10 +366,10 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
}
|
||||
// Set the content type if it isn't set already
|
||||
if ui.req.ContentType == nil {
|
||||
ui.req.ContentType = new(fs.MimeType(ctx, src))
|
||||
ui.req.ContentType = common.String(fs.MimeType(ctx, src))
|
||||
}
|
||||
if size >= 0 {
|
||||
ui.req.ContentLength = new(size)
|
||||
ui.req.ContentLength = common.Int64(size)
|
||||
}
|
||||
if md5sumBase64 != "" {
|
||||
ui.req.ContentMD5 = &md5sumBase64
|
||||
@@ -420,10 +420,10 @@ func (o *Object) createMultipartUpload(ctx context.Context, putReq *objectstorag
|
||||
}
|
||||
}
|
||||
req := objectstorage.CreateMultipartUploadRequest{
|
||||
NamespaceName: new(o.fs.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
NamespaceName: common.String(o.fs.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
}
|
||||
req.Object = new(bucketPath)
|
||||
req.Object = common.String(bucketPath)
|
||||
if o.fs.opt.StorageTier != "" {
|
||||
storageTier, ok := objectstorage.GetMappingStorageTierEnum(o.fs.opt.StorageTier)
|
||||
if !ok {
|
||||
|
||||
@@ -84,9 +84,9 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObjectResponse, err error) {
|
||||
bucketName, objectPath := o.split()
|
||||
req := objectstorage.HeadObjectRequest{
|
||||
NamespaceName: new(o.fs.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
ObjectName: new(objectPath),
|
||||
NamespaceName: common.String(o.fs.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
ObjectName: common.String(objectPath),
|
||||
}
|
||||
useBYOKHeadObject(o.fs, &req)
|
||||
var response objectstorage.HeadObjectResponse
|
||||
@@ -236,10 +236,10 @@ func (o *Object) SetTier(tier string) (err error) {
|
||||
}
|
||||
|
||||
req := objectstorage.UpdateObjectStorageTierRequest{
|
||||
NamespaceName: new(o.fs.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
NamespaceName: common.String(o.fs.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
UpdateObjectStorageTierDetails: objectstorage.UpdateObjectStorageTierDetails{
|
||||
ObjectName: new(bucketPath),
|
||||
ObjectName: common.String(bucketPath),
|
||||
StorageTier: tierEnum,
|
||||
},
|
||||
}
|
||||
@@ -322,9 +322,9 @@ func (o *Object) Storable() bool {
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
bucketName, bucketPath := o.split()
|
||||
req := objectstorage.DeleteObjectRequest{
|
||||
NamespaceName: new(o.fs.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
ObjectName: new(bucketPath),
|
||||
NamespaceName: common.String(o.fs.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
ObjectName: common.String(bucketPath),
|
||||
}
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.DeleteObject(ctx, req)
|
||||
@@ -337,9 +337,9 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
bucketName, bucketPath := o.split()
|
||||
req := objectstorage.GetObjectRequest{
|
||||
NamespaceName: new(o.fs.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
ObjectName: new(bucketPath),
|
||||
NamespaceName: common.String(o.fs.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
ObjectName: common.String(bucketPath),
|
||||
}
|
||||
o.applyGetObjectOptions(&req, options...)
|
||||
useBYOKGetObject(o.fs, &req)
|
||||
@@ -460,15 +460,15 @@ func (o *Object) applyPutOptions(req *objectstorage.PutObjectRequest, options ..
|
||||
case "":
|
||||
// ignore
|
||||
case "cache-control":
|
||||
req.CacheControl = new(value)
|
||||
req.CacheControl = common.String(value)
|
||||
case "content-disposition":
|
||||
req.ContentDisposition = new(value)
|
||||
req.ContentDisposition = common.String(value)
|
||||
case "content-encoding":
|
||||
req.ContentEncoding = new(value)
|
||||
req.ContentEncoding = common.String(value)
|
||||
case "content-language":
|
||||
req.ContentLanguage = new(value)
|
||||
req.ContentLanguage = common.String(value)
|
||||
case "content-type":
|
||||
req.ContentType = new(value)
|
||||
req.ContentType = common.String(value)
|
||||
default:
|
||||
if strings.HasPrefix(lowerKey, ociMetaPrefix) {
|
||||
req.OpcMeta[lowerKey] = value
|
||||
@@ -500,15 +500,15 @@ func (o *Object) applyGetObjectOptions(req *objectstorage.GetObjectRequest, opti
|
||||
case "":
|
||||
// ignore
|
||||
case "cache-control":
|
||||
req.HttpResponseCacheControl = new(value)
|
||||
req.HttpResponseCacheControl = common.String(value)
|
||||
case "content-disposition":
|
||||
req.HttpResponseContentDisposition = new(value)
|
||||
req.HttpResponseContentDisposition = common.String(value)
|
||||
case "content-encoding":
|
||||
req.HttpResponseContentEncoding = new(value)
|
||||
req.HttpResponseContentEncoding = common.String(value)
|
||||
case "content-language":
|
||||
req.HttpResponseContentLanguage = new(value)
|
||||
req.HttpResponseContentLanguage = common.String(value)
|
||||
case "content-type":
|
||||
req.HttpResponseContentType = new(value)
|
||||
req.HttpResponseContentType = common.String(value)
|
||||
case "range":
|
||||
// do nothing
|
||||
default:
|
||||
|
||||
@@ -257,8 +257,7 @@ to start uploading.`,
|
||||
// so that OSS keys are always valid file names
|
||||
Default: encoder.EncodeInvalidUtf8 |
|
||||
encoder.EncodeSlash |
|
||||
encoder.EncodeDot |
|
||||
encoder.EncodeCrLf,
|
||||
encoder.EncodeDot,
|
||||
}, {
|
||||
Name: "leave_parts_on_error",
|
||||
Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts for manual recovery.
|
||||
|
||||
@@ -6,7 +6,6 @@ package oracleobjectstorage
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -304,28 +303,17 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
// listFn is called from list to handle an object.
|
||||
type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error
|
||||
|
||||
// list calls fn for objects in bucket and directory.
|
||||
//
|
||||
// ctx controls the API requests.
|
||||
// bucket is the bucket to list.
|
||||
// directory is the path to list within bucket.
|
||||
// list the objects into the function supplied from
|
||||
// the bucket and root supplied
|
||||
// (bucket, directory) is the starting directory
|
||||
// If prefix is set then it is removed from all file names
|
||||
// If addBucket is set then it adds the bucket to the start of the remotes generated
|
||||
// If recurse is set the function will recursively list
|
||||
// If limit is > 0 then it limits to that many files (must be less than 1000)
|
||||
// includeDirectoryMarkers includes zero-length objects with names ending in a slash.
|
||||
// fn receives each listed object.
|
||||
func (f *Fs) list(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
directory string,
|
||||
prefix string,
|
||||
addBucket bool,
|
||||
recurse bool,
|
||||
limit int,
|
||||
includeDirectoryMarkers bool,
|
||||
fn listFn,
|
||||
) (err error) {
|
||||
// If hidden is set then it will list the hidden (deleted) files too.
|
||||
// if findFile is set it will look for files called (bucket, directory)
|
||||
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int,
|
||||
fn listFn) (err error) {
|
||||
if prefix != "" {
|
||||
prefix += "/"
|
||||
}
|
||||
@@ -342,14 +330,14 @@ func (f *Fs) list(
|
||||
chunkSize = limit
|
||||
}
|
||||
var request = objectstorage.ListObjectsRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucket),
|
||||
Prefix: new(directory),
|
||||
Limit: new(chunkSize),
|
||||
Fields: new("name,size,etag,timeCreated,md5,timeModified,storageTier,archivalState"),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
BucketName: common.String(bucket),
|
||||
Prefix: common.String(directory),
|
||||
Limit: common.Int(chunkSize),
|
||||
Fields: common.String("name,size,etag,timeCreated,md5,timeModified,storageTier,archivalState"),
|
||||
}
|
||||
if delimiter != "" {
|
||||
request.Delimiter = new(delimiter)
|
||||
request.Delimiter = common.String(delimiter)
|
||||
}
|
||||
|
||||
for {
|
||||
@@ -420,7 +408,7 @@ func (f *Fs) list(
|
||||
remote = path.Join(bucket, remote)
|
||||
}
|
||||
// is this a directory marker?
|
||||
if isDirectory && object.Size != nil && *object.Size == 0 && !includeDirectoryMarkers {
|
||||
if isDirectory && object.Size != nil && *object.Size == 0 {
|
||||
continue // skip directory marker
|
||||
}
|
||||
if isDirectory && len(remote) > 1 {
|
||||
@@ -469,7 +457,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, false, fn)
|
||||
err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -484,8 +472,8 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
|
||||
return nil, fmt.Errorf("can't list buckets with %v provider, use a valid auth provider in config file", noAuth)
|
||||
}
|
||||
var request = objectstorage.ListBucketsRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
CompartmentId: new(f.opt.Compartment),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
CompartmentId: common.String(f.opt.Compartment),
|
||||
}
|
||||
var resp objectstorage.ListBucketsResponse
|
||||
for {
|
||||
@@ -582,12 +570,12 @@ func (f *Fs) makeBucket(ctx context.Context, bucketName string) error {
|
||||
}
|
||||
return f.cache.Create(bucketName, func() error {
|
||||
details := objectstorage.CreateBucketDetails{
|
||||
Name: new(bucketName),
|
||||
CompartmentId: new(f.opt.Compartment),
|
||||
Name: common.String(bucketName),
|
||||
CompartmentId: common.String(f.opt.Compartment),
|
||||
PublicAccessType: objectstorage.CreateBucketDetailsPublicAccessTypeNopublicaccess,
|
||||
}
|
||||
req := objectstorage.CreateBucketRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
CreateBucketDetails: details,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -614,8 +602,8 @@ func (f *Fs) makeBucket(ctx context.Context, bucketName string) error {
|
||||
// NB this can return incorrect results if called immediately after bucket deletion
|
||||
func (f *Fs) bucketExists(ctx context.Context, bucketName string) (bool, error) {
|
||||
req := objectstorage.HeadBucketRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.HeadBucket(ctx, req)
|
||||
@@ -640,8 +628,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
}
|
||||
return f.cache.Remove(bucketName, func() error {
|
||||
req := objectstorage.DeleteBucketRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
BucketName: common.String(bucketName),
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.DeleteBucket(ctx, req)
|
||||
@@ -654,72 +642,12 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
})
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Implement Purger is an optional interfaces for Fs
|
||||
//------------------------------------------------------------
|
||||
|
||||
// Purge deletes all the files in dir.
|
||||
//
|
||||
// It implements fs.Purger, using OCI batch requests rather than removing each
|
||||
// object returned by List. It also removes directory markers and, when dir is
|
||||
// a bucket root, removes the empty bucket.
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
bucketName, directory := f.split(dir)
|
||||
if bucketName == "" {
|
||||
return fs.ErrorListBucketRequired
|
||||
}
|
||||
const batchSize = 1000
|
||||
objects := make([]objectstorage.BatchDeleteObjectIdentifier, 0, batchSize)
|
||||
deleteObjects := func() error {
|
||||
if len(objects) == 0 {
|
||||
return nil
|
||||
}
|
||||
req := objectstorage.BatchDeleteObjectsRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
BatchDeleteObjectsDetails: objectstorage.BatchDeleteObjectsDetails{
|
||||
Objects: objects,
|
||||
IsSkipDeletedResult: new(true),
|
||||
},
|
||||
}
|
||||
var resp objectstorage.BatchDeleteObjectsResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.srv.BatchDeleteObjects(ctx, req)
|
||||
return shouldRetry(ctx, resp.HTTPResponse(), err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, failed := range resp.Failed {
|
||||
errs = append(errs, fmt.Errorf("failed to delete object %q: %s", *failed.ObjectName, *failed.ErrorMessage))
|
||||
}
|
||||
objects = objects[:0]
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
err := f.list(ctx, bucketName, directory, f.rootDirectory, false, true, 0, true, func(_ string, object *objectstorage.ObjectSummary, _ bool) error {
|
||||
objects = append(objects, objectstorage.BatchDeleteObjectIdentifier{ObjectName: object.Name})
|
||||
if len(objects) == batchSize {
|
||||
return deleteObjects()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = deleteObjects(); err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Rmdir(ctx, dir)
|
||||
}
|
||||
|
||||
func (f *Fs) abortMultiPartUpload(ctx context.Context, bucketName, bucketPath, uploadID *string) (err error) {
|
||||
if uploadID == nil || *uploadID == "" {
|
||||
return nil
|
||||
}
|
||||
request := objectstorage.AbortMultipartUploadRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
NamespaceName: common.String(f.opt.Namespace),
|
||||
BucketName: bucketName,
|
||||
ObjectName: bucketPath,
|
||||
UploadId: uploadID,
|
||||
@@ -801,7 +729,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
bucketName, directory := f.split(dir)
|
||||
list := list.NewHelper(callback)
|
||||
listR := func(bucket, directory, prefix string, addBucket bool) error {
|
||||
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, false, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
|
||||
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -874,13 +802,11 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Copier = &Fs{}
|
||||
_ fs.Mover = &Fs{}
|
||||
_ fs.PutStreamer = &Fs{}
|
||||
_ fs.ListRer = &Fs{}
|
||||
_ fs.ListPer = &Fs{}
|
||||
_ fs.Commander = &Fs{}
|
||||
_ fs.CleanUpper = &Fs{}
|
||||
_ fs.Purger = &Fs{}
|
||||
_ fs.OpenChunkWriter = &Fs{}
|
||||
|
||||
_ fs.Object = &Object{}
|
||||
|
||||
@@ -13,14 +13,10 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/oracle/oci-go-sdk/v65/objectstorage"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
@@ -136,99 +132,9 @@ func (f *Fs) InternalTestGzipEncoding(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// InternalTestPurgeBatches tests purging more objects than fit in one batch.
|
||||
func (f *Fs) InternalTestPurgeBatches(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const (
|
||||
dir = "purge-batches"
|
||||
objectCount = 1001
|
||||
uploadConcurrency = 16
|
||||
)
|
||||
defer func() { _ = f.Purge(ctx, dir) }()
|
||||
jobs := make(chan int)
|
||||
errs := make(chan error, objectCount)
|
||||
var wg sync.WaitGroup
|
||||
for range uploadConcurrency {
|
||||
wg.Go(func() {
|
||||
for i := range jobs {
|
||||
remote := fmt.Sprintf("%s/%04d", dir, i)
|
||||
src := object.NewStaticObjectInfo(remote, time.Time{}, 0, true, nil, f)
|
||||
_, err := f.Put(ctx, bytes.NewReader(nil), src)
|
||||
if err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
for i := range objectCount {
|
||||
jobs <- i
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, f.Purge(ctx, dir))
|
||||
entries, err := f.List(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
}
|
||||
|
||||
// InternalTestPurgeDirectoryMarkers tests purging directory marker objects.
|
||||
func (f *Fs) InternalTestPurgeDirectoryMarkers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const dir = "purge-directory-markers"
|
||||
bucketName, directory := f.split(dir)
|
||||
objectNames := []string{
|
||||
directory + "/",
|
||||
directory + "/child/",
|
||||
directory + "/child/object",
|
||||
}
|
||||
defer func() { _ = f.Purge(ctx, dir) }()
|
||||
for _, objectName := range objectNames {
|
||||
req := objectstorage.PutObjectRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
ObjectName: new(objectName),
|
||||
PutObjectBody: io.NopCloser(bytes.NewReader(nil)),
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.PutObject(ctx, req)
|
||||
return shouldRetry(ctx, resp.HTTPResponse(), err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
// Use the SDK directly because normal OOS listings hide directory markers.
|
||||
listObjectNames := func() []string {
|
||||
req := objectstorage.ListObjectsRequest{
|
||||
NamespaceName: new(f.opt.Namespace),
|
||||
BucketName: new(bucketName),
|
||||
Prefix: new(directory + "/"),
|
||||
}
|
||||
var resp objectstorage.ListObjectsResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.srv.ListObjects(ctx, req)
|
||||
return shouldRetry(ctx, resp.HTTPResponse(), err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(resp.Objects))
|
||||
for _, object := range resp.Objects {
|
||||
names = append(names, *object.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
assert.ElementsMatch(t, objectNames, listObjectNames())
|
||||
require.NoError(t, f.Purge(ctx, dir))
|
||||
assert.Empty(t, listObjectNames())
|
||||
}
|
||||
|
||||
// InternalTest is called by fstests.Run to extra tests
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("GzipEncoding", f.InternalTestGzipEncoding)
|
||||
t.Run("PurgeBatches", f.InternalTestPurgeBatches)
|
||||
t.Run("PurgeDirectoryMarkers", f.InternalTestPurgeDirectoryMarkers)
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
@@ -22,7 +23,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -307,9 +307,7 @@ func readGcid(in io.Reader, size, threshold int64) (gcid string, out io.Reader,
|
||||
cleanup = func() {}
|
||||
|
||||
// don't cache small files on disk to reduce wear of the disk
|
||||
//
|
||||
// files of unknown size go to disk as they may not fit in memory
|
||||
if size < 0 || size > threshold {
|
||||
if size > threshold {
|
||||
var tempFile *os.File
|
||||
|
||||
// create the cache file
|
||||
@@ -342,16 +340,13 @@ func readGcid(in io.Reader, size, threshold int64) (gcid string, out io.Reader,
|
||||
// replace the already read source with a reader of our cached file
|
||||
out = tempFile
|
||||
} else {
|
||||
rw := multipart.NewRW()
|
||||
cleanup = func() {
|
||||
_ = rw.Close()
|
||||
}
|
||||
teeReader := io.TeeReader(in, rw)
|
||||
buf := &bytes.Buffer{}
|
||||
teeReader := io.TeeReader(in, buf)
|
||||
|
||||
if gcid, err = calcGcid(teeReader, size); err != nil {
|
||||
return
|
||||
}
|
||||
out = rw
|
||||
out = buf
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestReadGcid checks readGcid hashes and replays its input from a pooled
|
||||
// buffer for small known sizes, and from a temp file for large or unknown
|
||||
// sizes, releasing the buffer on cleanup.
|
||||
func TestReadGcid(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("pikpak"), 1024)
|
||||
want, err := calcGcid(bytes.NewReader(content), int64(len(content)))
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int64
|
||||
threshold int64
|
||||
wantFile bool
|
||||
}{
|
||||
{"memory", int64(len(content)), int64(len(content)), false},
|
||||
{"file", int64(len(content)), int64(len(content)) - 1, true},
|
||||
{"unknown size", -1, int64(len(content)), true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
inUse := pool.Global().InUse()
|
||||
gcid, out, cleanup, err := readGcid(bytes.NewReader(content), test.size, test.threshold)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, gcid)
|
||||
_, isFile := out.(*os.File)
|
||||
assert.Equal(t, test.wantFile, isFile, "expected temp file %v", test.wantFile)
|
||||
got, err := io.ReadAll(out)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, got)
|
||||
cleanup()
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "buffer should be returned to the pool")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
@@ -17,11 +18,38 @@ import (
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
bufferSize = 1024 * 1024 // default size of the pages used in the reader
|
||||
bufferCacheSize = 64 // max number of buffers to keep in cache
|
||||
bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long
|
||||
)
|
||||
|
||||
// bufferPool is a global pool of buffers
|
||||
var (
|
||||
bufferPool *pool.Pool
|
||||
bufferPoolOnce sync.Once
|
||||
)
|
||||
|
||||
// get a buffer pool
|
||||
func getPool() *pool.Pool {
|
||||
bufferPoolOnce.Do(func() {
|
||||
ci := fs.GetConfig(context.Background())
|
||||
// Initialise the buffer pool when used
|
||||
bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)
|
||||
})
|
||||
return bufferPool
|
||||
}
|
||||
|
||||
// NewRW gets a pool.RW using the multipart pool
|
||||
func NewRW() *pool.RW {
|
||||
return pool.NewRW(getPool())
|
||||
}
|
||||
|
||||
// Upload does a multipart upload in parallel
|
||||
func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
|
||||
// make concurrency machinery
|
||||
@@ -52,7 +80,7 @@ func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
|
||||
for partNum := int64(0); !finished; partNum++ {
|
||||
// Get a block of memory from the pool and token which limits concurrency.
|
||||
tokens.Get()
|
||||
rw := multipart.NewRW()
|
||||
rw := NewRW()
|
||||
if acc != nil {
|
||||
rw.SetAccounting(acc.AccountRead)
|
||||
}
|
||||
|
||||
@@ -224,7 +224,7 @@ Fill in for rclone to use a non root folder as its starting point.
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "hash_memory_limit",
|
||||
Help: "Files bigger than this will be cached on disk to calculate hash if required.\n\nFiles of unknown size are always cached on disk.",
|
||||
Help: "Files bigger than this will be cached on disk to calculate hash if required.",
|
||||
Default: fs.SizeSuffix(10 * 1024 * 1024),
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -433,7 +433,8 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
|
||||
|
||||
// traceback to possible api.Error wrapped in err, and re-authorize if necessary
|
||||
// "unauthenticated" (16): when access_token is invalid, but should be handled by oauthutil
|
||||
if terr, ok := errors.AsType[*oauth2.RetrieveError](err); ok {
|
||||
var terr *oauth2.RetrieveError
|
||||
if errors.As(err, &terr) {
|
||||
apiErr := new(api.Error)
|
||||
if err := json.Unmarshal(terr.Body, apiErr); err == nil {
|
||||
if apiErr.Reason == "invalid_grant" {
|
||||
|
||||
@@ -262,7 +262,8 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if err == nil {
|
||||
return false, nil
|
||||
}
|
||||
if apiErr, ok := errors.AsType[*proton.APIError](err); ok {
|
||||
var apiErr *proton.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
// Code 200501 is a generic Drive operation-failure code. Proton also
|
||||
// returns it with an HTTP 422 for permanent validation failures (for
|
||||
// example a content key packet that cannot be verified, or an upload
|
||||
@@ -455,9 +456,9 @@ func isDecimalString(value string) bool {
|
||||
// and is captured by --log-file.
|
||||
type protonLogger struct{ f *Fs }
|
||||
|
||||
func (l protonLogger) Errorf(format string, v ...any) { fs.Errorf(l.f, format, v...) }
|
||||
func (l protonLogger) Warnf(format string, v ...any) { fs.Logf(l.f, format, v...) }
|
||||
func (l protonLogger) Debugf(format string, v ...any) { fs.Debugf(l.f, format, v...) }
|
||||
func (l protonLogger) Errorf(format string, v ...interface{}) { fs.Errorf(l.f, format, v...) }
|
||||
func (l protonLogger) Warnf(format string, v ...interface{}) { fs.Logf(l.f, format, v...) }
|
||||
func (l protonLogger) Debugf(format string, v ...interface{}) { fs.Debugf(l.f, format, v...) }
|
||||
|
||||
func newProtonDrive(ctx context.Context, f *Fs, opt *Options, m configmap.Mapper) (*protonDriveAPI.ProtonDrive, error) {
|
||||
config := protonDriveAPI.NewDefaultConfig()
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestShouldRetry(t *testing.T) {
|
||||
{"rate limit Status=429 (handled by SDK, not retried here)", ctx, apiErr(429, 0), false},
|
||||
{"client error Status=400", ctx, apiErr(400, 0), false},
|
||||
{"client error Status=404", ctx, apiErr(404, 0), false},
|
||||
{"wrapped API error retried via errors.As", ctx, fmt.Errorf("wrapped: %w", apiErr(500, 0)), true},
|
||||
{"wrapped API error retried via errors.As", ctx, fmt.Errorf("wrapped: %w", &proton.APIError{Status: 500}), true},
|
||||
{"non-API error falls back to fserrors.ShouldRetry", ctx, errors.New("plain error"), false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
+5
-32
@@ -21,7 +21,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
@@ -331,31 +330,17 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
|
||||
}
|
||||
var clientOffset int64
|
||||
var offsetMismatch bool
|
||||
buf := make([]byte, defaultChunkSize)
|
||||
for clientOffset < size {
|
||||
chunkSize := min(size-clientOffset, int64(defaultChunkSize))
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
|
||||
chunkStart := clientOffset
|
||||
reqSize := chunkSize
|
||||
transferOffset := clientOffset
|
||||
fs.Debugf(f, "chunkStart: %d, reqSize: %d", chunkStart, reqSize)
|
||||
|
||||
// Buffer the chunk in memory from the global pool so reads are
|
||||
// repeatable for retries
|
||||
rw := multipart.NewRW()
|
||||
_, err = io.CopyN(rw, in, chunkSize)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
if err == io.EOF {
|
||||
err = fmt.Errorf("short read of chunk at %d: %w", chunkStart, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if offsetMismatch {
|
||||
// Get file offset and seek to the position
|
||||
offset, err := f.getServerOffset(ctx, location)
|
||||
@@ -364,7 +349,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
|
||||
}
|
||||
sentBytes := offset - chunkStart
|
||||
fs.Debugf(f, "sentBytes: %d", sentBytes)
|
||||
_, err = rw.Seek(sentBytes, io.SeekStart)
|
||||
_, err = chunk.Seek(sentBytes, io.SeekStart)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
@@ -374,7 +359,7 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
|
||||
}
|
||||
fs.Debugf(f, "Sending chunk. transferOffset: %d length: %d", transferOffset, reqSize)
|
||||
var serverOffset int64
|
||||
serverOffset, fileID, err = f.transferChunk(ctx, location, transferOffset, rw, reqSize)
|
||||
serverOffset, fileID, err = f.transferChunk(ctx, location, transferOffset, chunk, reqSize)
|
||||
if cerr, ok := err.(*statusCodeError); ok && cerr.response.StatusCode == 409 {
|
||||
offsetMismatch = true
|
||||
return true, err
|
||||
@@ -385,13 +370,9 @@ func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.
|
||||
}
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if closeErr != nil {
|
||||
return 0, closeErr
|
||||
}
|
||||
|
||||
clientOffset += chunkSize
|
||||
}
|
||||
@@ -455,19 +436,11 @@ func (f *Fs) makeUploadHeadRequest(ctx context.Context, location string) (*http.
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// makeUploadPatchRequest makes the tus PATCH request sending length bytes
|
||||
// of in at offset.
|
||||
//
|
||||
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
|
||||
// to an io.Closer and close it after each attempt — the chunk buffer is
|
||||
// pool-backed, so an early close would free its pages and a retry would then
|
||||
// read a dead buffer. The upload loop owns the buffer's lifetime.
|
||||
func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.Reader, offset, length int64) (*http.Request, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "PATCH", location, readers.NoCloser(in))
|
||||
req, err := http.NewRequestWithContext(ctx, "PATCH", location, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.ContentLength = length
|
||||
req.Header.Set("tus-resumable", "1.0.0")
|
||||
req.Header.Set("upload-offset", strconv.FormatInt(offset, 10))
|
||||
req.Header.Set("content-length", strconv.FormatInt(length, 10))
|
||||
|
||||
@@ -3,7 +3,6 @@ description: Fastly Object Storage
|
||||
region:
|
||||
au-east-1: AU East 1
|
||||
eu-central: EU Central
|
||||
eu-central-1: EU Central 1
|
||||
eu-south-1: EU South 1
|
||||
eu-west-1: EU West 1
|
||||
jp-central-1: JP Central 1
|
||||
@@ -16,7 +15,6 @@ region:
|
||||
endpoint:
|
||||
au-east-1.object.fastlystorage.app: AU East 1
|
||||
eu-central.object.fastlystorage.app: EU Central
|
||||
eu-central-1.object.fastlystorage.app: EU Central 1
|
||||
eu-south-1.object.fastlystorage.app: EU South 1
|
||||
eu-west-1.object.fastlystorage.app: EU West 1
|
||||
jp-central-1.object.fastlystorage.app: JP Central 1
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
name: RelAix
|
||||
description: RelAix.Storage
|
||||
endpoint:
|
||||
s3-gw1.relaix.net: Aachen-1, Germany
|
||||
s3-gw2.relaix.net: Aachen-2, Germany
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
@@ -4,12 +4,10 @@ region:
|
||||
nl-ams: Amsterdam, The Netherlands
|
||||
fr-par: Paris, France
|
||||
pl-waw: Warsaw, Poland
|
||||
it-mil: Milan, Italy
|
||||
endpoint:
|
||||
s3.nl-ams.scw.cloud: Amsterdam Endpoint
|
||||
s3.fr-par.scw.cloud: Paris Endpoint
|
||||
s3.pl-waw.scw.cloud: Warsaw Endpoint
|
||||
s3.it-mil.scw.cloud: Milan Endpoint
|
||||
acl: {}
|
||||
storage_class:
|
||||
'': Default.
|
||||
|
||||
Loaded 100 of 361 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user