Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be8d6044ea | ||
|
|
bd07f481e2 | ||
|
|
56059c47c8 | ||
|
|
35ab21109b | ||
|
|
fc7a57b447 | ||
|
|
028f2b4568 | ||
|
|
62e770eb54 | ||
|
|
652bbb0908 | ||
|
|
4df376e2db | ||
|
|
68e43e5b88 | ||
|
|
1dcc246e03 | ||
|
|
4cc69c2a62 | ||
|
|
8d97a50c4a | ||
|
|
7c2405c035 | ||
|
|
10335538d1 | ||
|
|
a7b1599204 | ||
|
|
518f43dee9 | ||
|
|
43d3374465 | ||
|
|
10875eec70 | ||
|
|
40eaa114c1 | ||
|
|
a34b606abf | ||
|
|
37aa0cb629 | ||
|
|
a7f0259e55 | ||
|
|
323b0b4559 | ||
|
|
26175553a7 | ||
|
|
4a944dff0f | ||
|
|
e9b9b48a68 | ||
|
|
3a14789eb4 | ||
|
|
47027efd12 | ||
|
|
2f43b99abe | ||
|
|
133ced103a | ||
|
|
317c885857 | ||
|
|
a34f1f153a | ||
|
|
e7d4d425bc | ||
|
|
01a9d67e06 | ||
|
|
aece53963a | ||
|
|
291016ea75 | ||
|
|
7e57142de1 | ||
|
|
722c14d806 | ||
|
|
89b8a59e8b | ||
|
|
1b500594d3 | ||
|
|
9b495f2717 | ||
|
|
f6290108df | ||
|
|
ed9f1d836e | ||
|
|
59ea591bcb | ||
|
|
2cc9088374 | ||
|
|
d8dbeb9de4 | ||
|
|
9cf1af177a | ||
|
|
bb42b076af | ||
|
|
0992024791 | ||
|
|
48ac01c2b8 | ||
|
|
b928e8ddd0 | ||
|
|
9ac7855da9 | ||
|
|
5d20bd2832 | ||
|
|
2f84013eec | ||
|
|
675f9c3684 | ||
|
|
b8a697888c | ||
|
|
dddf808b47 | ||
|
|
451346f01c | ||
|
|
6fadaa46aa | ||
|
|
563919b0c0 | ||
|
|
1ecdeb4466 | ||
|
|
b85a681721 | ||
|
|
61b8a90db8 | ||
|
|
419c5f9844 | ||
|
|
50d2430b4a | ||
|
|
cd6941ed93 | ||
|
|
8290a478c8 | ||
|
|
d89f30a136 | ||
|
|
f8899ca982 | ||
|
|
d77c1bf5c6 | ||
|
|
19bf14aea4 | ||
|
|
353167fd38 | ||
|
|
09e1967c73 | ||
|
|
1d1eb94c34 | ||
|
|
fd3b9bd383 | ||
|
|
6676f2673e | ||
|
|
f6af51420f | ||
|
|
a755455609 | ||
|
|
0e6ed13131 | ||
|
|
96d2633836 | ||
|
|
f625ac98a6 | ||
|
|
19068b6608 | ||
|
|
ee40507d38 | ||
|
|
b4cb0426a3 | ||
|
|
5977137b3b | ||
|
|
512f6193a2 | ||
|
|
46cf167c22 | ||
|
|
2d24951658 | ||
|
|
ad9ad335f4 | ||
|
|
7d3e148212 | ||
|
|
2ff47e8274 | ||
|
|
f3bec5b1aa | ||
|
|
d418a5a882 | ||
|
|
db96e30c5e | ||
|
|
111fda1287 | ||
|
|
10d999e3ee | ||
|
|
6a7dcf12c3 | ||
|
|
fecd1d578a | ||
|
|
af92ea943d | ||
|
|
b46bb7647b | ||
|
|
3f6b38c9d9 | ||
|
|
a1fba2e8c1 | ||
|
|
dca3902fef | ||
|
|
06387f878a | ||
|
|
13ae25b561 | ||
|
|
5ea1851d52 | ||
|
|
2b65691945 | ||
|
|
5de8a01717 | ||
|
|
4d50f6d878 | ||
|
|
34ad19a1eb | ||
|
|
5bc26e57b4 | ||
|
|
8d15b2bb75 | ||
|
|
c062ec1b12 | ||
|
|
7b647c729b | ||
|
|
4347404119 | ||
|
|
366b188a5a | ||
|
|
0dd95c2840 | ||
|
|
efba10c065 | ||
|
|
cc53a4cdac | ||
|
|
89c6cb0911 | ||
|
|
a5f9653807 | ||
|
|
620ce60d00 | ||
|
|
fa75e2b178 | ||
|
|
b5055efecf | ||
|
|
ab8a00e159 | ||
|
|
70700de356 | ||
|
|
277d3ea146 | ||
|
|
fc65a1b9cb | ||
|
|
b1677718cd | ||
|
|
4dde74364b | ||
|
|
bb7b312b19 | ||
|
|
43162d20e0 | ||
|
|
ba3a6c17ce | ||
|
|
7c1a79e071 | ||
|
|
0d582bfffe | ||
|
|
459501873b | ||
|
|
f1590144a4 | ||
|
|
9d25ce63a3 | ||
|
|
18c39dc283 | ||
|
|
79ccf3d26b | ||
|
|
d659d10afd | ||
|
|
b8431d4f50 | ||
|
|
6c566ca014 | ||
|
|
cd7cef72db | ||
|
|
33ee8c0ef8 | ||
|
|
bb0633ee43 | ||
|
|
347738843f | ||
|
|
4d547c4b7b | ||
|
|
ab95b42328 | ||
|
|
409ba3b1f4 | ||
|
|
274342326c | ||
|
|
97eee1fecf | ||
|
|
98274d294a | ||
|
|
34aeb94efa | ||
|
|
de09fecf1c | ||
|
|
b7b89b0708 | ||
|
|
4079b6f493 | ||
|
|
bc482e79ff | ||
|
|
7ef04886b9 | ||
|
|
b70ebaba7b | ||
|
|
b03a9f8ea3 | ||
|
|
8a2bed2f71 | ||
|
|
df2a2d1127 | ||
|
|
2e18447255 | ||
|
|
ef6bae24c5 | ||
|
|
687d264b68 | ||
|
|
4158f63d2f | ||
|
|
c47832ad34 | ||
|
|
60e96416df | ||
|
|
c841c3bba3 | ||
|
|
79fbc0842f | ||
|
|
22859b7e69 | ||
|
|
925fb4fb21 | ||
|
|
96f298bdec | ||
|
|
60fb55dc6a | ||
|
|
181ef190ce | ||
|
|
ad8cd41c49 | ||
|
|
5dae3adbf5 | ||
|
|
6507e13d5a | ||
|
|
e4913c9e91 | ||
|
|
28bf49d66f | ||
|
|
17b0c03338 | ||
|
|
a7ab39d3d1 | ||
|
|
7a594b09c6 | ||
|
|
f376c64784 | ||
|
|
afeb26ae74 | ||
|
|
c6af0b57c2 | ||
|
|
90595f34f2 | ||
|
|
b3cf0444a2 | ||
|
|
11ff6e49bb | ||
|
|
bcef98db3f | ||
|
|
f18aacf2c7 | ||
|
|
ab1f458013 | ||
|
|
7c1dfd99f3 | ||
|
|
57842c5ee4 | ||
|
|
739403963a | ||
|
|
1bec2133c5 | ||
|
|
64de81e6a0 | ||
|
|
0b187f3d7a | ||
|
|
0e5100203c | ||
|
|
ba3d34b9f8 | ||
|
|
77e6390f5b | ||
|
|
f190b34d95 | ||
|
|
49e7016d12 | ||
|
|
6240cbb694 | ||
|
|
66bc465d1a | ||
|
|
b576cbdf40 | ||
|
|
66dba8239f | ||
|
|
92ef010fd2 | ||
|
|
feb0664b64 | ||
|
|
61427e9729 | ||
|
|
3d1fd51f61 | ||
|
|
6c44400bf7 | ||
|
|
041b766428 | ||
|
|
3c505b1e99 | ||
|
|
6317faccc0 | ||
|
|
5a490a31c5 | ||
|
|
b8470e4cac | ||
|
|
e7ae39f42d | ||
|
|
43f107e049 | ||
|
|
748eaf9b28 | ||
|
|
11dd394670 | ||
|
|
21483522a6 | ||
|
|
f92262d48c | ||
|
|
ad87bf6c93 | ||
|
|
f1c06a047e | ||
|
|
d6fb148d9b | ||
|
|
930de88f69 | ||
|
|
fad0359e6b | ||
|
|
c14e507609 | ||
|
|
0720a194cf | ||
|
|
194d22ce71 | ||
|
|
7995d87cd5 | ||
|
|
2d261879dd | ||
|
|
2e0718167f | ||
|
|
9718ed3fc2 | ||
|
|
645e1ea5be | ||
|
|
40dab7ecdd | ||
|
|
43a2a92950 | ||
|
|
62f8f944e2 | ||
|
|
841ff2d418 | ||
|
|
33ab81ce6d | ||
|
|
632ff74375 | ||
|
|
39b926a2ce | ||
|
|
31f4c78c5e | ||
|
|
fcb317e226 | ||
|
|
82a4a63b92 | ||
|
|
d407018995 | ||
|
|
b5f0ea9b7b | ||
|
|
d166eaf635 | ||
|
|
62a1df7271 | ||
|
|
e171f996d6 | ||
|
|
2bf9e0d20e | ||
|
|
afb1daa8fa | ||
|
|
5775f0430b | ||
|
|
d0f00a22d5 | ||
|
|
ad7e0c64af | ||
|
|
674d7a8a70 | ||
|
|
1274301521 | ||
|
|
840755543f | ||
|
|
2472718b7c | ||
|
|
a4f1c1558e | ||
|
|
76d5c49ae6 | ||
|
|
9fec073ab7 | ||
|
|
a9c4548d92 | ||
|
|
d038b8b1c9 | ||
|
|
3ceea42329 | ||
|
|
6beb4a5bd2 | ||
|
|
7d70921bf5 | ||
|
|
76c6992260 | ||
|
|
f1297d6298 | ||
|
|
9a112f40c1 | ||
|
|
a3dcf7f0ad | ||
|
|
3084177049 | ||
|
|
e4b778b5e3 | ||
|
|
ee363b1826 | ||
|
|
e7f9c7bc10 | ||
|
|
10eb7b5b10 | ||
|
|
e0faeeb5d6 | ||
|
|
dcf15b5f9c | ||
|
|
c93cc1e1ae | ||
|
|
21c98ff5ff | ||
|
|
5fbc677ed4 | ||
|
|
00347ecf36 |
No files matched your search
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(git status:*)", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)",
|
||||
"Bash(git rev-parse:*)", "Bash(git add:*)", "Bash(git commit:*)",
|
||||
"Bash(go mod download:*)", "Bash(go build:*)", "Bash(go test:*)", "Bash(go vet:*)",
|
||||
"Bash(gofmt -l:*)",
|
||||
"Bash(golangci-lint run:*)"
|
||||
],
|
||||
"ask": [
|
||||
"Bash(git push:*)", "Bash(git merge:*)", "Bash(git rebase:*)", "Bash(git switch:*)",
|
||||
"Bash(git restore:*)", "Bash(git commit --amend:*)", "Bash(rm:*)",
|
||||
"Bash(go get:*)", "Bash(go mod tidy:*)", "Bash(go install:*)",
|
||||
"Bash(make:*)", "Bash(docker:*)"
|
||||
],
|
||||
"deny": [
|
||||
"Bash(git push --force:*)", "Bash(git push -f:*)", "Bash(git push --force-with-lease:*)",
|
||||
"Bash(git reset --hard:*)", "Bash(git clean:*)", "Bash(git add -f:*)", "Bash(git add --force:*)",
|
||||
"Bash(git commit --no-verify:*)", "Bash(git commit -n:*)",
|
||||
"Bash(rm -r:*)", "Bash(rm -R:*)", "Bash(rm -rf:*)", "Bash(rm -fr:*)", "Bash(rm -Rf:*)",
|
||||
"Bash(rm -fR:*)", "Bash(rm --recursive:*)", "Bash(sudo:*)",
|
||||
"Read(./.env)", "Read(./.env.*)", "Read(./secrets/**)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
sandbox_mode = "workspace-write"
|
||||
approval_policy = "on-request"
|
||||
@@ -1,20 +0,0 @@
|
||||
# Rendered from docs/PERMISSIONS.md - change both in the same commit.
|
||||
# Deny
|
||||
prefix_rule(pattern = ["git", "push", ["--force", "-f", "--force-with-lease"]], decision = "forbidden",
|
||||
justification = "Force-push rewrites shared history.")
|
||||
prefix_rule(pattern = ["git", "reset", "--hard"], decision = "forbidden", justification = "Discards work.")
|
||||
prefix_rule(pattern = ["git", "clean"], decision = "forbidden", justification = "Deletes untracked files.")
|
||||
prefix_rule(pattern = ["git", "add", ["-f", "--force"]], decision = "forbidden",
|
||||
justification = "Stages ignored files, e.g. secrets.")
|
||||
prefix_rule(pattern = ["git", "commit", ["--no-verify", "-n"]], decision = "forbidden", justification = "Skips hooks.")
|
||||
prefix_rule(pattern = ["rm", ["-r", "-R", "-rf", "-fr", "-Rf", "-fR", "--recursive"]], decision = "forbidden",
|
||||
justification = "Recursive delete.")
|
||||
prefix_rule(pattern = ["sudo"], decision = "forbidden", justification = "No privilege escalation.")
|
||||
# Ask
|
||||
prefix_rule(pattern = ["git", ["push", "merge", "rebase", "switch", "restore"]], decision = "prompt")
|
||||
prefix_rule(pattern = ["git", "commit", "--amend"], decision = "prompt")
|
||||
prefix_rule(pattern = ["rm"], decision = "prompt")
|
||||
prefix_rule(pattern = ["go", ["get", "install"]], decision = "prompt")
|
||||
prefix_rule(pattern = ["go", "mod", "tidy"], decision = "prompt")
|
||||
prefix_rule(pattern = ["make"], decision = "prompt")
|
||||
prefix_rule(pattern = ["docker"], decision = "prompt")
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"tools": {
|
||||
"allowed": [
|
||||
"run_shell_command(git status)", "run_shell_command(git diff)", "run_shell_command(git log)",
|
||||
"run_shell_command(git show)", "run_shell_command(git branch --list)",
|
||||
"run_shell_command(git add)", "run_shell_command(git commit)",
|
||||
"run_shell_command(go mod download)", "run_shell_command(go build)",
|
||||
"run_shell_command(go test)", "run_shell_command(go vet)", "run_shell_command(gofmt -l)",
|
||||
"run_shell_command(golangci-lint run)"
|
||||
],
|
||||
"exclude": [
|
||||
"run_shell_command(git push --force)", "run_shell_command(git push -f)",
|
||||
"run_shell_command(git push --force-with-lease)", "run_shell_command(git reset --hard)",
|
||||
"run_shell_command(git clean)", "run_shell_command(git add -f)", "run_shell_command(git add --force)",
|
||||
"run_shell_command(git commit --no-verify)", "run_shell_command(git commit -n)",
|
||||
"run_shell_command(rm -r)", "run_shell_command(rm -R)", "run_shell_command(rm -rf)",
|
||||
"run_shell_command(rm -fr)", "run_shell_command(rm -Rf)", "run_shell_command(sudo)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,9 @@
|
||||
go.mod text eol=lf
|
||||
go.sum text eol=lf
|
||||
|
||||
# Embedded templates must render the same on every OS
|
||||
lib/http/templates/** text eol=lf
|
||||
|
||||
# Ignore generated files in GitHub language statistics and diffs
|
||||
/MANUAL.* linguist-generated=true
|
||||
/rclone.1 linguist-generated=true
|
||||
cmd/bisync/testdata/*/golden/** linguist-generated=true
|
||||
|
||||
# Don't fiddle with the line endings of test data
|
||||
**/testdata/** -text
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
# Fork CI for the Proton Drive backend. Every step is hermetic: the
|
||||
# tests run against the fake server in backend/protondrive/protontest.
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
env:
|
||||
# Never pick up a config holding a real Proton remote.
|
||||
RCLONE_CONFIG: /notfound
|
||||
|
||||
jobs:
|
||||
protondrive:
|
||||
runs-on: ubuntu-latest
|
||||
# The race suite takes about 6 minutes on a fast machine and 7 on
|
||||
# two cores; the slow runner may take several times that.
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
# The line the code was verified with, not go.mod's minimum,
|
||||
# so govulncheck sees a patched standard library.
|
||||
go-version: '1.27.x'
|
||||
# Caches the module and build caches, keyed on go.sum.
|
||||
cache: true
|
||||
|
||||
- name: Build
|
||||
run: go build ./...
|
||||
|
||||
- name: Vet
|
||||
run: go vet ./backend/protondrive/...
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
with:
|
||||
version: v2.15.0
|
||||
args: ./backend/protondrive/...
|
||||
|
||||
- name: Test
|
||||
# Go's default 10 minute limit per test binary is too short
|
||||
# for the race suite on a slow runner.
|
||||
run: go test -race -timeout 40m ./backend/protondrive/...
|
||||
|
||||
- name: Fuzz
|
||||
run: |
|
||||
for target in $(go test -list '^Fuzz' ./backend/protondrive/ | grep '^Fuzz'); do
|
||||
go test -run '^$' -fuzz "^${target}\$" -fuzztime 20s ./backend/protondrive/
|
||||
done
|
||||
|
||||
- name: Vulnerability check
|
||||
run: go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./backend/protondrive/...
|
||||
+12
-15
@@ -37,7 +37,7 @@ jobs:
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.3'
|
||||
go: '~1.26.9'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +48,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.3'
|
||||
go: '~1.26.9'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.27.3'
|
||||
go: '~1.26.9'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +64,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.27.3'
|
||||
go: '~1.26.9'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.27.3'
|
||||
go: '~1.26.9'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,14 +81,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.3'
|
||||
go: '~1.26.9'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.26
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.10'
|
||||
go: '~1.26.9'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -177,12 +177,9 @@ jobs:
|
||||
# https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
|
||||
# https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
|
||||
unset HOMEBREW_NO_INSTALL_FROM_API
|
||||
# brew untap fails if the tap isn't installed, which varies between runner images
|
||||
for tap in homebrew/core homebrew/cask aws/tap; do
|
||||
if brew tap | grep -qx "$tap"; then
|
||||
brew untap --force "$tap"
|
||||
fi
|
||||
done
|
||||
brew untap --force homebrew/core
|
||||
brew untap --force homebrew/cask
|
||||
brew untap --force aws/tap
|
||||
brew update
|
||||
brew install --cask macfuse
|
||||
brew install git-annex git-annex-remote-rclone
|
||||
@@ -287,7 +284,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.27.3'
|
||||
go-version: '~1.26.9'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
@@ -402,7 +399,7 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '~1.27.3'
|
||||
go-version: '~1.26.9'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
@@ -19,12 +19,4 @@ Thumbs.db
|
||||
__pycache__
|
||||
.DS_Store
|
||||
resource_windows_*.syso
|
||||
/librclone.h
|
||||
.devcontainer
|
||||
/cmd/bisync/testdata/bisync_vscode_debuggers_launch.json
|
||||
.env
|
||||
.env.*
|
||||
CLAUDE.local.md
|
||||
.claude/settings.local.json
|
||||
.worktrees/
|
||||
.superpowers/
|
||||
@@ -20,8 +20,6 @@ The core parts of rclone under `fs` and `vfs` need to work with all backends and
|
||||
|
||||
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
|
||||
|
||||
Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
|
||||
|
||||
## Build and Test Commands
|
||||
|
||||
```bash
|
||||
@@ -134,16 +132,3 @@ Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck,
|
||||
- Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
|
||||
- Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
|
||||
- Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
|
||||
|
||||
## Fork Workflow
|
||||
|
||||
This fork carries agent tooling that upstream does not. Never include `docs/specs/`, `docs/plans/`, `docs/{ARCHITECTURE,DECISIONS,TODO,PERMISSIONS}.md`, `GEMINI.md`, `.claude/`, `.codex/`, `.gemini/` or this section in a PR to rclone/rclone.
|
||||
|
||||
- Required skill: `superpowers` - check at session start, install if missing. There is no product UI here, so `impeccable` is not needed.
|
||||
- Specs go in `docs/specs/NNN-slug.md` (permanent). Plans go in `docs/plans/NNN-slug.md` and are removed with `git rm` before merge. This overrides the superpowers default paths.
|
||||
- Paused or in-flight work is listed under "Now" in `docs/TODO.md` with the handoff to read first.
|
||||
- Non-trivial work: brainstorm → spec → plan → TDD → review. For trivial fixes say "trivial, skipping spec" and proceed.
|
||||
- Log things noticed but deliberately not fixed in `docs/TODO.md`, record decisions in `docs/DECISIONS.md`, and keep `docs/ARCHITECTURE.md` in step with structural changes - in the same commit as the code.
|
||||
- `docs/PERMISSIONS.md` is the source for the `.claude/`, `.codex/` and `.gemini/` permission files - change them together.
|
||||
- Never push, force-push or commit directly to `master`. Ask before adding or upgrading a dependency (`go get`, `go mod tidy`).
|
||||
- Done means run in this session and seen passing: `go build`, `go test` on the touched packages, and `golangci-lint run` on them.
|
||||
+1
-4
@@ -35,9 +35,6 @@ RUN echo "**** Print Version Binary ****" && \
|
||||
# Begin final image
|
||||
FROM alpine:latest
|
||||
|
||||
ARG RCLONE_UID=1009
|
||||
ARG RCLONE_GID=1009
|
||||
|
||||
RUN echo "**** Install Dependencies ****" && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
@@ -49,7 +46,7 @@ RUN echo "**** Install Dependencies ****" && \
|
||||
|
||||
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
|
||||
|
||||
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
|
||||
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
|
||||
|
||||
ENTRYPOINT [ "rclone" ]
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# GEMINI.md
|
||||
|
||||
**`AGENTS.md` is the single source of truth for this repository.** Read
|
||||
`./AGENTS.md` before doing anything else and follow it exactly.
|
||||
|
||||
This file exists only so Gemini CLI finds the rules. It contains no project
|
||||
rules of its own - add those to `AGENTS.md`. If anything here ever conflicts
|
||||
with `AGENTS.md`, `AGENTS.md` wins.
|
||||
Generated
+2503
-5994
File diff suppressed because it is too large.
Load diff
Generated
+482
-3557
File diff suppressed because it is too large.
Load diff
@@ -38,9 +38,6 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
|
||||
BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
|
||||
BETA_UPLOAD_ROOT := beta.rclone.org:
|
||||
BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
|
||||
# Update the beta site's directory listings after an upload - add
|
||||
# --changed for the uploaded directories
|
||||
BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
# Pass in GOTAGS=xyz on the make command line to set build tags
|
||||
ifdef GOTAGS
|
||||
BUILDTAGS=-tags "$(GOTAGS)"
|
||||
@@ -237,7 +234,6 @@ check_sign:
|
||||
upload:
|
||||
rclone -P copy build/ downloads.rclone.org:/$(TAG)
|
||||
rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
|
||||
rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
|
||||
upload_github:
|
||||
./bin/upload-github $(TAG)
|
||||
@@ -270,7 +266,6 @@ ci_upload:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
|
||||
endif
|
||||
./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
|
||||
@echo Beta release ready at $(BETA_URL)/testbuilds
|
||||
|
||||
ci_beta:
|
||||
@@ -280,12 +275,11 @@ ci_beta:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
|
||||
endif
|
||||
rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
|
||||
@echo Beta release ready at $(BETA_URL)
|
||||
|
||||
# Fetch the binary builds from GitHub actions
|
||||
fetch_binaries:
|
||||
rclone -P sync --exclude "/testbuilds/**" --exclude "/index.{html,json}" --delete-excluded $(BETA_UPLOAD) build/
|
||||
rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
|
||||
|
||||
serve: website
|
||||
cd docs && hugo server --logLevel info -w --disableFastRender --ignoreCache
|
||||
@@ -302,26 +296,21 @@ retag:
|
||||
@echo "Version is $(VERSION)"
|
||||
git tag -f -s -m "Version $(VERSION)" $(VERSION)
|
||||
|
||||
# Start development of the next version after a release: the next
|
||||
# minor version on master or the next patch version on a stable branch.
|
||||
startdev:
|
||||
@branch=$$(git rev-parse --abbrev-ref HEAD); \
|
||||
case "$$branch" in \
|
||||
master) next="$(NEXT_VERSION)" ;; \
|
||||
v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
|
||||
*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
|
||||
echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
|
||||
fi; \
|
||||
echo "Version is $(VERSION)"; \
|
||||
echo "Next version is $$next"; \
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
|
||||
echo -n "$$next" > docs/layouts/partials/version.html; \
|
||||
echo "$$next" > VERSION; \
|
||||
git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next version is $(NEXT_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
startstable: startdev
|
||||
startstable:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_PATCH_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
winzip:
|
||||
zip -9 rclone-$(TAG).zip rclone.exe
|
||||
|
||||
@@ -24,25 +24,21 @@ directories to and from different cloud storage providers.
|
||||
## Storage providers
|
||||
|
||||
- 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
|
||||
- Abrha Storage [:page_facing_up:](https://rclone.org/s3/#abrhastorage)
|
||||
- Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
|
||||
- Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
|
||||
- Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
|
||||
- ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
|
||||
- Autonomys Auto Drive [:page_facing_up:](https://rclone.org/s3/#autonomys)
|
||||
- Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
|
||||
- Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
|
||||
- Box [:page_facing_up:](https://rclone.org/box/)
|
||||
- Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
|
||||
- China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
|
||||
- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
|
||||
- Clever Cloud Cellar [:page_facing_up:](https://rclone.org/s3/#clever-cloud-cellar)
|
||||
- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
|
||||
- Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
|
||||
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
|
||||
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
|
||||
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
|
||||
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
|
||||
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
@@ -74,11 +70,8 @@ directories to and from different cloud storage providers.
|
||||
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
|
||||
- Infomaniak Swiss Backup (S3) [:page_facing_up:](https://rclone.org/s3/#infomaniak)
|
||||
- Infomaniak Swiss Backup (Swift) [:page_facing_up:](https://rclone.org/swift/)
|
||||
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
|
||||
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
|
||||
- kDrive [:page_facing_up:](https://rclone.org/kdrive/)
|
||||
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
|
||||
- Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
|
||||
- Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
|
||||
@@ -94,7 +87,6 @@ directories to and from different cloud storage providers.
|
||||
- Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
|
||||
- Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
|
||||
- Minio [:page_facing_up:](https://rclone.org/s3/#minio)
|
||||
- Neon Object Storage [:page_facing_up:](https://rclone.org/s3/#neon)
|
||||
- Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
|
||||
- Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
|
||||
- OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
|
||||
@@ -118,7 +110,6 @@ directories to and from different cloud storage providers.
|
||||
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
|
||||
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
|
||||
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
|
||||
- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
|
||||
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
|
||||
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
|
||||
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
|
||||
|
||||
+22
-81
@@ -6,41 +6,27 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
- [gh the github cli](https://github.com/cli/cli) for uploading packages
|
||||
- pandoc for making the html and man pages
|
||||
- hugo for building the website
|
||||
- golangci-lint and docker for `make check` (markdownlint runs in docker)
|
||||
- a gpg key for signing the tag and the checksums
|
||||
- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
|
||||
`www.rclone.org:` and `test-rclone-org:`
|
||||
- the sponsor logos - see [Sponsor logos](#sponsor-logos)
|
||||
|
||||
## Making a release
|
||||
|
||||
- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
|
||||
- git checkout master # see below for stable branch
|
||||
- git pull # IMPORTANT
|
||||
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
|
||||
- git status - make sure everything is checked in
|
||||
- Check GitHub actions build for master is Green
|
||||
- Check the integration tests at <https://integration.rclone.org>
|
||||
- cat VERSION # check this is the version being released
|
||||
- make tag # signed tag on HEAD + new changelog entry + docs
|
||||
- edit docs/content/changelog.md # entries already in point releases are
|
||||
removed automatically
|
||||
- make test # see integration test server or run locally
|
||||
- make tag
|
||||
- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
|
||||
releases
|
||||
- make tidy
|
||||
- make doc # again to pick up the changelog edits
|
||||
- git status - to check for new command docs in docs/content/commands - git add them
|
||||
- make doc
|
||||
- git status - to check for new man pages - git add them
|
||||
- git commit -a -v -m "Version v1.XX.0"
|
||||
- make check # if this fails, fix, git commit --amend and run it again
|
||||
- make retag # moves the tag onto the Version commit
|
||||
- git push origin # without --follow-tags so it doesn't push the tag yet
|
||||
- Wait for the GitHub actions build of master to complete and check it is
|
||||
green - this is the first time CI sees any commits which were only local,
|
||||
such as security fixes, so it must pass before the tag is pushed
|
||||
- if it fails: fix, git commit, make retag, git push origin and wait again
|
||||
- make check
|
||||
- make retag
|
||||
- git push origin # without --follow-tags so it doesn't push the tag if it fails
|
||||
- git push --follow-tags origin
|
||||
- \# Wait for the GitHub builds of the tag to complete then...
|
||||
- \# Don't commit or push anything until after make upload_github as these steps
|
||||
use the tag on HEAD to find the version
|
||||
- \# Wait for the GitHub builds to complete then...
|
||||
- make fetch_binaries
|
||||
- make tarball
|
||||
- make vendorball
|
||||
@@ -49,31 +35,19 @@ This file describes how to make the various kinds of releases
|
||||
- make upload
|
||||
- make upload_test_website # check sponsors are correct/unchanged
|
||||
- make upload_website
|
||||
- make upload_github # this triggers the docker plugin and winget workflows
|
||||
- Check the docker image, docker plugin and winget GitHub actions succeeded
|
||||
- make startdev # works out the next version for master or a stable branch
|
||||
- git push
|
||||
- Disable the release freeze rule
|
||||
- Release any security advisories
|
||||
- request a CVE for each advisory if not done already
|
||||
- set the patched version on each advisory and publish it
|
||||
- when the CVEs are assigned replace `CVE-PENDING` in the changelog on
|
||||
master and the stable branch then `make upload_website`
|
||||
- \# announce with forum post, twitter post, and post to announcements list
|
||||
- make upload_github
|
||||
- make startdev # make startstable for stable branch
|
||||
- \# announce with forum post, twitter post, and post to annoucements list
|
||||
|
||||
## Update dependencies
|
||||
|
||||
Early in the next release cycle update the dependencies.
|
||||
|
||||
- Active Pins
|
||||
- `github.com/coreos/go-systemd/v22` v22.6.0 - v22.7.0 fails to compile on netbsd
|
||||
- `github.com/hanwen/go-fuse/v2` v2.11.0 - v2.12.0 fails to compile on 32
|
||||
bit linux, awaiting fix for
|
||||
[go-fuse/issues/628](https://github.com/hanwen/go-fuse/issues/628)
|
||||
- `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
|
||||
- See: [#9539](https://github.com/rclone/rclone/issues/9539)
|
||||
- Review Active Pins in go.mod and remove if possible
|
||||
- `make updatedirect`
|
||||
- The update ignores the pins so put them back, eg
|
||||
`go get github.com/coreos/go-systemd/v22@v22.6.0` then `go mod tidy`
|
||||
- `make GOTAGS=cmount`
|
||||
- `make compiletest`
|
||||
- Fix anything which doesn't compile at this point and commit changes here
|
||||
@@ -216,53 +190,20 @@ First make the release branch. If this is a second point release then
|
||||
this will be done already.
|
||||
|
||||
- git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
|
||||
- make startdev # sets the version to ${BASE_TAG}.1
|
||||
- git push -u origin ${BASE_TAG}-stable
|
||||
- make startstable
|
||||
|
||||
Now
|
||||
|
||||
- Enable release freeze rule
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- make fetch-gui-and-commit # update the GUI on master first
|
||||
- git push
|
||||
- git co ${BASE_TAG}-stable
|
||||
- git pull
|
||||
- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
|
||||
- `git cherry-pick -x` any fixes, including the GUI update if there was one
|
||||
- for security releases consider updating the Go patch version in
|
||||
`.github/workflows/build.yml` and dependencies with CVE fixes
|
||||
- fixes may depend on commits which weren't picked so push to CI
|
||||
and check it is green before tagging
|
||||
- Do the steps in "Making a release" above, except
|
||||
- stay on ${BASE_TAG}-stable - don't checkout master or run
|
||||
make fetch-gui-and-commit
|
||||
- check the GitHub actions build for ${BASE_TAG}-stable not master
|
||||
- run the integration tests on ${BASE_TAG}-stable as described below
|
||||
- the release freeze rule is already enabled - leave it until the end
|
||||
- `git cherry-pick -x` any fixes
|
||||
- make startstable
|
||||
- Do the steps as above
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- `#` cherry pick the changes to the changelog - check the diff to make sure it
|
||||
is correct
|
||||
- git checkout ${BASE_TAG}-stable docs/content/changelog.md
|
||||
- git diff --cached # check only the ${NEW_TAG} entry has changed
|
||||
- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
|
||||
- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
|
||||
- git push
|
||||
- Disable release freeze rule
|
||||
|
||||
Point releases are only made for the latest release. Once v1.(XX+1).0
|
||||
is out the ${BASE_TAG}-stable branch is abandoned.
|
||||
|
||||
### Running the integration tests on the stable branch
|
||||
|
||||
The integration tests at <https://integration.rclone.org> test master.
|
||||
To test the stable branch push it to the testing server and run the
|
||||
tests there in `screen` as they take a while.
|
||||
|
||||
```console
|
||||
git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
|
||||
ssh rclone@rclone-testing
|
||||
screen
|
||||
cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
|
||||
```
|
||||
|
||||
## Sponsor logos
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/compress"
|
||||
_ "github.com/rclone/rclone/backend/crypt"
|
||||
_ "github.com/rclone/rclone/backend/doi"
|
||||
_ "github.com/rclone/rclone/backend/dosya"
|
||||
_ "github.com/rclone/rclone/backend/drime"
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/dropbox"
|
||||
@@ -39,7 +38,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
_ "github.com/rclone/rclone/backend/internxt"
|
||||
_ "github.com/rclone/rclone/backend/jottacloud"
|
||||
_ "github.com/rclone/rclone/backend/kdrive"
|
||||
_ "github.com/rclone/rclone/backend/koofr"
|
||||
_ "github.com/rclone/rclone/backend/linkbox"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
|
||||
@@ -194,12 +194,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
}
|
||||
// Path to the archive
|
||||
archiveRemote := remote[:len(foundArchive.remote)]
|
||||
// Remote is archive leaf name. This must understand remote
|
||||
// names as the archive may be in the root, e.g. "remote:file.zip"
|
||||
_, foundArchive.remote, err = fspath.Split(archiveRemote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse archive path %q: %w", archiveRemote, err)
|
||||
}
|
||||
// Remote is archive leaf name
|
||||
foundArchive.remote = path.Base(archiveRemote)
|
||||
foundArchive.prefix = ""
|
||||
// Point remote to archive file
|
||||
remote = archiveRemote
|
||||
|
||||
@@ -12,19 +12,16 @@ import (
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/alias"
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -324,50 +321,6 @@ func TestArchiveUncleanRoot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestArchiveInRemoteRoot checks that an archive in the root of a
|
||||
// remote, with no "/" between the remote name and the archive name, can
|
||||
// be read, as can a path within it.
|
||||
func TestArchiveInRemoteRoot(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create("sub/a.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.zip"), buf.Bytes(), 0600))
|
||||
t.Chdir(dir)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
remote string
|
||||
}{
|
||||
{name: "plain", remote: ":local:test.zip"},
|
||||
{name: "slash in remote name", remote: ":alias,remote='" + dir + "':test.zip"},
|
||||
} {
|
||||
for _, sub := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{path: "", want: "sub"},
|
||||
{path: "/sub", want: "a.txt"},
|
||||
} {
|
||||
t.Run(test.name+sub.path, func(t *testing.T) {
|
||||
f, err := cache.Get(ctx, ":archive:"+test.remote+sub.path)
|
||||
require.NoError(t, err)
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, sub.want, entries[0].Remote())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// escapingObject is an object whose remote is not where it was asked for.
|
||||
type escapingObject struct {
|
||||
fs.Object
|
||||
@@ -462,103 +415,3 @@ func TestIsDirectChild(t *testing.T) {
|
||||
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
|
||||
}
|
||||
}
|
||||
|
||||
// A symlink in a squashfs image is skipped unless -l/--links is in use,
|
||||
// as it is on the other backends. With the flag it is exposed under its
|
||||
// name with the link suffix and reads back as its target.
|
||||
//
|
||||
// Note that this uses mksquashfs as an external binary.
|
||||
func TestArchiveSquashfsSymlink(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "mksquashfs")
|
||||
ctx := context.Background()
|
||||
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "input")
|
||||
require.NoError(t, os.Mkdir(input, 0777))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
|
||||
require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
|
||||
// a regular file which happens to carry the link suffix must stay
|
||||
// readable under the flag, not be taken for a symlink
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
|
||||
image := filepath.Join(dir, "test.sqfs")
|
||||
run(t, "mksquashfs", input, image)
|
||||
|
||||
remotes := func(t *testing.T, f fs.Fs) []string {
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
names = append(names, entry.Remote())
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
read := func(t *testing.T, f fs.Fs, remote string) string {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
require.NoError(t, err)
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
contents, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, o.Size(), int64(len(contents)))
|
||||
return string(contents)
|
||||
}
|
||||
|
||||
f, err := fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
|
||||
assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
|
||||
}
|
||||
|
||||
// A hostile archive must not be able to write outside the destination.
|
||||
// A symlink pointing above the root with a member inside it is the
|
||||
// classic way to try it, and the refusal comes from the local backend,
|
||||
// so pin it from end to end here rather than trusting it stays.
|
||||
func TestArchiveZipSymlinkEscape(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
hdr := &zip.FileHeader{Name: "a"}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err := zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("../outside"))
|
||||
require.NoError(t, err)
|
||||
w, err = zw.Create("a/pwned.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("pwned"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
zipPath := filepath.Join(dir, "evil.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
dst := filepath.Join(dir, "dst")
|
||||
require.NoError(t, os.Mkdir(dst, 0777))
|
||||
|
||||
fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
|
||||
require.NoError(t, err)
|
||||
fdst, err := fs.NewFs(ctx, dst)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
|
||||
|
||||
_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
|
||||
assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
|
||||
}
|
||||
@@ -43,7 +43,6 @@ type Fs struct {
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
file string // remote of the single file root points at, if any
|
||||
links bool // read symlinks as .rclonelink files
|
||||
}
|
||||
|
||||
// recoverParsePanic converts a panic raised while parsing a squashfs image
|
||||
@@ -100,7 +99,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
|
||||
c: c,
|
||||
remote: remote,
|
||||
root: strings.Trim(root, "/"),
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -222,30 +220,6 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
|
||||
}
|
||||
}
|
||||
|
||||
// Convert a symlink entry into an Object holding its target
|
||||
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
|
||||
size: int64(len(target)),
|
||||
modTime: item.ModTime(),
|
||||
isLink: true,
|
||||
link: target,
|
||||
}
|
||||
}
|
||||
|
||||
// readlink returns the target of the symlink the info describes.
|
||||
//
|
||||
// A symlink may carry an empty target, so the caller decides what is a
|
||||
// symlink from the mode rather than from the target being set.
|
||||
func readlink(info os.FileInfo) (string, error) {
|
||||
st, ok := info.Sys().(*squashfs.StatT)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
|
||||
}
|
||||
return st.LinkTarget, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
@@ -275,7 +249,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, item := range items {
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
// Skip entry names that aren't safe
|
||||
@@ -298,20 +271,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = f.objectFromFileInfo(nativeDir, info)
|
||||
} else if item.Type()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry = f.objectFromLink(nativeDir, info, target)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
@@ -327,9 +286,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
@@ -351,17 +307,7 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
// Name the entry as List does, so that the two agree on
|
||||
// which names exist
|
||||
isLink := fi.Type()&os.ModeSymlink != 0
|
||||
if isLink && !f.links {
|
||||
continue
|
||||
}
|
||||
name := fi.Name()
|
||||
if isLink {
|
||||
name += fs.LinkSuffix
|
||||
}
|
||||
if name == leaf {
|
||||
if fi.Name() == leaf {
|
||||
if fi.IsDir() {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
@@ -369,14 +315,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
|
||||
}
|
||||
if isLink {
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o = f.objectFromLink(dir, info, target)
|
||||
break
|
||||
}
|
||||
o = f.objectFromFileInfo(dir, info)
|
||||
break
|
||||
}
|
||||
@@ -455,8 +393,6 @@ type Object struct {
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
isLink bool // whether this is a symlink
|
||||
link string // target of the symlink
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
@@ -528,17 +464,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
}
|
||||
}
|
||||
|
||||
if o.isLink {
|
||||
// Clamp offset into range to avoid panic
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset > int64(len(o.link)) {
|
||||
offset = int64(len(o.link))
|
||||
}
|
||||
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
|
||||
}
|
||||
|
||||
remote, err := o.fs.toNative(o.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -43,7 +43,6 @@ type Fs struct {
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
links bool // read symlinks as .rclonelink files
|
||||
dt dirtree.DirTree // read from zipfile
|
||||
}
|
||||
|
||||
@@ -68,7 +67,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -140,7 +138,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
}
|
||||
dt := dirtree.New()
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, file := range zr.File {
|
||||
// Skip entries whose name escapes the archive's own namespace
|
||||
remote, err := sanitize.Path(file.Name)
|
||||
@@ -155,15 +152,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !isDir && file.Mode()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
// The member holds the link target, so it reads back
|
||||
// like a .rclonelink file
|
||||
remote += fs.LinkSuffix
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root, requiring a path
|
||||
@@ -203,9 +191,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -37,26 +36,6 @@ func writeZip(t *testing.T, dir, name string, names ...string) string {
|
||||
return name
|
||||
}
|
||||
|
||||
// writeZipSymlink builds a zip holding file and a symlink entry link
|
||||
// pointing at target, and returns its leaf name.
|
||||
func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create(file)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data for " + file))
|
||||
require.NoError(t, err)
|
||||
hdr := &zip.FileHeader{Name: link}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err = zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(target))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// allRemotes returns every Object remote in the mounted archive's dirtree.
|
||||
func allRemotes(t *testing.T, f fs.Fs) []string {
|
||||
zf, ok := f.(*Fs)
|
||||
@@ -191,37 +170,3 @@ func TestListCacheNotAliased(t *testing.T) {
|
||||
}
|
||||
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A symlink stored in the archive is skipped unless -l/--links is in
|
||||
// use, as it is on the other backends. With the flag it is exposed
|
||||
// under its name with the link suffix and reads back as its target.
|
||||
func TestReadZipSymlink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
|
||||
|
||||
o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len("file.txt")), o.Size())
|
||||
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
target, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, "file.txt", string(target))
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Parallel listing support for azureblob, built on the Apache Arrow
|
||||
// listing format.
|
||||
@@ -62,7 +62,7 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
|
||||
|
||||
// Schedule every shard; SetLimit bounds concurrency, so more shards than
|
||||
// workers gives free load balancing as workers pick up the next shard.
|
||||
var total atomic.Int64
|
||||
var total int64
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(f.opt.ListParallelism)
|
||||
for _, sh := range shards {
|
||||
@@ -85,12 +85,12 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
|
||||
shardOpts.EndBefore = nil
|
||||
}
|
||||
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
|
||||
total.Add(int64(n))
|
||||
atomic.AddInt64(&total, int64(n))
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := g.Wait()
|
||||
found := int(total.Load())
|
||||
found := int(atomic.LoadInt64(&total))
|
||||
if isEndBeforeUnsupported(err) {
|
||||
// The account can't honour the shards' endBefore bounds, so parallel
|
||||
// listing can't be used on it.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
+38
-171
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -365,22 +364,7 @@ creates an empty object ending with "/", to persist the folder.
|
||||
|
||||
This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
|
||||
the Microsoft standard.
|
||||
|
||||
This is not needed on storage accounts with a hierarchical namespace
|
||||
(see the hns option) as these always support empty directories.
|
||||
`,
|
||||
}, {
|
||||
Name: "hns",
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
Help: `Set if the storage account has a hierarchical namespace.
|
||||
|
||||
Storage accounts with hierarchical namespace enabled (Azure Data Lake
|
||||
Storage Gen2) have real directories which can be empty.
|
||||
|
||||
If this is unset then rclone will read it from the storage account
|
||||
and save it in the config file.
|
||||
`,
|
||||
}, {
|
||||
Name: "no_check_container",
|
||||
Help: `If set, don't attempt to check the container exists or create it.
|
||||
@@ -448,7 +432,6 @@ type Options struct {
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
PublicAccess string `config:"public_access"`
|
||||
DirectoryMarkers bool `config:"directory_markers"`
|
||||
HNS fs.Tristate `config:"hns"`
|
||||
NoCheckContainer bool `config:"no_check_container"`
|
||||
NoHeadObject bool `config:"no_head_object"`
|
||||
DeleteSnapshots string `config:"delete_snapshots"`
|
||||
@@ -472,7 +455,6 @@ type Fs struct {
|
||||
rootContainer string // container part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
isLimited bool // if limited to one container
|
||||
hns bool // if the storage account has a hierarchical namespace
|
||||
cache *bucket.Cache // cache for container creation status
|
||||
pacer *fs.Pacer // To pace and retry the API calls
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
@@ -539,7 +521,7 @@ func parsePath(path string) (root string) {
|
||||
// relative to f.root
|
||||
func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
|
||||
containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
|
||||
if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
|
||||
if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
|
||||
containerPath = containerPath[:len(containerPath)-1]
|
||||
}
|
||||
return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
|
||||
@@ -587,7 +569,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
// General errors from:
|
||||
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
|
||||
// Blob specific errors from:
|
||||
@@ -706,6 +689,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ServerSideAcrossConfigs: true,
|
||||
DoubleSlash: true,
|
||||
}).Fill(ctx, f)
|
||||
if opt.DirectoryMarkers {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
|
||||
DefaultBaseURL: storageDefaultBaseURL,
|
||||
@@ -738,28 +725,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.isLimited = true
|
||||
}
|
||||
|
||||
if opt.HNS.Valid {
|
||||
f.hns = opt.HNS.Value
|
||||
} else if !f.anonymous {
|
||||
containerName := f.rootContainer
|
||||
if containerName == "" {
|
||||
containerName = res.Container
|
||||
}
|
||||
f.hns, err = f.readHNS(ctx, containerName)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
|
||||
} else {
|
||||
m.Set("hns", strconv.FormatBool(f.hns))
|
||||
}
|
||||
}
|
||||
if f.hns {
|
||||
fs.Debugf(f, "Storage account has a hierarchical namespace")
|
||||
}
|
||||
if f.dirMarkers() {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
if f.rootContainer != "" && f.rootDirectory != "" {
|
||||
// Check to see if the (container,directory) is actually an existing file
|
||||
oldRoot := f.root
|
||||
@@ -780,39 +745,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// readHNS reads whether the storage account has a hierarchical namespace
|
||||
//
|
||||
// If containerName is set then the container client is used which works
|
||||
// with credentials limited to that container.
|
||||
func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
|
||||
var isHNS *bool
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
if containerName != "" {
|
||||
var resp container.GetAccountInfoResponse
|
||||
resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
} else {
|
||||
var resp service.GetAccountInfoResponse
|
||||
resp, err = f.svc.GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if isHNS == nil {
|
||||
return false, errors.New("not returned by server")
|
||||
}
|
||||
return *isHNS, nil
|
||||
}
|
||||
|
||||
// dirMarkers returns true if directories are persisted with directory markers
|
||||
func (f *Fs) dirMarkers() bool {
|
||||
return f.opt.DirectoryMarkers || f.hns
|
||||
}
|
||||
|
||||
// return the container client for the container passed in
|
||||
func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
|
||||
f.cntSVCcacheMu.Lock()
|
||||
@@ -850,12 +782,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
o, err := f.newObjectWithInfo(ctx, remote, nil)
|
||||
if err == fs.ErrorNotAFile {
|
||||
// The blob is a directory
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, err
|
||||
return f.newObjectWithInfo(ctx, remote, nil)
|
||||
}
|
||||
|
||||
// getBlobSVC creates a blob client
|
||||
@@ -924,15 +851,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
|
||||
lowerKey := strings.ToLower(k)
|
||||
switch lowerKey {
|
||||
case "cache-control":
|
||||
headers.BlobCacheControl = new(v)
|
||||
headers.BlobCacheControl = pString(v)
|
||||
case "content-disposition":
|
||||
headers.BlobContentDisposition = new(v)
|
||||
headers.BlobContentDisposition = pString(v)
|
||||
case "content-encoding":
|
||||
headers.BlobContentEncoding = new(v)
|
||||
headers.BlobContentEncoding = pString(v)
|
||||
case "content-language":
|
||||
headers.BlobContentLanguage = new(v)
|
||||
headers.BlobContentLanguage = pString(v)
|
||||
case "content-type":
|
||||
headers.BlobContentType = new(v)
|
||||
headers.BlobContentType = pString(v)
|
||||
case "x-ms-tags":
|
||||
parsed, perr := parseXMsTags(v)
|
||||
if perr != nil {
|
||||
@@ -1169,33 +1096,18 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
|
||||
if endsWithSlash || remote == "" {
|
||||
return true
|
||||
}
|
||||
return hasDirMetadata(metadata)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether the metadata marks the blob as a directory
|
||||
func hasDirMetadata(metadata map[string]*string) bool {
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether props are for a directory on a storage account with
|
||||
// a hierarchical namespace
|
||||
//
|
||||
// These ignore a trailing "/" on the name so a file can be returned
|
||||
// when looking for a directory.
|
||||
func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
|
||||
return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
|
||||
}
|
||||
|
||||
// listFn is called from list to handle an object
|
||||
type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
|
||||
|
||||
@@ -1254,18 +1166,15 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f.dirMarkers() && foundItems == 0 && directory != "" {
|
||||
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
|
||||
// Determine whether the directory exists or not by whether it has a marker
|
||||
props, err := f.readMetaData(ctx, containerName, directory)
|
||||
_, err := f.readMetaData(ctx, containerName, directory)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
if f.hns && !isHNSDirectory(props) {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1318,13 +1227,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
fs.Debugf(f, "Nil name received")
|
||||
continue
|
||||
}
|
||||
// Storage accounts with a hierarchical namespace return
|
||||
// the file "dir" when listing "dir/"
|
||||
if !strings.HasPrefix(*file.Name, directory) {
|
||||
fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
|
||||
foundItems--
|
||||
continue
|
||||
}
|
||||
remote := f.opt.Enc.ToStandardPath(*file.Name)
|
||||
if !strings.HasPrefix(remote, prefix) {
|
||||
fs.Debugf(f, "Odd name received %q", remote)
|
||||
@@ -1586,7 +1488,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
|
||||
func (f *Fs) listContainersToFn(fn listContainerFn) error {
|
||||
max := int32(f.opt.ListChunkSize)
|
||||
pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
|
||||
Include: service.ListContainersInclude{Metadata: true},
|
||||
Include: service.ListContainersInclude{Metadata: true, Deleted: true},
|
||||
MaxResults: &max,
|
||||
})
|
||||
ctx := context.Background()
|
||||
@@ -1636,7 +1538,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
|
||||
// Create directory marker file and parents
|
||||
func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
|
||||
if !f.dirMarkers() || container == "" {
|
||||
if !f.opt.DirectoryMarkers || container == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1693,11 +1595,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
|
||||
// mkdirParent creates the parent bucket/directory if it doesn't exist
|
||||
func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
|
||||
if f.hns {
|
||||
// Parent directories are created by the upload
|
||||
container, _ := f.split(remote)
|
||||
return f.makeContainer(ctx, container)
|
||||
}
|
||||
remote, _ = strings.CutSuffix(remote, "/")
|
||||
dir := path.Dir(remote)
|
||||
if dir == "/" || dir == "." {
|
||||
@@ -1803,9 +1700,6 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
|
||||
// Returns an error if it isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
container, directory := f.split(dir)
|
||||
if f.hns && container != "" && directory != "" {
|
||||
return f.rmdirHNS(ctx, container, directory)
|
||||
}
|
||||
// Remove directory marker file
|
||||
if f.opt.DirectoryMarkers && container != "" && directory != "" {
|
||||
o := &Object{
|
||||
@@ -1828,39 +1722,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return f.deleteContainer(ctx, container)
|
||||
}
|
||||
|
||||
// rmdirHNS removes the directory on a storage account with a
|
||||
// hierarchical namespace
|
||||
//
|
||||
// The directory must be deleted without a trailing "/" and the server
|
||||
// refuses to delete it if it isn't empty.
|
||||
func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
|
||||
directory, _ = strings.CutSuffix(directory, "/")
|
||||
// Check it is a directory as deleting a file would succeed
|
||||
props, err := f.readMetaData(ctx, container, directory)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isHNSDirectory(props) {
|
||||
return fs.ErrorIsFile
|
||||
}
|
||||
blb := f.getBlobSVC(container, directory)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err := blb.Delete(ctx, nil)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if storageErr.StatusCode == http.StatusNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Precision of the remote
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Nanosecond
|
||||
@@ -2699,6 +2560,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return downloadResponse.Body, nil
|
||||
}
|
||||
|
||||
// Converts a string into a pointer to a string
|
||||
func pString(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
|
||||
type readSeekCloser struct {
|
||||
io.Reader
|
||||
@@ -2853,7 +2719,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
// isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
|
||||
// returning true if it is found
|
||||
func isInvalidBlockOrBlob(err error) bool {
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
|
||||
}
|
||||
return false
|
||||
@@ -3225,7 +3092,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
|
||||
// Start with default content-type based on source
|
||||
ui.httpHeaders = blob.HTTPHeaders{
|
||||
BlobContentType: new(fs.MimeType(ctx, src)),
|
||||
BlobContentType: pString(fs.MimeType(ctx, src)),
|
||||
}
|
||||
|
||||
// Apply mapped metadata/headers/tags if requested
|
||||
@@ -3271,15 +3138,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
o.tags[parts[0]] = parts[1]
|
||||
}
|
||||
case "cache-control":
|
||||
ui.httpHeaders.BlobCacheControl = new(value)
|
||||
ui.httpHeaders.BlobCacheControl = pString(value)
|
||||
case "content-disposition":
|
||||
ui.httpHeaders.BlobContentDisposition = new(value)
|
||||
ui.httpHeaders.BlobContentDisposition = pString(value)
|
||||
case "content-encoding":
|
||||
ui.httpHeaders.BlobContentEncoding = new(value)
|
||||
ui.httpHeaders.BlobContentEncoding = pString(value)
|
||||
case "content-language":
|
||||
ui.httpHeaders.BlobContentLanguage = new(value)
|
||||
ui.httpHeaders.BlobContentLanguage = pString(value)
|
||||
case "content-type":
|
||||
ui.httpHeaders.BlobContentType = new(value)
|
||||
ui.httpHeaders.BlobContentType = pString(value)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
@@ -325,14 +325,6 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
|
||||
}
|
||||
|
||||
// helper to read blob tags for an object
|
||||
// skipIfHNS skips the test if the storage account has a hierarchical
|
||||
// namespace as these don't support blob tags
|
||||
func skipIfHNS(t *testing.T, f *Fs) {
|
||||
if f.hns {
|
||||
t.Skip("blob tags not supported with hierarchical namespace")
|
||||
}
|
||||
}
|
||||
|
||||
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
||||
ao := o.(*Object)
|
||||
blb := ao.getBlobSVC()
|
||||
@@ -418,7 +410,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart upload
|
||||
t.Run("PutSinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -434,7 +425,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart upload
|
||||
t.Run("PutMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize + 2048))
|
||||
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -622,7 +612,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart copy
|
||||
t.Run("CopySinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
@@ -648,7 +637,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart copy
|
||||
t.Run("CopyMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
||||
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || solaris || js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -79,7 +79,7 @@ This is required and is the name of the share to access.
|
||||
Help: `Upload chunk size.
|
||||
|
||||
Note that this is stored in memory and there may be up to
|
||||
"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
|
||||
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
|
||||
in memory.`,
|
||||
Default: defaultChunkSize,
|
||||
Advanced: true,
|
||||
@@ -95,7 +95,7 @@ links and these uploads do not fully utilize your bandwidth, then
|
||||
increasing this may help to speed up the transfers.
|
||||
|
||||
Note that chunks are stored in memory and there may be up to
|
||||
"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
|
||||
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
|
||||
in memory.`,
|
||||
Default: 16,
|
||||
Advanced: true,
|
||||
@@ -650,6 +650,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
|
||||
@@ -813,8 +818,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
|
||||
}
|
||||
opt := file.RenameOptions{
|
||||
IgnoreReadOnly: new(true),
|
||||
ReplaceIfExists: new(true),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
ReplaceIfExists: ptr(true),
|
||||
}
|
||||
dstAbsPath := f.absPath(remote)
|
||||
fc := srcObj.fileClient()
|
||||
@@ -859,8 +864,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
opt := directory.RenameOptions{
|
||||
IgnoreReadOnly: new(false),
|
||||
ReplaceIfExists: new(false),
|
||||
IgnoreReadOnly: ptr(false),
|
||||
ReplaceIfExists: ptr(false),
|
||||
}
|
||||
dstAbsPath := dstFs.absPath(dstRemote)
|
||||
dirClient := srcFs.dirClient(srcRemote)
|
||||
@@ -899,8 +904,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
ChangeTime: file.SourceCopyFileChangeTime{},
|
||||
CreationTime: file.SourceCopyFileCreationTime{},
|
||||
LastWriteTime: file.SourceCopyFileLastWriteTime{},
|
||||
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: new(true),
|
||||
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
},
|
||||
}
|
||||
srcURL := srcObj.fileClient().URL()
|
||||
|
||||
+2
-2
@@ -277,7 +277,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
|
||||
Name: "sse_customer_key",
|
||||
Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
|
||||
|
||||
Alternatively you can provide --b2-sse-customer-key-base64.`,
|
||||
Alternatively you can provide --sse-customer-key-base64.`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
@@ -288,7 +288,7 @@ Alternatively you can provide --b2-sse-customer-key-base64.`,
|
||||
Name: "sse_customer_key_base64",
|
||||
Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
|
||||
|
||||
Alternatively you can provide --b2-sse-customer-key.`,
|
||||
Alternatively you can provide --sse-customer-key.`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
|
||||
+9
-10
@@ -27,7 +27,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/golang-jwt/jwt/v4"
|
||||
"github.com/rclone/rclone/backend/box/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
@@ -77,10 +77,7 @@ var (
|
||||
)
|
||||
|
||||
type boxCustomClaims struct {
|
||||
jwt.RegisteredClaims
|
||||
// Audience overrides RegisteredClaims.Audience so it is sent as
|
||||
// a single string rather than an array of strings
|
||||
Audience string `json:"aud,omitempty"`
|
||||
jwt.StandardClaims
|
||||
BoxSubType string `json:"box_sub_type,omitempty"`
|
||||
}
|
||||
|
||||
@@ -226,7 +223,7 @@ func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
|
||||
|
||||
if configFileCredentials == "" {
|
||||
configFile, _ := m.Get("box_config_file")
|
||||
configFileBytes, err = os.ReadFile(env.ShellExpand(configFile))
|
||||
configFileBytes, err = os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("box: failed to read Box config: %w", err)
|
||||
}
|
||||
@@ -246,13 +243,15 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
|
||||
}
|
||||
|
||||
claims = &boxCustomClaims{
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ID: val,
|
||||
//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
|
||||
//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
|
||||
StandardClaims: jwt.StandardClaims{
|
||||
Id: val,
|
||||
Issuer: boxConfig.BoxAppSettings.ClientID,
|
||||
Subject: boxConfig.EnterpriseID,
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
|
||||
Audience: tokenURL,
|
||||
ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
|
||||
},
|
||||
Audience: tokenURL,
|
||||
BoxSubType: boxSubType,
|
||||
}
|
||||
return claims, nil
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
package box
|
||||
|
||||
import (
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/box/api"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/lib/jwtutil"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// decodeJWTPart decodes one base64url encoded part of a JWT into a map
|
||||
func decodeJWTPart(t *testing.T, part string) map[string]any {
|
||||
data, err := base64.RawURLEncoding.DecodeString(part)
|
||||
require.NoError(t, err)
|
||||
var result map[string]any
|
||||
require.NoError(t, json.Unmarshal(data, &result))
|
||||
return result
|
||||
}
|
||||
|
||||
// Test the JWT assertion sent to the token endpoint is in the format Box expects
|
||||
func TestJWTAssertion(t *testing.T) {
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
require.NoError(t, err)
|
||||
boxConfig := &api.ConfigJSON{
|
||||
BoxAppSettings: api.AppSettings{
|
||||
ClientID: "client-id",
|
||||
ClientSecret: "client-secret",
|
||||
AppAuth: api.AppAuth{
|
||||
PublicKeyID: "public-key-id",
|
||||
},
|
||||
},
|
||||
EnterpriseID: "enterprise-id",
|
||||
}
|
||||
|
||||
var form map[string][]string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "POST", r.Method)
|
||||
assert.NoError(t, r.ParseForm())
|
||||
form = r.PostForm
|
||||
_, _ = w.Write([]byte(`{"access_token":"access-token","token_type":"bearer","expires_in":3600}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
claims, err := getClaims(boxConfig, "enterprise")
|
||||
require.NoError(t, err)
|
||||
m := configmap.Simple{}
|
||||
start := time.Now()
|
||||
err = jwtutil.Config("box", "test", ts.URL, *claims, getSigningHeaders(boxConfig), getQueryParams(boxConfig), privateKey, m, ts.Client())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check the token was stored
|
||||
assert.Contains(t, m["token"], `"access_token":"access-token"`)
|
||||
|
||||
// Check the form parameters
|
||||
assert.Equal(t, []string{"urn:ietf:params:oauth:grant-type:jwt-bearer"}, form["grant_type"])
|
||||
assert.Equal(t, []string{"client-id"}, form["client_id"])
|
||||
assert.Equal(t, []string{"client-secret"}, form["client_secret"])
|
||||
require.Len(t, form["assertion"], 1)
|
||||
parts := strings.Split(form["assertion"][0], ".")
|
||||
require.Len(t, parts, 3)
|
||||
|
||||
// Check the signature independently of the JWT library
|
||||
signature, err := base64.RawURLEncoding.DecodeString(parts[2])
|
||||
require.NoError(t, err)
|
||||
hash := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
|
||||
require.NoError(t, rsa.VerifyPKCS1v15(&privateKey.PublicKey, crypto.SHA256, hash[:], signature))
|
||||
|
||||
// Check the header
|
||||
assert.Equal(t, map[string]any{
|
||||
"alg": "RS256",
|
||||
"typ": "JWT",
|
||||
"kid": "public-key-id",
|
||||
}, decodeJWTPart(t, parts[0]))
|
||||
|
||||
// Check the claims - aud must be a string, not an array
|
||||
payload := decodeJWTPart(t, parts[1])
|
||||
exp, ok := payload["exp"].(float64)
|
||||
require.True(t, ok, "exp should be a number")
|
||||
assert.InDelta(t, float64(start.Add(45*time.Second).Unix()), exp, 5)
|
||||
jti, ok := payload["jti"].(string)
|
||||
require.True(t, ok, "jti should be a string")
|
||||
assert.Len(t, jti, 40)
|
||||
delete(payload, "exp")
|
||||
delete(payload, "jti")
|
||||
assert.Equal(t, map[string]any{
|
||||
"aud": tokenURL,
|
||||
"iss": "client-id",
|
||||
"sub": "enterprise-id",
|
||||
"box_sub_type": "enterprise",
|
||||
}, payload)
|
||||
}
|
||||
|
||||
// Test box_config_file has ~ and environment variables expanded
|
||||
func TestGetBoxConfigShellExpand(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"enterpriseID":"enterprise-id"}`), 0600))
|
||||
t.Setenv("RCLONE_TEST_BOX_CONFIG_DIR", dir)
|
||||
m := configmap.Simple{
|
||||
"box_config_file": "$RCLONE_TEST_BOX_CONFIG_DIR/config.json",
|
||||
}
|
||||
boxConfig, err := getBoxConfig(m)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "enterprise-id", boxConfig.EnterpriseID)
|
||||
}
|
||||
+15
-23
@@ -3,6 +3,7 @@
|
||||
package box
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
@@ -19,7 +20,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
|
||||
}
|
||||
|
||||
// uploadPart uploads a part in an upload session
|
||||
//
|
||||
// chunk must contain exactly the part's data and is read from the start
|
||||
// on each attempt.
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
hasher := sha1.New()
|
||||
chunkSize, err := io.Copy(hasher, chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
chunkSize := int64(len(chunk))
|
||||
sha1sum := sha1.Sum(chunk)
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/files/upload_sessions/" + SessionID,
|
||||
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
|
||||
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Digest": sha1Digest(hasher.Sum(nil)),
|
||||
"Digest": sha1Digest(sha1sum[:]),
|
||||
},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = chunk.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = wrap(chunk)
|
||||
opts.Body = wrap(bytes.NewReader(chunk))
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -223,25 +213,27 @@ outer:
|
||||
|
||||
reqSize := min(remaining, chunkSize)
|
||||
|
||||
// Read the chunk into memory from the global pool, hashing
|
||||
// it for the whole file hash (must be done sequentially)
|
||||
rw := multipart.NewRW().Reserve(reqSize)
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
|
||||
// Make a block of memory
|
||||
buf := make([]byte, reqSize)
|
||||
|
||||
// Read the chunk
|
||||
_, err = io.ReadFull(in, buf)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
err = fmt.Errorf("multipart upload failed to read source: %w", err)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Make the global hash (must be done sequentially)
|
||||
_, _ = hash.Write(buf)
|
||||
|
||||
// Transfer the chunk
|
||||
wg.Add(1)
|
||||
o.fs.uploadToken.Get()
|
||||
go func(part int, position int64) {
|
||||
defer wg.Done()
|
||||
defer o.fs.uploadToken.Put()
|
||||
defer func() { _ = rw.Close() }()
|
||||
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
|
||||
select {
|
||||
|
||||
@@ -1502,21 +1502,11 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {
|
||||
func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
|
||||
oldFsObject, err := f.NewObject(ctx, remote)
|
||||
if err == nil {
|
||||
removeStaleChunks(ctx, oldFsObject.(*Object), nil)
|
||||
}
|
||||
}
|
||||
|
||||
func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {
|
||||
currentRemotes := make(map[string]struct{}, len(currentChunks))
|
||||
for _, chunk := range currentChunks {
|
||||
currentRemotes[chunk.Remote()] = struct{}{}
|
||||
}
|
||||
for _, chunk := range oldObject.chunks {
|
||||
if _, found := currentRemotes[chunk.Remote()]; found {
|
||||
continue
|
||||
}
|
||||
if err := chunk.Remove(ctx); err != nil {
|
||||
fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
|
||||
oldObject := oldFsObject.(*Object)
|
||||
for _, chunk := range oldObject.chunks {
|
||||
if err := chunk.Remove(ctx); err != nil {
|
||||
fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1697,20 +1687,12 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
|
||||
// metadata format which might involve unsupported chunk types.
|
||||
return nil, fmt.Errorf("can't %s this file: %w", opName, err)
|
||||
}
|
||||
// Snapshot destination chunks before its metadata is replaced.
|
||||
var oldObject *Object
|
||||
if oldFsObject, err := f.NewObject(ctx, remote); err == nil {
|
||||
oldObject = oldFsObject.(*Object)
|
||||
}
|
||||
if !o.isComposite() {
|
||||
fs.Debugf(o, "%s non-chunked object...", opName)
|
||||
oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if oldObject != nil {
|
||||
removeStaleChunks(ctx, oldObject, nil)
|
||||
}
|
||||
return f.newObject("", oResult, nil), nil
|
||||
}
|
||||
|
||||
@@ -1776,9 +1758,6 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
|
||||
silentlyRemove(ctx, newObj)
|
||||
return nil, err
|
||||
}
|
||||
if oldObject != nil {
|
||||
removeStaleChunks(ctx, oldObject, newChunks)
|
||||
}
|
||||
return newObj, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
@@ -44,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
@@ -878,104 +877,6 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
|
||||
_ = operations.Purge(ctx, f.base, dir)
|
||||
}
|
||||
|
||||
func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {
|
||||
ctx := context.Background()
|
||||
const dir = "serveroverwrite"
|
||||
remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)
|
||||
fsResult, err := fs.NewFs(ctx, remote)
|
||||
require.NoError(t, err)
|
||||
copyFs, ok := fsResult.(*Fs)
|
||||
require.True(t, ok)
|
||||
oldMoveChunkSize := moveFs.opt.ChunkSize
|
||||
moveFs.opt.ChunkSize = fs.SizeSuffix(3)
|
||||
defer func() {
|
||||
moveFs.opt.ChunkSize = oldMoveChunkSize
|
||||
_ = operations.Purge(ctx, copyFs.base, "")
|
||||
_ = operations.Purge(ctx, moveFs.base, dir)
|
||||
}()
|
||||
|
||||
type operation struct {
|
||||
name string
|
||||
fs *Fs
|
||||
enabled bool
|
||||
do copyMoveFn
|
||||
}
|
||||
operations := []operation{
|
||||
{name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},
|
||||
{name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},
|
||||
}
|
||||
testCases := []struct {
|
||||
name string
|
||||
sourceContents string
|
||||
targetContents string
|
||||
}{
|
||||
{name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},
|
||||
{name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},
|
||||
}
|
||||
transactionModes := []struct {
|
||||
name string
|
||||
useNoRename bool
|
||||
}{
|
||||
{name: "Rename"},
|
||||
{name: "NoRename", useNoRename: true},
|
||||
}
|
||||
|
||||
for _, operation := range operations {
|
||||
t.Run(operation.name, func(t *testing.T) {
|
||||
if !operation.enabled {
|
||||
t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))
|
||||
}
|
||||
for _, transactionMode := range transactionModes {
|
||||
t.Run(transactionMode.name, func(t *testing.T) {
|
||||
f := operation.fs
|
||||
if transactionMode.useNoRename && !f.useMeta {
|
||||
t.Skip("Can't test norename transactions without metadata")
|
||||
}
|
||||
oldUseNoRename := f.useNoRename
|
||||
f.useNoRename = transactionMode.useNoRename
|
||||
defer func() { f.useNoRename = oldUseNoRename }()
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))
|
||||
source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)
|
||||
target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)
|
||||
targetObject := target.(*Object)
|
||||
require.NotEmpty(t, targetObject.chunks)
|
||||
|
||||
oldChunks := append([]fs.Object(nil), targetObject.chunks...)
|
||||
result, err := operation.do(ctx, source, target.Remote())
|
||||
require.NoError(t, err)
|
||||
|
||||
reader, err := result.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
contents, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, reader.Close())
|
||||
assert.Equal(t, testCase.sourceContents, string(contents))
|
||||
|
||||
resultObject := result.(*Object)
|
||||
currentChunks := make(map[string]struct{}, len(resultObject.chunks))
|
||||
for _, chunk := range resultObject.chunks {
|
||||
currentChunks[chunk.Remote()] = struct{}{}
|
||||
}
|
||||
removed := 0
|
||||
for _, chunk := range oldChunks {
|
||||
if _, found := currentChunks[chunk.Remote()]; found {
|
||||
continue
|
||||
}
|
||||
_, err := f.base.NewObject(ctx, chunk.Remote())
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
removed++
|
||||
}
|
||||
assert.Positive(t, removed)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Test that md5all creates metadata even for small files
|
||||
func testMD5AllSlow(t *testing.T, f *Fs) {
|
||||
ctx := context.Background()
|
||||
@@ -1049,9 +950,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("ChunkerServerSideMove", func(t *testing.T) {
|
||||
testChunkerServerSideMove(t, f)
|
||||
})
|
||||
t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {
|
||||
testChunkerServerSideOverwrite(t, f)
|
||||
})
|
||||
t.Run("MD5AllSlow", func(t *testing.T) {
|
||||
testMD5AllSlow(t, f)
|
||||
})
|
||||
|
||||
@@ -472,8 +472,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
if updateOptions, ok := option.(*api.UpdateOptions); ok {
|
||||
if updateOptions.PublicID != "" {
|
||||
updateObject = true
|
||||
params.Overwrite = new(true)
|
||||
params.Invalidate = new(true)
|
||||
params.Overwrite = SDKApi.Bool(true)
|
||||
params.Invalidate = SDKApi.Bool(true)
|
||||
params.PublicID = updateOptions.PublicID
|
||||
params.ResourceType = updateOptions.ResourceType
|
||||
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -564,19 +563,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
|
||||
|
||||
// cache small files in memory and do normal upload
|
||||
rw := multipart.NewRW()
|
||||
defer fs.CheckClose(rw, &err)
|
||||
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
|
||||
if err == io.EOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, rw, src, options...)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
buf := make([]byte, f.opt.RAMCacheLimit)
|
||||
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
|
||||
}
|
||||
|
||||
// Need to include what we already read
|
||||
in = &ReadCloserWrapper{
|
||||
Reader: io.MultiReader(rw, in),
|
||||
Reader: io.MultiReader(bytes.NewReader(buf), in),
|
||||
Closer: in,
|
||||
}
|
||||
|
||||
@@ -1157,15 +1152,6 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return objErr
|
||||
}
|
||||
|
||||
// countingDiscard is an io.Writer which discards its input and counts
|
||||
// the bytes written
|
||||
type countingDiscard int64
|
||||
|
||||
func (c *countingDiscard) Write(p []byte) (int, error) {
|
||||
*c += countingDiscard(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
|
||||
type ReadCloserWrapper struct {
|
||||
io.Reader
|
||||
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type gzipModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
|
||||
// PutStream. It reads each body twice when it is seekable to model a
|
||||
// backend retrying an upload.
|
||||
type putRecorder struct {
|
||||
fs.Fs
|
||||
t *testing.T
|
||||
stream bool // expose PutStream from the wrapped Fs
|
||||
size int64
|
||||
seekable bool
|
||||
body []byte
|
||||
streamed bool
|
||||
}
|
||||
|
||||
func (r *putRecorder) Features() *fs.Features {
|
||||
features := *r.Fs.Features()
|
||||
features.PutStream = nil
|
||||
if r.stream {
|
||||
features.PutStream = r.PutStream
|
||||
}
|
||||
return &features
|
||||
}
|
||||
|
||||
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
|
||||
r.size = src.Size()
|
||||
body, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
r.seekable = true
|
||||
_, err = seeker.Seek(0, io.SeekStart)
|
||||
require.NoError(r.t, err)
|
||||
again, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
assert.Equal(r.t, body, again, "retried body should be identical")
|
||||
}
|
||||
r.body = body
|
||||
return body
|
||||
}
|
||||
|
||||
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
body := r.record(in, src)
|
||||
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
|
||||
}
|
||||
|
||||
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
r.streamed = true
|
||||
return r.Put(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// TestRcat checks the three upload paths rcat takes: small files buffered
|
||||
// in memory and uploaded with a known size and a seekable body, large
|
||||
// files streamed when the wrapped remote supports it, and large files
|
||||
// spooled to a temporary file when it doesn't. Pool pages taken for the
|
||||
// buffer must be returned on every path.
|
||||
func TestRcat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const limit = 1024
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
stream bool
|
||||
wantSize int64
|
||||
}{
|
||||
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
|
||||
{name: "Empty", size: 0, stream: true, wantSize: 0},
|
||||
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
|
||||
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
|
||||
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
|
||||
require.NoError(t, err)
|
||||
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
|
||||
f := &Fs{Fs: rec}
|
||||
f.opt.RAMCacheLimit = limit
|
||||
content := []byte(random.String(test.size))
|
||||
inUse := pool.Global().InUse()
|
||||
|
||||
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
|
||||
|
||||
assert.Equal(t, test.wantSize, rec.size)
|
||||
assert.Equal(t, content, rec.body)
|
||||
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
|
||||
if test.size < limit {
|
||||
assert.True(t, rec.seekable, "small file body should be seekable")
|
||||
}
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, content, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsCompressible checks the compressibility heuristic on data at
|
||||
// both ends of the scale for every compression mode which has one.
|
||||
func TestIsCompressible(t *testing.T) {
|
||||
compressible := bytes.Repeat([]byte("compress me "), 4096)
|
||||
incompressible := make([]byte, len(compressible))
|
||||
_, err := io.ReadFull(rand.Reader, incompressible)
|
||||
require.NoError(t, err)
|
||||
for name, handler := range map[string]compressionModeHandler{
|
||||
"gzip": &gzipModeHandler{},
|
||||
"zstd": &zstdModeHandler{},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type zstdModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1197,7 +1197,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
// if this is wrapping a local object then we work out the hash
|
||||
if srcObj != nil && srcObj.Fs().Features().IsLocal {
|
||||
if srcObj.Fs().Features().IsLocal {
|
||||
// Read the data and encrypt it to calculate the hash
|
||||
fs.Debugf(o, "Computing %v hash of encrypted source", hash)
|
||||
return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)
|
||||
|
||||
@@ -93,16 +93,6 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
|
||||
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
|
||||
}
|
||||
|
||||
// Test ObjectInfo.Hash doesn't panic when the source is an
|
||||
// fs.OverrideRemote which wraps an fs.ObjectInfo that isn't an fs.Object
|
||||
func TestObjectInfoHashOverrideRemoteNotObject(t *testing.T) {
|
||||
src := object.NewStaticObjectInfo("file", time.Now(), 3, true, nil, nil)
|
||||
oi := (&Fs{}).newObjectInfo(fs.NewOverrideRemote(src, "new_remote"), nonce{})
|
||||
gotHash, err := oi.Hash(context.Background(), hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", gotHash)
|
||||
}
|
||||
|
||||
func testComputeHash(t *testing.T, f *Fs) {
|
||||
var (
|
||||
contents = random.String(100)
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
// Package api has type definitions for dosya.dev
|
||||
package api
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Response is the base response wrapper
|
||||
type Response struct {
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Error is an error response from the API.
|
||||
//
|
||||
// StatusCode and Status are filled in from the HTTP response. The API
|
||||
// sets ErrorCode on the errors a client is expected to act on rather
|
||||
// than just report, e.g. "concurrent_upload_limit".
|
||||
type Error struct {
|
||||
StatusCode int `json:"-"`
|
||||
Status string `json:"-"`
|
||||
OK bool `json:"ok"`
|
||||
Message string `json:"error"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
|
||||
}
|
||||
|
||||
// Error satisfies the error interface
|
||||
func (e *Error) Error() string {
|
||||
out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
|
||||
if e.Message != "" {
|
||||
out += ": " + e.Message
|
||||
}
|
||||
if e.ErrorCode != "" {
|
||||
out += " (" + e.ErrorCode + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileItem represents a file from the dosya API
|
||||
type FileItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
UpdatedAt float64 `json:"updated_at"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
|
||||
// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
|
||||
// are never re-read to hash the whole object - the same limitation S3 has).
|
||||
ContentHash string `json:"content_hash"`
|
||||
}
|
||||
|
||||
// FolderItem represents a folder from the dosya API
|
||||
type FolderItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
FileCount int `json:"file_count"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
}
|
||||
|
||||
// ListResponse is the response from GET /api/files
|
||||
type ListResponse struct {
|
||||
Response
|
||||
Files []FileItem `json:"files"`
|
||||
Folders []FolderItem `json:"folders"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
// Pagination describes which page of files a ListResponse holds
|
||||
type Pagination struct {
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest is the request body for POST /api/folders
|
||||
type CreateFolderRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse is the response from POST /api/folders
|
||||
type CreateFolderResponse struct {
|
||||
Response
|
||||
Folder struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Workspace string `json:"workspace_id"`
|
||||
} `json:"folder"`
|
||||
}
|
||||
|
||||
// DeleteFolderResponse is the response from DELETE /api/folders/:id
|
||||
type DeleteFolderResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
Complete bool `json:"complete"`
|
||||
Remaining int `json:"remaining"`
|
||||
FilesAffected int `json:"files_affected"`
|
||||
FoldersRemoved int `json:"folders_removed"`
|
||||
}
|
||||
|
||||
// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
|
||||
type RenameFolderRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFolderRequest is the request body for PUT /api/folders/:id/move
|
||||
type MoveFolderRequest struct {
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// MoveFolderResponse is the response from PUT /api/folders/:id/move
|
||||
type MoveFolderResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// UploadInitRequest is the request body for POST /api/upload/init
|
||||
type UploadInitRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Region string `json:"region,omitempty"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
FileID *string `json:"file_id,omitempty"`
|
||||
}
|
||||
|
||||
// UploadInitResponse is the response from POST /api/upload/init
|
||||
type UploadInitResponse struct {
|
||||
Response
|
||||
SessionID string `json:"session_id"`
|
||||
UploadURL string `json:"upload_url"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Resumable *struct {
|
||||
PartSize int64 `json:"part_size"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
PartUploadURL string `json:"part_upload_url"`
|
||||
CompleteURL string `json:"complete_url"`
|
||||
StatusURL string `json:"status_url"`
|
||||
} `json:"resumable"`
|
||||
}
|
||||
|
||||
// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
|
||||
type UploadCompleteResponse struct {
|
||||
Response
|
||||
File struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Version int `json:"version"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
|
||||
// "" for a multipart upload, which the server does not hash.
|
||||
ContentHash string `json:"content_hash"`
|
||||
} `json:"file"`
|
||||
}
|
||||
|
||||
// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
|
||||
type UploadPartResponse struct {
|
||||
Response
|
||||
PartNumber int `json:"part_number"`
|
||||
ETag string `json:"etag"`
|
||||
BytesUploaded int64 `json:"bytes_uploaded"`
|
||||
PartsCompleted int `json:"parts_completed"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
}
|
||||
|
||||
// DeleteFileResponse is the response from DELETE /api/files/:id
|
||||
type DeleteFileResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
}
|
||||
|
||||
// RenameFileRequest is the request body for PUT /api/files/:id/rename
|
||||
type RenameFileRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// RenameFileResponse is the response from PUT /api/files/:id/rename
|
||||
type RenameFileResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFileRequest is the request body for PUT /api/files/:id/move
|
||||
type MoveFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest is the request body for POST /api/files/:id/copy
|
||||
type CopyFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileResponse is the response from POST /api/files/:id/copy
|
||||
type CopyFileResponse struct {
|
||||
Response
|
||||
FileID string `json:"file_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// ShareLinkRequest is the request body for POST /api/files/:id/share
|
||||
type ShareLinkRequest struct {
|
||||
ExpiresInDays *int `json:"expires_in_days,omitempty"`
|
||||
}
|
||||
|
||||
// ShareLinkResponse is the response from POST /api/files/:id/share
|
||||
type ShareLinkResponse struct {
|
||||
Response
|
||||
Link struct {
|
||||
ID string `json:"id"`
|
||||
Token string `json:"token"`
|
||||
URL string `json:"url"`
|
||||
ExpiresAt *int64 `json:"expires_at"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
} `json:"link"`
|
||||
}
|
||||
|
||||
// Workspace is one entry of the workspace list
|
||||
type Workspace struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// WorkspaceListResponse is the response from GET /api/workspaces
|
||||
type WorkspaceListResponse struct {
|
||||
Response
|
||||
Workspaces []Workspace `json:"workspaces"`
|
||||
}
|
||||
|
||||
// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
|
||||
type WorkspaceInfoResponse struct {
|
||||
Response
|
||||
Storage struct {
|
||||
Used int64 `json:"used"`
|
||||
Total int64 `json:"total"`
|
||||
Free int64 `json:"free"`
|
||||
} `json:"storage"`
|
||||
}
|
||||
@@ -1,772 +0,0 @@
|
||||
// Package dosya provides an interface to the dosya.dev cloud storage system.
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
rootID = "root"
|
||||
defaultBaseURL = "https://api.dosya.dev"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2
|
||||
attackConstant = 0
|
||||
)
|
||||
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "dosya",
|
||||
Description: "dosya.dev",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
|
||||
Name: "api_key",
|
||||
Sensitive: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "workspace_id",
|
||||
Help: `Your workspace ID.
|
||||
|
||||
Leave this blank to use the one workspace the API key can reach, which
|
||||
is what a key restricted to a single workspace does. If the key can
|
||||
reach several, rclone will refuse and list their IDs for you to choose
|
||||
from.`,
|
||||
}, {
|
||||
Help: "API base URL.",
|
||||
Name: "api_url",
|
||||
Default: defaultBaseURL,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
Help: `Concurrency for multipart uploads.
|
||||
|
||||
This is the number of parts of the same file that are uploaded
|
||||
concurrently. The first part is always uploaded on its own, because
|
||||
the server creates the upload when it receives it.
|
||||
|
||||
Each part in flight is held in memory, so memory use grows with
|
||||
this value times the part size the server chooses.
|
||||
|
||||
If you are uploading large files over high-speed links and these
|
||||
uploads do not fully utilize your bandwidth, then increasing this
|
||||
may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeSingleQuote |
|
||||
encoder.EncodeDoubleQuote |
|
||||
encoder.EncodeLtGt |
|
||||
encoder.EncodeCtl |
|
||||
encoder.EncodeLeftSpace |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
APIKey string `config:"api_key"`
|
||||
WorkspaceID string `config:"workspace_id"`
|
||||
APIURL string `config:"api_url"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dosya.dev filesystem
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
features *fs.Features
|
||||
rest *rest.Client
|
||||
pacer *fs.Pacer
|
||||
dirCache *dircache.DirCache
|
||||
}
|
||||
|
||||
// Object represents a dosya.dev file
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
file api.FileItem
|
||||
}
|
||||
|
||||
// findWorkspace fills in the workspace ID when the config leaves it blank.
|
||||
//
|
||||
// The ID is not shown anywhere in the dosya.dev web interface, so requiring
|
||||
// it meant asking people to find it in a developer console. A key restricted
|
||||
// to one workspace can only mean that one, and an account with a single
|
||||
// workspace has only one possible answer; anything else genuinely has to be
|
||||
// chosen, so list the choices in the error.
|
||||
func (f *Fs) findWorkspace(ctx context.Context) error {
|
||||
workspaces, err := f.listWorkspaces(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch len(workspaces) {
|
||||
case 0:
|
||||
return errors.New("no workspaces found for this API key")
|
||||
case 1:
|
||||
f.opt.WorkspaceID = workspaces[0].ID
|
||||
fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
|
||||
return nil
|
||||
}
|
||||
var choices strings.Builder
|
||||
for _, w := range workspaces {
|
||||
_, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
|
||||
}
|
||||
return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
|
||||
len(workspaces), choices.String())
|
||||
}
|
||||
|
||||
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
||||
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
|
||||
listing, err := f.listFilesAndFolders(ctx, pathID)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, folder := range listing.Folders {
|
||||
if f.opt.Enc.ToStandardName(folder.Name) == leaf {
|
||||
return folder.ID, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// CreateDir makes a directory with pathID as parent and name leaf
|
||||
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
|
||||
name := f.opt.Enc.FromStandardName(leaf)
|
||||
resp, err := f.createFolder(ctx, name, pathID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.Folder.ID, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("dosya.dev root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
// Modification times are stored and returned to the second (the API
|
||||
// keeps them as Unix seconds, sent on upload via the source-mtime header).
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash types of the filesystem
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
// The server computes and stores the SHA-256 of a single-shot upload's
|
||||
// bytes; multipart uploads have none (see Object.Hash).
|
||||
return hash.NewHashSet(hash.SHA256)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(config, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
|
||||
}
|
||||
|
||||
// No DuplicateFiles: an upload that names no existing file but whose
|
||||
// (workspace, folder, name) is already taken becomes a new VERSION of
|
||||
// that file rather than a second row, so a name is unique within a
|
||||
// folder and the same upload twice returns the same file id.
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
ReadMimeType: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
|
||||
f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
|
||||
|
||||
if f.opt.WorkspaceID == "" {
|
||||
err = f.findWorkspace(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, rootID, f)
|
||||
|
||||
// Find the current root
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// Assume it is a file
|
||||
newRoot, remote := dircache.SplitPath(root)
|
||||
tempF := *f
|
||||
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
|
||||
tempF.root = newRoot
|
||||
// Make new Fs which is the parent
|
||||
err = tempF.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// No root so return old f
|
||||
return f, nil
|
||||
}
|
||||
_, err := tempF.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// File doesn't exist so return old f
|
||||
return f, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
f.features.Fill(ctx, &tempF)
|
||||
f.dirCache = tempF.dirCache
|
||||
f.root = tempF.root
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, folder := range listing.Folders {
|
||||
folderName := f.opt.Enc.ToStandardName(folder.Name)
|
||||
fullPath := getRemote(dir, folderName)
|
||||
modTime := time.Unix(int64(folder.CreatedAt), 0)
|
||||
d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
|
||||
entries = append(entries, d)
|
||||
f.dirCache.Put(fullPath, folder.ID)
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
fileName := f.opt.Enc.ToStandardName(file.Name)
|
||||
remote := getRemote(dir, fileName)
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}
|
||||
entries = append(entries, o)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
if f.opt.Enc.ToStandardName(file.Name) == leaf {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
// Put uploads a new file
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
existingObj, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return existingObj, existingObj.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// put uploads src as a new file
|
||||
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
remote := src.Remote()
|
||||
|
||||
_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: fileFromUpload(resp, modTime),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fileFromUpload builds the FileItem for the object an upload just created.
|
||||
// The single-shot upload response omits updated_at, so the modification time
|
||||
// is taken from what was sent (modTime) rather than defaulting to the create
|
||||
// time - it is the time the server stored via the source-mtime header. The
|
||||
// SHA-256 comes straight from the response (empty for a multipart upload).
|
||||
func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
|
||||
updatedAt := resp.File.CreatedAt
|
||||
if !modTime.IsZero() {
|
||||
updatedAt = float64(modTime.Unix())
|
||||
}
|
||||
return api.FileItem{
|
||||
ID: resp.File.ID,
|
||||
Name: resp.File.Name,
|
||||
SizeBytes: resp.File.SizeBytes,
|
||||
MimeType: resp.File.MimeType,
|
||||
Extension: resp.File.Extension,
|
||||
Region: resp.File.Region,
|
||||
CreatedAt: resp.File.CreatedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
ContentHash: resp.File.ContentHash,
|
||||
}
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
return err
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The server checks emptiness itself (?if_empty=1) and answers
|
||||
// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
|
||||
err = f.removeFolder(ctx, directoryID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move src to this remote using server-side move operations
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't move - not same remote type")
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
|
||||
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find destination directory
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
needsMove := srcDirID != dstDirID
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
|
||||
// Moving and renaming in the same request (rather than a moveFile call
|
||||
// followed by a separate renameFile call) matters when both are needed:
|
||||
// a two-step move-then-rename can transiently place the file's CURRENT
|
||||
// name into a destination folder that already holds an unrelated file
|
||||
// with that name, even though the FINAL name is free - the server only
|
||||
// allows one active name per folder, so that intermediate state 409s.
|
||||
if needsMove || needsRename {
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Return new object. file.Name must reflect the destination leaf, not
|
||||
// the source's - a caller that reuses this object as the src of a
|
||||
// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
|
||||
// recomputes srcLeaf from file.Name; a stale name there makes that
|
||||
// second move see srcLeaf == dstLeaf and skip the rename entirely,
|
||||
// silently leaving the object under the wrong name.
|
||||
newFile := srcObj.file
|
||||
newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: newFile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
|
||||
srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srcLeaf := srcRemote
|
||||
if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
|
||||
srcLeaf = srcRemote[idx+1:]
|
||||
}
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
|
||||
// Move folder to new parent only if parent changed. The new name goes
|
||||
// in the same request, as for files in Move: a move followed by a
|
||||
// separate rename can collide with an unrelated folder sharing the
|
||||
// CURRENT name in the destination even though the final name is free.
|
||||
if srcDirectoryID != dstDirectoryID {
|
||||
renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move directory: %w", err)
|
||||
}
|
||||
if renamed {
|
||||
needsRename = false
|
||||
}
|
||||
}
|
||||
|
||||
if needsRename {
|
||||
err = f.renameFolder(ctx, srcID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side copy operations
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The copy API creates a new row named after the source file when
|
||||
// copying across folders, which collides with one already at the
|
||||
// destination. Copy under a temporary name, then delete the old
|
||||
// destination, then move the copy onto the name - so a copy that fails
|
||||
// leaves the destination as it was. Deleting it up front lost it
|
||||
// (prior art: backend/box).
|
||||
existingObj, err := f.NewObject(ctx, remote)
|
||||
switch err {
|
||||
case nil:
|
||||
tempRemote := remote + "-rclone-copy-" + random.String(8)
|
||||
fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
|
||||
tempObj, err := f.Copy(ctx, src, tempRemote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs.Debugf(tempRemote, "moving to real name %v", remote)
|
||||
if err := existingObj.Remove(ctx); err != nil {
|
||||
return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
|
||||
}
|
||||
return f.Move(ctx, tempObj, remote)
|
||||
case fs.ErrorObjectNotFound:
|
||||
// nothing at the destination, so copy straight onto the name
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dstName := f.opt.Enc.FromStandardName(dstLeaf)
|
||||
|
||||
resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
|
||||
// Servers that predate the name field ignore it and name the copy after
|
||||
// the source, or "Copy of <original>" in the same folder. Rename to the
|
||||
// requested destination name if it differs.
|
||||
if resp.Name != dstName {
|
||||
err = f.renameFile(ctx, resp.FileID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't rename copied file: %w", err)
|
||||
}
|
||||
resp.Name = dstName
|
||||
}
|
||||
|
||||
// A server-side copy is byte-identical to the source and the API
|
||||
// preserves the source's modification time and content hash onto the
|
||||
// copy, so the returned object must report the SOURCE's modtime and hash,
|
||||
// not the copy's wall-clock time. Reading "now" here made the object
|
||||
// rclone holds in memory disagree with the one read back from the remote
|
||||
// (its Fingerprint check compares the two exactly).
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: api.FileItem{
|
||||
ID: resp.FileID,
|
||||
Name: resp.Name,
|
||||
SizeBytes: srcObj.file.SizeBytes,
|
||||
MimeType: srcObj.file.MimeType,
|
||||
Extension: srcObj.file.Extension,
|
||||
Region: srcObj.file.Region,
|
||||
CreatedAt: srcObj.file.CreatedAt,
|
||||
UpdatedAt: srcObj.file.UpdatedAt,
|
||||
ContentHash: srcObj.file.ContentHash,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Purge deletes all the files and the container
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
|
||||
err = f.removeFolder(ctx, directoryID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
obj := o.(*Object)
|
||||
|
||||
resp, err := f.createShareLink(ctx, obj.file.ID, expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return resp.Link.URL, nil
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing as an
|
||||
// optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
info, err := f.getWorkspaceInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage := &fs.Usage{
|
||||
Used: fs.NewUsageValue(info.Storage.Used),
|
||||
Total: fs.NewUsageValue(info.Storage.Total),
|
||||
Free: fs.NewUsageValue(info.Storage.Free),
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Object methods
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// String returns a description of the Object
|
||||
func (o *Object) String() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// ModTime returns the modification date of the file
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
if o.file.UpdatedAt > 0 {
|
||||
return time.Unix(int64(o.file.UpdatedAt), 0)
|
||||
}
|
||||
return time.Unix(int64(o.file.CreatedAt), 0)
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return o.file.SizeBytes
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.SHA256 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// May be "" - a multipart-uploaded object carries no server-side hash,
|
||||
// which rclone treats as "hash unavailable for this object" (as it does
|
||||
// for S3 multipart objects).
|
||||
return o.file.ContentHash, nil
|
||||
}
|
||||
|
||||
// Storable says whether this object can be stored
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
return fs.ErrorCantSetModTime
|
||||
}
|
||||
|
||||
// Open opens the file for read
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
fs.FixRangeOption(options, o.file.SizeBytes)
|
||||
return o.fs.downloadFile(ctx, o.file.ID, options)
|
||||
}
|
||||
|
||||
// Update replaces the file content
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
// Find the directory ID for this file
|
||||
_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload as new version of existing file
|
||||
fileID := o.file.ID
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update object metadata
|
||||
o.file = fileFromUpload(resp, modTime)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove removes this object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.deleteFile(ctx, o.file.ID)
|
||||
}
|
||||
|
||||
// MimeType of an Object if known, "" otherwise
|
||||
func (o *Object) MimeType(ctx context.Context) string {
|
||||
return o.file.MimeType
|
||||
}
|
||||
|
||||
// ID returns the ID of the Object if known, or "" if not
|
||||
func (o *Object) ID() string {
|
||||
return o.file.ID
|
||||
}
|
||||
|
||||
func getRemote(dir, fileName string) string {
|
||||
if dir == "" {
|
||||
return fileName
|
||||
}
|
||||
return dir + "/" + fileName
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ dircache.DirCacher = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
)
|
||||
@@ -1,880 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// sourceMTimeHeader is the request header the upload endpoints read to set a
|
||||
// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
|
||||
const sourceMTimeHeader = "x-dosya-source-mtime"
|
||||
|
||||
// modTimeHeaders returns the ExtraHeaders that carry a source modification time
|
||||
// on an upload, or nil when there is nothing plausible to send. The server
|
||||
// clamps implausible values itself, but a zero time is never worth a header.
|
||||
func modTimeHeaders(modTime time.Time) map[string]string {
|
||||
if modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
|
||||
}
|
||||
|
||||
// workspaceHintHeader names the workspace a by-id request is about. The API
|
||||
// maps an id to its workspace through an index the workspace fills about a
|
||||
// second after a write, so an id this client was handed moments ago can still
|
||||
// miss; the hint lets the server look for it in the named workspace instead.
|
||||
// It only locates: every route still checks membership against the workspace
|
||||
// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
|
||||
const workspaceHintHeader = "X-Dosya-Workspace"
|
||||
|
||||
// byIDHeaders returns the ExtraHeaders every /api/files/:id and
|
||||
// /api/folders/:id call carries, or nil when no workspace is configured.
|
||||
func (f *Fs) byIDHeaders() map[string]string {
|
||||
if f.opt.WorkspaceID == "" {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
|
||||
}
|
||||
|
||||
// retryErrorCodes is a slice of error codes that we will retry
|
||||
var retryErrorCodes = []int{
|
||||
429, // Too Many Requests
|
||||
500, // Internal Server Error
|
||||
502, // Bad Gateway
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
|
||||
if retry {
|
||||
if d, ok := retryAfter(resp); ok {
|
||||
// The API answers 503 + Retry-After while a workspace is being
|
||||
// moved between storage homes, and Cloudflare sends it with 429.
|
||||
// The pacer's own backoff tops out at maxSleep, far short of that.
|
||||
fs.Debugf(nil, "Server asked to retry after %v", d)
|
||||
err = pacer.RetryAfterError(err, d)
|
||||
}
|
||||
}
|
||||
return retry, err
|
||||
}
|
||||
|
||||
// maxRetryAfter caps how long a single Retry-After is honoured for
|
||||
const maxRetryAfter = 5 * time.Minute
|
||||
|
||||
// retryAfter reads a 429 or 503 response's Retry-After header, which the API
|
||||
// sends as whole seconds
|
||||
func retryAfter(resp *http.Response) (time.Duration, bool) {
|
||||
if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
|
||||
return 0, false
|
||||
}
|
||||
seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
|
||||
if err != nil || seconds <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
|
||||
}
|
||||
|
||||
// errorHandler decodes a non-2xx response into an *api.Error so callers can
|
||||
// act on the machine-readable error_code rather than the message text.
|
||||
func errorHandler(resp *http.Response) error {
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading error out of body: %w", err)
|
||||
}
|
||||
apiErr := &api.Error{
|
||||
StatusCode: resp.StatusCode,
|
||||
Status: resp.Status,
|
||||
}
|
||||
if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
|
||||
// Not the API's JSON shape (e.g. an edge response that never
|
||||
// reached the API) - keep the body text so it isn't lost.
|
||||
apiErr.ErrorCode = ""
|
||||
apiErr.Message = strings.TrimSpace(string(body))
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
||||
// The workspace caps the number of in-flight uploads per user. The API
|
||||
// refuses an upload/init past that cap with a 400 carrying
|
||||
// error_code "concurrent_upload_limit", and documents it as an error to
|
||||
// wait on rather than give up on: slots free as uploads finish, and the
|
||||
// server marks sessions idle for 10 minutes as failed, so waiting up to
|
||||
// that long guarantees progress unless the workspace really is saturated.
|
||||
// A 400 is deliberately outside the pacer's retry codes, so the wait
|
||||
// lives in initUpload instead of shouldRetry.
|
||||
var (
|
||||
concurrentUploadRetryDelay = 5 * time.Second
|
||||
concurrentUploadWaitMax = 10 * time.Minute
|
||||
)
|
||||
|
||||
// abortUploadTimeout bounds the abort of a session the client is giving up on,
|
||||
// which runs outside the transfer's context so a cancelled transfer still
|
||||
// sends it
|
||||
const abortUploadTimeout = 10 * time.Second
|
||||
|
||||
// isConcurrentUploadLimit reports whether err is the API refusing an
|
||||
// upload because the workspace's concurrent upload limit is reached
|
||||
func isConcurrentUploadLimit(err error) bool {
|
||||
var apiErr *api.Error
|
||||
return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
|
||||
}
|
||||
|
||||
// listPageSize is the largest page GET /api/files will serve
|
||||
const listPageSize = 500
|
||||
|
||||
// listFilesAndFolders lists files and folders in a directory
|
||||
//
|
||||
// The API pages files (at most listPageSize per page) but returns every
|
||||
// folder on each page, so the files from all pages are gathered and the
|
||||
// folders are taken from the first page only. Pages are ordered by name,
|
||||
// which the API tie-breaks on id, so offsets are stable between requests.
|
||||
func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
|
||||
var listing *api.ListResponse
|
||||
for page := 1; ; page++ {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files",
|
||||
Parameters: map[string][]string{
|
||||
"workspace_id": {f.opt.WorkspaceID},
|
||||
"per_page": {strconv.Itoa(listPageSize)},
|
||||
"page": {strconv.Itoa(page)},
|
||||
"sort": {"name"},
|
||||
"dir": {"asc"},
|
||||
},
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
opts.Parameters["folder_id"] = []string{folderID}
|
||||
}
|
||||
|
||||
var result api.ListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list files: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if listing == nil {
|
||||
listing = &result
|
||||
} else {
|
||||
listing.Files = append(listing.Files, result.Files...)
|
||||
}
|
||||
if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
|
||||
return listing, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// createFolder creates a folder
|
||||
func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
|
||||
request := api.CreateFolderRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
Name: name,
|
||||
}
|
||||
if parentID != "" && parentID != rootID {
|
||||
request.ParentID = &parentID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/folders",
|
||||
}
|
||||
|
||||
var result api.CreateFolderResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// removeFolder deletes a folder
|
||||
//
|
||||
// ?permanent=1 trashes and purges a live folder in one request on today's
|
||||
// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
|
||||
// HTTP 409 "Folder is not empty" instead of purging a folder that still
|
||||
// holds a live file or a live child folder; that 409 is turned into
|
||||
// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
|
||||
// error, and it is never retried (409 is not one of retryErrorCodes). A big
|
||||
// subtree is purged in bounded passes (202, complete=false), so keep
|
||||
// calling until the purge is complete. An older server that ignores
|
||||
// permanent=1 only trashes on the first call (answering permanent=false),
|
||||
// so the loop's second pass sends the plain DELETE that purges it.
|
||||
func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
|
||||
parameters := map[string][]string{"permanent": {"1"}}
|
||||
if ifEmpty {
|
||||
parameters["if_empty"] = []string{"1"}
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/folders/" + folderID,
|
||||
Parameters: parameters,
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
for pass := 0; ; pass++ {
|
||||
var result api.DeleteFolderResponse
|
||||
var statusCode int
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
if resp != nil {
|
||||
statusCode = resp.StatusCode
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if ifEmpty && statusCode == http.StatusConflict {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if pass == 0 {
|
||||
return fmt.Errorf("couldn't remove folder: %w", err)
|
||||
}
|
||||
return fmt.Errorf("couldn't permanently remove folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if result.Permanent && result.Complete {
|
||||
return nil
|
||||
}
|
||||
if pass >= maxFolderPurgePasses {
|
||||
return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
|
||||
const maxFolderPurgePasses = 1000
|
||||
|
||||
// renameFolder renames a folder
|
||||
func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
|
||||
request := api.RenameFolderRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFolder moves a folder to a new parent
|
||||
func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
|
||||
request := api.MoveFolderRequest{Name: newName}
|
||||
if newParentID != "" && newParentID != rootID {
|
||||
request.ParentID = &newParentID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.MoveFolderResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("couldn't move folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return false, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
// Servers that predate the name field ignore it and answer without a
|
||||
// name, leaving the caller to rename separately
|
||||
return newName != "" && result.Name == newName, nil
|
||||
}
|
||||
|
||||
// deleteFile deletes a file
|
||||
//
|
||||
// ?permanent=1 trashes and purges it in one request on today's API. An
|
||||
// older server that ignores the flag only soft-deletes it, answering
|
||||
// permanent=false, so a second DELETE on the now soft-deleted file purges
|
||||
// it from the database and R2, as it always has.
|
||||
func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/files/" + fileID,
|
||||
Parameters: map[string][]string{"permanent": {"1"}},
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.DeleteFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't delete file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
|
||||
// Answered into its own value so a field the second response omits
|
||||
// can't be read off the first.
|
||||
if !result.Permanent {
|
||||
var permanent api.DeleteFileResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't permanently delete file: %w", err)
|
||||
}
|
||||
if !permanent.OK {
|
||||
return fmt.Errorf("API error: %s", permanent.Error)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renameFile renames a file
|
||||
func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
|
||||
request := api.RenameFileRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.RenameFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFile moves a file to a new folder and, if newName is non-empty,
|
||||
// renames it in the same request
|
||||
func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
|
||||
request := api.MoveFileRequest{}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
if newName != "" {
|
||||
request.Name = &newName
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies a file to a folder
|
||||
func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
|
||||
request := api.CopyFileRequest{Name: name}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/copy",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.CopyFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// downloadFile downloads a file by its ID
|
||||
//
|
||||
// The download endpoint answers 302 with a presigned R2 URL in Location, so
|
||||
// this is two calls. Both go through the rest client, which is built on
|
||||
// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
|
||||
// --user-agent, --bind, --dump and the proxy settings apply to downloads as
|
||||
// they do to every other call, and errorHandler and shouldRetry decide the
|
||||
// errors and the retries uniformly.
|
||||
func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
|
||||
// NoRedirect stops the client following the 302 with the API's
|
||||
// Authorization header attached. A 302 is not a 2xx, so Call reports it
|
||||
// as an error (having read and closed the body); the Location header
|
||||
// is what actually matters, and finding it clears the error, as in
|
||||
// backend/linkbox.
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files/" + fileID + "/download",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
NoRedirect: true,
|
||||
}
|
||||
var downloadURL string
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
if resp != nil {
|
||||
if location := resp.Header.Get("Location"); location != "" {
|
||||
downloadURL, err = location, nil
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get download URL: %w", err)
|
||||
}
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("no download URL in redirect response")
|
||||
}
|
||||
|
||||
// Fetch the object from the presigned URL. It carries its own
|
||||
// credentials and R2 refuses a request that also has an Authorization
|
||||
// header, so the client-level bearer token is removed by setting the
|
||||
// header empty: the rest client skips a header with an empty value.
|
||||
opts = rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: downloadURL,
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{"Authorization": ""},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't download file: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// uploadSmallFile uploads a file smaller than 10MB in a single request
|
||||
func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
Body: in,
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
// The body is a stream which can't be rewound, so a low level retry
|
||||
// would resend whatever is left of it - nothing - and the server
|
||||
// would store that as an empty file. Don't retry here; rclone's high
|
||||
// level retries redo the whole transfer instead.
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadPart uploads a single part of a multipart upload
|
||||
func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
|
||||
size := int64(len(part))
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
var result api.UploadPartResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
// A fresh reader per attempt so a retry resends the whole part
|
||||
opts.Body = bytes.NewReader(part)
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// completeUpload finalizes a multipart upload
|
||||
func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/" + sessionID + "/complete",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't complete upload: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// initUpload initializes a new upload session
|
||||
func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
|
||||
request := api.UploadInitRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
FileName: fileName,
|
||||
FileSize: fileSize,
|
||||
MimeType: mimeType,
|
||||
FileID: fileID,
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/init",
|
||||
}
|
||||
|
||||
var result api.UploadInitResponse
|
||||
deadline := time.Now().Add(concurrentUploadWaitMax)
|
||||
for {
|
||||
result = api.UploadInitResponse{}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", err)
|
||||
}
|
||||
fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
|
||||
case <-time.After(concurrentUploadRetryDelay):
|
||||
}
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadParts reads the source part by part and uploads the parts.
|
||||
//
|
||||
// Part 1 goes on its own: the server creates the R2 multipart upload (and a
|
||||
// new file's id and key) when it receives the first part, so parts sent
|
||||
// before that has finished would each start an upload of their own. The
|
||||
// rest go out opt.UploadConcurrency at a time. A part is only read from the
|
||||
// source once a buffer is free, so at most that many parts are held in
|
||||
// memory, and a short source is still found by the reader before the part
|
||||
// it could not fill is sent.
|
||||
func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
|
||||
readPart := func(buf []byte, partNum int) ([]byte, error) {
|
||||
currentPartSize := partSize
|
||||
if partNum == totalParts {
|
||||
currentPartSize = size - partSize*int64(totalParts-1)
|
||||
}
|
||||
part := buf[:currentPartSize]
|
||||
if _, err := io.ReadFull(in, part); err != nil {
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
|
||||
}
|
||||
return part, nil
|
||||
}
|
||||
|
||||
concurrency := max(f.opt.UploadConcurrency, 1)
|
||||
buffers := make(chan []byte, concurrency)
|
||||
buffers <- make([]byte, partSize)
|
||||
allocated := 1
|
||||
|
||||
first := <-buffers
|
||||
part, err := readPart(first, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
buffers <- first
|
||||
|
||||
// A failed read has to take the parts already in flight down with it,
|
||||
// or they upload to completion while the error waits on Wait.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var readErr error
|
||||
for partNum := 2; partNum <= totalParts; partNum++ {
|
||||
// Take a free buffer, allocating one while under the limit.
|
||||
var buf []byte
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
default:
|
||||
if allocated < concurrency {
|
||||
buf = make([]byte, partSize)
|
||||
allocated++
|
||||
} else {
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
case <-gCtx.Done():
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf == nil {
|
||||
break // a part failed; its error comes back from Wait
|
||||
}
|
||||
part, err := readPart(buf, partNum)
|
||||
if err != nil {
|
||||
readErr = err
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
defer func() { buffers <- buf }()
|
||||
_, err := f.uploadPart(gCtx, part, sessionID, partNum)
|
||||
return err
|
||||
})
|
||||
}
|
||||
err = g.Wait()
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// abortUpload retires an upload session the client is giving up on.
|
||||
//
|
||||
// Best effort: the server reclaims an abandoned session itself, so a failure
|
||||
// here - including an older server with no such route - costs the slot the
|
||||
// wait it would have cost anyway and is not worth failing an upload over.
|
||||
// Sent on a context of its own so a cancelled transfer still gives its slot
|
||||
// back rather than leaving it to the sweep.
|
||||
func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
|
||||
defer cancel()
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
NoResponse: true,
|
||||
}
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadFile handles the full upload flow (small or multipart)
|
||||
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
|
||||
// Both refusals live here so a new file and a new version of an existing
|
||||
// one answer the same way: upload/init needs a byte count up front, and
|
||||
// the server stores no zero-length object.
|
||||
if size < 0 {
|
||||
return nil, errors.New("can't upload files with unknown size")
|
||||
}
|
||||
if size == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
leaf := f.opt.Enc.FromStandardName(path.Base(remote))
|
||||
|
||||
initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A session abandoned here holds one of the workspace's upload slots
|
||||
// until the server's sweep retires it ten minutes later, and rclone's
|
||||
// high-level retry opens a fresh session rather than resuming this one -
|
||||
// so without this, a few failed attempts use up every slot.
|
||||
defer func() {
|
||||
if err != nil {
|
||||
f.abortUpload(ctx, initResp.SessionID)
|
||||
}
|
||||
}()
|
||||
|
||||
// Count what the source actually supplies, so a source which ends
|
||||
// early is reported as an error rather than stored as a truncated
|
||||
// file.
|
||||
counter := readers.NewCountingReader(in)
|
||||
|
||||
// Small file upload (single PUT, no resumable)
|
||||
if initResp.Resumable == nil {
|
||||
resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := int64(counter.BytesRead()); got != size {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Multipart upload. Each part is read fully into memory before it is
|
||||
// sent, so a short source is caught before the part goes out and a
|
||||
// part which has to be retried is resent with all of its bytes.
|
||||
partSize := initResp.Resumable.PartSize
|
||||
totalParts := initResp.Resumable.TotalParts
|
||||
// Both bounds matter. partSize*(totalParts-1) >= size means a part would
|
||||
// have nothing to carry; size > partSize*totalParts means the last part
|
||||
// is bigger than partSize, and readPart would slice past its buffer and
|
||||
// panic.
|
||||
if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
|
||||
return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
|
||||
}
|
||||
if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.completeUpload(ctx, initResp.SessionID, modTime)
|
||||
}
|
||||
|
||||
// createShareLink creates a public share link for a file
|
||||
func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
|
||||
request := api.ShareLinkRequest{}
|
||||
if expire < fs.DurationOff {
|
||||
days := int(time.Duration(expire).Hours() / 24)
|
||||
if days < 1 {
|
||||
days = 1
|
||||
}
|
||||
request.ExpiresInDays = &days
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/share",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.ShareLinkResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create share link: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// getWorkspaceInfo fetches workspace storage info
|
||||
// listWorkspaces lists the workspaces this credential can reach. An API key
|
||||
// pinned to a workspace sees exactly one: its own.
|
||||
func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces",
|
||||
}
|
||||
|
||||
var result api.WorkspaceListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list workspaces: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return result.Workspaces, nil
|
||||
}
|
||||
|
||||
func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces/" + f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
var result api.WorkspaceInfoResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get workspace info: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
header string
|
||||
want time.Duration
|
||||
ok bool
|
||||
}{
|
||||
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
|
||||
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
|
||||
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
|
||||
{"missing header", http.StatusServiceUnavailable, "", 0, false},
|
||||
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
|
||||
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
|
||||
if tc.header != "" {
|
||||
resp.Header.Set("Retry-After", tc.header)
|
||||
}
|
||||
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
|
||||
assert.True(t, retry)
|
||||
d, ok := pacer.IsRetryAfter(err)
|
||||
assert.Equal(t, tc.ok, ok)
|
||||
if tc.ok {
|
||||
assert.Equal(t, tc.want, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
|
||||
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
|
||||
resp.Header.Set("Retry-After", "30")
|
||||
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
|
||||
assert.False(t, retry)
|
||||
_, ok := pacer.IsRetryAfter(err)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestRemoveFolderPurgesPermanently(t *testing.T) {
|
||||
// a big subtree purges in bounded passes: 202 complete=false repeats
|
||||
// the same request until complete=true
|
||||
answers := []struct {
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
|
||||
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
|
||||
}
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "DELETE", r.Method)
|
||||
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
|
||||
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
|
||||
a := answers[calls]
|
||||
calls++
|
||||
return jsonResp(a.status, a.body), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
|
||||
assert.Equal(t, len(answers), calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "locked file")
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", true)
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRmdirMakesOneRequest(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
require.NoError(t, f.Rmdir(context.Background(), "sub"))
|
||||
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
|
||||
}
|
||||
|
||||
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
err := f.Rmdir(context.Background(), "sub")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
}
|
||||
|
||||
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
permanent bool
|
||||
wantCalls int
|
||||
}{
|
||||
{"today's API purges in one call", true, 1},
|
||||
{"an older server ignores the flag and still gets a second DELETE", false, 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
|
||||
assert.Equal(t, tc.wantCalls, calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// decodeBody reads a JSON request body into a map
|
||||
func decodeBody(t *testing.T, r *http.Request) map[string]any {
|
||||
b, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
var m map[string]any
|
||||
require.NoError(t, json.Unmarshal(b, &m))
|
||||
return m
|
||||
}
|
||||
|
||||
func TestMoveFolderSendsName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
newName string
|
||||
respBody string
|
||||
wantRenamed bool
|
||||
}{
|
||||
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
|
||||
{"older server ignores name", "dst", `{"ok":true}`, false},
|
||||
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_parent", body["parent_id"])
|
||||
if tc.newName == "" {
|
||||
assert.NotContains(t, body, "name")
|
||||
} else {
|
||||
assert.Equal(t, tc.newName, body["name"])
|
||||
}
|
||||
return jsonResp(http.StatusOK, tc.respBody), nil
|
||||
}))
|
||||
|
||||
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantRenamed, renamed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyFileSendsName(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_dst", body["folder_id"])
|
||||
assert.Equal(t, "final.txt", body["name"])
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
|
||||
}))
|
||||
|
||||
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_2", resp.FileID)
|
||||
assert.Equal(t, "final.txt", resp.Name)
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
|
||||
f := &Fs{}
|
||||
assert.Equal(t, time.Second, f.Precision())
|
||||
assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
|
||||
// Exactly SHA-256, nothing else.
|
||||
assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
|
||||
}
|
||||
|
||||
func TestObjectHashReturnsContentHash(t *testing.T) {
|
||||
o := &Object{file: api.FileItem{ContentHash: "abc123"}}
|
||||
|
||||
got, err := o.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "abc123", got)
|
||||
|
||||
// A multipart object with no server-side hash yields "" and no error,
|
||||
// exactly as an S3 multipart object does.
|
||||
empty := &Object{file: api.FileItem{ContentHash: ""}}
|
||||
got, err = empty.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", got)
|
||||
|
||||
// Any other algorithm is unsupported.
|
||||
_, err = o.Hash(context.Background(), hash.MD5)
|
||||
assert.ErrorIs(t, err, hash.ErrUnsupported)
|
||||
}
|
||||
|
||||
func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
|
||||
assert.True(t, mt.Equal(o.ModTime(context.Background())))
|
||||
}
|
||||
|
||||
func TestModTimeHeaders(t *testing.T) {
|
||||
assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
|
||||
h := modTimeHeaders(mt)
|
||||
require.NotNil(t, h)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
|
||||
}
|
||||
|
||||
func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
|
||||
resp := &api.UploadCompleteResponse{}
|
||||
resp.File.ID = "fil_1"
|
||||
resp.File.Name = "x.txt"
|
||||
resp.File.SizeBytes = 10
|
||||
resp.File.CreatedAt = 1_000
|
||||
resp.File.ContentHash = "deadbeef"
|
||||
|
||||
mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
|
||||
item := fileFromUpload(resp, mt)
|
||||
assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
|
||||
assert.Equal(t, "deadbeef", item.ContentHash)
|
||||
assert.Equal(t, float64(1_000), item.CreatedAt)
|
||||
|
||||
// A zero modtime falls back to the create time.
|
||||
item = fileFromUpload(resp, time.Time{})
|
||||
assert.Equal(t, float64(1_000), item.UpdatedAt)
|
||||
}
|
||||
|
||||
// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
|
||||
// fake transport and asserts the source-mtime header actually rides on the PUT.
|
||||
func TestUploadSendsSourceMTimeHeader(t *testing.T) {
|
||||
var putMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
|
||||
}
|
||||
switch {
|
||||
case r.Method == "POST" && r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
|
||||
putMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "h", resp.File.ContentHash)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
|
||||
// the completion request for a multipart upload (the server stores the modtime
|
||||
// there, not on the individual parts).
|
||||
func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
|
||||
var completeMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body)
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
|
||||
case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
case r.URL.Path == "/api/upload/upl_1/complete":
|
||||
completeMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
|
||||
// server-side Copy returns: it must report the SOURCE's modtime and hash (the
|
||||
// copy is byte-identical and the API preserves both), not the copy's wall
|
||||
// clock. Reading "now" here made rclone's in-memory object disagree with the
|
||||
// one read back from the remote and failed FsCopy's fingerprint check.
|
||||
func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case r.Method == "GET" && r.URL.Path == "/api/files":
|
||||
// NewObject's existence probe on the destination: nothing there.
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
|
||||
case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
|
||||
mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
|
||||
ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
|
||||
ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
|
||||
CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
|
||||
}}
|
||||
|
||||
dst, err := f.Copy(context.Background(), src, "f-copy.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
|
||||
assert.Equal(t, int64(100), dst.Size())
|
||||
got, err := dst.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "srchash", got, "copy keeps the source hash")
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
|
||||
|
||||
func TestErrorHandlerDecodesAPIError(t *testing.T) {
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Status: "400 Bad Request",
|
||||
Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
|
||||
assert.Contains(t, err.Error(), "uploads in progress")
|
||||
assert.Contains(t, err.Error(), "concurrent_upload_limit")
|
||||
}
|
||||
|
||||
func TestErrorHandlerNonJSONBody(t *testing.T) {
|
||||
// e.g. an edge (Cloudflare) response that never reached the API
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Status: "429 Too Many Requests",
|
||||
Body: io.NopCloser(strings.NewReader("error code: 1015\n")),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
|
||||
assert.Equal(t, "", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "429")
|
||||
assert.Contains(t, err.Error(), "error code: 1015")
|
||||
}
|
||||
|
||||
func TestShouldRetryAPIErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// The concurrent upload limit is handled by initUpload's slot wait,
|
||||
// not by the pacer's low level retries.
|
||||
limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
|
||||
retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
|
||||
assert.False(t, retry)
|
||||
assert.Equal(t, limitErr, err)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
|
||||
assert.True(t, retry)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
|
||||
assert.False(t, retry)
|
||||
}
|
||||
|
||||
// swapWaitTimings shortens the slot wait timings for a test and returns a
|
||||
// function which restores them
|
||||
func swapWaitTimings(delay, max time.Duration) func() {
|
||||
oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
|
||||
concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
|
||||
return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
|
||||
}
|
||||
|
||||
// newTestFs makes an Fs which talks to srv
|
||||
func newTestFs(srv *httptest.Server) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// limitServer answers the first n init calls with the concurrent upload
|
||||
// limit error and the rest with success
|
||||
func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/upload/init", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if calls.Add(1) <= n {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(concurrentLimitBody))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
}
|
||||
|
||||
func TestInitUploadWaitsForFreeSlot(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 2, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, resp.OK)
|
||||
assert.Equal(t, int32(3), calls.Load())
|
||||
}
|
||||
|
||||
func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "couldn't init upload")
|
||||
assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
|
||||
}
|
||||
|
||||
func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
|
||||
defer swapWaitTimings(time.Minute, time.Hour)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeListServer serves GET /api/files the way the API does: files paged at
|
||||
// per_page, every folder repeated on each page
|
||||
func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/files", r.URL.Path)
|
||||
q := r.URL.Query()
|
||||
assert.Equal(t, "name", q.Get("sort"))
|
||||
assert.Equal(t, "asc", q.Get("dir"))
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
perPage, _ := strconv.Atoi(q.Get("per_page"))
|
||||
require.Positive(t, page)
|
||||
require.Positive(t, perPage)
|
||||
*pages = append(*pages, page)
|
||||
|
||||
var files []string
|
||||
for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
|
||||
files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
|
||||
}
|
||||
totalPages := max(1, (totalFiles+perPage-1)/perPage)
|
||||
body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
|
||||
`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
|
||||
strings.Join(files, ","), page, perPage, totalFiles, totalPages)
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersFollowsPages(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1, 2, 3}, pages)
|
||||
require.Len(t, listing.Files, 2*listPageSize+7)
|
||||
assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
|
||||
// folders repeat on every page and must not be duplicated
|
||||
assert.Len(t, listing.Folders, 1)
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersSinglePage(t *testing.T) {
|
||||
for _, total := range []int{0, 1, listPageSize} {
|
||||
t.Run(strconv.Itoa(total), func(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, total, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), rootID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1}, pages)
|
||||
assert.Len(t, listing.Files, total)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// parallelUploadServer answers the multipart endpoints without holding a
|
||||
// lock across a request, so parts really do overlap, and records what the
|
||||
// ordering rules below are about: how many parts were in flight at once, and
|
||||
// whether any part started before part 1 had finished.
|
||||
//
|
||||
// Part 1 matters because the API creates the R2 multipart upload (and, for a
|
||||
// new file, its id and key) on the first part it sees. Parts that race ahead
|
||||
// of part 1 would each create their own upload, and only one of those would
|
||||
// be completed.
|
||||
type parallelUploadServer struct {
|
||||
t *testing.T
|
||||
resumable string
|
||||
partDelay time.Duration
|
||||
failPart int // answer this part with a 400, 0 for none
|
||||
mu sync.Mutex
|
||||
inFlight int
|
||||
maxFlight int
|
||||
part1Done bool
|
||||
earlyParts []int // parts that started before part 1 finished
|
||||
parts map[int]string
|
||||
completes int
|
||||
aborts int
|
||||
}
|
||||
|
||||
func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
|
||||
s.mu.Lock()
|
||||
if n != 1 && !s.part1Done {
|
||||
s.earlyParts = append(s.earlyParts, n)
|
||||
}
|
||||
s.inFlight++
|
||||
s.maxFlight = max(s.maxFlight, s.inFlight)
|
||||
s.mu.Unlock()
|
||||
|
||||
time.Sleep(s.partDelay)
|
||||
|
||||
s.mu.Lock()
|
||||
s.inFlight--
|
||||
if n == 1 {
|
||||
s.part1Done = true
|
||||
}
|
||||
s.parts[n] = string(body)
|
||||
fail := n == s.failPart
|
||||
s.mu.Unlock()
|
||||
|
||||
if fail {
|
||||
return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.mu.Lock()
|
||||
s.completes++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.mu.Lock()
|
||||
s.aborts++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
|
||||
return ¶llelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
|
||||
}
|
||||
|
||||
const tenParts = `{"part_size":2,"total_parts":10}`
|
||||
|
||||
func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
content := "aabbccddeeffgghhiijj"
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
|
||||
assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
for i := 1; i <= 10; i++ {
|
||||
assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 1
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, srv.maxFlight)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
srv.failPart = 6
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "part 6")
|
||||
assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
|
||||
// 20 bytes declared, 15 supplied: the short read is found by the
|
||||
// reader, whichever parts are already in flight, and nothing completes.
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Equal(t, 0, srv.completes)
|
||||
assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Test Dosya filesystem interface
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestDosya:",
|
||||
})
|
||||
}
|
||||
@@ -1,247 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// rtFunc adapts a function into an http.RoundTripper so the upload path
|
||||
// can be driven without a network. Unlike a real server it always reads
|
||||
// the whole request body and answers, which is exactly the misbehaviour
|
||||
// (accepting a short body as a complete upload) these tests guard against.
|
||||
type rtFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func jsonResp(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{"Content-Type": {"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
ContentLength: int64(len(body)),
|
||||
}
|
||||
}
|
||||
|
||||
func newFakeFs(rt http.RoundTripper) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// fakeUploadServer records what an upload sends and lets a test inject
|
||||
// failures per request
|
||||
type fakeUploadServer struct {
|
||||
t *testing.T
|
||||
mu sync.Mutex
|
||||
resumable string // JSON for the init "resumable" field, "" for a single PUT
|
||||
puts [][]byte // bodies of PUT /api/upload/<id>
|
||||
parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
|
||||
completes int
|
||||
aborts int // DELETE /api/upload/<id>
|
||||
failFirst map[string]bool // path -> answer the first request with a 502
|
||||
seenBefore map[string]bool
|
||||
}
|
||||
|
||||
func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
|
||||
return &fakeUploadServer{
|
||||
t: t,
|
||||
resumable: resumable,
|
||||
parts: map[int][][]byte{},
|
||||
failFirst: map[string]bool{},
|
||||
seenBefore: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
// Bodies are recorded for every attempt, including the ones answered
|
||||
// with a 502, so tests can see exactly what each attempt carried.
|
||||
fail := s.failFirst[path] && !s.seenBefore[path]
|
||||
s.seenBefore[path] = true
|
||||
bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
if s.resumable != "" {
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && path == "/api/upload/upl_1":
|
||||
s.puts = append(s.puts, body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
s.parts[n] = append(s.parts[n], body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.completes++
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.aborts++
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestUploadSmallSendsWholeBody(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := strings.Repeat("a", 100)
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
require.Len(t, srv.puts, 1)
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
func TestUploadSmallRejectsShortSource(t *testing.T) {
|
||||
// The source declares 100 bytes but only supplies 60. A server which
|
||||
// accepts that as a complete upload must not turn into a success.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
|
||||
}
|
||||
|
||||
func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
|
||||
// A retried PUT would resend an already-consumed reader, i.e. an
|
||||
// empty body, which the server would store as an empty file. The
|
||||
// single PUT therefore must not be retried at the low level.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
srv.failFirst["/api/upload/upl_1"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
content := strings.Repeat("a", 100)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
const threeParts = `{"part_size":4,"total_parts":3}`
|
||||
|
||||
func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := "0123456789"
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
require.Len(t, srv.parts, 3)
|
||||
assert.Equal(t, "0123", string(srv.parts[1][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
}
|
||||
|
||||
func TestUploadMultipartRejectsShortSource(t *testing.T) {
|
||||
// 10 bytes declared, 8 supplied: the last part must never be sent
|
||||
// and the upload must not be completed.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
|
||||
assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
|
||||
assert.Equal(t, 0, srv.completes, "upload must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
|
||||
// A part which fails transiently must be resent with all its bytes,
|
||||
// not with whatever is left of a consumed reader.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
srv.failFirst["/api/upload/upl_1/part/2"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][1]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
// An upload that fails after init has to hand its session back: the session
|
||||
// holds one of the workspace's upload slots for ten minutes otherwise, and
|
||||
// rclone's high-level retry opens a new one rather than resuming it.
|
||||
func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
resumable string
|
||||
content string
|
||||
size int64
|
||||
}{
|
||||
{"small file short source", "", "0123", 10},
|
||||
{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, tc.resumable)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
|
||||
"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, srv.aborts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A successful upload must not abort anything
|
||||
func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
|
||||
"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, srv.aborts)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Every by-id call names the workspace, so the API can find an id it minted
|
||||
// moments ago before its file_home index row lands.
|
||||
func TestByIDCallsSendWorkspaceHint(t *testing.T) {
|
||||
seen := map[string]string{}
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
|
||||
switch r.URL.Path {
|
||||
case "/api/folders/fld_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
case "/api/files/fil_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
|
||||
case "/api/folders/fld_1/move":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
|
||||
default:
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
}))
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.removeFolder(ctx, "fld_1", false))
|
||||
require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
|
||||
_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.deleteFile(ctx, "fil_1"))
|
||||
require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
|
||||
require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
|
||||
_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
_, err = f.createShareLink(ctx, "fil_1", 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
for call, header := range seen {
|
||||
assert.Equal(t, "ws_test", header, call)
|
||||
}
|
||||
assert.Len(t, seen, 8)
|
||||
}
|
||||
|
||||
// The download's API request carries the hint; the presigned R2 fetch must not.
|
||||
func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
|
||||
var apiHint, blobHint string
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/files/fil_1/download" {
|
||||
apiHint = r.Header.Get("X-Dosya-Workspace")
|
||||
http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
|
||||
return
|
||||
}
|
||||
blobHint = r.Header.Get("X-Dosya-Workspace")
|
||||
_, _ = w.Write([]byte("bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
f.opt.APIURL = srv.URL
|
||||
|
||||
rc, err := f.downloadFile(context.Background(), "fil_1", nil)
|
||||
require.NoError(t, err)
|
||||
_ = rc.Close()
|
||||
|
||||
assert.Equal(t, "ws_test", apiHint)
|
||||
assert.Equal(t, "", blobHint)
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeWorkspaceServer answers GET /api/workspaces with the given body
|
||||
func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/workspaces", r.URL.Path)
|
||||
*calls++
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.NoError(t, f.findWorkspace(context.Background()))
|
||||
assert.Equal(t, "ws_only", f.opt.WorkspaceID)
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceListsTheChoices(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
|
||||
`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
err := f.findWorkspace(context.Background())
|
||||
require.Error(t, err)
|
||||
// the ids have to be in the message: they are the only place the user
|
||||
// can read them, which is the whole point of listing them
|
||||
assert.Contains(t, err.Error(), "ws_one (One)")
|
||||
assert.Contains(t, err.Error(), "ws_two (Two)")
|
||||
assert.Empty(t, f.opt.WorkspaceID)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceWithNone(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
|
||||
}
|
||||
@@ -321,13 +321,10 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
|
||||
}
|
||||
return false
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return info, nil
|
||||
return info, err
|
||||
}
|
||||
|
||||
// errorHandler parses a non 2xx error response into an error
|
||||
@@ -549,11 +546,6 @@ OUTER:
|
||||
if err != nil {
|
||||
return found, fmt.Errorf("couldn't list files: %w", err)
|
||||
}
|
||||
// Drime won't page past a fixed number of entries (20,000 for
|
||||
// rclone) and serves the last page it allows again instead
|
||||
if result.CurrentPage != page {
|
||||
return found, errors.New("couldn't list files: directory has more entries than drime will list")
|
||||
}
|
||||
for _, item := range result.Data {
|
||||
if item.Type == api.ItemTypeFolder {
|
||||
if filesOnly {
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
package drime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/drime/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testPerPage = 5 // entries per page the test server returns
|
||||
testLimit = 20 // entries the test server will page through
|
||||
)
|
||||
|
||||
// listingServer serves /drive/file-entries from entries the way Drime
|
||||
// does.
|
||||
//
|
||||
// It returns testPerPage entries per page and won't page past testLimit
|
||||
// entries. Asking for a later page returns the last page it allows
|
||||
// again, with last_page one beyond it.
|
||||
//
|
||||
// It counts the requests made in requests.
|
||||
func listingServer(t *testing.T, entries []api.Item, requests *atomic.Int64) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
assert.Equal(t, "/drive/file-entries", r.URL.Path)
|
||||
params := r.URL.Query()
|
||||
page, err := strconv.Atoi(params.Get("page"))
|
||||
require.NoError(t, err)
|
||||
maxPage := testLimit / testPerPage
|
||||
currentPage := min(page, maxPage)
|
||||
start := min((currentPage-1)*testPerPage, len(entries))
|
||||
end := min(start+testPerPage, len(entries))
|
||||
lastPage := currentPage
|
||||
if page > maxPage || end < len(entries) {
|
||||
lastPage = currentPage + 1
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
|
||||
"current_page": currentPage,
|
||||
"last_page": lastPage,
|
||||
"per_page": testPerPage,
|
||||
"total": (currentPage + 1) * testPerPage,
|
||||
"data": entries[start:end],
|
||||
}))
|
||||
}))
|
||||
}
|
||||
|
||||
// makeEntries makes n file entries
|
||||
func makeEntries(n int) []api.Item {
|
||||
entries := make([]api.Item, n)
|
||||
for i := range entries {
|
||||
entries[i] = api.Item{
|
||||
ID: json.Number(strconv.Itoa(1000 + i)),
|
||||
Name: fmt.Sprintf("file%05d", i),
|
||||
Type: "text",
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// testFs makes an Fs which uses server as its API
|
||||
func testFs(t *testing.T, server *httptest.Server) *Fs {
|
||||
t.Helper()
|
||||
f := &Fs{
|
||||
opt: Options{ListChunk: 1000},
|
||||
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
|
||||
}
|
||||
f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
|
||||
f.srv.SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// listNames lists a directory with listAll and returns the names found
|
||||
func listNames(ctx context.Context, f *Fs) (names []string, err error) {
|
||||
_, err = f.listAll(ctx, "1", false, false, "", func(item *api.Item) bool {
|
||||
names = append(names, item.Name)
|
||||
return false
|
||||
})
|
||||
return names, err
|
||||
}
|
||||
|
||||
// TestListAll checks that listAll returns every entry exactly once
|
||||
func TestListAll(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
entries int
|
||||
}{
|
||||
{name: "Empty", entries: 0},
|
||||
{name: "OnePage", entries: 3},
|
||||
{name: "ExactPages", entries: 10},
|
||||
{name: "PartialLastPage", entries: 13},
|
||||
{name: "AtLimit", entries: testLimit},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
var requests atomic.Int64
|
||||
entries := makeEntries(test.entries)
|
||||
server := listingServer(t, entries, &requests)
|
||||
defer server.Close()
|
||||
f := testFs(t, server)
|
||||
|
||||
names, err := listNames(ctx, f)
|
||||
require.NoError(t, err)
|
||||
var want []string
|
||||
for _, entry := range entries {
|
||||
want = append(want, entry.Name)
|
||||
}
|
||||
assert.ElementsMatch(t, want, names)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestListAllTooManyEntries checks that listAll returns an error for a
|
||||
// directory with more entries than the server will list, rather than
|
||||
// looping forever.
|
||||
func TestListAllTooManyEntries(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
var requests atomic.Int64
|
||||
server := listingServer(t, makeEntries(2*testLimit), &requests)
|
||||
defer server.Close()
|
||||
f := testFs(t, server)
|
||||
|
||||
_, err := listNames(ctx, f)
|
||||
require.Error(t, err)
|
||||
assert.NoError(t, ctx.Err(), "listing didn't finish")
|
||||
assert.LessOrEqual(t, requests.Load(), int64(testLimit/testPerPage+1))
|
||||
}
|
||||
|
||||
// TestGetItemListError checks that getItem returns listing errors
|
||||
// rather than fs.ErrorObjectNotFound.
|
||||
func TestGetItemListError(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"message":"access denied"}`, http.StatusForbidden)
|
||||
}))
|
||||
defer server.Close()
|
||||
f := testFs(t, server)
|
||||
|
||||
_, err := f.getItem(context.Background(), "1000", "1", "file00000")
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
assert.Contains(t, err.Error(), "access denied")
|
||||
}
|
||||
+9
-22
@@ -568,7 +568,7 @@ recommended usage is using the flag form --drive-size-as-quota when
|
||||
doing rclone ls/lsl/lsf/lsjson/etc only.
|
||||
|
||||
If you do use this flag for syncing (not recommended) then you will
|
||||
need to use --ignore-size also.`,
|
||||
need to use --ignore size also.`,
|
||||
Advanced: true,
|
||||
Hide: fs.OptionHideConfigurator,
|
||||
}, {
|
||||
@@ -2298,9 +2298,15 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
wg.Wait()
|
||||
// if the input channel overflowed add the collected entries to the channel now
|
||||
for len(overflow) > 0 {
|
||||
if !refillListRInput(&mu, &in, &overflow, &wg) {
|
||||
return
|
||||
mu.Lock()
|
||||
// only fill half of the channel to prevent entries being put into overflow again
|
||||
l := min(len(overflow), listRInputBuffer/2)
|
||||
wg.Add(l)
|
||||
for _, d := range overflow[:l] {
|
||||
in <- d
|
||||
}
|
||||
overflow = overflow[l:]
|
||||
mu.Unlock()
|
||||
|
||||
// wait again for the completion of all directories
|
||||
wg.Wait()
|
||||
@@ -2348,25 +2354,6 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
return nil
|
||||
}
|
||||
|
||||
// refillListRInput moves queued directory jobs back onto in.
|
||||
// It returns false when a listing error has already closed in.
|
||||
// Sending on that nil channel would block forever while holding mu.
|
||||
func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if *in == nil {
|
||||
return false
|
||||
}
|
||||
// only fill half of the channel to prevent entries being put into overflow again
|
||||
l := min(len(*overflow), listRInputBuffer/2)
|
||||
wg.Add(l)
|
||||
for _, d := range (*overflow)[:l] {
|
||||
*in <- d
|
||||
}
|
||||
*overflow = (*overflow)[l:]
|
||||
return true
|
||||
}
|
||||
|
||||
const shortcutSeparator = '\t'
|
||||
|
||||
// joinID adds an actual drive ID to the shortcut ID it came from
|
||||
|
||||
@@ -8,14 +8,10 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
stdsync "sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -29,113 +25,13 @@ import (
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
service, err := drive.NewService(
|
||||
context.Background(),
|
||||
option.WithHTTPClient(server.Client()),
|
||||
option.WithEndpoint(server.URL+"/"),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return &Fs{
|
||||
svc: service,
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
permissionsMu: new(stdsync.Mutex),
|
||||
permissions: make(map[string]*drive.Permission),
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionConcurrent(t *testing.T) {
|
||||
const requestCount = 2
|
||||
arrived := make(chan string, requestCount)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce stdsync.Once
|
||||
releaseRequests := func() {
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
}
|
||||
defer releaseRequests()
|
||||
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
arrived <- permissionID
|
||||
<-release
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
type permissionResult struct {
|
||||
permission *drive.Permission
|
||||
inherited bool
|
||||
err error
|
||||
}
|
||||
results := make(chan permissionResult, requestCount)
|
||||
permissionIDs := []string{"permission-1", "permission-2"}
|
||||
for _, permissionID := range permissionIDs {
|
||||
go func() {
|
||||
permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
|
||||
results <- permissionResult{permission: permission, inherited: inherited, err: err}
|
||||
}()
|
||||
}
|
||||
|
||||
arrivedIDs := make(map[string]bool, requestCount)
|
||||
timer := time.NewTimer(time.Second)
|
||||
defer timer.Stop()
|
||||
for len(arrivedIDs) < requestCount {
|
||||
select {
|
||||
case permissionID := <-arrived:
|
||||
arrivedIDs[permissionID] = true
|
||||
case <-timer.C:
|
||||
releaseRequests()
|
||||
t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
|
||||
}
|
||||
}
|
||||
releaseRequests()
|
||||
|
||||
assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
|
||||
returnedIDs := make(map[string]bool, requestCount)
|
||||
for range requestCount {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
require.NotNil(t, result.permission)
|
||||
assert.False(t, result.inherited)
|
||||
returnedIDs[result.permission.Id] = true
|
||||
}
|
||||
assert.Equal(t, arrivedIDs, returnedIDs)
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionCacheHit(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
|
||||
assert.Same(t, first, second)
|
||||
assert.Equal(t, int32(1), requests.Load())
|
||||
}
|
||||
|
||||
func TestDriveScopes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
@@ -916,51 +812,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {
|
||||
mu := stdsync.Mutex{}
|
||||
wg := stdsync.WaitGroup{}
|
||||
in := make(chan listREntry, listRInputBuffer)
|
||||
overflow := []listREntry{{id: "folder", path: "folder"}}
|
||||
|
||||
mu.Lock()
|
||||
close(in)
|
||||
in = nil
|
||||
mu.Unlock()
|
||||
|
||||
done := make(chan bool, 1)
|
||||
go func() {
|
||||
done <- refillListRInput(&mu, &in, &overflow, &wg)
|
||||
}()
|
||||
select {
|
||||
case ok := <-done:
|
||||
require.False(t, ok, "refill continued after the input channel was closed")
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("refill blocked after the input channel was closed")
|
||||
}
|
||||
require.Len(t, overflow, 1)
|
||||
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(finished)
|
||||
}()
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("refill incremented the wait group after the input channel was closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {
|
||||
mu := stdsync.Mutex{}
|
||||
wg := stdsync.WaitGroup{}
|
||||
in := make(chan listREntry, listRInputBuffer)
|
||||
overflow := []listREntry{{id: "folder", path: "folder"}}
|
||||
|
||||
require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")
|
||||
require.Empty(t, overflow)
|
||||
got := <-in
|
||||
require.Equal(t, listREntry{id: "folder", path: "folder"}, got)
|
||||
wg.Done()
|
||||
}
|
||||
@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
|
||||
|
||||
// getPermission returns permissions for the fileID and permissionID passed in
|
||||
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
|
||||
f.permissionsMu.Lock()
|
||||
defer f.permissionsMu.Unlock()
|
||||
if useCache {
|
||||
f.permissionsMu.Lock()
|
||||
perm = f.permissions[permissionID]
|
||||
f.permissionsMu.Unlock()
|
||||
if perm != nil {
|
||||
return perm, false, nil
|
||||
}
|
||||
@@ -131,9 +131,7 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
|
||||
cleanPermission(perm)
|
||||
|
||||
// cache the permission
|
||||
f.permissionsMu.Lock()
|
||||
f.permissions[permissionID] = perm
|
||||
f.permissionsMu.Unlock()
|
||||
|
||||
return perm, inherited, err
|
||||
}
|
||||
|
||||
+18
-30
@@ -11,6 +11,7 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -21,7 +22,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
@@ -112,13 +112,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
|
||||
}
|
||||
|
||||
// Make an http.Request for the range passed in
|
||||
//
|
||||
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
|
||||
// to an io.Closer and close it after each attempt — the chunk buffer is
|
||||
// pool-backed, so an early close would free its pages and a retry would then
|
||||
// read a dead buffer. The upload loop owns the buffer's lifetime.
|
||||
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
|
||||
req.ContentLength = reqSize
|
||||
totalSize := "*"
|
||||
if rx.ContentLength >= 0 {
|
||||
@@ -173,40 +168,37 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
start := int64(0)
|
||||
var StatusCode int
|
||||
var err error
|
||||
chunkSize := int64(rx.f.opt.ChunkSize)
|
||||
buf := make([]byte, int(rx.f.opt.ChunkSize))
|
||||
for finished := false; !finished; {
|
||||
reqSize := chunkSize
|
||||
var reqSize int64
|
||||
var chunk io.ReadSeeker
|
||||
if rx.ContentLength >= 0 {
|
||||
// If size known use repeatable reader for smoother bwlimit
|
||||
if start >= rx.ContentLength {
|
||||
break
|
||||
}
|
||||
reqSize = min(rx.ContentLength-start, chunkSize)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool so reads are
|
||||
// repeatable for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, rx.Media, reqSize)
|
||||
if rx.ContentLength < 0 {
|
||||
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
|
||||
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
|
||||
} else {
|
||||
// If size unknown read into buffer
|
||||
var n int
|
||||
n, err = readers.ReadFill(rx.Media, buf)
|
||||
if err == io.EOF {
|
||||
// Send the last chunk with the correct ContentLength
|
||||
// otherwise Google doesn't know we've finished
|
||||
rx.ContentLength = start + n
|
||||
rx.ContentLength = start + int64(n)
|
||||
finished = true
|
||||
err = nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqSize = n
|
||||
}
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
reqSize = int64(n)
|
||||
chunk = bytes.NewReader(buf[:reqSize])
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
err = rx.f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
|
||||
again, err := rx.f.shouldRetry(ctx, err)
|
||||
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
|
||||
again = false
|
||||
@@ -214,13 +206,9 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
}
|
||||
return again, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
|
||||
start += reqSize
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
|
||||
// retried successfully. The chunk buffer is pool-backed and implements
|
||||
// io.Closer, so if it reaches the http transport unwrapped the transport
|
||||
// closes it after the failed attempt, returning its pages to the pool, and
|
||||
// the retry then reads a freed buffer.
|
||||
func TestResumableUploadRetry(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("resumable"), 512)
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
if attempts.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
client: server.Client(),
|
||||
}
|
||||
f.opt.ChunkSize = fs.SizeSuffix(len(content))
|
||||
rx := &resumableUpload{
|
||||
f: f,
|
||||
remote: "remote",
|
||||
URI: server.URL,
|
||||
Media: bytes.NewReader(content),
|
||||
MediaType: "application/octet-stream",
|
||||
ContentLength: int64(len(content)),
|
||||
}
|
||||
info, err := rx.Upload(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, info)
|
||||
assert.Equal(t, "fake-id", info.Id)
|
||||
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
|
||||
}
|
||||
+10
-17
@@ -51,7 +51,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -2083,6 +2082,11 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
|
||||
// write chunks
|
||||
in := readers.NewCountingReader(in0)
|
||||
bufSize := chunkSize
|
||||
if size >= 0 && size < bufSize {
|
||||
bufSize = size
|
||||
}
|
||||
buf := make([]byte, int(bufSize))
|
||||
cursor := files.UploadSessionCursor{
|
||||
SessionId: res.SessionId,
|
||||
Offset: 0,
|
||||
@@ -2102,21 +2106,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
}
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
|
||||
skip := int64(0)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// seek to the start in case this is a retry
|
||||
if _, err = rw.Seek(skip, io.SeekStart); err != nil {
|
||||
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, chunk)
|
||||
// after session is started, we retry everything
|
||||
if err != nil {
|
||||
// Check for incorrect offset error and retry with new offset
|
||||
@@ -2128,10 +2125,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
|
||||
if skip < 0 {
|
||||
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
|
||||
} else if skip == n {
|
||||
} else if skip == chunkSize {
|
||||
fs.Debugf(o, "%s: chunk received OK - continuing", what)
|
||||
return false, nil
|
||||
} else if skip > n {
|
||||
} else if skip > chunkSize {
|
||||
// This error should never happen
|
||||
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
|
||||
}
|
||||
@@ -2148,13 +2145,9 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
return err != nil, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if size >= 0 {
|
||||
// Check for sources which truncate early
|
||||
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
|
||||
|
||||
@@ -16,16 +16,13 @@ import (
|
||||
|
||||
// multipartInit starts a new multipart upload and returns server details.
|
||||
func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
|
||||
encodedFolderPath := f.fromStandardPath(folderPath)
|
||||
encodedFilename := f.fromStandardPath(filename)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/multipart/init",
|
||||
Parameters: url.Values{
|
||||
"key": {f.opt.Key},
|
||||
"filename": {encodedFilename},
|
||||
"folder_path": {encodedFolderPath},
|
||||
"filename": {filename},
|
||||
"folder_path": {folderPath},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -10,11 +11,9 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
rclonemultipart "github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -48,32 +47,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
server := initResp.Result.Server
|
||||
objectPath := initResp.Result.ObjectPath
|
||||
|
||||
chunkSize := int64(f.opt.ChunkSize)
|
||||
if chunkSize <= 0 {
|
||||
return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)
|
||||
}
|
||||
for partNo := 1; ; partNo++ {
|
||||
// Buffer the part in memory from the global pool so it can be
|
||||
// re-sent on retry
|
||||
rw := rclonemultipart.NewRW()
|
||||
n, err := io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("read failed: %w", err)
|
||||
}
|
||||
chunkSize := int(f.opt.ChunkSize)
|
||||
buf := make([]byte, 0, chunkSize)
|
||||
tmp := make([]byte, 1024*1024)
|
||||
partNo := 1
|
||||
|
||||
for {
|
||||
n, errRead := in.Read(tmp)
|
||||
if n > 0 {
|
||||
uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)
|
||||
if uploadErr != nil {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)
|
||||
buf = append(buf, tmp[:n]...)
|
||||
|
||||
// If buffer reached chunkSize, upload a full part
|
||||
if len(buf) >= chunkSize {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
partNo++
|
||||
buf = buf[:0]
|
||||
}
|
||||
}
|
||||
if closeErr := rw.Close(); closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if err == io.EOF {
|
||||
|
||||
if errRead == io.EOF {
|
||||
break
|
||||
}
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read failed: %w", errRead)
|
||||
}
|
||||
}
|
||||
|
||||
if len(buf) > 0 {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
|
||||
@@ -84,40 +91,31 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadPart sends a single multipart chunk of size bytes to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: server,
|
||||
Parameters: url.Values{
|
||||
"partNumber": {strconv.Itoa(partNo)},
|
||||
"uploadId": {uploadID},
|
||||
},
|
||||
Body: r,
|
||||
ContentLength: &size,
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-RC-Upload-Id": uploadID,
|
||||
"X-RC-Part-No": strconv.Itoa(partNo),
|
||||
"X-Sess-ID": sessID,
|
||||
"X-Object-Path": objectPath,
|
||||
},
|
||||
NoResponse: true,
|
||||
// uploadPart sends a single multipart chunk to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
|
||||
url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
// rewind the pooled buffer so each attempt sends the whole chunk
|
||||
_, err := r.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)
|
||||
}
|
||||
return shouldRetry(err), err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
req.Header.Set("X-RC-Upload-Id", uploadID)
|
||||
req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
|
||||
req.Header.Set("X-Sess-ID", sessID)
|
||||
req.Header.Set("X-Object-Path", objectPath)
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("uploadPart failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFile uploads a file to FileLu
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
@@ -12,8 +12,5 @@ func TestIntegration(t *testing.T) {
|
||||
RemoteName: "TestFileLu:",
|
||||
NilObject: nil,
|
||||
SkipInvalidUTF8: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: 1,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -353,7 +353,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
|
||||
for it.Next() {
|
||||
item := new(it.File())
|
||||
item := ptr(it.File())
|
||||
remote := f.opt.Enc.ToStandardPath(item.DisplayName)
|
||||
remote = path.Join(dir, remote)
|
||||
if remote == dir {
|
||||
@@ -424,7 +424,7 @@ func (f *Fs) mkdir(ctx context.Context, path string) error {
|
||||
|
||||
params := files_sdk.FolderCreateParams{
|
||||
Path: path,
|
||||
MkdirParents: new(true),
|
||||
MkdirParents: ptr(true),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -466,7 +466,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
|
||||
params := files_sdk.FileDeleteParams{
|
||||
Path: path,
|
||||
Recursive: new(!check),
|
||||
Recursive: ptr(!check),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -537,7 +537,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.
|
||||
params := files_sdk.FileCopyParams{
|
||||
Path: srcPath,
|
||||
Destination: dstPath,
|
||||
Overwrite: new(true),
|
||||
Overwrite: ptr(true),
|
||||
}
|
||||
|
||||
var action files_sdk.FileAction
|
||||
@@ -680,7 +680,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
Paths: []string{f.absPath(remote)},
|
||||
}
|
||||
if expire < fs.DurationOff {
|
||||
params.ExpiresAt = new(time.Now().Add(time.Duration(expire)))
|
||||
params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
|
||||
}
|
||||
|
||||
var bundle files_sdk.Bundle
|
||||
@@ -842,6 +842,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
func isFolderNotEmpty(err error) bool {
|
||||
var re files_sdk.ResponseError
|
||||
ok := errors.As(err, &re)
|
||||
|
||||
+14
-32
@@ -584,9 +584,8 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
|
||||
//
|
||||
// It nils the pointed to connection out so it can't be reused
|
||||
//
|
||||
// if err is not nil then it discards the connection unless the error
|
||||
// carries an FTP reply code, in which case it checks the connection
|
||||
// is alive using a NOOP request
|
||||
// if err is not nil then it checks the connection is alive using a
|
||||
// NOOP request
|
||||
func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
|
||||
if f.opt.Concurrency > 0 {
|
||||
defer f.tokens.Put()
|
||||
@@ -600,17 +599,14 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
|
||||
}
|
||||
*pc = nil
|
||||
if err != nil {
|
||||
// An error without an FTP reply code means the control
|
||||
// connection can no longer be trusted, so discard it.
|
||||
if textprotoError(err) == nil {
|
||||
fs.Debugf(f, "Discarding connection after non-FTP error: %v", err)
|
||||
_ = c.Quit()
|
||||
return
|
||||
}
|
||||
if nopErr := c.NoOp(); nopErr != nil {
|
||||
fs.Debugf(f, "Connection failed, closing: %v", nopErr)
|
||||
_ = c.Quit()
|
||||
return
|
||||
// If not a regular FTP error code then check the connection
|
||||
if tpErr := textprotoError(err); tpErr != nil {
|
||||
nopErr := c.NoOp()
|
||||
if nopErr != nil {
|
||||
fs.Debugf(f, "Connection failed, closing: %v", nopErr)
|
||||
_ = c.Quit()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
f.poolMu.Lock()
|
||||
@@ -1269,18 +1265,15 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// ftpReadCloser implements io.ReadCloser for FTP objects.
|
||||
type ftpReadCloser struct {
|
||||
rc io.ReadCloser
|
||||
c *ftp.ServerConn
|
||||
f *Fs
|
||||
err error // errors found during read
|
||||
closed bool // set by Close so a second call is a no-op
|
||||
read int64 // bytes returned by Read
|
||||
rc io.ReadCloser
|
||||
c *ftp.ServerConn
|
||||
f *Fs
|
||||
err error // errors found during read
|
||||
}
|
||||
|
||||
// Read bytes into p
|
||||
func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
|
||||
n, err = f.rc.Read(p)
|
||||
f.read += int64(n)
|
||||
if err != nil && err != io.EOF {
|
||||
f.err = err // store any errors for Close to examine
|
||||
}
|
||||
@@ -1289,10 +1282,6 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
|
||||
|
||||
// Close the FTP reader and return the connection to the pool
|
||||
func (f *ftpReadCloser) Close() error {
|
||||
if f.closed {
|
||||
return nil
|
||||
}
|
||||
f.closed = true
|
||||
var err error
|
||||
errchan := make(chan error, 1)
|
||||
go func() {
|
||||
@@ -1316,7 +1305,6 @@ func (f *ftpReadCloser) Close() error {
|
||||
// if errors while reading or closing, dump the connection
|
||||
if err != nil || f.err != nil {
|
||||
_ = f.c.Quit()
|
||||
f.c = nil // never pool a connection after Quit
|
||||
f.f.putFtpConnection(nil, nil)
|
||||
} else {
|
||||
f.f.putFtpConnection(&f.c, nil)
|
||||
@@ -1328,12 +1316,6 @@ func (f *ftpReadCloser) Close() error {
|
||||
switch errX.Code {
|
||||
case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend, ftp.StatusRequestedFileActionOK:
|
||||
err = nil
|
||||
case ftp.StatusCanNotOpenDataConnection:
|
||||
// Egnyte answers 425 when a RETR is closed before the first
|
||||
// read; a transfer cut after data was read stays an error
|
||||
if f.read == 0 {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return err
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
@@ -50,7 +43,7 @@ func TestCommandEncodingNeutralisesCRLF(t *testing.T) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
@@ -151,159 +144,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// fakeFTPServer is a minimal FTP control-connection server which answers
|
||||
// every command with 200 OK.
|
||||
type fakeFTPServer struct {
|
||||
ln net.Listener
|
||||
mu sync.Mutex
|
||||
conn net.Conn
|
||||
}
|
||||
|
||||
func startFakeFTPServer(t *testing.T) *fakeFTPServer {
|
||||
t.Helper()
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
s := &fakeFTPServer{ln: ln}
|
||||
go s.serve()
|
||||
t.Cleanup(func() { s.close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *fakeFTPServer) serve() {
|
||||
conn, err := s.ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.conn = conn
|
||||
s.mu.Unlock()
|
||||
defer func() { _ = conn.Close() }()
|
||||
_, _ = fmt.Fprint(conn, "220 fake FTP server\r\n")
|
||||
scanner := bufio.NewScanner(conn)
|
||||
for scanner.Scan() {
|
||||
_, _ = fmt.Fprint(conn, "200 OK\r\n")
|
||||
}
|
||||
}
|
||||
|
||||
// killConnection closes the server side of the control connection,
|
||||
// simulating a server that dropped an idle connection.
|
||||
func (s *fakeFTPServer) killConnection() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.conn != nil {
|
||||
_ = s.conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeFTPServer) close() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.conn != nil {
|
||||
_ = s.conn.Close()
|
||||
}
|
||||
_ = s.ln.Close()
|
||||
}
|
||||
|
||||
// TestPutFtpConnectionDiscardsDeadConnection checks that a connection
|
||||
// returned with a raw network error is closed and never re-pooled, while
|
||||
// a connection returned with a plain FTP reply-code error is probed with
|
||||
// NOOP and pooled while the probe succeeds.
|
||||
func TestPutFtpConnectionDiscardsDeadConnection(t *testing.T) {
|
||||
t.Run("network error discards", func(t *testing.T) {
|
||||
server := startFakeFTPServer(t)
|
||||
c, err := ftp.Dial(server.ln.Addr().String())
|
||||
require.NoError(t, err)
|
||||
f := &Fs{opt: Options{}}
|
||||
f.putFtpConnection(&c, errors.New("unexpected EOF"))
|
||||
assert.Nil(t, c)
|
||||
assert.Empty(t, f.pool)
|
||||
})
|
||||
|
||||
t.Run("FTP reply error with live connection pools", func(t *testing.T) {
|
||||
server := startFakeFTPServer(t)
|
||||
c, err := ftp.Dial(server.ln.Addr().String())
|
||||
require.NoError(t, err)
|
||||
f := &Fs{opt: Options{}}
|
||||
f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
|
||||
assert.Nil(t, c)
|
||||
require.Len(t, f.pool, 1)
|
||||
assert.NoError(t, f.pool[0].NoOp())
|
||||
_ = f.pool[0].Quit()
|
||||
})
|
||||
|
||||
t.Run("FTP reply error with dead connection discards", func(t *testing.T) {
|
||||
server := startFakeFTPServer(t)
|
||||
c, err := ftp.Dial(server.ln.Addr().String())
|
||||
require.NoError(t, err)
|
||||
server.killConnection()
|
||||
// Let the FIN propagate so the NOOP probe observes the dead connection.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
f := &Fs{opt: Options{}}
|
||||
f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
|
||||
assert.Nil(t, c)
|
||||
assert.Empty(t, f.pool)
|
||||
})
|
||||
}
|
||||
|
||||
// fakeDataReader stands in for the RETR data stream of ftpReadCloser.
|
||||
type fakeDataReader struct {
|
||||
data []byte
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (r *fakeDataReader) Read(p []byte) (int, error) {
|
||||
if len(r.data) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, r.data)
|
||||
r.data = r.data[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *fakeDataReader) Close() error { return r.closeErr }
|
||||
|
||||
// TestFtpReadCloserClose checks that an early-closed RETR answered with 425
|
||||
// is masked only when nothing was read, that a connection which got QUIT is
|
||||
// never pooled, and that a second Close does not touch the pool.
|
||||
func TestFtpReadCloserClose(t *testing.T) {
|
||||
err425 := &textproto.Error{Code: ftp.StatusCanNotOpenDataConnection, Msg: "Can't open data connection."}
|
||||
newReader := func(t *testing.T, data []byte, closeErr error) (*ftpReadCloser, *Fs) {
|
||||
server := startFakeFTPServer(t)
|
||||
c, err := ftp.Dial(server.ln.Addr().String())
|
||||
require.NoError(t, err)
|
||||
f := &Fs{opt: Options{}}
|
||||
return &ftpReadCloser{rc: &fakeDataReader{data: data, closeErr: closeErr}, c: c, f: f}, f
|
||||
}
|
||||
|
||||
t.Run("425 before any read is masked and the connection is not pooled", func(t *testing.T) {
|
||||
rc, f := newReader(t, nil, err425)
|
||||
assert.NoError(t, rc.Close())
|
||||
assert.Empty(t, f.pool)
|
||||
assert.NoError(t, rc.Close())
|
||||
assert.Empty(t, f.pool)
|
||||
})
|
||||
|
||||
t.Run("425 after data was read stays an error", func(t *testing.T) {
|
||||
rc, f := newReader(t, []byte("partial"), err425)
|
||||
buf := make([]byte, 4)
|
||||
n, err := rc.Read(buf)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 4, n)
|
||||
err = rc.Close()
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, ftp.StatusCanNotOpenDataConnection, textprotoError(err).Code)
|
||||
assert.Empty(t, f.pool)
|
||||
})
|
||||
|
||||
t.Run("clean close pools the connection once", func(t *testing.T) {
|
||||
rc, f := newReader(t, []byte("data"), nil)
|
||||
_, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, rc.Close())
|
||||
require.Len(t, f.pool, 1)
|
||||
assert.NoError(t, rc.Close())
|
||||
assert.Len(t, f.pool, 1)
|
||||
_ = f.pool[0].Quit()
|
||||
})
|
||||
}
|
||||
@@ -194,7 +194,8 @@ var retryErrorCodes = []int{
|
||||
|
||||
// Return true if the api error has the status given
|
||||
func isAPIErr(err error, status string) bool {
|
||||
if apiErr, ok := errors.AsType[api.Error](err); ok {
|
||||
var apiErr api.Error
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.Status == status
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -122,49 +122,6 @@ func (f *Fs) testUpdateStoresHash(t *testing.T) {
|
||||
_ = operations.Purge(ctx, f, dirName)
|
||||
}
|
||||
|
||||
func (f *Fs) testSetModTimeKeepsHash(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
const dirName = "setmodtime_hash_1"
|
||||
const fileName = dirName + "/file_setmodtime_1"
|
||||
const longTime = fs.ModTimeNotSupported
|
||||
hashType := f.keepHashes.GetOne()
|
||||
|
||||
// upload a file to hasher via Put
|
||||
src := putFile(ctx, t, f, fileName, "setmodtime content")
|
||||
dst, err := f.NewObject(ctx, fileName)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, dst)
|
||||
_ = src
|
||||
|
||||
// verify hash was stored after Put
|
||||
if f.opt.MaxAge <= 0 {
|
||||
t.Skip("hash cache is disabled")
|
||||
}
|
||||
hash1, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, hash1)
|
||||
|
||||
// change only the modification time, as operations does when it
|
||||
// syncs the modtime of an otherwise unchanged destination file
|
||||
newTime := fstest.Time("2002-03-04T05:06:07.499999999Z")
|
||||
require.NoError(t, dst.SetModTime(ctx, newTime))
|
||||
|
||||
// the cached hash must survive a pure modtime change
|
||||
hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, hash1, hash2, "hash should be kept after SetModTime")
|
||||
|
||||
// and it must be found via the fingerprint-checked lookup too
|
||||
dst2, err := f.NewObject(ctx, fileName)
|
||||
require.NoError(t, err)
|
||||
hashVal, err := dst2.Hash(ctx, hashType)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, hash1, hashVal, "hash should be returned after SetModTime")
|
||||
|
||||
_ = operations.Purge(ctx, f, dirName)
|
||||
}
|
||||
|
||||
// InternalTest dispatches all internal tests
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
if !kv.Supported() {
|
||||
@@ -172,7 +129,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
t.Run("UploadFromCrypt", f.testUploadFromCrypt)
|
||||
t.Run("UpdateStoresHash", f.testUpdateStoresHash)
|
||||
t.Run("SetModTimeKeepsHash", f.testSetModTimeKeepsHash)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
@@ -206,34 +206,6 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
// kvUpdateFingerprint: rewrite the fingerprint of an existing record
|
||||
type kvUpdateFingerprint struct {
|
||||
key string
|
||||
oldFp string
|
||||
newFp string
|
||||
age time.Duration
|
||||
}
|
||||
|
||||
func (op *kvUpdateFingerprint) Do(ctx context.Context, b kv.Bucket) error {
|
||||
data := b.Get([]byte(op.key))
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
var r hashRecord
|
||||
if err := r.decode(op.key, data); err != nil {
|
||||
return nil
|
||||
}
|
||||
if r.Fp != op.oldFp || time.Since(r.Created) > op.age {
|
||||
return nil
|
||||
}
|
||||
r.Fp = op.newFp
|
||||
data, err := r.encode(op.key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.Put([]byte(op.key), data)
|
||||
}
|
||||
|
||||
// kvDump: dump the database.
|
||||
// Note: long dump can cause concurrent operations to fail.
|
||||
type kvDump struct {
|
||||
|
||||
@@ -56,23 +56,6 @@ func (o *Object) putHashes(ctx context.Context, rawHashes hashMap) error {
|
||||
return o.f.putRawHashes(ctx, key, fp, hashes)
|
||||
}
|
||||
|
||||
// updateFingerprint rewrites the fingerprint of the cached hash
|
||||
// record for remote from oldFp to newFp, keeping the hashes. It is
|
||||
// used when only the modification time of an object changes, so the
|
||||
// cached hashes remain valid under the new fingerprint.
|
||||
func (f *Fs) updateFingerprint(ctx context.Context, remote, oldFp, newFp string) error {
|
||||
if oldFp == "" || newFp == "" || oldFp == newFp || f.opt.MaxAge <= 0 {
|
||||
return nil
|
||||
}
|
||||
key := path.Join(f.Fs.Root(), remote)
|
||||
return f.db.Do(true, &kvUpdateFingerprint{
|
||||
key: key,
|
||||
oldFp: oldFp,
|
||||
newFp: newFp,
|
||||
age: time.Duration(f.opt.MaxAge),
|
||||
})
|
||||
}
|
||||
|
||||
// set hashes for a path without any validation
|
||||
func (f *Fs) putRawHashes(ctx context.Context, key, fp string, hashes operations.HashSums) error {
|
||||
return f.db.Do(true, &kvPut{
|
||||
@@ -213,27 +196,14 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the file.
|
||||
//
|
||||
// When the fingerprint includes the modtime, the contents have not
|
||||
// changed, so the cached hashes are carried over to the fingerprint
|
||||
// for the new modtime. Otherwise the cache entry is pruned when the
|
||||
// modtime changes so that touching a file will trigger checksum
|
||||
// recalculation even on backends that don't provide modTime with
|
||||
// fingerprint.
|
||||
// Also prunes the cache entry when modtime changes so that
|
||||
// touching a file will trigger checksum recalculation even
|
||||
// on backends that don't provide modTime with fingerprint.
|
||||
func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
|
||||
if mtime == o.Object.ModTime(ctx) {
|
||||
return o.Object.SetModTime(ctx, mtime)
|
||||
}
|
||||
if !o.f.fpTime {
|
||||
if mtime != o.Object.ModTime(ctx) {
|
||||
_ = o.f.pruneHash(o.Remote())
|
||||
return o.Object.SetModTime(ctx, mtime)
|
||||
}
|
||||
oldFp := o.fingerprint(ctx)
|
||||
if err := o.Object.SetModTime(ctx, mtime); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = o.f.updateFingerprint(ctx, o.Remote(), oldFp, o.fingerprint(ctx))
|
||||
return nil
|
||||
return o.Object.SetModTime(ctx, mtime)
|
||||
}
|
||||
|
||||
// Open opens the file for read.
|
||||
|
||||
+61
-82
@@ -10,6 +10,7 @@ package hidrive
|
||||
// to be resolved prior to execution with resolvePath(...).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -23,9 +24,8 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/ranges"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/sync/semaphore"
|
||||
@@ -454,17 +454,17 @@ func (f *Fs) deleteObject(ctx context.Context, path string) error {
|
||||
}
|
||||
|
||||
// createFile creates a file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// This guarantees that existing files will not be overwritten.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
// This returns ErrorFileExists if a file already exists at the specified path.
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
if onExist == AutoNameOnExist {
|
||||
@@ -479,14 +479,12 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -512,16 +510,16 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
|
||||
// overwriteFile updates the content of the file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the file does not exist it will be created.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
|
||||
@@ -533,14 +531,12 @@ func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, m
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -638,18 +634,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
}
|
||||
|
||||
// Read a chunk of data.
|
||||
var (
|
||||
chunkReader *pool.RW
|
||||
bytesRead int64
|
||||
)
|
||||
chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)
|
||||
if bytesRead < int64(chunkSize) {
|
||||
chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
|
||||
if bytesRead < chunkSize {
|
||||
startMoreTransfers = false
|
||||
}
|
||||
if readErr != nil || bytesRead <= 0 {
|
||||
if chunkReader != nil {
|
||||
_ = chunkReader.Close()
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
@@ -659,12 +648,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// After this upload is done,
|
||||
// signal that another transfer can be started.
|
||||
defer transferSemaphore.Release(1)
|
||||
defer func() { _ = chunkReader.Close() }()
|
||||
uploadErr := f.patchFile(gCtx, path, chunkReader, chunkOffset, zeroTime)
|
||||
uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
|
||||
if uploadErr == nil {
|
||||
// Remember successfully written chunks.
|
||||
okChunksMu.Lock()
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: bytesRead})
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
|
||||
okChunksMu.Unlock()
|
||||
fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
|
||||
} else {
|
||||
@@ -708,10 +696,10 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// starting at the given offset.
|
||||
//
|
||||
// Replaces the file contents starting from the given byte offset
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the offset is beyond the file end, the file is extended up to the offset.
|
||||
// The maximum size of the update is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
@@ -723,7 +711,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// usually expand the holes into regions of 0-bytes.
|
||||
//
|
||||
// This returns fs.ErrorObjectNotFound if the object is not found.
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offset uint64, modTime time.Time) error {
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetPath(path)
|
||||
parameters.Set("offset", strconv.FormatUint(offset, 10))
|
||||
@@ -735,15 +723,13 @@ func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offse
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
}
|
||||
|
||||
var resp *http.Response
|
||||
@@ -853,48 +839,41 @@ func createHiDriveScopes(role string, access string) []string {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
// unwrapAccounting splits any transfer accounting off reader,
|
||||
// returning the raw stream and the accounting (nil if there is none),
|
||||
// so the stream can be buffered and accounted when the buffer is uploaded.
|
||||
func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Account) {
|
||||
// Any kind of accounter re-wraps into the one type UnWrapAccounting
|
||||
// knows how to take the *Account out of.
|
||||
unwrapped, wrap := accounting.UnWrap(reader)
|
||||
_, acc = accounting.UnWrapAccounting(wrap(unwrapped))
|
||||
return unwrapped, acc
|
||||
}
|
||||
|
||||
// readerForChunk reads up to length bytes from reader into a buffer
|
||||
// from the global memory pool and returns it
|
||||
// along with the number of bytes that have been read from reader.
|
||||
// Reaching the end of reader early is not an error.
|
||||
//
|
||||
// If accountUpload is set, any accounting on reader is applied
|
||||
// when the buffer is read instead, so the upload is what gets accounted.
|
||||
//
|
||||
// This must not be set for a buffer which creates or replaces a file
|
||||
// in a single request. Accounting can fail part way through the upload,
|
||||
// for example when --max-transfer is reached, and HiDrive keeps
|
||||
// what it has received of an aborted request as the file.
|
||||
//
|
||||
// The caller must Close the buffer when done with it.
|
||||
func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {
|
||||
chunkReader = multipart.NewRW()
|
||||
if accountUpload {
|
||||
var acc *accounting.Account
|
||||
reader, acc = unwrapAccounting(reader)
|
||||
if acc != nil {
|
||||
chunkReader.SetAccounting(acc.AccountRead)
|
||||
}
|
||||
// cachedReader returns a version of the reader that caches its contents and
|
||||
// can therefore be reset using Seek.
|
||||
func cachedReader(reader io.Reader) io.ReadSeeker {
|
||||
bytesReader, ok := reader.(*bytes.Reader)
|
||||
if ok {
|
||||
return bytesReader
|
||||
}
|
||||
|
||||
bytesRead, err = io.CopyN(chunkReader, reader, length)
|
||||
if err == io.EOF {
|
||||
repeatableReader, ok := reader.(*readers.RepeatableReader)
|
||||
if ok {
|
||||
return repeatableReader
|
||||
}
|
||||
|
||||
return readers.NewRepeatableReader(reader)
|
||||
}
|
||||
|
||||
// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
|
||||
// Returns a new io.Reader (chunkReader) for that chunk
|
||||
// and the number of bytes that have been read from reader.
|
||||
func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
|
||||
// Unwrap any accounting from the input if present.
|
||||
reader, wrap := accounting.UnWrap(reader)
|
||||
|
||||
// Read a chunk of data.
|
||||
buffer := make([]byte, length)
|
||||
bytesRead, err = io.ReadFull(reader, buffer)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
_ = chunkReader.Close()
|
||||
return nil, bytesRead, err
|
||||
}
|
||||
return chunkReader, bytesRead, nil
|
||||
// Truncate unused capacity.
|
||||
buffer = buffer[:bytesRead]
|
||||
|
||||
// Use wrap to put any accounting back for chunkReader.
|
||||
return wrap(bytes.NewReader(buffer)), bytesRead, nil
|
||||
}
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -515,14 +514,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
// (i.e. everything up to the cutoff) in the first request,
|
||||
// avoids files being created on upload failure for small files.
|
||||
// (As opposed to creating an empty file and then uploading the content.)
|
||||
//
|
||||
// Only buffer as much as the source declares it has,
|
||||
// so small files do not cost the whole cutoff in memory.
|
||||
prefixSize := int64(f.opt.UploadCutoff)
|
||||
if size := src.Size(); size >= 0 && size < prefixSize {
|
||||
prefixSize = size
|
||||
}
|
||||
cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)
|
||||
tmpReader, bytesRead, err := readerForChunk(in, int(f.opt.UploadCutoff))
|
||||
cutoffReader := cachedReader(tmpReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -547,7 +540,6 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
return false, createErr
|
||||
})
|
||||
_ = cutoffReader.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -564,7 +556,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
// If there is more left to write, o.Update needs to skip ahead.
|
||||
// Use a fs.SeekOption with the current offset to do this.
|
||||
options = append(options, &fs.SeekOption{Offset: bytesRead})
|
||||
options = append(options, &fs.SeekOption{Offset: int64(bytesRead)})
|
||||
err = o.Update(ctx, in, src, options...)
|
||||
|
||||
if err == nil {
|
||||
@@ -979,12 +971,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Metadata should be updated even if the upload fails.
|
||||
info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
|
||||
} else {
|
||||
var content *pool.RW
|
||||
content, _, err = readerForChunk(in, src.Size(), false)
|
||||
if err == nil {
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
|
||||
_ = content.Close()
|
||||
}
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, cachedReader(in), modTime)
|
||||
metaErr = err
|
||||
}
|
||||
|
||||
|
||||
+11
-44
@@ -63,8 +63,7 @@ The headers are only sent to the host in the configured URL. If the
|
||||
server redirects to another host (including a subdomain or a different
|
||||
port) the headers are not sent to it, or to any further hop in that
|
||||
redirect chain. When headers are set, a redirect from https to http is
|
||||
refused as it would send them in cleartext, even with
|
||||
--allow-https-downgrade.`,
|
||||
refused as it would send them in cleartext.`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -101,30 +100,6 @@ that directory listings are much quicker, but rclone won't have the times or
|
||||
sizes of any files, and some files that don't exist may be in the listing.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "no_head_size",
|
||||
Help: `Set the reported file size for --http-no-head.
|
||||
|
||||
When HEAD requests are disabled, rclone normally reports an unknown file size.
|
||||
Set this option to a non-negative value to report that size for every file,
|
||||
including 0. Leave it unset (the default, shown as "off") to keep reporting
|
||||
an unknown size.
|
||||
|
||||
The configured size is authoritative. If it does not match the actual file
|
||||
size, transfers may fail their size check and retry unless --ignore-size is
|
||||
used. It can also make multi-thread transfers request incorrect ranges and
|
||||
make mounted files unreadable past the reported size.`,
|
||||
Default: fs.SizeSuffix(-1),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "no_head_time",
|
||||
Help: `Set a default modification time for --http-no-head.
|
||||
|
||||
When HEAD requests are disabled, rclone normally reports an unset modification
|
||||
time. Set this option to report the specified time for every file instead.
|
||||
Changing the configured time later makes every file appear modified to a sync.`,
|
||||
Default: fs.Time{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "no_escape",
|
||||
Help: "Do not escape URL metacharacters in path names.",
|
||||
@@ -170,13 +145,11 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Endpoint string `config:"url"`
|
||||
NoSlash bool `config:"no_slash"`
|
||||
NoHead bool `config:"no_head"`
|
||||
NoHeadSize fs.SizeSuffix `config:"no_head_size"`
|
||||
NoHeadTime fs.Time `config:"no_head_time"`
|
||||
Headers fs.CommaSepList `config:"headers"`
|
||||
NoEscape bool `config:"no_escape"`
|
||||
Endpoint string `config:"url"`
|
||||
NoSlash bool `config:"no_slash"`
|
||||
NoHead bool `config:"no_head"`
|
||||
Headers fs.CommaSepList `config:"headers"`
|
||||
NoEscape bool `config:"no_escape"`
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote HTTP files
|
||||
@@ -316,9 +289,8 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
|
||||
}
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
// Without configured headers keep the default policy so
|
||||
// --allow-https-downgrade can follow public mirrors that
|
||||
// redirect from https to http.
|
||||
// Without configured headers keep the default policy which
|
||||
// public mirrors that redirect from https to http rely on.
|
||||
if len(opt.Headers) > 0 {
|
||||
client.CheckRedirect = checkRedirect(opt)
|
||||
}
|
||||
@@ -350,9 +322,7 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
|
||||
// the host specified in the config file.
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := &Options{
|
||||
NoHeadSize: fs.SizeSuffix(-1),
|
||||
}
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -603,7 +573,7 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
|
||||
return nil, fmt.Errorf("failed to readDir: %w", err)
|
||||
}
|
||||
|
||||
contentType, _, _ := strings.Cut(res.Header.Get("Content-Type"), ";")
|
||||
contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
|
||||
switch contentType {
|
||||
case "text/html":
|
||||
names, err = parse(u, res.Body)
|
||||
@@ -747,11 +717,8 @@ func (o *Object) url() string {
|
||||
// head sends a HEAD request to update info fields in the Object
|
||||
func (o *Object) head(ctx context.Context) error {
|
||||
if o.fs.opt.NoHead {
|
||||
o.size = int64(o.fs.opt.NoHeadSize)
|
||||
o.size = -1
|
||||
o.modTime = timeUnset
|
||||
if o.fs.opt.NoHeadTime.IsSet() {
|
||||
o.modTime = time.Time(o.fs.opt.NoHeadTime)
|
||||
}
|
||||
o.contentType = fs.MimeType(ctx, o)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -314,54 +314,6 @@ func TestOpen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoHeadMetadataDefaults(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
size string
|
||||
modTime string
|
||||
wantSize int64
|
||||
wantModTime time.Time
|
||||
}{
|
||||
{
|
||||
name: "defaults",
|
||||
wantSize: -1,
|
||||
wantModTime: timeUnset,
|
||||
}, {
|
||||
name: "zero size",
|
||||
size: "0B",
|
||||
wantSize: 0,
|
||||
wantModTime: timeUnset,
|
||||
}, {
|
||||
name: "configured metadata",
|
||||
size: "100B",
|
||||
modTime: "2024-08-05T07:36:07Z",
|
||||
wantSize: 100,
|
||||
wantModTime: time.Date(2024, 8, 5, 7, 36, 7, 0, time.UTC),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
m := prepareServer(t)
|
||||
m.Set("no_head", "true")
|
||||
if test.size != "" {
|
||||
m.Set("no_head_size", test.size)
|
||||
}
|
||||
if test.modTime != "" {
|
||||
m.Set("no_head_time", test.modTime)
|
||||
}
|
||||
|
||||
f, err := NewFs(context.Background(), remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
o, err := f.NewObject(context.Background(), "four/under four.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, test.wantSize, o.Size())
|
||||
assert.Equal(t, test.wantModTime, o.ModTime(context.Background()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMimeType(t *testing.T) {
|
||||
f := prepare(t)
|
||||
|
||||
@@ -688,7 +640,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
|
||||
|
||||
// open makes a remote pointing at ts with or without headers
|
||||
// and opens file.txt on it
|
||||
open := func(t *testing.T, withHeaders, allowDowngrade bool) (io.ReadCloser, error) {
|
||||
open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
|
||||
configfile.Install()
|
||||
m := configmap.Simple{
|
||||
"type": "http",
|
||||
@@ -700,7 +652,6 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
|
||||
}
|
||||
ctx, ci := fs.AddConfig(context.Background())
|
||||
ci.InsecureSkipVerify = true
|
||||
ci.AllowHTTPSDowngrade = allowDowngrade
|
||||
f, err := NewFs(ctx, remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -712,25 +663,13 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
|
||||
|
||||
t.Run("WithHeaders", func(t *testing.T) {
|
||||
// Headers could leak so the downgrade must be refused
|
||||
_, err := open(t, true, false)
|
||||
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
|
||||
})
|
||||
|
||||
t.Run("WithHeadersAllowDowngrade", func(t *testing.T) {
|
||||
// Headers could leak so the downgrade is refused even when allowed
|
||||
_, err := open(t, true, true)
|
||||
_, err := open(t, true)
|
||||
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
|
||||
})
|
||||
|
||||
t.Run("WithoutHeaders", func(t *testing.T) {
|
||||
// The transport refuses downgrades by default
|
||||
_, err := open(t, false, false)
|
||||
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
|
||||
})
|
||||
|
||||
t.Run("WithoutHeadersAllowDowngrade", func(t *testing.T) {
|
||||
// Nothing to leak so the downgrade is followed when allowed
|
||||
fd, err := open(t, false, true)
|
||||
// Nothing to leak so the downgrade is followed
|
||||
fd, err := open(t, false)
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(fd)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -37,44 +37,44 @@ type FileList struct {
|
||||
|
||||
// File represents a file or folder in Huawei Drive
|
||||
type File struct {
|
||||
ID string `json:"id"`
|
||||
FileName string `json:"fileName"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
EditedTime time.Time `json:"editedTime"`
|
||||
EditedByMeTime time.Time `json:"editedByMeTime"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Owners []User `json:"owners"`
|
||||
LastEditor User `json:"lastEditor"`
|
||||
Permissions []Permission `json:"permissions"`
|
||||
PermissionIDs []string `json:"permissionIds"`
|
||||
Capabilities FileCapabilities `json:"capabilities"`
|
||||
OwnedByMe bool `json:"ownedByMe"`
|
||||
EditedByMe bool `json:"editedByMe"`
|
||||
ViewedByMe bool `json:"viewedByMe"`
|
||||
HasShared bool `json:"hasShared"`
|
||||
Recycled bool `json:"recycled"`
|
||||
DirectlyRecycled bool `json:"directlyRecycled"`
|
||||
Favorite bool `json:"favorite"`
|
||||
ExistThumbnail bool `json:"existThumbnail"`
|
||||
ThumbnailVersion int64 `json:"thumbnailVersion"`
|
||||
IconDownloadLink string `json:"iconDownloadLink,omitempty"`
|
||||
ContentDownloadLink string `json:"contentDownloadLink,omitempty"`
|
||||
ContentVersion string `json:"contentVersion,omitempty"`
|
||||
LastHistoryVersionID string `json:"lastHistoryVersionId,omitempty"`
|
||||
OccupiedSpace int64 `json:"occupiedSpace,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
WritersHasSharePermission bool `json:"writersHasSharePermission"`
|
||||
WriterHasCopyPermission bool `json:"writerHasCopyPermission"`
|
||||
Containers []string `json:"containers"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
ContentExtras *ContentExtras `json:"contentExtras,omitempty"`
|
||||
ID string `json:"id"`
|
||||
FileName string `json:"fileName"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
EditedTime time.Time `json:"editedTime"`
|
||||
EditedByMeTime time.Time `json:"editedByMeTime,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Owners []User `json:"owners"`
|
||||
LastEditor User `json:"lastEditor"`
|
||||
Permissions []Permission `json:"permissions"`
|
||||
PermissionIDs []string `json:"permissionIds"`
|
||||
Capabilities FileCapabilities `json:"capabilities"`
|
||||
OwnedByMe bool `json:"ownedByMe"`
|
||||
EditedByMe bool `json:"editedByMe"`
|
||||
ViewedByMe bool `json:"viewedByMe"`
|
||||
HasShared bool `json:"hasShared"`
|
||||
Recycled bool `json:"recycled"`
|
||||
DirectlyRecycled bool `json:"directlyRecycled"`
|
||||
Favorite bool `json:"favorite"`
|
||||
ExistThumbnail bool `json:"existThumbnail"`
|
||||
ThumbnailVersion int64 `json:"thumbnailVersion"`
|
||||
IconDownloadLink string `json:"iconDownloadLink,omitempty"`
|
||||
ContentDownloadLink string `json:"contentDownloadLink,omitempty"`
|
||||
ContentVersion string `json:"contentVersion,omitempty"`
|
||||
LastHistoryVersionID string `json:"lastHistoryVersionId,omitempty"`
|
||||
OccupiedSpace int64 `json:"occupiedSpace,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
WritersHasSharePermission bool `json:"writersHasSharePermission"`
|
||||
WriterHasCopyPermission bool `json:"writerHasCopyPermission"`
|
||||
Containers []string `json:"containers"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
ContentExtras *ContentExtras `json:"contentExtras,omitempty"`
|
||||
}
|
||||
|
||||
// IsDir returns true if the file is a directory
|
||||
@@ -125,41 +125,41 @@ type Thumbnail struct {
|
||||
|
||||
// CreateFolderRequest represents request to create a folder
|
||||
type CreateFolderRequest struct {
|
||||
FileName string `json:"fileName"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Favorite bool `json:"favorite,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
AutoRename int `json:"autoRename,omitempty"`
|
||||
FileName string `json:"fileName"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Favorite bool `json:"favorite,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AutoRename int `json:"autoRename,omitempty"`
|
||||
}
|
||||
|
||||
// UpdateFileRequest represents request to update a file
|
||||
type UpdateFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Favorite *bool `json:"favorite,omitempty"`
|
||||
Recycled *bool `json:"recycled,omitempty"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
WriterHasCopyPermission *bool `json:"writerHasCopyPermission,omitempty"`
|
||||
WritersHasSharePermission *bool `json:"writersHasSharePermission,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
AddParentFolder []string `json:"addParentFolder,omitempty"`
|
||||
RemoveParentFolder []string `json:"removeParentFolder,omitempty"`
|
||||
CreatedTime *time.Time `json:"createdTime,omitempty"`
|
||||
EditedTime *time.Time `json:"editedTime,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Favorite *bool `json:"favorite,omitempty"`
|
||||
Recycled *bool `json:"recycled,omitempty"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
WriterHasCopyPermission *bool `json:"writerHasCopyPermission,omitempty"`
|
||||
WritersHasSharePermission *bool `json:"writersHasSharePermission,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AddParentFolder []string `json:"addParentFolder,omitempty"`
|
||||
RemoveParentFolder []string `json:"removeParentFolder,omitempty"`
|
||||
CreatedTime *time.Time `json:"createdTime,omitempty"`
|
||||
EditedTime *time.Time `json:"editedTime,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest represents request to copy a file
|
||||
type CopyFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Properties map[string]any `json:"properties,omitempty"`
|
||||
AppSettings map[string]any `json:"appSettings,omitempty"`
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
}
|
||||
|
||||
// ResumeUploadInitResponse represents response to resume upload init
|
||||
@@ -224,6 +224,11 @@ var GlobalDomains = map[string]bool{
|
||||
"drive.cloud.hicloud.com": true,
|
||||
}
|
||||
|
||||
// BoolPtr returns a pointer to a bool value (helper for UpdateFileRequest)
|
||||
func BoolPtr(b bool) *bool {
|
||||
return &b
|
||||
}
|
||||
|
||||
// StartCursor represents response to Changes.getStartCursor endpoint
|
||||
type StartCursor struct {
|
||||
Category string `json:"category"` // "drive#startCursor"
|
||||
|
||||
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -50,10 +49,6 @@ const (
|
||||
defaultChunkSize = 8 * fs.Mebi
|
||||
)
|
||||
|
||||
// jsonContentType is sent with JSON request bodies as the API doesn't
|
||||
// decode non-ASCII file names in them as UTF-8 unless told the charset.
|
||||
const jsonContentType = "application/json; charset=UTF-8"
|
||||
|
||||
// OAuth2 configuration
|
||||
var oauthConfig = &oauthutil.Config{
|
||||
Scopes: []string{
|
||||
@@ -551,7 +546,6 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
|
||||
Parameters: url.Values{
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
|
||||
}
|
||||
|
||||
var info *api.File
|
||||
@@ -1020,7 +1014,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
Parameters: url.Values{
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
|
||||
}
|
||||
|
||||
copyReq := api.CopyFileRequest{
|
||||
@@ -1038,7 +1031,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
fs.Debugf(f, "Copy: failed to get metadata options: %v", err)
|
||||
}
|
||||
if len(mergedMeta) > 0 {
|
||||
copyReq.Properties = make(map[string]any)
|
||||
copyReq.Properties = make(map[string]interface{})
|
||||
for key, value := range mergedMeta {
|
||||
switch key {
|
||||
case "content-type", "sha256", "btime", "mtime", "utime",
|
||||
@@ -1205,10 +1198,9 @@ func (f *Fs) updateFile(ctx context.Context, fileID string, parameters url.Value
|
||||
}
|
||||
parameters.Set("fields", "*")
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/files/" + fileID,
|
||||
Parameters: parameters,
|
||||
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
|
||||
Method: "PATCH",
|
||||
Path: "/files/" + fileID,
|
||||
Parameters: parameters,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, req, &info)
|
||||
@@ -1678,7 +1670,6 @@ func (o *Object) uploadEmpty(ctx context.Context, leaf, directoryID string, src
|
||||
Parameters: url.Values{
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
|
||||
}
|
||||
|
||||
var info *api.File
|
||||
@@ -1730,7 +1721,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
}
|
||||
|
||||
// Create metadata
|
||||
metadata := map[string]any{
|
||||
metadata := map[string]interface{}{
|
||||
"fileName": leaf,
|
||||
"mimeType": mimeType,
|
||||
}
|
||||
@@ -1750,10 +1741,9 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
metadata["autoRename"] = 3
|
||||
}
|
||||
|
||||
// Build the multipart body in pooled memory so it can be re-sent on retry
|
||||
rw := pool.NewRW(pool.Global())
|
||||
defer fs.CheckClose(rw, &err)
|
||||
writer := multipart.NewWriter(rw)
|
||||
// Create multipart form
|
||||
var buf bytes.Buffer
|
||||
writer := multipart.NewWriter(&buf)
|
||||
|
||||
// Add metadata part - use exact format from Huawei docs
|
||||
metadataWriter, err := writer.CreatePart(textproto.MIMEHeader{
|
||||
@@ -1799,17 +1789,15 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
return fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
bodySize := rw.Size()
|
||||
bodyBytes := buf.Bytes()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Parameters: url.Values{
|
||||
"uploadType": []string{"multipart"},
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
ContentLength: &bodySize,
|
||||
RootURL: o.fs.uploadURL,
|
||||
Body: rw,
|
||||
ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
|
||||
RootURL: o.fs.uploadURL,
|
||||
}
|
||||
|
||||
if o.id != "" {
|
||||
@@ -1823,10 +1811,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
|
||||
var info *api.File
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = bytes.NewReader(bodyBytes)
|
||||
resp, err := srv.CallJSON(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -1854,7 +1839,6 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ExtraHeaders: map[string]string{
|
||||
"Content-Type": jsonContentType,
|
||||
"X-Upload-Content-Length": strconv.FormatInt(size, 10),
|
||||
},
|
||||
RootURL: o.fs.uploadURL,
|
||||
@@ -1888,7 +1872,7 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
|
||||
// Prepare metadata for resumable upload initialization
|
||||
metadata := map[string]any{
|
||||
metadata := map[string]interface{}{
|
||||
"fileName": leaf,
|
||||
}
|
||||
|
||||
@@ -1921,26 +1905,25 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
|
||||
// Now upload the content in chunks
|
||||
// Note 256k is the minimum chunk size and 64M the maximum
|
||||
// single upload size according to the Huawei Drive API
|
||||
chunkSize := min(max(int64(o.fs.opt.ChunkSize), 256*1024), 64*1024*1024)
|
||||
chunkSize := int64(o.fs.opt.ChunkSize)
|
||||
if chunkSize < 256*1024 {
|
||||
chunkSize = 256 * 1024 // Minimum chunk size according to Huawei Drive API
|
||||
}
|
||||
if chunkSize > 64*1024*1024 {
|
||||
chunkSize = 64 * 1024 * 1024 // Maximum single upload size
|
||||
}
|
||||
|
||||
buf := make([]byte, chunkSize)
|
||||
var offset int64
|
||||
|
||||
for offset < size {
|
||||
// Buffer the chunk in pooled memory so it can be re-sent on retry
|
||||
rw := pool.NewRW(pool.Global())
|
||||
n, err := io.CopyN(rw, in, min(size-offset, chunkSize))
|
||||
if err != nil {
|
||||
// io.EOF here means the source ran out before the declared size
|
||||
if err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
_ = rw.Close()
|
||||
n, err := io.ReadFull(in, buf)
|
||||
if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
|
||||
return fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)
|
||||
}
|
||||
|
||||
end := offset + n - 1
|
||||
chunk := buf[:n]
|
||||
end := offset + int64(n) - 1
|
||||
|
||||
// Upload this chunk using the original client with OAuth authentication
|
||||
chunkOpts := rest.Opts{
|
||||
@@ -1948,20 +1931,17 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
RootURL: location,
|
||||
Path: "", // Empty path since RootURL contains the full URL
|
||||
ExtraHeaders: map[string]string{
|
||||
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
|
||||
"Content-Type": mimeType,
|
||||
"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
|
||||
"Content-Length": strconv.Itoa(n),
|
||||
"Content-Type": mimeType,
|
||||
},
|
||||
ContentLength: &n,
|
||||
Body: rw,
|
||||
}
|
||||
|
||||
var info *api.File
|
||||
var chunkResp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err := rw.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
chunkOpts.Body = bytes.NewReader(chunk)
|
||||
var err error
|
||||
chunkResp, err = srv.CallJSON(ctx, &chunkOpts, nil, &info)
|
||||
// 308 means continue uploading (incomplete)
|
||||
if chunkResp != nil && chunkResp.StatusCode == 308 {
|
||||
@@ -1975,15 +1955,11 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
|
||||
}
|
||||
return shouldRetry(ctx, chunkResp, err)
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
|
||||
offset += n
|
||||
offset += int64(n)
|
||||
|
||||
// If we got file info back, we're done
|
||||
if info != nil && info.ID != "" {
|
||||
@@ -2114,8 +2090,8 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
|
||||
|
||||
// Prepare the update request payload
|
||||
updateReq := api.UpdateFileRequest{
|
||||
Properties: make(map[string]any),
|
||||
AppSettings: make(map[string]any),
|
||||
Properties: make(map[string]interface{}),
|
||||
AppSettings: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Process metadata and separate into properties and app settings
|
||||
@@ -2126,7 +2102,7 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
|
||||
updateReq.Description = value
|
||||
case "favorite":
|
||||
if favorite, err := strconv.ParseBool(value); err == nil {
|
||||
updateReq.Favorite = new(favorite)
|
||||
updateReq.Favorite = api.BoolPtr(favorite)
|
||||
}
|
||||
case "content-type":
|
||||
// Allow setting/overriding MIME type
|
||||
@@ -2154,7 +2130,6 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
|
||||
Parameters: url.Values{
|
||||
"fields": []string{"*"},
|
||||
},
|
||||
ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
|
||||
}
|
||||
|
||||
var info *api.File
|
||||
|
||||
@@ -19,49 +19,15 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
)
|
||||
|
||||
// Limits on the chunk size accepted by the resumable upload API
|
||||
const (
|
||||
minChunkSize = 256 * fs.Kibi
|
||||
maxChunkSize = 64 * fs.Mebi
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestHuaweiDrive:",
|
||||
NilObject: (*Object)(nil),
|
||||
SkipBadWindowsCharacters: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: minChunkSize,
|
||||
MaxChunkSize: maxChunkSize,
|
||||
CeilChunkSize: fstests.NextPowerOfTwo,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
|
||||
// TestNewFs tests the NewFs constructor
|
||||
func TestNewFs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@@ -1618,7 +1617,9 @@ func (ps *PhotosService) GetLibraryAlbumCounts(ctx context.Context) (map[string]
|
||||
if err := ps.requestForArea(ctx, area, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to get library album counts: %w", err)
|
||||
}
|
||||
maps.Copy(counts, response.toCounts(order))
|
||||
for k, v := range response.toCounts(order) {
|
||||
counts[k] = v
|
||||
}
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
@@ -2031,7 +2032,7 @@ func (album *Album) fetchPhotosParallel(ctx context.Context, totalPhotos int64)
|
||||
sem := make(chan struct{}, workers)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i := range numPartitions {
|
||||
for i := 0; i < numPartitions; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
|
||||
@@ -1191,7 +1191,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for range 100 {
|
||||
for i := 0; i < 100; i++ {
|
||||
lib.deltaMu.Lock()
|
||||
lib.pendingDelta = &deltaPayload{
|
||||
records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},
|
||||
@@ -1201,7 +1201,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
|
||||
lib.applyPendingDelta(context.Background())
|
||||
}
|
||||
}()
|
||||
for range 100 {
|
||||
for i := 0; i < 100; i++ {
|
||||
ps.FlushCaches()
|
||||
// Re-add the library since FlushCaches clears it
|
||||
ps.mu.Lock()
|
||||
|
||||
@@ -718,7 +718,7 @@ func (s *Session) GetAuthState(ctx context.Context) (*AuthStateResponse, error)
|
||||
Path: "",
|
||||
ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
|
||||
RootURL: authEndpoint,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
// Use srv.Call directly to capture the raw response body for debugging
|
||||
resp, err := s.srv.Call(ctx, &opts)
|
||||
@@ -828,7 +828,7 @@ func (s *Session) TrustSession(ctx context.Context) error {
|
||||
ExtraHeaders: s.GetAuthHeaders(map[string]string{}),
|
||||
RootURL: authEndpoint,
|
||||
NoResponse: true,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
|
||||
_, err := s.Request(ctx, opts, nil, nil)
|
||||
@@ -846,7 +846,7 @@ func (s *Session) ValidateSession(ctx context.Context) error {
|
||||
Path: "/validate",
|
||||
ExtraHeaders: s.GetHeaders(map[string]string{}),
|
||||
RootURL: setupEndpoint,
|
||||
ContentLength: new(int64(0)),
|
||||
ContentLength: int64Ptr(0),
|
||||
}
|
||||
_, err := s.Request(ctx, opts, nil, &s.AccountInfo)
|
||||
if err != nil {
|
||||
@@ -904,6 +904,8 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
|
||||
return headers
|
||||
}
|
||||
|
||||
func int64Ptr(v int64) *int64 { return &v }
|
||||
|
||||
// NewSession creates a new Session instance with default values
|
||||
func NewSession() *Session {
|
||||
session := &Session{
|
||||
|
||||
@@ -694,7 +694,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: true, // names are matched with EqualFold, see findLeafItem
|
||||
CanHaveEmptyDirectories: true,
|
||||
PartialUploads: false,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
@@ -376,7 +376,8 @@ func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
if libraryName, ok := strings.CutPrefix(pathID, "lib:"); ok {
|
||||
if strings.HasPrefix(pathID, "lib:") {
|
||||
libraryName := strings.TrimPrefix(pathID, "lib:")
|
||||
libraries, err := photosService.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
@@ -735,7 +736,10 @@ func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallb
|
||||
}
|
||||
close(jobCh)
|
||||
|
||||
workers := min(len(jobs), fs.GetConfig(ctx).Checkers)
|
||||
workers := fs.GetConfig(ctx).Checkers
|
||||
if len(jobs) < workers {
|
||||
workers = len(jobs)
|
||||
}
|
||||
errs := make(chan error, workers)
|
||||
|
||||
for range workers {
|
||||
|
||||
@@ -77,8 +77,6 @@ func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadPara
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
|
||||
}
|
||||
// Stop the form writer reading file if the request fails before sending it all
|
||||
defer func() { _ = formReader.Close() }()
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
|
||||
@@ -634,27 +634,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
|
||||
"x-archive-filemeta-rclone-update-track": updateTracker,
|
||||
}
|
||||
|
||||
mdata, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if mdata != nil {
|
||||
// IA ignores file metadata headers with the COPY directive.
|
||||
headers["x-amz-metadata-directive"] = "REPLACE"
|
||||
}
|
||||
for key, value := range mdata {
|
||||
key = strings.ToLower(key)
|
||||
if key == "mtime" {
|
||||
key = "rclone-mtime"
|
||||
} else if strings.HasPrefix(key, "rclone-") {
|
||||
// Keep the new copy tracker and mapped mtime, not the source internals.
|
||||
continue
|
||||
} else if _, readOnly := roMetadataKey[key]; readOnly {
|
||||
continue
|
||||
}
|
||||
headers["x-archive-filemeta-"+key] = value
|
||||
}
|
||||
|
||||
// make a PUT request at (IAS3)/:item/:path without body
|
||||
var resp *http.Response
|
||||
opts := rest.Opts{
|
||||
|
||||
@@ -54,7 +54,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
switch httpErr.StatusCode() {
|
||||
case 401:
|
||||
if !f.authFailed {
|
||||
@@ -1208,7 +1209,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
err := o.f.pacer.Call(func() (bool, error) {
|
||||
err := files.DeleteFile(ctx, o.f.cfg, backupUUID)
|
||||
if err != nil {
|
||||
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
if errors.As(err, &httpErr) {
|
||||
// Treat 404 (Not Found) and 204 (No Content) as success
|
||||
switch httpErr.StatusCode() {
|
||||
case 404, 204:
|
||||
@@ -1274,7 +1276,8 @@ func isEmptyFileLimitError(err error) bool {
|
||||
// fileTooLargeError extracts the SDK's FileTooLargeError from a wrapped error
|
||||
// chain, returning it (or nil) so callers can branch on the size limit.
|
||||
func fileTooLargeError(err error) *sdkerrors.FileTooLargeError {
|
||||
if tooLarge, ok := errors.AsType[*sdkerrors.FileTooLargeError](err); ok {
|
||||
var tooLarge *sdkerrors.FileTooLargeError
|
||||
if errors.As(err, &tooLarge) {
|
||||
return tooLarge
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -34,7 +34,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
@@ -86,8 +85,8 @@ func getServices() []service {
|
||||
{"jottacloud", "Jottacloud", "id.jottacloud.com", "jottacloud", "desktop", []string{"openid", "jotta-default", "offline_access"}}, // Chose client id "desktop" here, will be identified as "Jottacloud for Desktop" in "My logged in devices", but could have used "jottacli" here as well.
|
||||
{"elgiganten_dk", "Elgiganten Cloud (Denmark)", "cloud.elgiganten.dk", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"elgiganten_se", "Elgiganten Cloud (Sweden)", "cloud.elgiganten.se", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "offline_access"}},
|
||||
{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "offline_access"}},
|
||||
{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"gigantti", "Gigantti Cloud (Finland)", "cloud.gigantti.fi", "gigantti", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
|
||||
{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
@@ -258,7 +257,6 @@ service, for the official service on https://www.jottacloud.com/web/secure.`)
|
||||
}
|
||||
m.Set("configVersion", fmt.Sprint(configVersion))
|
||||
m.Set(configClientID, service.clientID)
|
||||
m.Set(configClientSecret, "")
|
||||
m.Set(configTokenURL, wellKnown.TokenEndpoint)
|
||||
return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
|
||||
OAuth2Config: &oauthutil.Config{
|
||||
@@ -1501,9 +1499,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
|
||||
|
||||
if err == nil {
|
||||
createTime := srcObj.createTime
|
||||
var createTime time.Time
|
||||
var createTimeMeta bool
|
||||
modTime := srcObj.modTime
|
||||
var modTime time.Time
|
||||
var modTimeMeta bool
|
||||
if meta != nil {
|
||||
createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
|
||||
@@ -1519,11 +1517,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
|
||||
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
|
||||
info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
|
||||
// The restored file keeps the name it was trashed with, which may differ in case
|
||||
if err == nil && info != nil && info.Name != f.opt.Enc.FromStandardName(path.Base(remote)) {
|
||||
fs.Debugf(src, "Renaming restored %q to %q", info.Name, path.Base(remote))
|
||||
info, err = f.copyOrMove(ctx, "mv", f.filePath(remote), remote)
|
||||
}
|
||||
} else if createTimeMeta || modTimeMeta {
|
||||
info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
|
||||
}
|
||||
@@ -1931,9 +1924,8 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
|
||||
// nothing to clean up by default
|
||||
cleanup = func() {}
|
||||
|
||||
// don't cache small files on disk to reduce wear of the disk, but
|
||||
// spool unknown sized streams there as they can't be bounded in memory
|
||||
if size > threshold || size < 0 {
|
||||
// don't cache small files on disk to reduce wear of the disk
|
||||
if size > threshold {
|
||||
var tempFile *os.File
|
||||
|
||||
// create the cache file
|
||||
@@ -1963,15 +1955,15 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
|
||||
// replace the already read source with a reader of our cached file
|
||||
out = tempFile
|
||||
} else {
|
||||
// that's a small file, just read it into memory from the global pool
|
||||
rw := multipart.NewRW()
|
||||
cleanup = func() {
|
||||
_ = rw.Close()
|
||||
}
|
||||
if _, err = io.Copy(rw, teeReader); err != nil {
|
||||
// that's a small file, just read it into memory
|
||||
var inData []byte
|
||||
inData, err = io.ReadAll(teeReader)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
out = rw
|
||||
|
||||
// set the reader to our read memory block
|
||||
out = bytes.NewReader(inData)
|
||||
}
|
||||
return hex.EncodeToString(md5Hasher.Sum(nil)), out, cleanup, nil
|
||||
}
|
||||
|
||||
@@ -5,14 +5,12 @@ import (
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -30,16 +28,11 @@ func TestReadMD5(t *testing.T) {
|
||||
wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
for _, threshold := range []int64{512, 1024, 10 * 1024, 20 * 1024} {
|
||||
t.Run(fmt.Sprintf("%d", threshold), func(t *testing.T) {
|
||||
inUse := pool.Global().InUse()
|
||||
in := readers.NewPatternReader(size)
|
||||
gotMD5, out, cleanup, err := readMD5(in, size, threshold)
|
||||
defer cleanup()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
if size > threshold {
|
||||
assert.IsType(t, (*os.File)(nil), out, "big files should be spooled to disk")
|
||||
} else {
|
||||
assert.IsType(t, (*pool.RW)(nil), out, "small files should be buffered in the pool")
|
||||
}
|
||||
|
||||
// check md5hash of out
|
||||
hasher := md5.New()
|
||||
@@ -48,36 +41,10 @@ func TestReadMD5(t *testing.T) {
|
||||
assert.Equal(t, n, size)
|
||||
outMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
assert.Equal(t, wantMD5, outMD5)
|
||||
|
||||
// check cleanup returns the buffer to the pool
|
||||
cleanup()
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Unknown sized streams can't be bounded in memory so must be spooled to disk
|
||||
t.Run("UnknownSize", func(t *testing.T) {
|
||||
const size = 1024
|
||||
inUse := pool.Global().InUse()
|
||||
hasher := md5.New()
|
||||
_, err := io.Copy(hasher, readers.NewPatternReader(size))
|
||||
require.NoError(t, err)
|
||||
wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
|
||||
gotMD5, out, cleanup, err := readMD5(readers.NewPatternReader(size), -1, 10*size)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantMD5, gotMD5)
|
||||
assert.IsType(t, (*os.File)(nil), out)
|
||||
hasher.Reset()
|
||||
n, err := io.Copy(hasher, out)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(size), n)
|
||||
assert.Equal(t, wantMD5, fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
cleanup()
|
||||
assert.Equal(t, inUse, pool.Global().InUse())
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestMetadata(t *testing.T) {
|
||||
|
||||
@@ -1,235 +0,0 @@
|
||||
// Package api has type definitions for kDrive
|
||||
//
|
||||
// Converted from the API docs with help from https://mholt.github.io/json-to-go/
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Time represents unix timestamp date and time information for the
|
||||
// kDrive API
|
||||
type Time time.Time
|
||||
|
||||
// UnmarshalJSON turns JSON into a Time
|
||||
func (t *Time) UnmarshalJSON(data []byte) error {
|
||||
if string(data) == "null" {
|
||||
return nil
|
||||
}
|
||||
|
||||
timestamp, err := strconv.ParseInt(string(data), 10, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newT := time.Unix(timestamp, 0)
|
||||
*t = Time(newT)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResultStatus return error details from kdrive when things go wrong
|
||||
//
|
||||
// If Status is "success" then everything is OK
|
||||
type ResultStatus struct {
|
||||
Status string `json:"result"`
|
||||
ErrorDetail struct {
|
||||
Result string `json:"code"`
|
||||
ErrorString string `json:"description"`
|
||||
Context struct {
|
||||
Values []string `json:"values"`
|
||||
} `json:"context"`
|
||||
Errors []struct {
|
||||
Result string `json:"code"`
|
||||
ErrorString string `json:"description"`
|
||||
} `json:"errors"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// Error returns a string for the error and satisfies the error interface
|
||||
func (e *ResultStatus) Error() string {
|
||||
var details string
|
||||
for i := range e.ErrorDetail.Errors {
|
||||
details += "|" + e.ErrorDetail.Errors[i].Result
|
||||
}
|
||||
|
||||
return fmt.Sprintf("kDrive error: %s (%s: %s %s)", e.Status, e.ErrorDetail.Result, e.ErrorDetail.ErrorString, details)
|
||||
}
|
||||
|
||||
// IsError returns true if there is an error
|
||||
func (e *ResultStatus) IsError() bool {
|
||||
return e.Status != "success"
|
||||
}
|
||||
|
||||
// Update returns err directly if it was != nil, otherwise it returns
|
||||
// an Error or nil if no error was detected
|
||||
func (e *ResultStatus) Update(err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if e.IsError() {
|
||||
return e
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check ResultStatus satisfies the error interface
|
||||
var _ error = (*ResultStatus)(nil)
|
||||
|
||||
// Item describes a folder or a file as returned by Get Folder Items and others
|
||||
type Item struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
FullPath string `json:"path"`
|
||||
Status string `json:"status"`
|
||||
Hash string `json:"hash"`
|
||||
Size int64 `json:"size"`
|
||||
Visibility string `json:"visibility"`
|
||||
DriveID int `json:"drive_id"`
|
||||
Depth int `json:"depth"`
|
||||
CreatedBy int `json:"created_by"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
AddedAt int `json:"added_at"`
|
||||
LastModifiedAt Time `json:"last_modified_at"`
|
||||
LastModifiedBy int `json:"last_modified_by"`
|
||||
RevisedAt int `json:"revised_at"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
MimeType string `json:"mime_type"`
|
||||
ParentID int `json:"parent_id"`
|
||||
Color string `json:"color"`
|
||||
}
|
||||
|
||||
// ItemResult is returned when an item is queried
|
||||
type ItemResult struct {
|
||||
ResultStatus
|
||||
Data Item `json:"data"`
|
||||
}
|
||||
|
||||
// SearchResult is returned when a list of items is requested
|
||||
type SearchResult struct {
|
||||
ResultStatus
|
||||
Data []Item `json:"data"`
|
||||
Cursor string `json:"cursor"`
|
||||
HasMore bool `json:"has_more"`
|
||||
ResponseAt int `json:"response_at"`
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the item
|
||||
func (i *Item) ModTime() (t time.Time) {
|
||||
t = time.Time(i.LastModifiedAt)
|
||||
if t.IsZero() {
|
||||
t = time.Time(i.CreatedAt)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// CancelResource is a kdrive resource that can be cancelled after some action
|
||||
type CancelResource struct {
|
||||
CancelID string `json:"cancel_id"`
|
||||
ValidUntil int `json:"valid_until"`
|
||||
}
|
||||
|
||||
// CancellableResponse is returned from kdrive when an action can be cancelled afterwards
|
||||
type CancellableResponse struct {
|
||||
ResultStatus
|
||||
Data CancelResource `json:"data"`
|
||||
}
|
||||
|
||||
// CreateDirResult is returned from kdrive after a call to MkDir
|
||||
type CreateDirResult struct {
|
||||
ResultStatus
|
||||
Data Item `json:"data"`
|
||||
}
|
||||
|
||||
// FileCopyResponse is returned from kdrive after a call to Copy
|
||||
type FileCopyResponse struct {
|
||||
ResultStatus
|
||||
Data Item `json:"data"`
|
||||
}
|
||||
|
||||
// UploadFileResponse is returned from kdrive after a call to Upload
|
||||
type UploadFileResponse struct {
|
||||
ResultStatus
|
||||
Data Item `json:"data"`
|
||||
}
|
||||
|
||||
// ChecksumFileResult is returned from kdrive after a call to Hash
|
||||
type ChecksumFileResult struct {
|
||||
ResultStatus
|
||||
Data struct {
|
||||
Hash string `json:"hash"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// PubLinkResult is returned from kdrive after a call to public link API
|
||||
type PubLinkResult struct {
|
||||
ResultStatus
|
||||
Data struct {
|
||||
URL string `json:"url"`
|
||||
FileID int `json:"file_id"`
|
||||
Right string `json:"right"`
|
||||
ValidUntil int `json:"valid_until"`
|
||||
CreatedBy int `json:"created_by"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
Capabilities struct {
|
||||
CanEdit bool `json:"can_edit"`
|
||||
CanSeeStats bool `json:"can_see_stats"`
|
||||
CanSeeInfo bool `json:"can_see_info"`
|
||||
CanDownload bool `json:"can_download"`
|
||||
CanComment bool `json:"can_comment"`
|
||||
CanRequestAccess bool `json:"can_request_access"`
|
||||
} `json:"capabilities"`
|
||||
AccessBlocked bool `json:"access_blocked"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// PubLinkUpdateResult is returned from kdrive after a updating a public link
|
||||
type PubLinkUpdateResult struct {
|
||||
ResultStatus
|
||||
Data bool `json:"data"`
|
||||
}
|
||||
|
||||
// QuotaInfo is return from kdrive after a call get drive info
|
||||
type QuotaInfo struct {
|
||||
ResultStatus
|
||||
Data struct {
|
||||
Size int64 `json:"size"`
|
||||
UsedSize int64 `json:"used_size"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// SessionStartResponse is used to start a multichunk upload session
|
||||
type SessionStartResponse struct {
|
||||
ResultStatus
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
UploadURL string `json:"upload_url"` // URL pour uploader les chunks
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// ChunkUploadResponse is returned by kdrive after a chunk upload
|
||||
type ChunkUploadResponse struct {
|
||||
ResultStatus
|
||||
Data struct {
|
||||
Hash string `json:"hash,omitempty"` // Only present when requested with with:'hash'
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// SessionFinishResponse is used to end a multichunk upload session
|
||||
type SessionFinishResponse struct {
|
||||
ResultStatus
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
File Item `json:"file"`
|
||||
Result bool `json:"result"`
|
||||
Message string `json:"message"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// SessionCancelResponse is used to cancel a multichunk upload session
|
||||
type SessionCancelResponse struct {
|
||||
ResultStatus
|
||||
Data bool `json:"data"`
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,551 +0,0 @@
|
||||
// Unit tests for the kdrive backend
|
||||
package kdrive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/kdrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestCheckChunkSize checks chunk_size validation against the API limits
|
||||
func TestCheckChunkSize(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in fs.SizeSuffix
|
||||
wantErr string
|
||||
}{
|
||||
{in: 20 * fs.Mebi, wantErr: ""}, // default
|
||||
{in: 1000 * 1000, wantErr: ""}, // API minimum
|
||||
{in: 1000 * 1000 * 1000, wantErr: ""}, // API maximum
|
||||
{in: 0, wantErr: "too small"}, // panics without validation
|
||||
{in: 999 * 1000, wantErr: "too small"}, // below API minimum
|
||||
{in: 1000*1000*1000 + 1, wantErr: "too large"}, // above API maximum
|
||||
{in: 2 * 1000 * 1000 * 1000, wantErr: "too large"}, // way above API maximum
|
||||
} {
|
||||
t.Run(test.in.String(), func(t *testing.T) {
|
||||
err := checkChunkSize(test.in)
|
||||
if test.wantErr == "" {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), test.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenChunkWriterClampsCallerChunkSize checks that a caller
|
||||
// supplied chunk size hint below the API minimum is raised to it
|
||||
func TestOpenChunkWriterClampsCallerChunkSize(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"token":"tok","upload_url":"http://unused"}}`)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
f.dirCache = dircache.New("", "1", f)
|
||||
src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0)}
|
||||
|
||||
info, _, err := f.OpenChunkWriter(context.Background(), "file.txt", src, &fs.ChunkOption{ChunkSize: 512 * 1024})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(minChunkSize), info.ChunkSize, "a chunk size hint below the API minimum must be raised to it")
|
||||
}
|
||||
|
||||
// TestObjectSetMetaDataResetsStaleHash checks that setMetaData always
|
||||
// replaces the hash (a chunked upload finish response carries none)
|
||||
func TestObjectSetMetaDataResetsStaleHash(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
staleHash string
|
||||
staleNested bool
|
||||
newHash string
|
||||
wantHash string
|
||||
wantIsNested bool
|
||||
wantHashKnown bool
|
||||
}{
|
||||
{
|
||||
name: "empty hash clears stale plain hash",
|
||||
staleHash: "deadbeef",
|
||||
staleNested: false,
|
||||
newHash: "",
|
||||
wantHash: "",
|
||||
wantIsNested: false,
|
||||
wantHashKnown: false,
|
||||
},
|
||||
{
|
||||
name: "empty hash clears stale nested hash",
|
||||
staleHash: "deadbeef",
|
||||
staleNested: true,
|
||||
newHash: "",
|
||||
wantHash: "",
|
||||
wantIsNested: false,
|
||||
wantHashKnown: false,
|
||||
},
|
||||
{
|
||||
name: "plain hash is set",
|
||||
staleHash: "deadbeef",
|
||||
staleNested: false,
|
||||
newHash: "xxh3:12345678",
|
||||
wantHash: "12345678",
|
||||
wantIsNested: false,
|
||||
wantHashKnown: true,
|
||||
},
|
||||
{
|
||||
name: "nested hash is set",
|
||||
staleHash: "deadbeef",
|
||||
staleNested: false,
|
||||
newHash: "N:xxh3:87654321",
|
||||
wantHash: "87654321",
|
||||
wantIsNested: true,
|
||||
wantHashKnown: true,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
o := &Object{
|
||||
remote: "file.txt",
|
||||
hasMetaData: true,
|
||||
xxh3: test.staleHash,
|
||||
nestedHash: test.staleNested,
|
||||
}
|
||||
info := &api.Item{
|
||||
ID: 42,
|
||||
Name: "file.txt",
|
||||
Type: "file",
|
||||
Hash: test.newHash,
|
||||
}
|
||||
|
||||
err := o.setMetaData(info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.wantHash, o.xxh3, "hash after setMetaData")
|
||||
assert.Equal(t, test.wantIsNested, o.nestedHash, "nested flag after setMetaData")
|
||||
assert.True(t, o.hasMetaData)
|
||||
|
||||
// Hash() must reflect the new state. Only checked when a
|
||||
// hash is known: with no hash Hash() would query the API.
|
||||
// Nested hashes are not comparable with plain XXH3, so
|
||||
// Hash() returns "" for them.
|
||||
if test.wantHashKnown {
|
||||
got, gotErr := o.Hash(context.Background(), hash.XXH3)
|
||||
require.NoError(t, gotErr)
|
||||
if test.wantIsNested {
|
||||
assert.Equal(t, "", got, "nested hash must not be returned as XXH3")
|
||||
} else {
|
||||
assert.Equal(t, test.wantHash, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// newTestFs creates an Fs wired to a test server with a plain client
|
||||
// (no OAuth) and a fast pacer
|
||||
func newTestFs(t *testing.T, endpoint string) *Fs {
|
||||
t.Helper()
|
||||
return &Fs{
|
||||
name: "TestKdrive",
|
||||
root: "",
|
||||
opt: Options{DriveID: "1234", Endpoint: endpoint},
|
||||
srv: rest.NewClient(&http.Client{}).SetRoot(endpoint),
|
||||
pacer: fs.NewPacer(context.Background(),
|
||||
pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond), pacer.DecayConstant(2))),
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteChunkDetectsResultError checks that a 200 with result:error
|
||||
// is reported as an error
|
||||
func TestWriteChunkDetectsResultError(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"chunk_error","description":"chunk upload failed"}}`)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
u := &uploadSession{
|
||||
f: f,
|
||||
token: "tok",
|
||||
uploadURL: ts.URL,
|
||||
fileSize: 4,
|
||||
chunkSize: 4,
|
||||
chunkHashes: make(map[int64]string),
|
||||
}
|
||||
|
||||
_, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
|
||||
require.Error(t, err, "result:error with HTTP 200 must not be treated as success")
|
||||
assert.Contains(t, err.Error(), "chunk upload failed")
|
||||
}
|
||||
|
||||
// TestWriteChunkRetriesAndRewinds checks that transient failures are
|
||||
// retried with the chunk re-sent from the start of the reader
|
||||
func TestWriteChunkRetriesAndRewinds(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var attempts int
|
||||
var bodies []string
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
attempts++
|
||||
bodies = append(bodies, string(body))
|
||||
current := attempts
|
||||
mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if current == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"chunk_number":1,"status":"OK"}}`)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
u := &uploadSession{
|
||||
f: f,
|
||||
token: "tok",
|
||||
uploadURL: ts.URL,
|
||||
fileSize: 4,
|
||||
chunkSize: 4,
|
||||
chunkHashes: make(map[int64]string),
|
||||
}
|
||||
|
||||
written, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(4), written)
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
require.Len(t, bodies, 2, "chunk upload must be retried once after a 5xx")
|
||||
assert.Equal(t, "test", bodies[0], "first attempt sends the chunk")
|
||||
assert.Equal(t, "test", bodies[1], "retry must re-send the chunk from the start of the reader")
|
||||
}
|
||||
|
||||
// kdriveMockItem returns a JSON body for a single item result
|
||||
func kdriveMockItem(id int, name string) string {
|
||||
return fmt.Sprintf(`{"result":"success","data":{"id":%d,"name":%q,"type":"file","path":%q,"last_modified_at":1700000000}}`, id, name, "/"+name)
|
||||
}
|
||||
|
||||
// mockObjectInfo is a minimal fs.ObjectInfo for testing
|
||||
type mockObjectInfo struct {
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
hash string
|
||||
}
|
||||
|
||||
func (m *mockObjectInfo) Fs() fs.Info { return nil }
|
||||
func (m *mockObjectInfo) String() string { return m.remote }
|
||||
func (m *mockObjectInfo) Remote() string { return m.remote }
|
||||
func (m *mockObjectInfo) Size() int64 { return m.size }
|
||||
func (m *mockObjectInfo) ModTime(_ context.Context) time.Time { return m.modTime }
|
||||
func (m *mockObjectInfo) Storable() bool { return true }
|
||||
func (m *mockObjectInfo) MimeType(_ context.Context) string { return "application/octet-stream" }
|
||||
|
||||
func (m *mockObjectInfo) Hash(_ context.Context, t hash.Type) (string, error) {
|
||||
if m.hash == "" || t != hash.XXH3 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
return m.hash, nil
|
||||
}
|
||||
|
||||
// TestUpdateDirectNoRetryOnServerError checks that a failed single
|
||||
// part upload is not retried here: the input must not be buffered, the
|
||||
// higher layers retry the whole transfer
|
||||
func TestUpdateDirectNoRetryOnServerError(t *testing.T) {
|
||||
var attempts int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&attempts, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
o := &Object{fs: f, remote: "file.txt"}
|
||||
src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: "12345678"}
|
||||
|
||||
err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&attempts), "single part uploads must not be retried here")
|
||||
}
|
||||
|
||||
// TestUpdateDirect checks the upload parameters and metadata of a
|
||||
// successful single part upload, and that total_chunk_hash is only
|
||||
// sent when the source hash is known
|
||||
func TestUpdateDirect(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
srcHash string
|
||||
wantTotalHash string
|
||||
}{
|
||||
{name: "hash known", srcHash: "12345678", wantTotalHash: "xxh3:12345678"},
|
||||
{name: "hash unknown", srcHash: "", wantTotalHash: ""},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var gotTotalHash, gotBody string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
gotTotalHash = r.URL.Query().Get("total_chunk_hash")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
gotBody = string(body)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"id":55,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:99998888","last_modified_at":1700000000}}`)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
o := &Object{fs: f, remote: "file.txt"}
|
||||
src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: test.srcHash}
|
||||
|
||||
err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "test", gotBody, "the content must be streamed as-is")
|
||||
assert.Equal(t, test.wantTotalHash, gotTotalHash, "total_chunk_hash")
|
||||
assert.Equal(t, "55", o.id)
|
||||
gotHash, hashErr := o.Hash(context.Background(), hash.XXH3)
|
||||
require.NoError(t, hashErr)
|
||||
assert.Equal(t, "99998888", gotHash)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewFsRequiresDriveID checks that NewFs fails fast when drive_id
|
||||
// is missing
|
||||
func TestNewFsRequiresDriveID(t *testing.T) {
|
||||
m := configmap.Simple{"chunk_size": "20Mi"}
|
||||
_, err := NewFs(context.Background(), "kdrive", "", m)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "drive_id")
|
||||
}
|
||||
|
||||
// TestUploadSessionStringMasksToken checks that the upload session
|
||||
// String does not leak the session token in debug logs
|
||||
func TestUploadSessionStringMasksToken(t *testing.T) {
|
||||
u := &uploadSession{token: "very-secret-uuid"}
|
||||
assert.NotContains(t, u.String(), "very-secret-uuid")
|
||||
}
|
||||
|
||||
// TestListNonRecursiveSingleRoundTrip checks that a non recursive
|
||||
// listing needs a single API request: without depth=unlimited the API
|
||||
// returns direct children only, so neither the parent item lookup nor
|
||||
// a child filter is needed
|
||||
func TestListNonRecursiveSingleRoundTrip(t *testing.T) {
|
||||
var requests int32
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&requests, 1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
switch {
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/files"):
|
||||
// Direct children only, as returned without depth=unlimited
|
||||
_, _ = fmt.Fprintln(w, `{"result":"success","data":[`+
|
||||
`{"id":10,"name":"sub","type":"dir","path":"/sub"},`+
|
||||
`{"id":11,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:12345678","last_modified_at":1700000000}],"has_more":false}`)
|
||||
default:
|
||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
f.dirCache = dircache.New("", "1", f)
|
||||
|
||||
entries, err := f.List(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&requests), "non recursive listing must make a single API request")
|
||||
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
names = append(names, entry.Remote())
|
||||
}
|
||||
assert.ElementsMatch(t, []string{"sub", "file.txt"}, names)
|
||||
}
|
||||
|
||||
// TestMoveConflictReturnsError checks that moving onto an occupied
|
||||
// destination reports the API conflict instead of resolving it: rclone
|
||||
// removes the destination itself before calling Move
|
||||
func TestMoveConflictReturnsError(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"conflict_error","description":"destination already exists"}}`)
|
||||
default:
|
||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
f.dirCache = dircache.New("", "1", f)
|
||||
srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
|
||||
|
||||
_, err := f.Move(context.Background(), srcObj, "file2")
|
||||
require.Error(t, err, "a move onto an occupied destination must fail")
|
||||
assert.Contains(t, err.Error(), "conflict", "the API conflict must be reported")
|
||||
}
|
||||
|
||||
// TestMoveDetectsForeignObjectAtDestination checks that Move does not
|
||||
// return a file re-created by another writer at the destination
|
||||
func TestMoveDetectsForeignObjectAtDestination(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
|
||||
// The move API reports success...
|
||||
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
|
||||
// ...but the file at the destination is not the moved
|
||||
// file: another writer re-created it
|
||||
_, _ = fmt.Fprintln(w, kdriveMockItem(999, "file2"))
|
||||
default:
|
||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
f.dirCache = dircache.New("", "1", f)
|
||||
srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
|
||||
|
||||
_, err := f.Move(context.Background(), srcObj, "file2")
|
||||
require.Error(t, err, "Move must not report success when the destination is not the moved file")
|
||||
}
|
||||
|
||||
// TestMoveSuccess checks the nominal move: the object returned is the
|
||||
// moved file (same ID as the source).
|
||||
func TestMoveSuccess(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
|
||||
_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
|
||||
// After the move, the moved file is at the destination
|
||||
_, _ = fmt.Fprintln(w, kdriveMockItem(111, "file2"))
|
||||
default:
|
||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
||||
_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
f := newTestFs(t, ts.URL)
|
||||
f.dirCache = dircache.New("", "1", f)
|
||||
srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
|
||||
|
||||
dstObj, err := f.Move(context.Background(), srcObj, "file2")
|
||||
require.NoError(t, err)
|
||||
moved, ok := dstObj.(*Object)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "111", moved.id, "the returned object must be the moved file")
|
||||
}
|
||||
|
||||
// TestFormatModTime checks that pre-1970 times format as negative
|
||||
// unix timestamps instead of wrapping through a uint64 cast
|
||||
func TestFormatModTime(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in time.Time
|
||||
want string
|
||||
}{
|
||||
{in: time.Unix(1700000000, 0), want: "1700000000"},
|
||||
{in: time.Unix(0, 0), want: "0"},
|
||||
{in: time.Unix(-1577923200, 0), want: "-1577923200"}, // Sat Jan 2 1965
|
||||
} {
|
||||
t.Run(test.want, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, formatModTime(test.in))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestObjectSetMetaDataIsDir checks that directories are rejected
|
||||
func TestObjectSetMetaDataIsDir(t *testing.T) {
|
||||
o := &Object{remote: "dir"}
|
||||
info := &api.Item{ID: 42, Name: "dir", Type: "dir"}
|
||||
err := o.setMetaData(info)
|
||||
assert.Equal(t, fs.ErrorIsDir, err)
|
||||
}
|
||||
|
||||
// TestShouldRetryLockError checks that transient locks held by server
|
||||
// side copies are retried while locks held by applications and other
|
||||
// errors are not
|
||||
func TestShouldRetryLockError(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
body string
|
||||
status int
|
||||
wantRetry bool
|
||||
}{
|
||||
{
|
||||
name: "lock held by internal copy",
|
||||
body: `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy","context":{"values":["internal_copy"]}}}`,
|
||||
status: http.StatusForbidden,
|
||||
wantRetry: true,
|
||||
},
|
||||
{
|
||||
name: "lock held by an application",
|
||||
body: `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by onlyoffice","context":{"values":["onlyoffice"]}}}`,
|
||||
status: http.StatusForbidden,
|
||||
wantRetry: false,
|
||||
},
|
||||
{
|
||||
name: "lock without context",
|
||||
body: `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy"}}`,
|
||||
status: http.StatusForbidden,
|
||||
wantRetry: false,
|
||||
},
|
||||
{
|
||||
name: "other api error",
|
||||
body: `{"result":"error","error":{"code":"object_not_found","description":"File not found"}}`,
|
||||
status: http.StatusNotFound,
|
||||
wantRetry: false,
|
||||
},
|
||||
{
|
||||
name: "server error",
|
||||
status: http.StatusServiceUnavailable,
|
||||
wantRetry: true,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var apiErr error
|
||||
if test.body != "" {
|
||||
e := new(api.ResultStatus)
|
||||
require.NoError(t, json.Unmarshal([]byte(test.body), e))
|
||||
apiErr = e
|
||||
}
|
||||
resp := &http.Response{StatusCode: test.status, Header: http.Header{}}
|
||||
got, gotErr := shouldRetry(context.Background(), resp, apiErr)
|
||||
assert.Equal(t, test.wantRetry, got)
|
||||
assert.Equal(t, apiErr, gotErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
// Test kdrive filesystem interface
|
||||
package kdrive
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestKdrive:",
|
||||
NilObject: (*Object)(nil),
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: minChunkSize,
|
||||
MaxChunkSize: maxChunkSize,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
return f.setChunkSize(cs)
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
// Package khash provides the kDrive specific hash handling.
|
||||
//
|
||||
// kDrive uses XXH3-64 for file hashing. Files uploaded in chunks (by
|
||||
// rclone, or by the kDrive apps which each use their own chunk size)
|
||||
// get a "nested" hash: the XXH3 of the concatenated lowercase hex
|
||||
// strings of the individual chunk hashes. Such a nested hash can't be
|
||||
// compared with a plain XXH3 of the file contents.
|
||||
//
|
||||
// Hash formats returned by the kDrive API:
|
||||
// - "xxh3:HASH" for a plain XXH3 of the file contents
|
||||
// - "N:xxh3:HASH" for the nested hash of a chunked upload
|
||||
package khash
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
// ParseHash parses a kDrive hash string from the API and returns the hex hash value
|
||||
// along with whether it is a nested (chunked) hash.
|
||||
func ParseHash(hashStr string) (hexHash string, isNested bool, err error) {
|
||||
if hashStr == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// Check for nested hash format: "N:xxh3:HASH"
|
||||
if strings.HasPrefix(hashStr, "N:xxh3:") {
|
||||
hashStr = strings.TrimPrefix(hashStr, "N:xxh3:")
|
||||
isNested = true
|
||||
} else {
|
||||
hashStr = strings.TrimPrefix(hashStr, "xxh3:")
|
||||
}
|
||||
|
||||
// Validate hex string
|
||||
if _, decodeErr := hex.DecodeString(hashStr); decodeErr != nil {
|
||||
return "", false, fmt.Errorf("invalid hash format: %w", decodeErr)
|
||||
}
|
||||
|
||||
return hashStr, isNested, nil
|
||||
}
|
||||
|
||||
// NestedChunkHash computes the kDrive nested hash from the hashes of the
|
||||
// file chunks, in order.
|
||||
//
|
||||
// kDrive stores the nested hash of a chunked file as the XXH3 of the
|
||||
// concatenated lowercase hex strings of the individual chunk hashes.
|
||||
// Each chunk hash may be passed with or without the "xxh3:" prefix.
|
||||
func NestedChunkHash(chunkHashes []string) (string, error) {
|
||||
if len(chunkHashes) == 0 {
|
||||
return "", errors.New("no chunk hashes to nest")
|
||||
}
|
||||
hexHashes := make([]string, len(chunkHashes))
|
||||
for i, chunkHash := range chunkHashes {
|
||||
hexHash, _, err := ParseHash(chunkHash)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid chunk hash %q: %w", chunkHash, err)
|
||||
}
|
||||
hexHashes[i] = hexHash
|
||||
}
|
||||
hasher := xxh3.New()
|
||||
_, _ = hasher.Write([]byte(strings.Join(hexHashes, "")))
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package khash
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseHash(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expectedHash string
|
||||
isNested bool
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "simple hash with prefix",
|
||||
input: "xxh3:1db797b96febd334",
|
||||
expectedHash: "1db797b96febd334",
|
||||
isNested: false,
|
||||
},
|
||||
{
|
||||
name: "nested hash",
|
||||
input: "N:xxh3:877abf3579f0a5c0",
|
||||
expectedHash: "877abf3579f0a5c0",
|
||||
isNested: true,
|
||||
},
|
||||
{
|
||||
name: "plain hex",
|
||||
input: "1db797b96febd334",
|
||||
expectedHash: "1db797b96febd334",
|
||||
isNested: false,
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
expectedHash: "",
|
||||
isNested: false,
|
||||
},
|
||||
{
|
||||
name: "invalid hex",
|
||||
input: "xxh3:notahexvalue",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
hash, isNested, err := ParseHash(tt.input)
|
||||
|
||||
if tt.expectError {
|
||||
if err == nil {
|
||||
t.Error("Expected error but got none")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if hash != tt.expectedHash {
|
||||
t.Errorf("Hash: got %q, expected %q", hash, tt.expectedHash)
|
||||
}
|
||||
if isNested != tt.isNested {
|
||||
t.Errorf("IsNested: got %t, expected %t", isNested, tt.isNested)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedChunkHash(t *testing.T) {
|
||||
// Golden values: the nested hash is the XXH3 of the concatenated
|
||||
// lowercase hex strings of the chunk hashes, in order
|
||||
chunk1 := "xxh3:d447b1ea40e6988b" // XXH3 of "hello world"
|
||||
chunk2 := "2d06800538d394c2" // XXH3 of ""
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
chunkHashes []string
|
||||
expected string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "two chunks in order",
|
||||
chunkHashes: []string{chunk1, chunk2},
|
||||
expected: "6d51edccc9c05a8b",
|
||||
},
|
||||
{
|
||||
name: "order matters",
|
||||
chunkHashes: []string{chunk2, chunk1},
|
||||
expected: "596c127c7bcc8820",
|
||||
},
|
||||
{
|
||||
name: "single chunk",
|
||||
chunkHashes: []string{chunk1},
|
||||
expected: "b73081dff8fe443c",
|
||||
},
|
||||
{
|
||||
name: "mixed prefixes",
|
||||
chunkHashes: []string{chunk1, "xxh3:" + chunk2, chunk1},
|
||||
expected: "ca0b13abcdcd1661",
|
||||
},
|
||||
{
|
||||
name: "no chunks",
|
||||
chunkHashes: []string{},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid chunk hash",
|
||||
chunkHashes: []string{chunk1, "notahexvalue"},
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
nested, err := NestedChunkHash(tt.chunkHashes)
|
||||
|
||||
if tt.expectError {
|
||||
if err == nil {
|
||||
t.Error("Expected error but got none")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if nested != tt.expected {
|
||||
t.Errorf("NestedChunkHash: got %q, expected %q", nested, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNestedChunkHashAcceptsPrefixedHashes(t *testing.T) {
|
||||
plain := []string{"d447b1ea40e6988b", "2d06800538d394c2"}
|
||||
prefixed := []string{"xxh3:d447b1ea40e6988b", "N:xxh3:2d06800538d394c2"}
|
||||
|
||||
gotPlain, err := NestedChunkHash(plain)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
gotPrefixed, err := NestedChunkHash(prefixed)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
if gotPlain != gotPrefixed {
|
||||
t.Errorf("Prefixed hashes should give the same nested hash: %q != %q", gotPlain, gotPrefixed)
|
||||
}
|
||||
}
|
||||
@@ -1,383 +0,0 @@
|
||||
package kdrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/kdrive/api"
|
||||
"github.com/rclone/rclone/backend/kdrive/khash"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/zeebo/xxh3"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxUploadParts is the maximum number of chunks the kDrive API accepts per file
|
||||
maxUploadParts = 10000
|
||||
|
||||
// maxChunkSize is the largest chunk size the kDrive API accepts
|
||||
maxChunkSize = 1 * 1000 * 1000 * 1000
|
||||
|
||||
// minChunkSize is the smallest chunk size the kDrive API accepts
|
||||
minChunkSize = 1 * 1000 * 1000
|
||||
)
|
||||
|
||||
// checkChunkSize checks the user supplied chunk_size against the
|
||||
// limits the kDrive API accepts
|
||||
func checkChunkSize(chunkSize fs.SizeSuffix) error {
|
||||
if chunkSize < minChunkSize {
|
||||
return fmt.Errorf("chunk_size %s is too small: the kDrive API requires chunks of at least 1 MB", chunkSize)
|
||||
}
|
||||
if chunkSize > maxChunkSize {
|
||||
return fmt.Errorf("chunk_size %s is too large: the kDrive API accepts chunks of at most 1 GB (1000000000 bytes)", chunkSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// setChunkSize sets the chunk size, returning the old value
|
||||
func (f *Fs) setChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
||||
err = checkChunkSize(cs)
|
||||
if err == nil {
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// formatModTime formats the modification time of an upload as the
|
||||
// unix timestamp string the kDrive API expects
|
||||
func formatModTime(modTime time.Time) string {
|
||||
return fmt.Sprintf("%d", modTime.Unix())
|
||||
}
|
||||
|
||||
// uploadSession implements fs.ChunkWriter for kdrive multipart uploads
|
||||
type uploadSession struct {
|
||||
f *Fs
|
||||
token string
|
||||
uploadURL string
|
||||
fileInfo *api.Item
|
||||
fileSize int64 // total size of the file being uploaded
|
||||
chunkSize int64 // size of the chunks of this session
|
||||
|
||||
mu sync.Mutex
|
||||
chunkHashes map[int64]string // chunk number -> hash of the chunk contents
|
||||
}
|
||||
|
||||
// OpenChunkWriter returns the chunk size and a ChunkWriter
|
||||
//
|
||||
// Pass in the remote and the src object
|
||||
// You can also use options to hint at the desired chunk size
|
||||
// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
|
||||
func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
|
||||
fileSize := src.Size()
|
||||
if fileSize < 0 {
|
||||
return info, nil, errors.New("kdrive can't upload files with unknown size")
|
||||
}
|
||||
|
||||
dir, leaf := path.Split(remote)
|
||||
dir = strings.TrimSuffix(dir, "/")
|
||||
|
||||
// Create parent directories if they don't exist
|
||||
parentID, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to find parent directory: %w", err)
|
||||
}
|
||||
|
||||
// Calculate the chunk size, honouring any fs.ChunkOption hint
|
||||
chunkSize := f.opt.ChunkSize
|
||||
for _, opt := range options {
|
||||
if chunkOpt, ok := opt.(*fs.ChunkOption); ok && chunkOpt.ChunkSize > 0 {
|
||||
chunkSize = fs.SizeSuffix(chunkOpt.ChunkSize)
|
||||
}
|
||||
}
|
||||
chunkSize = chunksize.Calculator(src, fileSize, maxUploadParts, chunkSize)
|
||||
if chunkSize < minChunkSize {
|
||||
chunkSize = minChunkSize
|
||||
}
|
||||
if chunkSize > maxChunkSize {
|
||||
chunkSize = maxChunkSize
|
||||
}
|
||||
|
||||
// Calculate the total number of chunks
|
||||
totalChunks := max((fileSize+int64(chunkSize)-1)/int64(chunkSize), 1)
|
||||
|
||||
sessionReq := struct {
|
||||
Conflict string `json:"conflict"`
|
||||
DirectoryID string `json:"directory_id"`
|
||||
FileName string `json:"file_name"`
|
||||
LastModifiedAt string `json:"last_modified_at"`
|
||||
TotalChunks int64 `json:"total_chunks"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
}{
|
||||
Conflict: "version",
|
||||
DirectoryID: parentID,
|
||||
FileName: f.opt.Enc.FromStandardName(norm.NFC.String(leaf)),
|
||||
LastModifiedAt: formatModTime(src.ModTime(ctx)),
|
||||
TotalChunks: totalChunks,
|
||||
TotalSize: fileSize,
|
||||
}
|
||||
|
||||
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: fmt.Sprintf("/3/drive/%s/upload/session/start", f.opt.DriveID),
|
||||
}
|
||||
var sessionResp api.SessionStartResponse
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var callErr error
|
||||
resp, callErr = f.srv.CallJSON(ctx, &opts, &sessionReq, &sessionResp)
|
||||
callErr = sessionResp.ResultStatus.Update(callErr)
|
||||
return shouldRetry(ctx, resp, callErr)
|
||||
})
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to start upload session: %w", err)
|
||||
}
|
||||
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(chunkSize),
|
||||
Concurrency: f.opt.UploadConcurrency,
|
||||
}
|
||||
|
||||
writer = &uploadSession{
|
||||
f: f,
|
||||
token: sessionResp.Data.Token,
|
||||
uploadURL: sessionResp.Data.UploadURL,
|
||||
fileSize: fileSize,
|
||||
chunkSize: int64(chunkSize),
|
||||
chunkHashes: make(map[int64]string),
|
||||
}
|
||||
|
||||
fs.Debugf(f, "open chunk writer: started upload session for %d chunks of %v", totalChunks, chunkSize)
|
||||
return info, writer, nil
|
||||
}
|
||||
|
||||
// WriteChunk uploads a single chunk
|
||||
// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
|
||||
func (u *uploadSession) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
|
||||
if chunkNumber < 0 {
|
||||
return -1, fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
|
||||
}
|
||||
|
||||
// Calculate the size of this chunk from the session parameters so
|
||||
// the chunk can be streamed to the API in a single pass
|
||||
chunkLen := u.chunkSize
|
||||
if remaining := u.fileSize - int64(chunkNumber)*u.chunkSize; remaining < chunkLen {
|
||||
chunkLen = remaining
|
||||
}
|
||||
if chunkLen <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
sourceChunkNumber := chunkNumber + 1 // kDrive API uses 1-based numbering
|
||||
|
||||
// The chunk_hash parameter is optional, so the chunk is streamed
|
||||
// in a single pass, hashing the contents on the way with a
|
||||
// TeeReader. The server returns the hash of the chunk it received,
|
||||
// which is checked below. Retries seek back to the chunk start
|
||||
// and hash with a fresh hasher.
|
||||
var chunkHash string
|
||||
err = u.f.pacer.Call(func() (bool, error) {
|
||||
if _, seekErr := reader.Seek(0, io.SeekStart); seekErr != nil {
|
||||
return false, fmt.Errorf("failed to rewind chunk %d: %w", sourceChunkNumber, seekErr)
|
||||
}
|
||||
chunkHasher := xxh3.New()
|
||||
|
||||
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
|
||||
chunkOpts := rest.Opts{
|
||||
Method: "POST",
|
||||
RootURL: u.uploadURL,
|
||||
Path: fmt.Sprintf("/3/drive/%s/upload/session/%s/chunk", u.f.opt.DriveID, u.token),
|
||||
Parameters: url.Values{
|
||||
"chunk_number": {fmt.Sprintf("%d", sourceChunkNumber)},
|
||||
"chunk_size": {fmt.Sprintf("%d", chunkLen)},
|
||||
"with": {"hash"},
|
||||
},
|
||||
Body: io.TeeReader(reader, chunkHasher),
|
||||
ContentLength: &chunkLen,
|
||||
}
|
||||
|
||||
var chunkResp api.ChunkUploadResponse
|
||||
resp, callErr := u.f.srv.CallJSON(ctx, &chunkOpts, nil, &chunkResp)
|
||||
callErr = chunkResp.ResultStatus.Update(callErr)
|
||||
if callErr != nil {
|
||||
return shouldRetry(ctx, resp, callErr)
|
||||
}
|
||||
|
||||
chunkHash = hex.EncodeToString(chunkHasher.Sum(nil))
|
||||
|
||||
// The server hashed the chunk it received, so compare with what we
|
||||
// sent to detect any corruption immediately
|
||||
if chunkResp.Data.Hash != "" {
|
||||
if serverHash, _, parseErr := khash.ParseHash(chunkResp.Data.Hash); parseErr == nil {
|
||||
if !strings.EqualFold(chunkHash, serverHash) {
|
||||
return false, fmt.Errorf("chunk %d hash mismatch: client=%s, server=%s", sourceChunkNumber, chunkHash, serverHash)
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(u, "Failed to parse server chunk hash %q: %v", chunkResp.Data.Hash, parseErr)
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
u.mu.Lock()
|
||||
u.chunkHashes[int64(chunkNumber)] = chunkHash
|
||||
u.mu.Unlock()
|
||||
|
||||
fs.Debugf(u, "uploaded chunk %d (size: %d, hash: %s)", sourceChunkNumber, chunkLen, chunkHash)
|
||||
return chunkLen, nil
|
||||
}
|
||||
|
||||
// Close finalizes the upload session, verifies the hash of the uploaded
|
||||
// file and returns the created file info
|
||||
// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
|
||||
func (u *uploadSession) Close(ctx context.Context) error {
|
||||
// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: fmt.Sprintf("/3/drive/%s/upload/session/%s/finish", u.f.opt.DriveID, u.token),
|
||||
}
|
||||
var resp *http.Response
|
||||
var finishResp api.SessionFinishResponse
|
||||
err := u.f.pacer.Call(func() (bool, error) {
|
||||
var callErr error
|
||||
resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &finishResp)
|
||||
callErr = finishResp.ResultStatus.Update(callErr)
|
||||
return shouldRetry(ctx, resp, callErr)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to finish upload session: %w", err)
|
||||
}
|
||||
|
||||
u.fileInfo = &finishResp.Data.File
|
||||
fs.Debugf(u, "multipart upload completed: file id %d", finishResp.Data.File.ID)
|
||||
|
||||
return u.verifyHash(ctx)
|
||||
}
|
||||
|
||||
// verifyHash checks the hash of the uploaded file against the hashes of
|
||||
// the chunks we uploaded, deleting the file if it is corrupted
|
||||
func (u *uploadSession) verifyHash(ctx context.Context) (err error) {
|
||||
remoteHash := u.fileInfo.Hash
|
||||
if remoteHash == "" {
|
||||
// Hash might be missing in the finish response, try to fetch it
|
||||
obj := &Object{
|
||||
fs: u.f,
|
||||
id: strconv.Itoa(u.fileInfo.ID),
|
||||
}
|
||||
var hashErr error
|
||||
if remoteHash, hashErr = obj.retrieveHash(ctx); hashErr != nil {
|
||||
// skip verification
|
||||
fs.Debugf(u, "Failed to retrieve hash for verification: %v", hashErr)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
parsedRemoteHash, isNested, err := khash.ParseHash(remoteHash)
|
||||
if err != nil {
|
||||
fs.Debugf(u, "Failed to parse server hash %q: %v", remoteHash, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
// Rebuild the hash the server should have stored from the chunk
|
||||
// hashes we uploaded, in order
|
||||
chunkCount := int64(len(u.chunkHashes))
|
||||
if chunkCount == 0 {
|
||||
return nil
|
||||
}
|
||||
chunkHashes := make([]string, 0, chunkCount)
|
||||
for i := int64(0); i < chunkCount; i++ {
|
||||
chunkHash, ok := u.chunkHashes[i]
|
||||
if !ok {
|
||||
fs.Debugf(u, "Chunk %d hash missing, skipping hash verification", i)
|
||||
return nil
|
||||
}
|
||||
chunkHashes = append(chunkHashes, chunkHash)
|
||||
}
|
||||
|
||||
var localHash string
|
||||
if isNested {
|
||||
if localHash, err = khash.NestedChunkHash(chunkHashes); err != nil {
|
||||
fs.Debugf(u, "Failed to compute nested hash: %v", err)
|
||||
return nil
|
||||
}
|
||||
} else if chunkCount == 1 {
|
||||
localHash, _, _ = khash.ParseHash(chunkHashes[0])
|
||||
} else {
|
||||
// The server returned a simple hash for a multi chunk upload,
|
||||
// nothing to compare it against
|
||||
fs.Debugf(u, "Server returned a simple hash for a %d chunk upload, skipping hash verification", chunkCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
if !strings.EqualFold(localHash, parsedRemoteHash) {
|
||||
err = fmt.Errorf(
|
||||
"multipart upload hash mismatch: local=%s, remote=%s",
|
||||
localHash, parsedRemoteHash,
|
||||
)
|
||||
|
||||
// Remove the corrupted file
|
||||
obj := &Object{
|
||||
fs: u.f,
|
||||
id: strconv.Itoa(u.fileInfo.ID),
|
||||
}
|
||||
if delErr := obj.Remove(ctx); delErr != nil {
|
||||
fs.Errorf(nil, "Failed to remove corrupted file after hash mismatch: %v", delErr)
|
||||
} else {
|
||||
fs.Debugf(nil, "Removed corrupted file after hash mismatch")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
fs.Debugf(u, "Multipart upload hash verified: %s", localHash)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Abort the upload session
|
||||
// @see https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
|
||||
func (u *uploadSession) Abort(ctx context.Context) error {
|
||||
// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: fmt.Sprintf("/2/drive/%s/upload/session/%s", u.f.opt.DriveID, u.token),
|
||||
}
|
||||
var resp *http.Response
|
||||
var cancelResp api.SessionCancelResponse
|
||||
err := u.f.pacer.Call(func() (bool, error) {
|
||||
var callErr error
|
||||
resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &cancelResp)
|
||||
callErr = cancelResp.ResultStatus.Update(callErr)
|
||||
return shouldRetry(ctx, resp, callErr)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to cancel upload session: %w", err)
|
||||
}
|
||||
|
||||
fs.Debugf(u, "upload session cancelled")
|
||||
return nil
|
||||
}
|
||||
|
||||
// String implements fmt.Stringer
|
||||
func (u *uploadSession) String() string {
|
||||
return "kdrive upload session"
|
||||
}
|
||||
|
||||
// Verify interface compliance
|
||||
var _ fs.ChunkWriter = (*uploadSession)(nil)
|
||||
@@ -11,10 +11,10 @@ package linkbox
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -35,9 +35,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -48,8 +46,7 @@ const (
|
||||
pacerBurst = 1
|
||||
linkboxAPIURL = "https://www.linkbox.to/api/open/"
|
||||
linkboxWebAPIURL = "https://www.linkbox.to/api/"
|
||||
rootID = "0" // ID of root directory
|
||||
prefixSize = 10 * 1024 * 1024 // the API wants the MD5 of the first 10 MiB of a file
|
||||
rootID = "0" // ID of root directory
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -670,21 +667,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
// readPrefix reads up to prefixSize bytes of in into a buffer from the
|
||||
// global pool returning it along with the hex MD5 of what was read.
|
||||
//
|
||||
// The caller must Close the returned buffer when done with it.
|
||||
func readPrefix(in io.Reader) (rw *pool.RW, md5sum string, err error) {
|
||||
rw = multipart.NewRW()
|
||||
hasher := md5.New()
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hasher), prefixSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, "", err
|
||||
}
|
||||
return rw, hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
//
|
||||
// When called from outside an Fs by rclone, src.Size() will always be >= 0.
|
||||
@@ -719,13 +701,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
prefix, prefixMD5, err := readPrefix(in)
|
||||
first10m := io.LimitReader(in, 10_485_760)
|
||||
first10mBytes, err := io.ReadAll(first10m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Update err in reading file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = prefix.Close()
|
||||
}()
|
||||
|
||||
// get upload authorization (step 1)
|
||||
opts := &rest.Opts{
|
||||
@@ -735,7 +715,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
Parameters: url.Values{
|
||||
"token": {o.fs.opt.Token},
|
||||
"fileMd5ofPre10m": {prefixMD5},
|
||||
"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
|
||||
"fileSize": {itoa64(size)},
|
||||
},
|
||||
}
|
||||
@@ -777,7 +757,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return fmt.Errorf("head upload URL: %w", err)
|
||||
}
|
||||
|
||||
file := io.MultiReader(prefix, in)
|
||||
file := io.MultiReader(bytes.NewReader(first10mBytes), in)
|
||||
|
||||
opts.Method = "PUT"
|
||||
opts.Body = file
|
||||
@@ -817,7 +797,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
Parameters: url.Values{
|
||||
"token": {o.fs.opt.Token},
|
||||
"fileMd5ofPre10m": {prefixMD5},
|
||||
"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
|
||||
"fileSize": {itoa64(size)},
|
||||
"pid": {dirID},
|
||||
"diyName": {leaf},
|
||||
@@ -1039,7 +1019,8 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
|
||||
// intermittently returns an HTML challenge page with a 200
|
||||
// status instead of JSON. This surfaces as a JSON syntax error,
|
||||
// so retry it to let the pacer back off until the block lifts.
|
||||
if _, ok := errors.AsType[*json.SyntaxError](err); ok {
|
||||
var syntaxErr *json.SyntaxError
|
||||
if errors.As(err, &syntaxErr) {
|
||||
return true, err
|
||||
}
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
|
||||
Loaded 100 of 769 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user