Compare commits

..
Author SHA1 Message Date
Nick Craig-Wood 13a1d23bb8 mega: fix nil pointer panic when login with session keys fails - fixes #9232
When LoginWithKeys failed (e.g. after clearing Mega trash or with a
stale session), the error was silently ignored and only logged at debug
level. This left the filesystem uninitialized so GetRoot() returned
nil, causing a nil pointer dereference panic in findDir.

This returns the login error instead of swallowing it, and adds a nil
check for GetRoot() in findRoot, and adds defensive nil node checks in
findDir and findObject.
2026-03-05 12:30:43 +00:00
1177 changed files with 78016 additions and 172579 deletions

No files matched your search

-25
View File
@@ -1,25 +0,0 @@
{
"permissions": {
"allow": [
"Bash(git status:*)", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)",
"Bash(git rev-parse:*)", "Bash(git add:*)", "Bash(git commit:*)",
"Bash(go mod download:*)", "Bash(go build:*)", "Bash(go test:*)", "Bash(go vet:*)",
"Bash(gofmt -l:*)",
"Bash(golangci-lint run:*)"
],
"ask": [
"Bash(git push:*)", "Bash(git merge:*)", "Bash(git rebase:*)", "Bash(git switch:*)",
"Bash(git restore:*)", "Bash(git commit --amend:*)", "Bash(rm:*)",
"Bash(go get:*)", "Bash(go mod tidy:*)", "Bash(go install:*)",
"Bash(make:*)", "Bash(docker:*)"
],
"deny": [
"Bash(git push --force:*)", "Bash(git push -f:*)", "Bash(git push --force-with-lease:*)",
"Bash(git reset --hard:*)", "Bash(git clean:*)", "Bash(git add -f:*)", "Bash(git add --force:*)",
"Bash(git commit --no-verify:*)", "Bash(git commit -n:*)",
"Bash(rm -r:*)", "Bash(rm -R:*)", "Bash(rm -rf:*)", "Bash(rm -fr:*)", "Bash(rm -Rf:*)",
"Bash(rm -fR:*)", "Bash(rm --recursive:*)", "Bash(sudo:*)",
"Read(./.env)", "Read(./.env.*)", "Read(./secrets/**)"
]
}
}
-2
View File
@@ -1,2 +0,0 @@
sandbox_mode = "workspace-write"
approval_policy = "on-request"
-20
View File
@@ -1,20 +0,0 @@
# Rendered from docs/PERMISSIONS.md - change both in the same commit.
# Deny
prefix_rule(pattern = ["git", "push", ["--force", "-f", "--force-with-lease"]], decision = "forbidden",
justification = "Force-push rewrites shared history.")
prefix_rule(pattern = ["git", "reset", "--hard"], decision = "forbidden", justification = "Discards work.")
prefix_rule(pattern = ["git", "clean"], decision = "forbidden", justification = "Deletes untracked files.")
prefix_rule(pattern = ["git", "add", ["-f", "--force"]], decision = "forbidden",
justification = "Stages ignored files, e.g. secrets.")
prefix_rule(pattern = ["git", "commit", ["--no-verify", "-n"]], decision = "forbidden", justification = "Skips hooks.")
prefix_rule(pattern = ["rm", ["-r", "-R", "-rf", "-fr", "-Rf", "-fR", "--recursive"]], decision = "forbidden",
justification = "Recursive delete.")
prefix_rule(pattern = ["sudo"], decision = "forbidden", justification = "No privilege escalation.")
# Ask
prefix_rule(pattern = ["git", ["push", "merge", "rebase", "switch", "restore"]], decision = "prompt")
prefix_rule(pattern = ["git", "commit", "--amend"], decision = "prompt")
prefix_rule(pattern = ["rm"], decision = "prompt")
prefix_rule(pattern = ["go", ["get", "install"]], decision = "prompt")
prefix_rule(pattern = ["go", "mod", "tidy"], decision = "prompt")
prefix_rule(pattern = ["make"], decision = "prompt")
prefix_rule(pattern = ["docker"], decision = "prompt")
-20
View File
@@ -1,20 +0,0 @@
{
"tools": {
"allowed": [
"run_shell_command(git status)", "run_shell_command(git diff)", "run_shell_command(git log)",
"run_shell_command(git show)", "run_shell_command(git branch --list)",
"run_shell_command(git add)", "run_shell_command(git commit)",
"run_shell_command(go mod download)", "run_shell_command(go build)",
"run_shell_command(go test)", "run_shell_command(go vet)", "run_shell_command(gofmt -l)",
"run_shell_command(golangci-lint run)"
],
"exclude": [
"run_shell_command(git push --force)", "run_shell_command(git push -f)",
"run_shell_command(git push --force-with-lease)", "run_shell_command(git reset --hard)",
"run_shell_command(git clean)", "run_shell_command(git add -f)", "run_shell_command(git add --force)",
"run_shell_command(git commit --no-verify)", "run_shell_command(git commit -n)",
"run_shell_command(rm -r)", "run_shell_command(rm -R)", "run_shell_command(rm -rf)",
"run_shell_command(rm -fr)", "run_shell_command(rm -Rf)", "run_shell_command(sudo)"
]
}
}
-4
View File
@@ -2,13 +2,9 @@
go.mod text eol=lf
go.sum text eol=lf
# Embedded templates must render the same on every OS
lib/http/templates/** text eol=lf
# Ignore generated files in GitHub language statistics and diffs
/MANUAL.* linguist-generated=true
/rclone.1 linguist-generated=true
cmd/bisync/testdata/*/golden/** linguist-generated=true
# Don't fiddle with the line endings of test data
**/testdata/** -text
-56
View File
@@ -1,56 +0,0 @@
# Fork CI for the Proton Drive backend. Every step is hermetic: the
# tests run against the fake server in backend/protondrive/protontest.
name: ci
on:
push:
pull_request:
env:
# Never pick up a config holding a real Proton remote.
RCLONE_CONFIG: /notfound
jobs:
protondrive:
runs-on: ubuntu-latest
# The race suite takes about 6 minutes on a fast machine and 7 on
# two cores; the slow runner may take several times that.
timeout-minutes: 60
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install Go
uses: actions/setup-go@v5
with:
# The line the code was verified with, not go.mod's minimum,
# so govulncheck sees a patched standard library.
go-version: '1.27.x'
# Caches the module and build caches, keyed on go.sum.
cache: true
- name: Build
run: go build ./...
- name: Vet
run: go vet ./backend/protondrive/...
- name: Lint
uses: golangci/golangci-lint-action@v8
with:
version: v2.15.0
args: ./backend/protondrive/...
- name: Test
# Go's default 10 minute limit per test binary is too short
# for the race suite on a slow runner.
run: go test -race -timeout 40m ./backend/protondrive/...
- name: Fuzz
run: |
for target in $(go test -list '^Fuzz' ./backend/protondrive/ | grep '^Fuzz'); do
go test -run '^$' -fuzz "^${target}\$" -fuzztime 20s ./backend/protondrive/
done
- name: Vulnerability check
run: go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./backend/protondrive/...
-78
View File
@@ -1,78 +0,0 @@
# CODEOWNERS for rclone
#
# Maps paths to the maintainers auto-requested for review when a pull request
# touches matching files. Patterns are evaluated top to bottom and the LAST
# matching pattern wins, so the catch-all below is overridden by the more
# specific subsystem/backend lines that follow.
#
# Every owner listed here has write access to the repo (GitHub ignores owners
# without it). Assignments are derived from `git log` history per directory:
# each area lists its dominant contributor(s) alongside @ncw, the project lead
# and global fallback.
# Global fallback - everything not matched below
* @ncw
# -----------------------------------------------------------------------------
# Core subsystems
# -----------------------------------------------------------------------------
/fs/ @ncw
/vfs/ @ncw
/lib/ @ncw
/cmd/ @ncw
/librclone/ @ncw
/fstest/ @ncw
/lib/encoder/ @ncw @klauspost
/fs/sync/ @ncw @nielash
/fs/operations/ @ncw @nielash
/cmd/bisync/ @nielash @ivandeex @ncw
/cmd/serve/ @albertony @ncw
/cmd/selfupdate/ @ivandeex @ncw
/cmd/mount2/ @ncw @darthShadow
/cmd/gitannex/ @dmcardle @ncw
# -----------------------------------------------------------------------------
# Backends - keyed to the dominant / specialist maintainer per `git log`
# -----------------------------------------------------------------------------
# albertony
/backend/jottacloud/ @albertony @buengese @ncw
/backend/local/ @ncw @albertony @nielash
/backend/netstorage/ @albertony @ncw
/backend/hdfs/ @albertony @ncw
# buengese
/backend/yandex/ @ncw @buengese
/backend/zoho/ @buengese @ncw
/backend/fichier/ @ncw @buengese
/backend/compress/ @ncw @buengese
# ivandeex
/backend/chunker/ @ncw @ivandeex
/backend/mailru/ @ncw @ivandeex
/backend/ftp/ @ncw @ivandeex
/backend/sia/ @ncw @ivandeex
# Other maintainer-owned backends (see MAINTAINERS.md)
/backend/cache/ @ncw @remusb
/backend/union/ @ncw @Max-Sum @albertony
/backend/filescom/ @ncw @childish-sambino
# Microsoft Azure
/backend/azureblob/ @ncw @sandeepkru @darthShadow
# Single-maintainer specialist backends
/backend/onedrive/ @ncw @Cnly @nielash
/backend/iclouddrive/ @lostb1t @ncw
/backend/pikpak/ @wiserain @ncw
/backend/protondrive/ @henrybear327 @ncw
/backend/seafile/ @ncw @creativeprojects
/backend/filen/ @Enduriel @ncw
/backend/storj/ @ncw @calebcase
/backend/combine/ @ncw @nielash
# -----------------------------------------------------------------------------
# Repo plumbing
# -----------------------------------------------------------------------------
/.github/ @ncw
+31
View File
@@ -0,0 +1,31 @@
<!--
Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
https://forum.rclone.org/
instead of filing an issue for a quick response.
If you are reporting a bug or asking for a new feature then please use one of the templates here:
https://github.com/rclone/rclone/issues/new
otherwise fill in the form below.
Thank you
The Rclone Developers
-->
#### Output of `rclone version`
#### Describe the issue
+74
View File
@@ -0,0 +1,74 @@
---
name: Bug report
about: Report a problem with rclone
---
<!--
We understand you are having a problem with rclone; we want to help you with that!
**STOP and READ**
**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
Please show the effort you've put into solving the problem and please be specific.
People are volunteering their time to help! Low effort posts are not likely to get good answers!
If you think you might have found a bug, try to replicate it with the latest beta (or stable).
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
If you can still replicate it or just got a question then please use the rclone forum:
https://forum.rclone.org/
for a quick response instead of filing an issue on this repo.
If nothing else helps, then please fill in the info below which helps us help you.
**DO NOT REDACT** any information except passwords/keys/personal info.
You should use 3 backticks to begin and end your paste to make it readable.
Make sure to include a log obtained with '-vv'.
You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
Thank you
The Rclone Developers
-->
#### The associated forum post URL from `https://forum.rclone.org`
#### What is the problem you are having with rclone?
#### What is your rclone version (output from `rclone version`)
#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
#### Which cloud storage system are you using? (e.g. Google Drive)
#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
<!--- Please keep the note below for others who read your bug report. -->
#### How to use GitHub
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
* Subscribe to receive notifications on status change and new comments.
+51
View File
@@ -0,0 +1,51 @@
---
name: Feature request
about: Suggest a new feature or enhancement for rclone
---
<!--
Welcome :-)
So you've got an idea to improve rclone? We love that!
You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
Probably the latest beta (or stable) release has your feature, so try to update your rclone.
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
If it still isn't there, here is a checklist of things to do:
1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
2. Discuss on the forum: https://forum.rclone.org/
3. Make a feature request issue (this is the right place!).
4. Be prepared to get involved making the feature :-)
Looking forward to your great idea!
The Rclone Developers
-->
#### The associated forum post URL from `https://forum.rclone.org`
#### What is your current rclone version (output from `rclone version`)?
#### What problem are you are trying to solve?
#### How do you think rclone should be changed to solve that?
<!--- Please keep the note below for others who read your feature request. -->
#### How to use GitHub
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
* Subscribe to receive notifications on status change and new comments.
-124
View File
@@ -1,124 +0,0 @@
name: Bug report
description: Report a reproducible problem with rclone
labels: ["bug"]
body:
- type: markdown
attributes:
value: |
We want to help you with rclone - thank you for taking the time to fill this in!
**This tracker is for reproducible bugs only.** If you have a question, are
not sure whether something is a bug, or want help with a configuration, please
use the [rclone forum](https://forum.rclone.org/) instead - you will get a much
quicker answer there.
Before filing, please **test with the [latest beta or stable](https://rclone.org/commands/rclone_selfupdate/)**
(downloads at <https://beta.rclone.org/>) as your problem may already be fixed.
**Do not redact** any information except passwords, keys and personal info.
- type: checkboxes
id: acknowledgements
attributes:
label: Before you start
description: Please confirm the following. Issues that skip these steps may be closed.
options:
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues for this problem.
required: true
- label: I have tested with the latest beta or stable release and can still reproduce the problem.
required: true
- label: This is a reproducible bug, not a usage question (questions belong on the forum).
required: true
- type: input
id: forum-url
attributes:
label: Associated forum post URL
description: If you discussed this on the forum first (encouraged), paste the link here.
placeholder: https://forum.rclone.org/t/...
validations:
required: false
- type: textarea
id: problem
attributes:
label: What is the problem you are having with rclone?
description: Describe the problem and the exact steps to reproduce it.
validations:
required: true
- type: textarea
id: version
attributes:
label: rclone version
description: The full output of `rclone version`.
placeholder: |
rclone v1.75.0
- os/version: ubuntu 24.04 (64 bit)
- os/kernel: 6.8.0-111-generic (x86_64)
- os/type: linux
- os/arch: amd64
- go/version: go1.26.4
- go/linking: dynamic
- go/tags: none
validations:
required: true
- type: dropdown
id: os
attributes:
label: Operating system
description: Choose the operating system you are using with rclone.
options:
- Windows
- macOS
- Linux
- FreeBSD
- NetBSD
- OpenBSD
- Plan 9
- Solaris
- Android
- iOS
- Other
validations:
required: true
- type: input
id: backend
attributes:
label: Which cloud storage system are you using?
description: e.g. Google Drive, S3, Dropbox, local disk. List all involved.
validations:
required: true
- type: textarea
id: config
attributes:
label: Output of `rclone config redacted`
description: |
Run `rclone config redacted` (or `rclone config redacted <remote>` for a
single remote). This command automatically replaces passwords and tokens
with `XXX`, so it should be safe to paste but please double-check before posting.
render: ini
validations:
required: true
- type: textarea
id: command
attributes:
label: The command you were trying to run
description: e.g. `rclone copy /tmp remote:tmp`
render: shell
validations:
required: true
- type: textarea
id: log
attributes:
label: A log from the command with the `-vv` flag
description: |
e.g. output from `rclone -vv copy /tmp remote:tmp`. For long logs (> 100 lines) use
`-vv --log-file bug.log` and attach it here or send to a service such as
<https://gist.github.com/> or <https://pastebin.com/> and link it here.
render: shell
validations:
required: true
- type: textarea
id: extra
attributes:
label: Anything else?
description: Anything else that might help us, e.g. a minimal way to replicate the problem.
validations:
required: false
+2 -5
View File
@@ -1,8 +1,5 @@
blank_issues_enabled: false
contact_links:
- name: Rclone Forum - questions & support
- name: Rclone Forum Community Support
url: https://forum.rclone.org/
about: Have a question, need help, or not sure if it's a bug? Please ask here for a quick answer.
- name: Rclone Documentation
url: https://rclone.org/docs/
about: The docs answer many "how do I..." questions, including per-backend guides.
about: Please ask and answer questions here.
-68
View File
@@ -1,68 +0,0 @@
name: Feature request
description: Suggest a new feature or enhancement for rclone
labels: ["enhancement"]
body:
- type: markdown
attributes:
value: |
So you've got an idea to improve rclone? We love that - we've already
incorporated hundreds of ideas from contributors!
Please **search first**: your idea may already exist in the latest
[beta or stable](https://rclone.org/commands/rclone_selfupdate/), or already
be requested. It is often worth [discussing on the forum](https://forum.rclone.org/)
to refine the idea or checking to see if rclone can do it already before filing it here.
- type: checkboxes
id: acknowledgements
attributes:
label: Before you start
options:
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues and this hasn't already been requested.
required: true
- label: I have checked the latest beta and this feature doesn't already exist.
required: true
- type: input
id: forum-url
attributes:
label: Associated forum post URL
description: If you discussed this on the forum first (encouraged), paste the link here.
placeholder: https://forum.rclone.org/t/...
validations:
required: false
- type: textarea
id: version
attributes:
label: rclone version
description: The full output of `rclone version`, not just the version number.
placeholder: |
rclone v1.75.0
- os/version: ubuntu 24.04 (64 bit)
- os/kernel: 6.8.0-111-generic (x86_64)
- os/type: linux
- os/arch: amd64
- go/version: go1.26.4
- go/linking: dynamic
- go/tags: none
validations:
required: true
- type: textarea
id: problem
attributes:
label: What problem are you trying to solve?
description: Describe the use case and what is difficult or impossible today.
validations:
required: true
- type: textarea
id: solution
attributes:
label: How do you think rclone should be changed to solve that?
validations:
required: true
- type: checkboxes
id: involvement
attributes:
label: Getting involved
description: Features are much more likely to happen if someone helps make them.
options:
- label: I'm willing to help implement, test or fund this feature.
required: false
+11 -36
View File
@@ -1,54 +1,29 @@
<!--
Thank you for contributing to rclone!
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.
Please discuss anything more than a trivial fix in an issue FIRST - see the
"Linked issue" section below. Then fill in the questions to help us review.
You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->
#### What does this change do?
<!-- Describe the change here. -->
#### Linked issue
#### What is the purpose of this change?
<!--
Put `Fixes #1234` here if this closes an issue, or `#1234` to link without closing.
IMPORTANT: Anything beyond a trivial fix (typo, doc tweak, small obvious bug fix)
should be discussed and agreed in an issue BEFORE you open a pull request.
Pull requests for larger changes that have not been discussed in an issue first
may be closed. This saves everyone's time if the change needs a different approach
or isn't a good fit.
Describe the changes here
-->
Fixes #
#### For new or changed backends
#### Was the change discussed in an issue or in the forum before?
<!--
Backend pull requests are often sent without the integration tests having been run.
We cannot merge a backend change until it passes the integration tests.
To be merged, a backend change REQUIRES:
1. A clean run of `go run ./fstest/test_all -backends <remote>` (summarise the
result below or paste a screenshot of the test_all output in the web browser).
2. A test account so the maintainers can run the integration tests daily against
the backend on the integration test server.
If the tests aren't quite passing yet and you need help getting them
to pass, then submit the PR and we can help getting you over the line.
See https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#writing-a-new-backend
and https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests
Link issues and relevant forum posts here.
-->
#### Checklist
- [ ] This change is trivial **OR** it has been discussed and agreed in the linked issue.
- [ ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [ ] **(If I used AI tools to help write this code)** I have read and understood the [AI-assisted contributions guidance](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#ai-assisted-contributions), and I have tested and take ownership of this change myself.
- [ ] I have added tests for all changes in this PR if appropriate.
- [ ] I have added documentation for the changes if appropriate.
- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [ ] **(Backend changes only)** `test_all` passes for this backend and if submitting a new backend can provide a test account for the integration tester - see [CONTRIBUTING.md](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests).
- [ ] This Pull Request is ready for review.
- [ ] I'm done, this Pull Request is ready for review :-)
-2
View File
@@ -4,5 +4,3 @@ updates:
directory: "/"
schedule:
interval: "daily"
cooldown:
default-days: 7
+21 -144
View File
@@ -19,9 +19,6 @@ on:
type: boolean
default: true
permissions:
contents: read
jobs:
build:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
@@ -32,12 +29,12 @@ jobs:
strategy:
fail-fast: false
matrix:
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
include:
- job_name: linux
os: ubuntu-latest
go: '~1.27.3'
go: '~1.26.0'
gotags: cmount
build_flags: '-include "^linux/"'
check: true
@@ -48,14 +45,14 @@ jobs:
- job_name: linux_386
os: ubuntu-latest
go: '~1.27.3'
go: '~1.26.0'
goarch: 386
gotags: cmount
quicktest: true
- job_name: mac_amd64
os: macos-latest
go: '~1.27.3'
go: '~1.26.0'
gotags: 'cmount'
build_flags: '-include "^darwin/amd64" -cgo'
quicktest: true
@@ -64,14 +61,14 @@ jobs:
- job_name: mac_arm64
os: macos-latest
go: '~1.27.3'
go: '~1.26.0'
gotags: 'cmount'
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
deploy: true
- job_name: windows
os: windows-latest
go: '~1.27.3'
go: '~1.26.0'
gotags: cmount
cgo: '0'
build_flags: '-include "^windows/"'
@@ -81,14 +78,14 @@ jobs:
- job_name: other_os
os: ubuntu-latest
go: '~1.27.3'
go: '~1.26.0'
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
compile_all: true
deploy: true
- job_name: go1.26
- job_name: go1.25
os: ubuntu-latest
go: '~1.26.10'
go: '~1.25.7'
quicktest: true
racequicktest: true
@@ -97,58 +94,16 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: Get runner parameters
id: get-runner-parameters
run: |
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Go
id: setup-go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ${{ matrix.go }}
check-latest: true
# Caching is handled explicitly below. setup-go's built-in cache
# keys only on go.sum, so all matrix jobs collide on one key and
# only the first to finish saves - which loses the cross-compile
# build cache.
cache: false
- name: Get Go paths
id: go-paths
run: |
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
# The module cache holds downloaded source and is identical across
# jobs (it depends only on go.sum, not on OS, arch or Go version), so
# it is shared via a single key to avoid storing the same ~600 MiB in
# every job's cache and overflowing the repo cache limit.
- name: Module cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.mod-cache }}
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
# The build cache holds compiled artifacts which are specific to the
# OS, arch and Go version, so it is kept per job.
- name: Build cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.build-cache }}
key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
- name: Set environment variables
run: |
@@ -165,11 +120,6 @@ jobs:
sudo chown root:$USER /etc/fuse.conf
sudo apt-get update
sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
# The podman static bundle in runner image ubuntu24/20260726.254 and
# later ships a non-setuid fusermount3 in /usr/local/bin which
# shadows the setuid one from the fuse3 package and breaks FUSE
# mounts with "Operation not permitted", so remove it.
sudo rm -f /usr/local/bin/fusermount /usr/local/bin/fusermount3
if: matrix.os == 'ubuntu-latest'
- name: Install Libraries on macOS
@@ -177,12 +127,8 @@ jobs:
# https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
# https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
unset HOMEBREW_NO_INSTALL_FROM_API
# brew untap fails if the tap isn't installed, which varies between runner images
for tap in homebrew/core homebrew/cask aws/tap; do
if brew tap | grep -qx "$tap"; then
brew untap --force "$tap"
fi
done
brew untap --force homebrew/core
brew untap --force homebrew/cask
brew update
brew install --cask macfuse
brew install git-annex git-annex-remote-rclone
@@ -256,15 +202,6 @@ jobs:
# Deploy binaries if enabled in config && not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
lint:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -279,31 +216,23 @@ jobs:
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Go
id: setup-go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: '~1.27.3'
go-version: '~1.26.0'
check-latest: true
cache: false
- name: Module cache
uses: actions/cache@v6
with:
path: ~/go/pkg/mod
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
- name: Cache
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
@@ -370,15 +299,6 @@ jobs:
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
if: github.event_name == 'pull_request'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
android:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -386,50 +306,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Get runner parameters
id: get-runner-parameters
run: |
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
# Upgrade together with NDK version
- name: Set up Go
id: setup-go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: '~1.27.3'
# Caching is handled explicitly below to share the module cache
# with the other jobs - see the build job for the rationale.
cache: false
- name: Get Go paths
id: go-paths
run: |
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
- name: Module cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.mod-cache }}
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
- name: Build cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.build-cache }}
key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
go-version: '~1.26.0'
- name: Set global environment variables
run: |
@@ -505,12 +391,3 @@ jobs:
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
# Upload artifacts if not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
@@ -18,10 +18,6 @@ on:
type: boolean
default: true
permissions:
contents: read
packages: write
jobs:
build-image:
if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
@@ -56,7 +52,7 @@ jobs:
df -h .
- name: Checkout Repository
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
@@ -96,7 +92,7 @@ jobs:
# There's no way around this, because "ImageOS" is only available to
# processes, but the setup-go action uses it in its key.
id: imageos
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
result-encoding: string
script: |
@@ -104,7 +100,7 @@ jobs:
- name: Extract Metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
env:
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
with:
@@ -126,14 +122,14 @@ jobs:
type=raw,value=beta,enable={{is_default_branch}}
- name: Setup QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Load Go Build Cache for Docker
id: go-cache
uses: actions/cache@v6
uses: actions/cache@v5
with:
key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
@@ -152,7 +148,7 @@ jobs:
skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
# This is the user that triggered the Workflow. In this case, it will
@@ -163,7 +159,7 @@ jobs:
- name: Build and Publish Image Digest
id: build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
file: Dockerfile
context: .
@@ -214,7 +210,7 @@ jobs:
- name: Extract Metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
env:
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
with:
@@ -265,16 +261,16 @@ jobs:
env.write(f"ANNOTATIONS={annotations_string}\n")
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
# This is the user that triggered the Workflow. In this case, it will
@@ -14,9 +14,6 @@ on:
type: boolean
default: true
permissions:
contents: read
jobs:
build_docker_volume_plugin:
if: inputs.manual || github.repository == 'rclone/rclone'
@@ -33,7 +30,7 @@ jobs:
sudo rm -rf /usr/share/dotnet || true
df -h .
- name: Checkout master
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Build and publish docker plugin
-3
View File
@@ -4,9 +4,6 @@ on:
issues:
types: [labeled]
permissions:
contents: read
jobs:
notify:
runs-on: ubuntu-latest
-3
View File
@@ -3,9 +3,6 @@ on:
release:
types: [released]
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
-8
View File
@@ -19,12 +19,4 @@ Thumbs.db
__pycache__
.DS_Store
resource_windows_*.syso
/librclone.h
.devcontainer
/cmd/bisync/testdata/bisync_vscode_debuggers_launch.json
.env
.env.*
CLAUDE.local.md
.claude/settings.local.json
.worktrees/
.superpowers/
-5
View File
@@ -46,11 +46,6 @@ linters:
- -ST1022
# Disable quickfix checks
- -QF*
# Disabled because its dataflow analysis (new in staticcheck
# 0.8.0) makes linting large packages more than 10x slower,
# which takes the CI lint job past its timeout, and it
# produces false positives (eg on operations.Delete).
- -SA4023
gocritic:
# With gocritic there are different settings, but since enabled-checks
# and disabled-checks cannot both be set, for full customization the
+3 -3
View File
@@ -1,6 +1,6 @@
default: true
# Use specific styles, to be consistent across all documents.
# Use specific styles, to be consistent accross all documents.
# Default is to accept any as long as it is consistent within the same document.
heading-style: # MD003
style: atx
@@ -48,12 +48,12 @@ link-fragments: false # MD051
# https://gohugo.io/content-management/syntax-highlighting/#languages
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
# In addition, we only want to allow identifiers (aliases) that correspond to
# the same language in Hugo and GitHub, and preferably also VSCode and other
# the same language in Hugo and GitHub, and preferrably also VSCode and other
# commonly used tools, to avoid confusion. An example of this is that "shell"
# by some are considered an identifier for shell scripts, i.e. an alias for
# "sh", while others consider it an identifier for shell sessions, i.e. an
# alias for "console". Although Hugo and GitHub in this case are consistent and
# have chosen the former, using "sh" instead, and not allowing use of "shell",
# have choosen the former, using "sh" instead, and not allowing use of "shell",
# avoids the confusion entirely.
fenced-code-language: # MD040
allowed_languages:
-149
View File
@@ -1,149 +0,0 @@
# AGENTS.md
This file provides guidance to AI coding agents (e.g. Claude Code, Codex, Cursor, Gemini CLI, and similar tools) when working with code in this repository.
Rclone welcomes AI-assisted contributions, but the expectation is that you, the human submitter, understand every line you propose and have compiled and tested it against real rclone code - not just generated it. See the "AI-assisted contributions" section of [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request.
## Project Overview
Rclone is a command-line program to sync files and directories to and from cloud storage providers. It's written in Go and supports 70+ backends (cloud storage systems). Think "rsync for cloud storage".
## General Notes
**We take backwards compatibility very seriously.** PRs should not change the observable behaviour of existing commands, flags or rc API without very good reason. Rclone does not try to preserve a stable Go API but try not to change it gratuitously.
Rclone operates with a lot of different backends, so **compatibility is key**. It is the backend integration tests which guarantee that compatibility. Changes should consider both known and unknown backends and should take care not to break functionality of existing installations.
The core parts of rclone under `fs` and `vfs` need to work with all backends and **backend specific hacks won't be merged**. Fixes likely need to go in the relevant backend or if new behaviour is really needed, a new Feature flag needs to be added.
**Changes should be kept to the minimum.** Work hard to make the most elegant, smallest change you can. Do not refactor or re-order code unless necessary as this makes review more challenging. Re-use existing test scaffolding, existing or library routines (e.g. `lib`) where possible.
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
## Build and Test Commands
```bash
# Build rclone (simple)
go build
# Build with version info (preferred)
make
# Run all unit tests (no cloud credentials needed)
make quicktest
# or equivalently:
RCLONE_CONFIG="/notfound" go test ./...
# Run tests for a specific package
cd backend/memory && go test -v
# or from root:
go test -v ./backend/memory/
# Run a single test
go test -v -run TestIntegration/FsCheckWrap ./backend/memory/
# Run tests with race detector
make racequicktest
# Lint (requires golangci-lint)
golangci-lint run ./...
# Run backend integration tests (requires configured TestRemote remote)
cd backend/drive && go test -v
# Run sync/operations integration tests against a remote
cd fs/sync && go test -v -remote TestDrive:
cd fs/operations && go test -v -remote TestDrive:
# Run integration tests via test framework
go run ./fstest/test_all -backends drive
```
## Architecture
### Entry Point and Plugin Registration
`rclone.go` is the main entry point. It imports `backend/all` and `cmd/all` which use Go's `init()` pattern to register all backends and commands. Each backend calls `fs.Register()` with a `fs.RegInfo` struct during init.
### Core Interfaces (`fs/`)
The `fs` package defines the core abstractions:
- **`fs.Fs`** (`fs/types.go`): The filesystem interface every backend must implement (List, NewObject, Put, Mkdir, Rmdir).
- **`fs.Object`** (`fs/types.go`): Interface for a file/object (Open, Update, Remove, SetModTime).
- **`fs.Features`** (`fs/features.go`): Optional capabilities a backend can declare (Purge, Copy, Move, DirMove, etc.). Backends set function pointers for operations they support; nil means not supported.
- **`fs.RegInfo`** (`fs/registry.go`): Registration metadata for a backend including its name, config options, and NewFs constructor.
### Backend Structure (`backend/`)
Each backend is a single Go package (e.g., `backend/s3/`, `backend/drive/`). Key conventions:
- Main implementation in a single file (e.g., `s3.go`) - **do not** split into `fs.go`/`object.go`.
- API types go in a separate `api/types.go` file.
- Test file (e.g., `s3_test.go`) uses `fstests.Run()` from `fstest/fstests` for standardized integration tests.
- Register in `backend/all/all.go` via blank import.
- HTTP-based backends should use `lib/rest` for HTTP calls and `fs/fshttp` for the HTTP client.
- Use `lib/dircache` for directory-ID-based remotes, `lib/oauthutil` for OAuth, `lib/pacer` for rate limiting.
### Command Structure (`cmd/`)
Each command is a package under `cmd/` registered in `cmd/all/all.go` via blank import. Commands use cobra via `cmd.Main()`.
### Key Subsystems
- **`fs/operations/`**: Core file operations (Copy, Move, Delete, etc.)
- **`fs/sync/`**: Directory sync logic
- **`fs/march/`**: Parallel directory tree walker used by sync
- **`fs/filter/`**: Include/exclude filtering
- **`fs/accounting/`**: Transfer statistics and bandwidth limiting
- **`fs/config/`**: Configuration file management
- **`vfs/`**: Virtual filesystem layer (used by mount, serve)
- **`librclone/`**: C-compatible library interface for embedding rclone
- **`fstest/`**: Integration test framework; `fstest/fstests/` has the generic backend test suite
## Commit Message Convention
Prefix with the directory of the change, then a colon: `drive: add team drive support - fixes #885`. For cross-cutting changes use a broader prefix like `fs` or `operations`.
Make the first line of your commit message a summary of the change that a user (not a developer) of rclone would like to read. So write `drive: fix server side copy of big files` instead of `drive: no longer set the MimeType in Move or Copy`. This is important because these lines go into the change log which is read by users.
## Code Commenting Style
Comments describe the code as it is now, for a future reader who has no knowledge of the change that introduced it.
Every exported type, function, field, and constant has a **godoc comment** that starts with its name and is phrased as a present-tense statement of what it is or does (`// Mkdir makes the directory (container, bucket)`)
**Document the contract** callers need - preconditions, what's returned, which sentinel errors are returned and when, and any "shouldn't return an error if it already exists" style caveats - rather than the implementation.
**Keep comments terse**: a single line for most things, with extra paragraphs (separated by blank `//` lines) reserved for genuine subtlety.
Inline comments inside function bodies should **explain why** - a non-obvious API quirk, a workaround, a gotcha, or an ordering constraint - and may cite an external reference (forum thread, vendor docs, RFC) when that's what makes the behaviour non-obvious; skip comments that merely restate what the code plainly does.
Use `FIXME` and `TODO` for known shortcomings.
**Do not write comments that narrate the change itself** or compare against the previous behaviour (no "now we also handle...", "changed to...", "previously this returned...", or references to bug/PR numbers in the code) - that context belongs in the commit message, not in source that will outlive the change.
Code comments should only refer to **current** behaviour - and shouldn't describe behaviour that is trivially deducible from reading the code.
## Linting Configuration
Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck, govet, ineffassign, staticcheck, unused, gocritic, misspell, revive, unconvert. The `goimports` formatter is also enabled.
## Documentation
- Backend option docs come from `Help:` fields in the Go source Options structs, not from markdown files.
- Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
- Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
- Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
## Fork Workflow
This fork carries agent tooling that upstream does not. Never include `docs/specs/`, `docs/plans/`, `docs/{ARCHITECTURE,DECISIONS,TODO,PERMISSIONS}.md`, `GEMINI.md`, `.claude/`, `.codex/`, `.gemini/` or this section in a PR to rclone/rclone.
- Required skill: `superpowers` - check at session start, install if missing. There is no product UI here, so `impeccable` is not needed.
- Specs go in `docs/specs/NNN-slug.md` (permanent). Plans go in `docs/plans/NNN-slug.md` and are removed with `git rm` before merge. This overrides the superpowers default paths.
- Paused or in-flight work is listed under "Now" in `docs/TODO.md` with the handoff to read first.
- Non-trivial work: brainstorm → spec → plan → TDD → review. For trivial fixes say "trivial, skipping spec" and proceed.
- Log things noticed but deliberately not fixed in `docs/TODO.md`, record decisions in `docs/DECISIONS.md`, and keep `docs/ARCHITECTURE.md` in step with structural changes - in the same commit as the code.
- `docs/PERMISSIONS.md` is the source for the `.claude/`, `.codex/` and `.gemini/` permission files - change them together.
- Never push, force-push or commit directly to `master`. Ask before adding or upgrading a dependency (`go get`, `go mod tidy`).
- Done means run in this session and seen passing: `go build`, `go test` on the touched packages, and `golangci-lint run` on them.
-6
View File
@@ -1,6 +0,0 @@
# CLAUDE.md
Guidance for this repository lives in AGENTS.md, shared with all AI coding
agents. Claude Code reads it via the import below.
@AGENTS.md
+4 -69
View File
@@ -113,36 +113,6 @@ GitHub.
You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master)
or [squash your commits](#squashing-your-commits).
## AI-assisted contributions
You are welcome to use AI coding assistants (Claude Code, Codex, Cursor, Gemini
CLI, and similar) to help write your contribution. Rclone has an
[AGENTS.md](AGENTS.md) file at the top of the repository describing the project's
conventions; point your tool at it so the code it produces matches rclone's
style.
However, the same standard applies to every pull request whether or not a tool
was involved: **you are responsible for the code you submit.** Before you open a
pull request please make sure that:
- You understand every line of the change and can explain why it is correct. If
a reviewer asks a question about it, you should be able to answer without
going back to the tool.
- You have actually built and run it. At a minimum `go build` and
`make quicktest` must pass, and for a backend change you should run the
[backend tests](#backend-testing) against a real remote where you can.
- The change is a genuine fix or feature that you have verified solves the
problem, not a plausible-looking guess. Unverified, AI-generated pull requests
that do not compile, do not pass the tests, invent APIs that do not exist, or
do not actually do what the description claims waste maintainer time and are
likely to be closed.
- You have trimmed the comments. AI tools tend to add verbose comments that
restate what the code plainly does or narrate the change being made; please
cut these down to match the [code commenting style](AGENTS.md#code-commenting-style).
In short: an AI assistant is a tool to help *you* contribute, not a substitute
for understanding and testing your own work.
## Using Git and GitHub
### Committing your changes
@@ -630,37 +600,10 @@ remote or an fs.
- **Remember** we have >50 backends to maintain so keeping them as similar as
possible to each other is a high priority!
### Managing memory
Use rclone's memory pool for upload buffers rather than allocating them
yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
rclone's memory usage smaller.
- If the provider supports multipart or parallel chunk uploads, implement
[fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
and drive it with
[multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
This gives you pooled chunk buffers, concurrency, retries and
`--multi-thread-streams` support for free. See s3/b2 backends for examples.
- If you must buffer data yourself take the buffer from the global pool with
[multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
be rewound for a retry.
- When using a pooled buffer `Close()` it on all paths so its pages go back to
the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
attempt, set the request's `ContentLength` explicitly, and if the buffer
reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
so the transport can't close it and free its pages before the retry.
### Unit tests
- Create a config entry called `TestRemote` for the unit tests to use
- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
- If your backend does chunked uploads, set `ChunkedUpload` in the
`fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
`FsPutRetry` tests can run the chunked path with small chunks.
- Make sure all tests pass with `go test -v`
### Integration tests
@@ -680,22 +623,14 @@ Or if you want to run the integration tests manually:
- If your remote defines `ListR` check with this also
- `go test -v -remote TestRemote: -fast-list`
Before a new or changed backend can be merged we require:
- A clean run of `go run ./fstest/test_all -backends remote` (please include the
result in your pull request).
- A test account for the backend so the maintainers can add it to the
[integration test server](https://integration.rclone.org) and keep the backend
working as rclone evolves. A backend that we cannot test is likely to break and
may be removed.
See the [testing](#testing) section for more information on integration tests.
### Backend documentation
Add your backend to the docs. Keep lists of remotes in alphabetical
order of full name of remote (e.g. `drive` is ordered as `Google
Drive`) but with the local file system last.
Add your backend to the docs - you'll need to pick an icon for it from
[fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in
alphabetical order of full name of remote (e.g. `drive` is ordered as
`Google Drive`) but with the local file system last.
First add a data file about your backend in
`docs/data/backends/remote.yaml` - this is used to build the overview
+1 -5
View File
@@ -35,21 +35,17 @@ RUN echo "**** Print Version Binary ****" && \
# Begin final image
FROM alpine:latest
ARG RCLONE_UID=1009
ARG RCLONE_GID=1009
RUN echo "**** Install Dependencies ****" && \
apk add --no-cache \
ca-certificates \
fuse3 \
mailcap \
tzdata && \
echo "Enable user_allow_other in fuse" && \
echo "user_allow_other" >> /etc/fuse.conf
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
ENTRYPOINT [ "rclone" ]
-8
View File
@@ -1,8 +0,0 @@
# GEMINI.md
**`AGENTS.md` is the single source of truth for this repository.** Read
`./AGENTS.md` before doing anything else and follow it exactly.
This file exists only so Gemini CLI finds the rules. It contains no project
rules of its own - add those to `AGENTS.md`. If anything here ever conflicts
with `AGENTS.md`, `AGENTS.md` wins.
+21 -28
View File
@@ -2,34 +2,27 @@
Current active maintainers of rclone are:
| Name | GitHub ID |
| :--------------- | :---------------- |
| Nick Craig-Wood | @ncw |
| Stefan Breunig | @breunigs |
| Ishuah Kariuki | @ishuah |
| Remus Bunduc | @remusb |
| Fabian Möller | @B4dM4n |
| Alex Chen | @Cnly |
| Sandeep Ummadi | @sandeepkru |
| Sebastian Bünger | @buengese |
| Ivan Andreev | @ivandeex |
| Max Sum | @Max-Sum |
| Fred | @creativeprojects |
| Caleb Case | @calebcase |
| wiserain | @wiserain |
| albertony | @albertony |
| Chun-Hung Tseng | @henrybear327 |
| Hideo Aoyama | @boukendesho |
| nielash | @nielash |
| Dan McArdle | @dmcardle |
| Sam Harrison | @childish-sambino |
| Enduriel | @Enduriel |
Per-area ownership of code (which backend or command each maintainer looks
after) is defined in [.github/CODEOWNERS](.github/CODEOWNERS), which GitHub uses
to auto-request reviews. Two responsibilities don't map to a code path: @ncw
looks after overall project health, and @boukendesho maintains the snap
packaging.
| Name | GitHub ID | Specific Responsibilities |
| :--------------- | :---------------- | :-------------------------- |
| Nick Craig-Wood | @ncw | overall project health |
| Stefan Breunig | @breunigs | |
| Ishuah Kariuki | @ishuah | |
| Remus Bunduc | @remusb | cache backend |
| Fabian Möller | @B4dM4n | |
| Alex Chen | @Cnly | onedrive backend |
| Sandeep Ummadi | @sandeepkru | azureblob backend |
| Sebastian Bünger | @buengese | jottacloud, yandex & compress backends |
| Ivan Andreev | @ivandeex | chunker & mailru backends |
| Max Sum | @Max-Sum | union backend |
| Fred | @creativeprojects | seafile backend |
| Caleb Case | @calebcase | storj backend |
| wiserain | @wiserain | pikpak backend |
| albertony | @albertony | |
| Chun-Hung Tseng | @henrybear327 | Proton Drive Backend |
| Hideo Aoyama | @boukendesho | snap packaging |
| nielash | @nielash | bisync |
| Dan McArdle | @dmcardle | gitannex |
| Sam Harrison | @childish-sambino | filescom |
## This is a work in progress draft
Generated
+3256 -13365
View File
File diff suppressed because it is too large. Load diff
Generated
+1382 -9549
View File
File diff suppressed because it is too large. Load diff
Generated
+1417 -10321
View File
File diff suppressed because it is too large. Load diff
+17 -58
View File
@@ -38,9 +38,6 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
BETA_UPLOAD_ROOT := beta.rclone.org:
BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
# Update the beta site's directory listings after an upload - add
# --changed for the uploaded directories
BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
# Pass in GOTAGS=xyz on the make command line to set build tags
ifdef GOTAGS
BUILDTAGS=-tags "$(GOTAGS)"
@@ -48,7 +45,7 @@ LINTTAGS=--build-tags "$(GOTAGS)"
endif
LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
.PHONY: rclone test_all vars version fetch-gui fetch-gui-and-commit
.PHONY: rclone test_all vars version
rclone:
ifeq ($(GO_OS),windows)
@@ -62,12 +59,6 @@ endif
cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
fetch-gui:
$(SHELL) ./bin/fetch-gui-dist.sh
fetch-gui-and-commit:
$(SHELL) ./bin/fetch-gui-dist.sh --commit
test_all:
go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
@@ -96,15 +87,11 @@ test: rclone test_all
@echo "Written logs in test_all.log"
# Quick test
#
# The timeout is raised above the go test default of 10m as the
# cmd/gitannex end to end tests can take longer than that on slow CI
# runners.
quicktest:
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
racequicktest:
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
compiletest:
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
@@ -134,25 +121,6 @@ updatedirect:
go get $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
go mod tidy
# Update direct dependencies only but won't update the `go` line in go.mod
updatedirectnoupgrade:
@GO_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
for mod in $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all); do \
cp go.mod go.mod.bak && \
cp go.sum go.sum.bak && \
go get $$mod && \
NEW_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
if [ "$$NEW_VERSION" != "$$GO_VERSION" ]; then \
echo "SKIPPING $$mod (requires go $$NEW_VERSION)"; \
cp go.mod.bak go.mod; \
cp go.sum.bak go.sum; \
else \
echo "updated $$mod"; \
fi; \
done && \
rm -f go.mod.bak go.sum.bak && \
go mod tidy
# Update direct and indirect dependencies and test dependencies
update:
go get -u -t ./...
@@ -178,7 +146,6 @@ MANUAL.txt: MANUAL.md
commanddocs: rclone
go generate ./lib/transform
go generate ./cmd/bisync
-@rmdir -p '$$HOME/.config/rclone'
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
@@ -237,7 +204,6 @@ check_sign:
upload:
rclone -P copy build/ downloads.rclone.org:/$(TAG)
rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
upload_github:
./bin/upload-github $(TAG)
@@ -270,7 +236,6 @@ ci_upload:
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
endif
./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
@echo Beta release ready at $(BETA_URL)/testbuilds
ci_beta:
@@ -280,12 +245,11 @@ ci_beta:
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
endif
rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
@echo Beta release ready at $(BETA_URL)
# Fetch the binary builds from GitHub actions
fetch_binaries:
rclone -P sync --exclude "/testbuilds/**" --exclude "/index.{html,json}" --delete-excluded $(BETA_UPLOAD) build/
rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
serve: website
cd docs && hugo server --logLevel info -w --disableFastRender --ignoreCache
@@ -302,26 +266,21 @@ retag:
@echo "Version is $(VERSION)"
git tag -f -s -m "Version $(VERSION)" $(VERSION)
# Start development of the next version after a release: the next
# minor version on master or the next patch version on a stable branch.
startdev:
@branch=$$(git rev-parse --abbrev-ref HEAD); \
case "$$branch" in \
master) next="$(NEXT_VERSION)" ;; \
v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
esac; \
if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
fi; \
echo "Version is $(VERSION)"; \
echo "Next version is $$next"; \
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
echo -n "$$next" > docs/layouts/partials/version.html; \
echo "$$next" > VERSION; \
git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
@echo "Version is $(VERSION)"
@echo "Next version is $(NEXT_VERSION)"
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_VERSION)" > VERSION
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
startstable: startdev
startstable:
@echo "Version is $(VERSION)"
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_PATCH_VERSION)" > VERSION
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
winzip:
zip -9 rclone-$(TAG).zip rclone.exe
-15
View File
@@ -24,25 +24,21 @@ directories to and from different cloud storage providers.
## Storage providers
- 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
- Abrha Storage [:page_facing_up:](https://rclone.org/s3/#abrhastorage)
- Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
- Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
- Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
- ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
- Autonomys Auto Drive [:page_facing_up:](https://rclone.org/s3/#autonomys)
- Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
- Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
- Box [:page_facing_up:](https://rclone.org/box/)
- Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
- China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
- Clever Cloud Cellar [:page_facing_up:](https://rclone.org/s3/#clever-cloud-cellar)
- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
- Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
@@ -63,22 +59,16 @@ directories to and from different cloud storage providers.
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp)
- HTTP [:page_facing_up:](https://rclone.org/http/)
- Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
- Huawei Drive [:page_facing_up:](https://rclone.org/huaweidrive/)
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
- ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
- Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
- Internxt [:page_facing_up:](https://rclone.org/internxt/)
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
- Infomaniak Swiss Backup (S3) [:page_facing_up:](https://rclone.org/s3/#infomaniak)
- Infomaniak Swiss Backup (Swift) [:page_facing_up:](https://rclone.org/swift/)
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
- kDrive [:page_facing_up:](https://rclone.org/kdrive/)
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
- Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
- Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
@@ -94,7 +84,6 @@ directories to and from different cloud storage providers.
- Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
- Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
- Minio [:page_facing_up:](https://rclone.org/s3/#minio)
- Neon Object Storage [:page_facing_up:](https://rclone.org/s3/#neon)
- Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
- Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
- OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
@@ -118,10 +107,8 @@ directories to and from different cloud storage providers.
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
- Seafile [:page_facing_up:](https://rclone.org/seafile/)
- Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
- SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
@@ -136,12 +123,10 @@ directories to and from different cloud storage providers.
- Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
- Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
- Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
- US3 Object Storage [:page_facing_up:](https://rclone.org/s3/#us3)
- Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
- WebDAV [:page_facing_up:](https://rclone.org/webdav/)
- Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
- Zadara Object Storage [:page_facing_up:](https://rclone.org/s3/#zadara)
- Zero Services (ZERO-Z3) [:page_facing_up:](https://rclone.org/s3/#zero-z3)
- Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
- Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
- The local filesystem [:page_facing_up:](https://rclone.org/local/)
+44 -107
View File
@@ -6,74 +6,43 @@ This file describes how to make the various kinds of releases
- [gh the github cli](https://github.com/cli/cli) for uploading packages
- pandoc for making the html and man pages
- hugo for building the website
- golangci-lint and docker for `make check` (markdownlint runs in docker)
- a gpg key for signing the tag and the checksums
- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
`www.rclone.org:` and `test-rclone-org:`
- the sponsor logos - see [Sponsor logos](#sponsor-logos)
## Making a release
- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
- git checkout master # see below for stable branch
- git pull # IMPORTANT
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
- git status - make sure everything is checked in
- Check GitHub actions build for master is Green
- Check the integration tests at <https://integration.rclone.org>
- cat VERSION # check this is the version being released
- make tag # signed tag on HEAD + new changelog entry + docs
- edit docs/content/changelog.md # entries already in point releases are
removed automatically
- make test # see integration test server or run locally
- make tag
- edit docs/content/changelog.md # make sure to remove duplicate logs from point
releases
- make tidy
- make doc # again to pick up the changelog edits
- git status - to check for new command docs in docs/content/commands - git add them
- make doc
- git status - to check for new man pages - git add them
- git commit -a -v -m "Version v1.XX.0"
- make check # if this fails, fix, git commit --amend and run it again
- make retag # moves the tag onto the Version commit
- git push origin # without --follow-tags so it doesn't push the tag yet
- Wait for the GitHub actions build of master to complete and check it is
green - this is the first time CI sees any commits which were only local,
such as security fixes, so it must pass before the tag is pushed
- if it fails: fix, git commit, make retag, git push origin and wait again
- make check
- make retag
- git push origin # without --follow-tags so it doesn't push the tag if it fails
- git push --follow-tags origin
- \# Wait for the GitHub builds of the tag to complete then...
- \# Don't commit or push anything until after make upload_github as these steps
use the tag on HEAD to find the version
- \# Wait for the GitHub builds to complete then...
- make fetch_binaries
- make tarball
- make vendorball
- make sign_upload
- make check_sign
- make upload
- make upload_test_website # check sponsors are correct/unchanged
- make upload_website
- make upload_github # this triggers the docker plugin and winget workflows
- Check the docker image, docker plugin and winget GitHub actions succeeded
- make startdev # works out the next version for master or a stable branch
- git push
- Disable the release freeze rule
- Release any security advisories
- request a CVE for each advisory if not done already
- set the patched version on each advisory and publish it
- when the CVEs are assigned replace `CVE-PENDING` in the changelog on
master and the stable branch then `make upload_website`
- \# announce with forum post, twitter post, and post to announcements list
- make upload_github
- make startdev # make startstable for stable branch
- \# announce with forum post, twitter post, patreon post
## Update dependencies
Early in the next release cycle update the dependencies.
- Active Pins
- `github.com/coreos/go-systemd/v22` v22.6.0 - v22.7.0 fails to compile on netbsd
- `github.com/hanwen/go-fuse/v2` v2.11.0 - v2.12.0 fails to compile on 32
bit linux, awaiting fix for
[go-fuse/issues/628](https://github.com/hanwen/go-fuse/issues/628)
- Review Active Pins in go.mod and remove if possible
- Review any pinned packages in go.mod and remove if possible
- `make updatedirect`
- The update ignores the pins so put them back, eg
`go get github.com/coreos/go-systemd/v22@v22.6.0` then `go mod tidy`
- `make GOTAGS=cmount`
- `make compiletest`
- Fix anything which doesn't compile at this point and commit changes here
@@ -85,18 +54,32 @@ If the `make updatedirect` upgrades the version of go in the `go.mod`
go 1.22.0
```
then so this instead `make updatedirectnoupgrade`. This will try each
upgrade individually. Consider making an upstream bug report for any
`SKIPPING` lines as these only support the latest Go version not the
previous version too.
then go to manual mode. `go1.22` here is the lowest supported version
in the `go.mod`.
If `make updatedirect` or `make updatedirectnoupgrade` added a
`toolchain` directive then remove it. We don't want to force a
toolchain on our users. Linux packagers are often using a version of
Go that is a few versions out of date.
If `make updatedirect` added a `toolchain` directive then remove it.
We don't want to force a toolchain on our users. Linux packagers are
often using a version of Go that is a few versions out of date.
```console
go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
go get -d $(cat /tmp/potential-upgrades)
go mod tidy -go=1.22 -compat=1.22
```
If the `go mod tidy` fails use the output from it to remove the
package which can't be upgraded from `/tmp/potential-upgrades` when
done
```console
git co go.mod go.sum
```
And try again.
Optionally upgrade the direct and indirect dependencies. This is very
likely to fail if `make updatedirectnoupgrade` was used.
likely to fail if the manual method was used abve - in that case
ignore it as it is too time consuming to fix.
- `make update`
- `make GOTAGS=cmount`
@@ -114,19 +97,6 @@ build.
Once it compiles locally, push it on a test branch and commit fixes
until the tests pass.
### Pseudo versions
Go makes pseudo versions for untagged repos and repos not at a tag.
The pseudo versions on repos that have been tagged before do not get
updated automatically so need manually checking. These can be found with
```console
grep -E '[0-9]{14}-[0-9a-f]{12}' go.mod | grep -v indirect | grep -v 'v0\.0\.0'
```
These will need to be updated manually using the `go get ...@branch`
syntax.
### Major versions
The above procedure will not upgrade major versions, so v2 to v3.
@@ -216,53 +186,20 @@ First make the release branch. If this is a second point release then
this will be done already.
- git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
- make startdev # sets the version to ${BASE_TAG}.1
- git push -u origin ${BASE_TAG}-stable
- make startstable
Now
- Enable release freeze rule
- git co master
- git pull --rebase
- make fetch-gui-and-commit # update the GUI on master first
- git push
- git co ${BASE_TAG}-stable
- git pull
- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
- `git cherry-pick -x` any fixes, including the GUI update if there was one
- for security releases consider updating the Go patch version in
`.github/workflows/build.yml` and dependencies with CVE fixes
- fixes may depend on commits which weren't picked so push to CI
and check it is green before tagging
- Do the steps in "Making a release" above, except
- stay on ${BASE_TAG}-stable - don't checkout master or run
make fetch-gui-and-commit
- check the GitHub actions build for ${BASE_TAG}-stable not master
- run the integration tests on ${BASE_TAG}-stable as described below
- the release freeze rule is already enabled - leave it until the end
- git cherry-pick any fixes
- make startstable
- Do the steps as above
- git co master
- git pull --rebase
- `#` cherry pick the changes to the changelog - check the diff to make sure it
is correct
- git checkout ${BASE_TAG}-stable docs/content/changelog.md
- git diff --cached # check only the ${NEW_TAG} entry has changed
- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
- git push
- Disable release freeze rule
Point releases are only made for the latest release. Once v1.(XX+1).0
is out the ${BASE_TAG}-stable branch is abandoned.
### Running the integration tests on the stable branch
The integration tests at <https://integration.rclone.org> test master.
To test the stable branch push it to the testing server and run the
tests there in `screen` as they take a while.
```console
git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
ssh rclone@rclone-testing
screen
cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
```
## Sponsor logos
+1 -1
View File
@@ -1 +1 @@
v1.77.0
v1.74.0
-3
View File
@@ -16,7 +16,6 @@ import (
_ "github.com/rclone/rclone/backend/compress"
_ "github.com/rclone/rclone/backend/crypt"
_ "github.com/rclone/rclone/backend/doi"
_ "github.com/rclone/rclone/backend/dosya"
_ "github.com/rclone/rclone/backend/drime"
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/dropbox"
@@ -33,13 +32,11 @@ import (
_ "github.com/rclone/rclone/backend/hdfs"
_ "github.com/rclone/rclone/backend/hidrive"
_ "github.com/rclone/rclone/backend/http"
_ "github.com/rclone/rclone/backend/huaweidrive"
_ "github.com/rclone/rclone/backend/iclouddrive"
_ "github.com/rclone/rclone/backend/imagekit"
_ "github.com/rclone/rclone/backend/internetarchive"
_ "github.com/rclone/rclone/backend/internxt"
_ "github.com/rclone/rclone/backend/jottacloud"
_ "github.com/rclone/rclone/backend/kdrive"
_ "github.com/rclone/rclone/backend/koofr"
_ "github.com/rclone/rclone/backend/linkbox"
_ "github.com/rclone/rclone/backend/local"
+8 -48
View File
@@ -36,7 +36,6 @@ import (
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fspath"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/sanitize"
)
// Register with Fs
@@ -186,20 +185,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
foundArchive := subArchive(remote)
if foundArchive != nil {
fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
// Archive path, in canonical form so that it compares equal
// to the cleaned entry names inside the archive
foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
if foundArchive.root == "." {
foundArchive.root = ""
}
// Archive path
foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
// Path to the archive
archiveRemote := remote[:len(foundArchive.remote)]
// Remote is archive leaf name. This must understand remote
// names as the archive may be in the root, e.g. "remote:file.zip"
_, foundArchive.remote, err = fspath.Split(archiveRemote)
if err != nil {
return nil, fmt.Errorf("failed to parse archive path %q: %w", archiveRemote, err)
}
// Remote is archive leaf name
foundArchive.remote = path.Base(archiveRemote)
foundArchive.prefix = ""
// Point remote to archive file
remote = archiveRemote
@@ -500,50 +491,24 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, err
}
subEntries, err := subFs.List(ctx, dir)
entries, err = subFs.List(ctx, dir)
if err != nil {
return nil, err
}
entries = subEntries[:0]
skipped := 0
for _, entry := range subEntries {
remote := entry.Remote()
// Only pass on direct children of dir - anything else
// could escape the archive's namespace
if !isDirectChild(dir, remote) {
fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
skipped++
continue
}
for i, entry := range entries {
// Can only unarchive files
if o, ok := entry.(fs.Object); ok {
remote := o.Remote()
archive := f.findArchive(remote)
if archive != nil {
// Overwrite entry with directory
entry = fs.NewDir(remote, o.ModTime(ctx))
entries[i] = fs.NewDir(remote, o.ModTime(ctx))
}
}
entries = append(entries, entry)
}
if skipped > 0 {
fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
}
return entries, nil
}
// isDirectChild reports whether remote names an entry directly in dir
// ("" being the root), with no ".." or other components in between.
func isDirectChild(dir, remote string) bool {
if dir != "" {
var ok bool
remote, ok = strings.CutPrefix(remote, dir+"/")
if !ok {
return false
}
}
return sanitize.Leaf(remote) == nil
}
// NewObject creates a new remote archive file object
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
@@ -561,11 +526,6 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
if err != nil {
return nil, err
}
// Don't trust the archiver to have returned the object asked for
if o.Remote() != remote {
fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
return nil, fs.ErrorObjectNotFound
}
return o, nil
}
-343
View File
@@ -3,28 +3,22 @@
package archive
import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
_ "github.com/rclone/rclone/backend/alias"
"github.com/rclone/rclone/backend/archive/archiver"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/stretchr/testify/assert"
@@ -178,18 +172,6 @@ func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T,
checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
// Now check a single file
//
// The filter below applies to both sides, so check first that the
// archive exposes only that file without it
Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
require.NoError(t, err)
entries, err := Fsingle.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
_, err = Fsingle.List(ctx, subDir)
assert.Equal(t, fs.ErrorDirNotFound, err)
_, err = Fsingle.NewObject(ctx, subDir)
assert.Equal(t, fs.ErrorObjectNotFound, err)
fiCtx, fi := filter.AddConfig(ctx)
require.NoError(t, fi.AddRule("+ "+aFile))
require.NoError(t, fi.AddRule("- *"))
@@ -237,328 +219,3 @@ func TestArchiveSquashfs(t *testing.T) {
run(t, "mksquashfs", input, output)
})
}
// TestArchiveSquashfsIssue9004 lists and reads squashfs images that exercise
// two layouts go-diskfs used to choke on (fixed in go-diskfs v1.9.4):
//
// - 1.sqfs: a single empty directory, so the image has no fragment table
// (its fragment-table start holds the "not present" sentinel).
// - 2.sqfs: a small tree whose superblock has the NO_XATTRS flag set while
// inodes still carry a (non-sentinel) xattr index - the shape squashfs-
// tools-ng can emit. Built by packing a two-file tree with xattrs via
// `gensquashfs -x`, then setting the NO_XATTRS superblock flag; the tree
// content is trivial placeholder data.
//
// Both images used to fail to list. Regression test for #9004.
func TestArchiveSquashfsIssue9004(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
testdata, err := filepath.Abs(filepath.Join("squashfs", "testdata"))
require.NoError(t, err)
archiveFor := func(t *testing.T, name string) fs.Fs {
f, err := cache.Get(ctx, ":archive:"+filepath.Join(testdata, name))
require.NoError(t, err)
return f
}
t.Run("EmptyDir", func(t *testing.T) {
// 1.sqfs is a single empty directory - it must list without error.
entries, err := archiveFor(t, "1.sqfs").List(ctx, "")
require.NoError(t, err)
assert.Equal(t, 0, len(entries))
})
t.Run("NoXattrTree", func(t *testing.T) {
f := archiveFor(t, "2.sqfs")
entries, err := f.List(ctx, "")
require.NoError(t, err)
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, path.Base(e.Remote()))
}
assert.Contains(t, names, "alpha")
assert.Contains(t, names, "beta")
// A file in the tree must be readable with its real content.
obj, err := f.NewObject(ctx, "beta/sample.xml")
require.NoError(t, err)
assert.Greater(t, obj.Size(), int64(0))
rc, err := obj.Open(ctx)
require.NoError(t, err)
data, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, int(obj.Size()), len(data))
assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
})
}
// TestArchiveUncleanRoot checks that a path into an archive which isn't
// in canonical form (with "./" or doubled slashes) still finds its
// directory.
func TestArchiveUncleanRoot(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, err := zw.Create("sub/dir/a.txt")
require.NoError(t, err)
_, err = w.Write([]byte("data"))
require.NoError(t, err)
require.NoError(t, zw.Close())
zipPath := filepath.Join(t.TempDir(), "test.zip")
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
t.Run(root, func(t *testing.T) {
f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, "a.txt", entries[0].Remote())
})
}
}
// TestArchiveInRemoteRoot checks that an archive in the root of a
// remote, with no "/" between the remote name and the archive name, can
// be read, as can a path within it.
func TestArchiveInRemoteRoot(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, err := zw.Create("sub/a.txt")
require.NoError(t, err)
_, err = w.Write([]byte("data"))
require.NoError(t, err)
require.NoError(t, zw.Close())
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.zip"), buf.Bytes(), 0600))
t.Chdir(dir)
for _, test := range []struct {
name string
remote string
}{
{name: "plain", remote: ":local:test.zip"},
{name: "slash in remote name", remote: ":alias,remote='" + dir + "':test.zip"},
} {
for _, sub := range []struct {
path string
want string
}{
{path: "", want: "sub"},
{path: "/sub", want: "a.txt"},
} {
t.Run(test.name+sub.path, func(t *testing.T) {
f, err := cache.Get(ctx, ":archive:"+test.remote+sub.path)
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, sub.want, entries[0].Remote())
})
}
}
}
// escapingObject is an object whose remote is not where it was asked for.
type escapingObject struct {
fs.Object
remote string
}
func (o *escapingObject) Remote() string { return o.remote }
func (o *escapingObject) String() string { return o.remote }
// escapingFs stands in for a badly behaved archiver which exposes entry
// names outside the directory being listed.
type escapingFs struct {
fs.Fs
prefix string
}
func (f *escapingFs) Name() string { return "escaping" }
func (f *escapingFs) Root() string { return "" }
func (f *escapingFs) String() string { return "escaping" }
func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
return fs.DirEntries{
&escapingObject{remote: path.Join(dir, "good.txt")},
fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
&escapingObject{remote: path.Join(dir, "../escape.txt")},
&escapingObject{remote: "../../escape.txt"},
&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
}, nil
}
func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return &escapingObject{remote: "../escape.txt"}, nil
}
// TestArchiveEscapingArchiver checks that the archive backend does not
// pass on entries from an archiver which escape the directory being
// listed, whatever the archiver does.
func TestArchiveEscapingArchiver(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
archiver.Register(archiver.Archiver{
New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
return &escapingFs{prefix: prefix}, nil
},
Extension: ".escaping",
})
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
f, err := cache.Get(ctx, ":archive:"+dir)
require.NoError(t, err)
// Archives are discovered when their parent directory is listed
_, err = f.List(ctx, "")
require.NoError(t, err)
entries, err := f.List(ctx, "test.escaping")
require.NoError(t, err)
var remotes []string
for _, entry := range entries {
remotes = append(remotes, entry.Remote())
}
assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
_, err = f.NewObject(ctx, "test.escaping/file.txt")
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}
// TestIsDirectChild checks the guard which decides whether an entry
// returned by an archiver belongs directly in the directory listed.
func TestIsDirectChild(t *testing.T) {
for _, test := range []struct {
dir, remote string
want bool
}{
{"", "a.txt", true},
{"", "/a.txt", false},
{"", "a.txt/", false},
{"", "../a.txt", false},
{"", "sub/a.txt", false},
{"d", "d/a.txt", true},
{"d", "d", false},
{"d", "d/", false},
{"d", "d//a.txt", false},
{"d", "d/../a.txt", false},
{"d", "dd/a.txt", false},
{"d", "a.txt", false},
} {
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
}
}
// A symlink in a squashfs image is skipped unless -l/--links is in use,
// as it is on the other backends. With the flag it is exposed under its
// name with the link suffix and reads back as its target.
//
// Note that this uses mksquashfs as an external binary.
func TestArchiveSquashfsSymlink(t *testing.T) {
fstest.Initialise()
skipIfNoExe(t, "mksquashfs")
ctx := context.Background()
dir := t.TempDir()
input := filepath.Join(dir, "input")
require.NoError(t, os.Mkdir(input, 0777))
require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
// a regular file which happens to carry the link suffix must stay
// readable under the flag, not be taken for a symlink
require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
image := filepath.Join(dir, "test.sqfs")
run(t, "mksquashfs", input, image)
remotes := func(t *testing.T, f fs.Fs) []string {
entries, err := f.List(ctx, "")
require.NoError(t, err)
names := make([]string, 0, len(entries))
for _, entry := range entries {
names = append(names, entry.Remote())
}
sort.Strings(names)
return names
}
read := func(t *testing.T, f fs.Fs, remote string) string {
o, err := f.NewObject(ctx, remote)
require.NoError(t, err)
rc, err := o.Open(ctx)
require.NoError(t, err)
contents, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, o.Size(), int64(len(contents)))
return string(contents)
}
f, err := fs.NewFs(ctx, ":archive:"+image)
require.NoError(t, err)
assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
ctx, ci := fs.AddConfig(ctx)
ci.Links = true
f, err = fs.NewFs(ctx, ":archive:"+image)
require.NoError(t, err)
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
}
// A hostile archive must not be able to write outside the destination.
// A symlink pointing above the root with a member inside it is the
// classic way to try it, and the refusal comes from the local backend,
// so pin it from end to end here rather than trusting it stays.
func TestArchiveZipSymlinkEscape(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.Links = true
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
hdr := &zip.FileHeader{Name: "a"}
hdr.SetMode(os.ModeSymlink | 0777)
w, err := zw.CreateHeader(hdr)
require.NoError(t, err)
_, err = w.Write([]byte("../outside"))
require.NoError(t, err)
w, err = zw.Create("a/pwned.txt")
require.NoError(t, err)
_, err = w.Write([]byte("pwned"))
require.NoError(t, err)
require.NoError(t, zw.Close())
dir := t.TempDir()
zipPath := filepath.Join(dir, "evil.zip")
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
dst := filepath.Join(dir, "dst")
require.NoError(t, os.Mkdir(dst, 0777))
fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
require.NoError(t, err)
fdst, err := fs.NewFs(ctx, dst)
require.NoError(t, err)
assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
}
-5
View File
@@ -8,11 +8,6 @@ import (
)
// Archiver describes an archive package
//
// Entry names inside an archive are attacker controlled. An archiver
// must not expose an entry whose remote escapes the archive's own
// namespace (for example one with a ".." component), typically by
// validating names with lib/sanitize.
type Archiver struct {
// New constructs an Fs from the (wrappedFs, remote) with the objects
// prefix with prefix and rooted at root
+1 -1
View File
@@ -36,7 +36,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs
// FIXME vfs cache?
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
VFS := vfs.New(ctx, wrappedFs, nil)
VFS := vfs.New(wrappedFs, nil)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
-5
View File
@@ -161,10 +161,5 @@ func (c *cache) Writable() (backend.WritableFile, error) {
return nil, errCacheNotImplemented
}
// Path returns the path to the underlying storage - not applicable here
func (c *cache) Path() string {
return ""
}
// check interfaces
var _ backend.Storage = (*cache)(nil)
+32 -173
View File
@@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
"os"
"path"
"strings"
"time"
@@ -16,7 +15,6 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/sanitize"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
)
@@ -42,42 +40,17 @@ type Fs struct {
prefix string // position for objects
prefixSlash string // position for objects with a slash on
root string // position to read from within the archive
file string // remote of the single file root points at, if any
links bool // read symlinks as .rclonelink files
}
// recoverParsePanic converts a panic raised while parsing a squashfs image
// into an error assigned through err.
func recoverParsePanic(err *error) {
if r := recover(); r != nil {
*err = fmt.Errorf("invalid or corrupt squashfs image: %v", r)
}
}
// recoveringReadCloser wraps a reader returned by the go-diskfs parser so
// panics raised while reading become errors.
//
// The parser reads file data lazily, so an image which opens cleanly can
// still panic on the first Read.
type recoveringReadCloser struct {
io.ReadCloser
}
func (r recoveringReadCloser) Read(p []byte) (n int, err error) {
defer recoverParsePanic(&err)
return r.ReadCloser.Read(p)
}
// New constructs an Fs from the (wrappedFs, remote) with the objects
// prefix with prefix and rooted at root
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ fs.Fs, err error) {
defer recoverParsePanic(&err)
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
// FIXME vfs cache?
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
vfsOpt := vfscommon.Opt
vfsOpt.ReadWait = 0
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
VFS := vfs.New(wrappedFs, &vfsOpt)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
@@ -100,7 +73,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
c: c,
remote: remote,
root: strings.Trim(root, "/"),
links: fs.GetConfig(ctx).Links,
prefix: prefix,
prefixSlash: prefix + "/",
}
@@ -118,13 +90,10 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
_, err := f.newObjectNative(native)
if err == nil {
// If it pointed to a file, find the directory above
// and remember the file so that only it is listed
f.root = path.Dir(f.root)
if f.root == "." || f.root == "/" {
f.root = ""
}
f.file = f.prefixSlash + path.Base(native)
singleObject = true
}
}
}
@@ -156,9 +125,7 @@ func (f *Fs) Name() string {
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
// Include the file so the fs cache can tell this Fs apart from
// the one for the directory containing it
return path.Join(f.root, f.file)
return f.root
}
// Features returns the optional features of this Fs
@@ -189,17 +156,6 @@ func (f *Fs) toNative(remote string) (string, error) {
return native, nil
}
// This turns a native path with a leading / into a path suitable for
// the squashfs library which requires io/fs.ValidPath paths with no
// leading / and "." for the root directory.
func toIOFS(native string) string {
native = strings.Trim(native, "/")
if native == "" {
return "."
}
return native
}
// Turn a (nativeDir, leaf) into a remote
func (f *Fs) fromNative(nativeDir string, leaf string) string {
// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
@@ -213,39 +169,16 @@ func (f *Fs) fromNative(nativeDir string, leaf string) string {
}
// Convert a FileInfo into an Object from native dir
func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {
return &Object{
fs: f,
remote: f.fromNative(nativeDir, item.Name()),
size: item.Size(),
modTime: item.ModTime(),
item: item,
}
}
// Convert a symlink entry into an Object holding its target
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
return &Object{
fs: f,
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
size: int64(len(target)),
modTime: item.ModTime(),
isLink: true,
link: target,
}
}
// readlink returns the target of the symlink the info describes.
//
// A symlink may carry an empty target, so the caller decides what is a
// symlink from the mode rather than from the target being set.
func readlink(info os.FileInfo) (string, error) {
st, ok := info.Sys().(*squashfs.StatT)
if !ok {
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
}
return st.LinkTarget, nil
}
// List the objects and directories in dir into entries. The
// entries can be returned in any order but should be for a
// complete directory.
@@ -257,127 +190,70 @@ func readlink(info os.FileInfo) (string, error) {
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
defer recoverParsePanic(&err)
if f.file != "" && dir != "" {
return nil, fs.ErrorDirNotFound
}
nativeDir, err := f.toNative(dir)
if err != nil {
return nil, err
}
items, err := f.sqfs.ReadDir(toIOFS(nativeDir))
items, err := f.sqfs.ReadDir(nativeDir)
if err != nil {
return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
}
entries = make(fs.DirEntries, 0, len(items))
skipped := 0
skippedLinks := 0
for _, item := range items {
// fs.Debugf(item.Name(), "entry = %#v", item)
// Skip entry names that aren't safe
if err := sanitize.Leaf(item.Name()); err != nil {
fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
skipped++
continue
for _, fi := range items {
item, ok := fi.(squashfs.FileStat)
if !ok {
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
}
// fs.Debugf(item.Name(), "entry = %#v", item)
var entry fs.DirEntry
if err != nil {
return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)
}
if item.IsDir() {
remote := f.fromNative(nativeDir, item.Name())
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
entry = fs.NewDir(remote, info.ModTime())
} else if item.Type().IsRegular() {
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
entry = f.objectFromFileInfo(nativeDir, info)
} else if item.Type()&os.ModeSymlink != 0 {
if !f.links {
skippedLinks++
var remote = f.fromNative(nativeDir, item.Name())
entry = fs.NewDir(remote, item.ModTime())
} else {
if item.Mode().IsRegular() {
entry = f.objectFromFileInfo(nativeDir, item)
} else {
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
continue
}
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
target, err := readlink(info)
if err != nil {
return nil, err
}
entry = f.objectFromLink(nativeDir, info, target)
} else {
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
continue
}
if f.file != "" && entry.Remote() != f.file {
continue
}
entries = append(entries, entry)
}
// Log once per listing rather than per entry so a crafted image
// can't flood the log
if skipped > 0 {
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
}
if skippedLinks > 0 {
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
}
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
return entries, nil
}
// newObjectNative finds the object at the native path passed in
func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
defer recoverParsePanic(&err)
// get the path and filename
dir, leaf := path.Split(nativePath)
dir = strings.TrimRight(dir, "/")
leaf = strings.Trim(leaf, "/")
// FIXME need to detect directory not found
fis, err := f.sqfs.ReadDir(toIOFS(dir))
fis, err := f.sqfs.ReadDir(dir)
if err != nil {
return nil, fs.ErrorObjectNotFound
}
for _, fi := range fis {
// Name the entry as List does, so that the two agree on
// which names exist
isLink := fi.Type()&os.ModeSymlink != 0
if isLink && !f.links {
continue
}
name := fi.Name()
if isLink {
name += fs.LinkSuffix
}
if name == leaf {
if fi.Name() == leaf {
if fi.IsDir() {
return nil, fs.ErrorNotAFile
}
info, err := fi.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
item, ok := fi.(squashfs.FileStat)
if !ok {
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
}
if isLink {
target, err := readlink(info)
if err != nil {
return nil, err
}
o = f.objectFromLink(dir, info, target)
break
}
o = f.objectFromFileInfo(dir, info)
o = f.objectFromFileInfo(dir, item)
break
}
}
@@ -391,9 +267,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
if f.file != "" && remote != f.file {
return nil, fs.ErrorObjectNotFound
}
nativePath, err := f.toNative(remote)
if err != nil {
return nil, err
@@ -455,8 +328,7 @@ type Object struct {
remote string
size int64
modTime time.Time
isLink bool // whether this is a symlink
link string // target of the symlink
item squashfs.FileStat
}
// Fs returns read only access to the Fs that this object is part of
@@ -513,7 +385,6 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
defer recoverParsePanic(&err)
var offset, limit int64 = 0, -1
for _, option := range options {
switch x := option.(type) {
@@ -528,24 +399,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
}
}
if o.isLink {
// Clamp offset into range to avoid panic
if offset < 0 {
offset = 0
}
if offset > int64(len(o.link)) {
offset = int64(len(o.link))
}
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
}
remote, err := o.fs.toNative(o.remote)
if err != nil {
return nil, err
}
fs.Debugf(o, "Opening %q", remote)
fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)
//fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)
fh, err := o.item.Open()
if err != nil {
return nil, err
}
@@ -557,15 +418,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
return nil, err
}
}
rc = recoveringReadCloser{fh}
// If limited then don't return everything
if limit >= 0 {
fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
return readers.NewLimitedReadCloser(rc, limit), nil
return readers.NewLimitedReadCloser(fh, limit), nil
}
return rc, nil
return fh, nil
}
// Update in to the object with the modTime given of the given size
@@ -1,32 +0,0 @@
package squashfs
import (
"context"
"testing"
_ "github.com/rclone/rclone/backend/local"
fscache "github.com/rclone/rclone/fs/cache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A backslash is an ordinary character in a file name on the systems
// squashfs images are made on, so an entry containing one must be
// listed and openable like any other file.
func TestBackslashInName(t *testing.T) {
ctx := context.Background()
localFs, err := fscache.Get(ctx, "testdata")
require.NoError(t, err)
f, err := New(ctx, localFs, "backslash.sqfs", "", "")
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, `back\slash.txt`, entries[0].Remote())
o, err := f.NewObject(ctx, `back\slash.txt`)
require.NoError(t, err)
assert.Equal(t, int64(6), o.Size())
}
-25
View File
@@ -1,25 +0,0 @@
package squashfs
import (
"io"
"testing"
"github.com/stretchr/testify/assert"
)
// panickingReader panics on Read, standing in for the go-diskfs parser
// hitting an unchecked slice while reading a malformed image.
type panickingReader struct{}
func (panickingReader) Read([]byte) (int, error) { panic("boom") }
func (panickingReader) Close() error { return nil }
// Test that a panic raised while reading file data becomes an error. The
// parser reads lazily, so an image can open cleanly and only panic here.
func TestRecoveringReadCloser(t *testing.T) {
var rc io.ReadCloser = recoveringReadCloser{panickingReader{}}
_, err := rc.Read(make([]byte, 16))
assert.ErrorContains(t, err, "invalid or corrupt squashfs image")
assert.ErrorContains(t, err, "boom")
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,89 +0,0 @@
//go:build !plan9
package archive
import (
"context"
"encoding/binary"
"os"
"path/filepath"
"testing"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// squashfsSuperblock builds a minimal 96-byte squashfs superblock with the
// given attacker-controllable fields. It mirrors the on-disk layout the
// go-diskfs parser reads, and is only used to construct deliberately malformed
// images for the tests below.
func squashfsSuperblock(blockSize uint32, blockLog, compression uint16, rootInode, inodeTableStart uint64) []byte {
const u64max = ^uint64(0)
b := make([]byte, 96)
binary.LittleEndian.PutUint32(b[0:], 0x73717368) // "hsqs" magic
binary.LittleEndian.PutUint32(b[4:], 1) // inode count
binary.LittleEndian.PutUint32(b[8:], 0) // modtime
binary.LittleEndian.PutUint32(b[12:], blockSize)
binary.LittleEndian.PutUint32(b[16:], 0) // fragment count
binary.LittleEndian.PutUint16(b[20:], compression)
binary.LittleEndian.PutUint16(b[22:], blockLog)
binary.LittleEndian.PutUint16(b[24:], 0x0210) // no fragments/xattrs
binary.LittleEndian.PutUint16(b[26:], 0) // id count
binary.LittleEndian.PutUint16(b[28:], 4) // version major
binary.LittleEndian.PutUint16(b[30:], 0) // version minor
binary.LittleEndian.PutUint64(b[32:], rootInode)
binary.LittleEndian.PutUint64(b[40:], 96) // bytes used
binary.LittleEndian.PutUint64(b[48:], u64max)
binary.LittleEndian.PutUint64(b[56:], u64max)
binary.LittleEndian.PutUint64(b[64:], inodeTableStart)
binary.LittleEndian.PutUint64(b[72:], inodeTableStart)
binary.LittleEndian.PutUint64(b[80:], u64max)
binary.LittleEndian.PutUint64(b[88:], u64max)
return b
}
// TestArchiveSquashfsMalformed feeds the archive backend squashfs images whose
// superblock/metadata make the go-diskfs parser panic. See GHSA-6jcg-q3wp-x2f4.
func TestArchiveSquashfsMalformed(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
// Variant 1: zero block size reaches an integer division in the parser.
zeroBlock := squashfsSuperblock(0, 0, 1, 0, 96)
// Variant 2: an out-of-range root inode byte offset reaches an unchecked
// slice against an empty, uncompressed metadata block.
oob := append(squashfsSuperblock(65536, 16, 1, 12336, 96), 0x00, 0x80)
binary.LittleEndian.PutUint64(oob[40:], uint64(len(oob)))
for _, tc := range []struct {
name string
image []byte
}{
{"ZeroBlockSize", zeroBlock},
{"OutOfRangeInodeOffset", oob},
} {
t.Run(tc.name, func(t *testing.T) {
// A directory per image: the fs cache keys on the parent
// directory, so sharing one would hand the second subtest a
// cached listing which does not contain its image.
path := filepath.Join(t.TempDir(), tc.name+".sqfs")
require.NoError(t, os.WriteFile(path, tc.image, 0o644))
// The whole point is that this returns rather than panicking and
// killing the test binary (and, in production, the serve daemon).
f, err := cache.Get(ctx, ":archive:"+path)
if err == nil {
// The image parsed, so the panic must surface from a later
// lazily-parsed operation instead.
_, err = f.List(ctx, "")
}
require.Error(t, err, "a malformed squashfs image must error, not panic")
assert.Contains(t, err.Error(), "invalid or corrupt squashfs image",
"error should come from the recover, not an unrelated failure")
})
}
}
+8 -42
View File
@@ -9,7 +9,6 @@ import (
"io"
"os"
"path"
"slices"
"strings"
"time"
@@ -19,7 +18,6 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/sanitize"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
)
@@ -43,7 +41,6 @@ type Fs struct {
prefix string // position for objects
prefixSlash string // position for objects with a slash on
root string // position to read from within the archive
links bool // read symlinks as .rclonelink files
dt dirtree.DirTree // read from zipfile
}
@@ -55,7 +52,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
vfsOpt := vfscommon.Opt
vfsOpt.ReadWait = 0
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
VFS := vfs.New(wrappedFs, &vfsOpt)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
@@ -68,7 +65,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
node: node,
remote: remote,
root: root,
links: fs.GetConfig(ctx).Links,
prefix: prefix,
prefixSlash: prefix + "/",
}
@@ -139,37 +135,15 @@ func (f *Fs) readZip() (singleObject bool, err error) {
return singleObject, fmt.Errorf("failed to read zip file: %w", err)
}
dt := dirtree.New()
skipped := 0
skippedLinks := 0
for _, file := range zr.File {
// Skip entries whose name escapes the archive's own namespace
remote, err := sanitize.Path(file.Name)
if err != nil {
skipped++
continue
}
isDir := strings.HasSuffix(file.Name, "/")
// A file entry whose last component is "", "." or ".." names
// a directory rather than a file, so skip
if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
skipped++
continue
}
if !isDir && file.Mode()&os.ModeSymlink != 0 {
if !f.links {
skippedLinks++
continue
}
// The member holds the link target, so it reads back
// like a .rclonelink file
remote += fs.LinkSuffix
remote := strings.Trim(path.Clean(file.Name), "/")
if remote == "." {
remote = ""
}
remote = path.Join(f.prefix, remote)
if f.root != "" {
// Ignore all files outside the root, requiring a path
// boundary so that root "foo" does not also match a
// sibling entry such as "foobar"
if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
// Ignore all files outside the root
if !strings.HasPrefix(remote, f.root) {
continue
}
if remote == f.root {
@@ -178,12 +152,11 @@ func (f *Fs) readZip() (singleObject bool, err error) {
remote = strings.TrimPrefix(remote, f.root+"/")
}
}
if isDir {
if strings.HasSuffix(file.Name, "/") {
dir := fs.NewDir(remote, file.Modified)
dt.AddDir(dir)
} else {
if remote == "" {
// A file at the root is the archive member f.root points at
remote = path.Base(f.root)
singleObject = true
dt = dirtree.New()
@@ -200,12 +173,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
}
}
}
if skipped > 0 {
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
}
if skippedLinks > 0 {
fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
}
dt.CheckParents("")
dt.Sort()
f.dt = dt
@@ -233,8 +200,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, fs.ErrorDirNotFound
}
fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
// Return a copy as callers may filter the listing in place
return slices.Clone(entries), nil
return entries, nil
}
// NewObject finds the Object at remote.
-227
View File
@@ -1,227 +0,0 @@
package zip
import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// writeZip builds a zip file with the given entry names (a name ending
// in "/" is a directory) in dir and returns its leaf name.
func writeZip(t *testing.T, dir, name string, names ...string) string {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, n := range names {
w, err := zw.Create(n)
require.NoError(t, err)
if !strings.HasSuffix(n, "/") {
_, err = w.Write([]byte("data for " + n))
require.NoError(t, err)
}
}
require.NoError(t, zw.Close())
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
return name
}
// writeZipSymlink builds a zip holding file and a symlink entry link
// pointing at target, and returns its leaf name.
func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, err := zw.Create(file)
require.NoError(t, err)
_, err = w.Write([]byte("data for " + file))
require.NoError(t, err)
hdr := &zip.FileHeader{Name: link}
hdr.SetMode(os.ModeSymlink | 0777)
w, err = zw.CreateHeader(hdr)
require.NoError(t, err)
_, err = w.Write([]byte(target))
require.NoError(t, err)
require.NoError(t, zw.Close())
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
return name
}
// allRemotes returns every Object remote in the mounted archive's dirtree.
func allRemotes(t *testing.T, f fs.Fs) []string {
zf, ok := f.(*Fs)
require.True(t, ok)
var remotes []string
for _, entries := range zf.dt {
for _, entry := range entries {
if o, ok := entry.(*Object); ok {
remotes = append(remotes, o.Remote())
}
}
}
// dt iteration order is nondeterministic, so sort for stable comparison.
sort.Strings(remotes)
return remotes
}
// A malicious zip whose entry names escape the archive's own namespace
// must not be exposed by the zip backend (Zip Slip).
func TestReadZipSlip(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "evil.zip",
"good.txt",
"../../etc/cron.d/evil",
"../escape.txt",
"sub/../../above.txt",
`..\..\evil.exe`,
)
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "")
require.NoError(t, err)
remotes := allRemotes(t, f)
// The one benign entry must survive.
assert.Contains(t, remotes, "good.txt")
// No escaping entry may be exposed.
for _, remote := range remotes {
assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
"escaping remote exposed: %q", remote)
}
assert.Equal(t, []string{"good.txt"}, remotes)
}
// Mounting with a non-empty root must only expose entries within that
// root directory, not sibling directories that merely share a name
// prefix (root "foo" must not match "foobar").
func TestReadZipRootBoundary(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "test.zip",
"foo/a.txt",
"foobar/b.txt",
)
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "foo")
require.NoError(t, err)
remotes := allRemotes(t, f)
assert.Equal(t, []string{"a.txt"}, remotes)
}
// A file entry whose name refers to a directory (".", "/", "./", "" or
// "sub/.") must be skipped, not turn the archive or the directory
// mounted as its root into a single file which hides every other
// entry, or become an object named for the archive itself when the
// archive is found by listing its directory.
func TestReadZipDirNamedFileEntry(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "dot.zip",
".",
"/",
"./",
"",
"good.txt",
"sub/",
"sub/.",
"sub/good.txt",
)
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
for _, test := range []struct {
prefix, root string
want []string
}{
{"", "", []string{"good.txt", "sub/good.txt"}},
{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
{"", "sub", []string{"good.txt"}},
} {
t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
f, err := New(ctx, localFs, name, test.prefix, test.root)
require.NoError(t, err)
assert.Equal(t, test.want, allRemotes(t, f))
})
}
}
// Listings are served from a cache which must survive callers
// filtering the returned slice in place, as fs/list does.
func TestListCacheNotAliased(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "")
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 3)
// Compact in place, dropping the first entry
copy(entries, entries[1:])
entries, err = f.List(ctx, "")
require.NoError(t, err)
var remotes []string
for _, entry := range entries {
remotes = append(remotes, entry.Remote())
}
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
}
// A symlink stored in the archive is skipped unless -l/--links is in
// use, as it is on the other backends. With the flag it is exposed
// under its name with the link suffix and reads back as its target.
func TestReadZipSymlink(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "")
require.NoError(t, err)
assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
ctx, ci := fs.AddConfig(ctx)
ci.Links = true
f, err = New(ctx, localFs, name, "", "")
require.NoError(t, err)
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
require.NoError(t, err)
assert.Equal(t, int64(len("file.txt")), o.Size())
rc, err := o.Open(ctx)
require.NoError(t, err)
target, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, "file.txt", string(target))
}
-162
View File
@@ -1,162 +0,0 @@
//go:build !plan9 && !js
// Parallel listing support for azureblob, built on the Apache Arrow
// listing format.
//
// The Arrow listing exposes the server-side startFrom (inclusive) and endBefore
// (exclusive) name-range parameters, which let a listing be sharded by
// blob-name range and listed concurrently.
package azureblob
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/rclone/rclone/fs"
"golang.org/x/sync/errgroup"
)
// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
// response, used to tell whether the service honoured an Arrow request.
const arrowContentType = "application/vnd.apache.arrow.stream"
// listArrowParallel lists (containerName, directory) by splitting the blob name
// keyspace into shards and listing them concurrently. opts must already request
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
// returns the number of raw items seen. directory/prefix are already normalised
// by the caller.
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
// Split the keyspace by a single-case character ladder (see
// arrowLadderBoundaries for why single-case matters).
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
if len(points) < 1 {
// Couldn't usefully split - list sequentially.
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
// Build half-open [startFrom, endBefore) shards from the sorted boundary
// points p0..pk: ["", p0), [p0, p1), ..., [pk, "").
type shard struct{ startFrom, endBefore string }
shards := make([]shard, 0, len(points)+1)
prev := ""
for _, p := range points {
shards = append(shards, shard{prev, p})
prev = p
}
shards = append(shards, shard{prev, ""})
// fn (and the list.Helper behind it) is not concurrency-safe, so serialise
// delivery. Only the cheap callback is locked, not the HTTP/decode work.
var fnMu sync.Mutex
safeFn := func(remote string, item *container.BlobItem, isDir bool) error {
fnMu.Lock()
defer fnMu.Unlock()
return fn(remote, item, isDir)
}
// Schedule every shard; SetLimit bounds concurrency, so more shards than
// workers gives free load balancing as workers pick up the next shard.
var total atomic.Int64
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(f.opt.ListParallelism)
for _, sh := range shards {
if gCtx.Err() != nil {
break
}
g.Go(func() error {
// Clone opts per shard so concurrent shards don't share StartFrom/EndBefore.
shardOpts := *opts
if sh.startFrom != "" {
startFrom := sh.startFrom
shardOpts.StartFrom = &startFrom
} else {
shardOpts.StartFrom = nil
}
if sh.endBefore != "" {
endBefore := sh.endBefore
shardOpts.EndBefore = &endBefore
} else {
shardOpts.EndBefore = nil
}
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
total.Add(int64(n))
return err
})
}
err := g.Wait()
found := int(total.Load())
if isEndBeforeUnsupported(err) {
// The account can't honour the shards' endBefore bounds, so parallel
// listing can't be used on it.
f.arrowXMLFallback.Store(true)
if found == 0 {
// No shard delivered anything yet (the normal case - Arrow listing
// enablement is per account so every shard fails on its first
// page) so it is safe to relist sequentially.
fs.Debugf(f, "Parallel Arrow listing not supported by this account (%v) - listing sequentially", err)
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
// Some entries were already delivered so a rerun would duplicate them.
return found, fmt.Errorf("parallel listing aborted after partial results (retry, or unset list_parallelism): %w", err)
}
return found, err
}
// isEndBeforeUnsupported reports whether err means the service can't honour
// the endBefore listing bound, so sharded parallel listing must not be used.
//
// The service rejects the endBefore parameter with 400
// OperationNotSupportedWithFeatureMissing "The requested operation is not
// allowed as EndBefore parameter support is missing on this account" -
// endBefore is only supported on the Arrow listing path of accounts with
// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
// accept an XML page with endBefore set, but the service never sends one.)
func isEndBeforeUnsupported(err error) bool {
var respErr *azcore.ResponseError
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
}
// arrowShardTarget is the number of ladder boundaries to use for the given
// parallelism (a few per worker for load balancing).
func arrowShardTarget(parallelism int) int {
return parallelism * 4
}
// arrowLadderBoundaries returns up to target sorted ENCODED-space boundaries of
// the form directory+c, used to split the keyspace into shards.
//
// The partition is always correct (no dup/gap) regardless of the alphabet: keys
// whose next byte is outside the alphabet simply fall into an adjacent shard;
// the alphabet only affects balance. A single-case (digit+lowercase) alphabet
// is deliberate: the boundaries are then monotonic under BOTH byte order (the
// listing order) AND the service's case-insensitive startFrom/endBefore
// validation, so server-side endBefore is accepted. (A mixed-case ladder gives
// pairs like [".../Z", ".../a") which are byte-ascending but rejected 400 as
// "endbefore precedes startfrom".) directory is already encoded; the alphabet
// is ASCII (encoder-invariant) and ascending, so the result is sorted.
func arrowLadderBoundaries(directory string, target int) []string {
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
if target < 1 {
return nil
}
if target > len(alphabet) {
target = len(alphabet)
}
points := make([]string, 0, target)
last := -1
for i := 0; i < target; i++ {
idx := i * len(alphabet) / target
if idx == last {
continue
}
last = idx
points = append(points, directory+string(alphabet[idx]))
}
return points
}
-219
View File
@@ -1,219 +0,0 @@
//go:build !plan9 && !js
package azureblob
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/random"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// testArrowList exercises the experimental Apache Arrow listing.
//
// It is skipped automatically if Arrow listing is not enabled on the test
// account (the server returns XML, or 409 on HNS accounts).
func (f *Fs) testArrowList(t *testing.T) {
ctx := context.Background()
containerName, _ := f.split("probe")
if containerName == "" {
t.Skip("Arrow list test needs a container in the remote root")
}
// withArrow runs fn with UseArrowList temporarily set.
withArrow := func(on bool, fn func()) {
old := f.opt.UseArrowList
f.opt.UseArrowList = on
defer func() { f.opt.UseArrowList = old }()
fn()
}
// withParallel runs fn with ListParallelism temporarily set.
withParallel := func(n int, fn func()) {
old := f.opt.ListParallelism
f.opt.ListParallelism = n
defer func() { f.opt.ListParallelism = old }()
fn()
}
// Create a set of known objects under a unique directory: flat files plus
// one in a subdirectory (so we exercise both the flat and delimited paths).
dir := "arrow-list-" + random.String(8)
// Includes upper-case and digit first-characters: these byte-sort between
// the digit and lowercase ladder boundaries, so they verify the server's
// endBefore filtering is byte-ordered (not case-insensitive, which would
// drop them from a shard and make parallel != sequential).
objects := []string{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt", "sub/x.txt", "Foo.txt", "Zoo.txt", "0num.txt", "9end.txt"}
wantSizes := map[string]int64{}
created := make([]string, 0, len(objects))
for i, n := range objects {
contents := random.String(10 + i)
remote := dir + "/" + n
item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
_ = fstests.PutTestContents(ctx, t, f, &item, contents, true)
created = append(created, remote)
wantSizes[remote] = int64(len(contents))
}
defer func() {
for _, remote := range created {
if o, err := f.NewObject(ctx, remote); err == nil {
_ = o.Remove(ctx)
}
}
}()
// listing returns remote->size for dir (dirs reported as size -1).
listing := func(recurse bool) (map[string]int64, error) {
got := map[string]int64{}
add := func(entries fs.DirEntries) error {
for _, e := range entries {
if o, ok := e.(fs.Object); ok {
got[e.Remote()] = o.Size()
} else {
got[e.Remote()] = -1
}
}
return nil
}
if recurse {
return got, f.ListR(ctx, dir, add)
}
entries, err := f.List(ctx, dir)
if err != nil {
return nil, err
}
_ = add(entries)
return got, nil
}
// Probe: is Arrow available? If not, skip cleanly.
var probeErr error
withArrow(true, func() { _, probeErr = listing(true) })
if probeErr != nil {
if strings.Contains(probeErr.Error(), "OperationNotSupportedWithFeatureMissing") || strings.Contains(probeErr.Error(), "409") {
t.Skipf("Arrow listing not available on this account: %v", probeErr)
}
require.NoError(t, probeErr)
}
// Arrow listing must equal XML listing, recursively and per-directory
// (including the "sub" subdirectory).
t.Run("EqualsXML", func(t *testing.T) {
for _, recurse := range []bool{true, false} {
var xml, arrow map[string]int64
var err error
withArrow(false, func() { xml, err = listing(recurse) })
require.NoError(t, err)
withArrow(true, func() { arrow, err = listing(recurse) })
require.NoError(t, err)
assert.Equal(t, xml, arrow, "recurse=%v: Arrow listing differs from XML", recurse)
}
// Recursive Arrow listing contains every object with the right size.
var arrow map[string]int64
withArrow(true, func() { arrow, _ = listing(true) })
for remote, size := range wantSizes {
assert.Equal(t, size, arrow[remote], remote)
}
// Single-directory Arrow listing reports "sub" as a directory.
var entries fs.DirEntries
withArrow(true, func() {
var err error
entries, err = f.List(ctx, dir)
require.NoError(t, err)
})
foundSubDir := false
for _, e := range entries {
if e.Remote() == dir+"/sub" {
_, isObj := e.(fs.Object)
foundSubDir = !isObj
}
}
assert.True(t, foundSubDir, "expected %q to be listed as a directory", dir+"/sub")
})
// Parallel listing must equal sequential listing, recursively and
// per-directory, including the "sub" subdirectory.
t.Run("ParallelEqualsSequential", func(t *testing.T) {
for _, recurse := range []bool{true, false} {
var seq, par map[string]int64
var err error
withArrow(true, func() {
withParallel(0, func() { seq, err = listing(recurse) })
require.NoError(t, err)
withParallel(8, func() { par, err = listing(recurse) })
require.NoError(t, err)
})
assert.Equal(t, seq, par, "recurse=%v: parallel listing differs from sequential", recurse)
}
})
// A tiny list_chunk forces each shard to span multiple Arrow pages, so this
// exercises pagination (startFrom+endBefore+marker resent together) inside a
// shard. The "sub/" boundary puts a.txt..e.txt in one shard => >1 page at
// chunk size 2.
t.Run("ParallelMultiPage", func(t *testing.T) {
oldChunk := f.opt.ListChunkSize
f.opt.ListChunkSize = 2
defer func() { f.opt.ListChunkSize = oldChunk }()
var seq, par map[string]int64
var err error
withArrow(true, func() {
withParallel(0, func() { seq, err = listing(true) })
require.NoError(t, err)
withParallel(8, func() { par, err = listing(true) })
require.NoError(t, err)
})
assert.Equal(t, seq, par)
for remote, size := range wantSizes {
assert.Equal(t, size, par[remote], remote)
}
})
}
func TestArrowLadderBoundaries(t *testing.T) {
for _, tc := range []struct {
directory string
target int
}{
{"", 8},
{"dir/", 16},
{"dir/", 100}, // capped at alphabet size
{"dir/", 1},
} {
points := arrowLadderBoundaries(tc.directory, tc.target)
require.NotEmpty(t, points)
assert.LessOrEqual(t, len(points), 62)
for i, p := range points {
assert.True(t, strings.HasPrefix(p, tc.directory), "boundary %q lacks prefix %q", p, tc.directory)
if i > 0 {
assert.Less(t, points[i-1], p, "boundaries not strictly ascending")
}
}
}
assert.Nil(t, arrowLadderBoundaries("dir/", 0))
// alphabet is single-case (digit+lowercase) so boundaries are monotonic
// under both byte order and case-insensitive order
pts := arrowLadderBoundaries("dir/", 36)
for _, p := range pts {
assert.Equal(t, p, strings.ToLower(p), "boundary must be lowercase: %q", p)
}
}
func TestIsEndBeforeUnsupported(t *testing.T) {
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
assert.False(t, isEndBeforeUnsupported(nil))
}
-11
View File
@@ -9,7 +9,6 @@ import (
"net/http"
"net/url"
"os"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
@@ -284,9 +283,6 @@ type transporter struct {
// Make a new transporter
func newTransporter(ctx context.Context) transporter {
// Set the User-Agent to include the Microsoft Partner Network prefix
ctx, ci := fs.AddConfig(ctx)
ci.UserAgent = "APN/1.0 rclone/1.0 rclone/" + strings.TrimPrefix(fs.Version, "v")
return transporter{
RoundTripper: fshttp.NewTransport(ctx),
}
@@ -297,13 +293,6 @@ func (tr transporter) Do(req *http.Request) (*http.Response, error) {
return tr.RoundTripper.RoundTrip(req)
}
// Transporter returns the policy.Transporter rclone uses for Azure SDK
// clients (an fshttp based transport with the APN user agent), for callers
// which build their own azcore pipelines.
func Transporter(ctx context.Context) policy.Transporter {
return newTransporter(ctx)
}
// NewClientOpts should be passed to configure NewClient
type NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct {
DefaultBaseURL string // Base URL, eg blob.core.windows.net
+81 -380
View File
@@ -1,4 +1,4 @@
//go:build !plan9 && !js
//go:build !plan9 && !solaris && !js
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
package azureblob
@@ -22,7 +22,6 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
@@ -48,8 +47,6 @@ import (
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/rclone/rclone/lib/transferaccounter"
"golang.org/x/sync/errgroup"
)
@@ -68,19 +65,8 @@ const (
defaultChunkSize = 4 * fs.Mebi
defaultAccessTier = blob.AccessTier("") // FIXME AccessTierNone
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
sasCopyStartSkew = 15 * time.Minute // how far back SAS should start to allow for clock skew
)
// setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
// on every request. This prevents the Go HTTP transport from automatically
// decompressing gzip-encoded blobs on download.
type setAcceptEncodingGzip struct{}
func (p setAcceptEncodingGzip) Do(req *policy.Request) (*http.Response, error) {
req.Raw().Header.Set("Accept-Encoding", "gzip")
return req.Next()
}
var (
errCantUpdateArchiveTierBlobs = fserrors.NoRetryError(errors.New("can't update archive tier blob without --azureblob-archive-tier-delete"))
@@ -238,40 +224,6 @@ minutes per megabyte on average, it will time out (
avoid the time out.`,
Default: maxListChunkSize,
Advanced: true,
}, {
Name: "use_arrow_list",
Help: `Use the Apache Arrow listing format.
If set, directory listings are fetched using the ListBlobs Apache
Arrow response format instead of XML. Arrow responses are smaller and
much cheaper to parse, making listings of large containers several
times faster. Combine with "list_parallelism" for the biggest gains.
"Blob Listing with Apache Arrow" is in public preview at Microsoft and
is only supported on flat namespace accounts. On accounts with a
hierarchical namespace (ADLS Gen2), or where the feature is otherwise
unavailable, the server returns XML and the listing transparently
falls back to the normal XML path (logged at debug level).`,
Default: false,
Advanced: true,
}, {
Name: "list_parallelism",
Help: `Number of parallel shards to list a directory with.
If set greater than 1, the blob name keyspace of each directory is
split into ranges which are listed concurrently using the Arrow
startFrom/endBefore range parameters. This can dramatically speed up
listing containers with millions of objects, for both recursive
(ListR) and single directory listings. Speed keeps improving up to a
parallelism of around 30.
This has no effect unless "use_arrow_list" is also set, as Arrow is the
only listing path that supports server-side name ranges. If the
account does not support range listing (e.g. it has a hierarchical
namespace) the listing falls back to sequential. The default of 0 (or
1) lists sequentially.`,
Default: 0,
Advanced: true,
}, {
Name: "access_tier",
Help: `Access tier of blob: hot, cool, cold or archive.
@@ -365,22 +317,7 @@ creates an empty object ending with "/", to persist the folder.
This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
the Microsoft standard.
This is not needed on storage accounts with a hierarchical namespace
(see the hns option) as these always support empty directories.
`,
}, {
Name: "hns",
Default: fs.Tristate{},
Advanced: true,
Help: `Set if the storage account has a hierarchical namespace.
Storage accounts with hierarchical namespace enabled (Azure Data Lake
Storage Gen2) have real directories which can be empty.
If this is unset then rclone will read it from the storage account
and save it in the config file.
`,
}, {
Name: "no_check_container",
Help: `If set, don't attempt to check the container exists or create it.
@@ -413,19 +350,6 @@ rclone does if you know the container exists already.
Default: "",
Exclusive: true,
Advanced: true,
}, {
Name: "decompress",
Help: `If set this will decompress gzip encoded objects.
It is possible to upload objects to Azure Blob Storage with "Content-Encoding: gzip"
set. Normally rclone will download these files as compressed objects.
If this flag is set then rclone will decompress these files with
"Content-Encoding: gzip" as they are received. This means that rclone
can't check the size and hash but the file contents will be decompressed.
`,
Advanced: true,
Default: false,
}}),
})
}
@@ -440,19 +364,15 @@ type Options struct {
UseCopyBlob bool `config:"use_copy_blob"`
UploadConcurrency int `config:"upload_concurrency"`
ListChunkSize uint `config:"list_chunk"`
UseArrowList bool `config:"use_arrow_list"`
ListParallelism int `config:"list_parallelism"`
AccessTier string `config:"access_tier"`
ArchiveTierDelete bool `config:"archive_tier_delete"`
DisableCheckSum bool `config:"disable_checksum"`
Enc encoder.MultiEncoder `config:"encoding"`
PublicAccess string `config:"public_access"`
DirectoryMarkers bool `config:"directory_markers"`
HNS fs.Tristate `config:"hns"`
NoCheckContainer bool `config:"no_check_container"`
NoHeadObject bool `config:"no_head_object"`
DeleteSnapshots string `config:"delete_snapshots"`
Decompress bool `config:"decompress"`
}
// Fs represents a remote azure server
@@ -464,7 +384,6 @@ type Fs struct {
features *fs.Features // optional features
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache
cntSVCcache map[string]*container.Client // reference to containerClient per container
arrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retried
svc *service.Client // client to access azblob
cred azcore.TokenCredential // how to generate tokens (may be nil)
usingSharedKeyCred bool // set if using shared key credentials
@@ -472,15 +391,12 @@ type Fs struct {
rootContainer string // container part of root (if any)
rootDirectory string // directory part of root (if any)
isLimited bool // if limited to one container
hns bool // if the storage account has a hierarchical namespace
cache *bucket.Cache // cache for container creation status
pacer *fs.Pacer // To pace and retry the API calls
uploadToken *pacer.TokenDispenser // control concurrency
copyToken *pacer.TokenDispenser // global multipart copy concurrency limiter
publicAccess container.PublicAccessType // Container Public Access Level
warnCompressed sync.Once // warn once about compressed files
// user delegation cache
userDelegationMu sync.Mutex
userDelegation *service.UserDelegationCredential
@@ -489,16 +405,15 @@ type Fs struct {
// Object describes an azure object
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
modTime time.Time // The modified time of the object if known
md5 string // MD5 hash if known
size int64 // Size of the object
mimeType string // Content-Type of the object
accessTier blob.AccessTier // Blob Access Tier
meta map[string]string // blob metadata - take metadataMu when accessing
tags map[string]string // blob tags
contentEncoding *string // Content-Encoding of the object
fs *Fs // what this object is part of
remote string // The remote path
modTime time.Time // The modified time of the object if known
md5 string // MD5 hash if known
size int64 // Size of the object
mimeType string // Content-Type of the object
accessTier blob.AccessTier // Blob Access Tier
meta map[string]string // blob metadata - take metadataMu when accessing
tags map[string]string // blob tags
}
// ------------------------------------------------------------
@@ -539,7 +454,7 @@ func parsePath(path string) (root string) {
// relative to f.root
func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
containerPath = containerPath[:len(containerPath)-1]
}
return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
@@ -587,7 +502,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
var storageErr *azcore.ResponseError
if errors.As(err, &storageErr) {
// General errors from:
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
// Blob specific errors from:
@@ -664,9 +580,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if opt.ListChunkSize > maxListChunkSize {
return nil, fmt.Errorf("blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
}
if opt.ListParallelism > 1 && !opt.UseArrowList {
fs.Logf(nil, "azureblob: list_parallelism has no effect without use_arrow_list - listing sequentially")
}
if opt.AccessTier == "" {
opt.AccessTier = string(defaultAccessTier)
@@ -706,6 +619,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ServerSideAcrossConfigs: true,
DoubleSlash: true,
}).Fill(ctx, f)
if opt.DirectoryMarkers {
f.features.CanHaveEmptyDirectories = true
fs.Debugf(f, "Using directory markers")
}
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
DefaultBaseURL: storageDefaultBaseURL,
@@ -717,9 +634,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential,
NewSharedKeyCredential: service.NewSharedKeyCredential,
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
// Override the automatic decompression in the transport
// to download compressed files as-is
policyClientOptions.PerCallPolicies = append(policyClientOptions.PerCallPolicies, setAcceptEncodingGzip{})
options.ClientOptions = policyClientOptions
},
}
@@ -738,28 +652,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f.isLimited = true
}
if opt.HNS.Valid {
f.hns = opt.HNS.Value
} else if !f.anonymous {
containerName := f.rootContainer
if containerName == "" {
containerName = res.Container
}
f.hns, err = f.readHNS(ctx, containerName)
if err != nil {
fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
} else {
m.Set("hns", strconv.FormatBool(f.hns))
}
}
if f.hns {
fs.Debugf(f, "Storage account has a hierarchical namespace")
}
if f.dirMarkers() {
f.features.CanHaveEmptyDirectories = true
fs.Debugf(f, "Using directory markers")
}
if f.rootContainer != "" && f.rootDirectory != "" {
// Check to see if the (container,directory) is actually an existing file
oldRoot := f.root
@@ -780,39 +672,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return f, nil
}
// readHNS reads whether the storage account has a hierarchical namespace
//
// If containerName is set then the container client is used which works
// with credentials limited to that container.
func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
var isHNS *bool
err = f.pacer.Call(func() (bool, error) {
var err error
if containerName != "" {
var resp container.GetAccountInfoResponse
resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
isHNS = resp.IsHierarchicalNamespaceEnabled
} else {
var resp service.GetAccountInfoResponse
resp, err = f.svc.GetAccountInfo(ctx, nil)
isHNS = resp.IsHierarchicalNamespaceEnabled
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return false, err
}
if isHNS == nil {
return false, errors.New("not returned by server")
}
return *isHNS, nil
}
// dirMarkers returns true if directories are persisted with directory markers
func (f *Fs) dirMarkers() bool {
return f.opt.DirectoryMarkers || f.hns
}
// return the container client for the container passed in
func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
f.cntSVCcacheMu.Lock()
@@ -850,12 +709,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
// NewObject finds the Object at remote. If it can't be found
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
o, err := f.newObjectWithInfo(ctx, remote, nil)
if err == fs.ErrorNotAFile {
// The blob is a directory
return nil, fs.ErrorObjectNotFound
}
return o, err
return f.newObjectWithInfo(ctx, remote, nil)
}
// getBlobSVC creates a blob client
@@ -924,15 +778,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
lowerKey := strings.ToLower(k)
switch lowerKey {
case "cache-control":
headers.BlobCacheControl = new(v)
headers.BlobCacheControl = pString(v)
case "content-disposition":
headers.BlobContentDisposition = new(v)
headers.BlobContentDisposition = pString(v)
case "content-encoding":
headers.BlobContentEncoding = new(v)
headers.BlobContentEncoding = pString(v)
case "content-language":
headers.BlobContentLanguage = new(v)
headers.BlobContentLanguage = pString(v)
case "content-type":
headers.BlobContentType = new(v)
headers.BlobContentType = pString(v)
case "x-ms-tags":
parsed, perr := parseXMsTags(v)
if perr != nil {
@@ -1169,33 +1023,18 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
if endsWithSlash || remote == "" {
return true
}
return hasDirMetadata(metadata)
}
return false
}
// Returns whether the metadata marks the blob as a directory
func hasDirMetadata(metadata map[string]*string) bool {
// Note that metadata with hdi_isfolder = true seems to be a
// defacto standard for marking blobs as directories.
// Note also that the metadata hasn't been normalised to lower case yet
for k, v := range metadata {
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
return true
// Note that metadata with hdi_isfolder = true seems to be a
// defacto standard for marking blobs as directories.
// Note also that the metadata hasn't been normalised to lower case yet
for k, v := range metadata {
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
return true
}
}
}
return false
}
// Returns whether props are for a directory on a storage account with
// a hierarchical namespace
//
// These ignore a trailing "/" on the name so a file can be returned
// when looking for a directory.
func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
}
// listFn is called from list to handle an object
type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
@@ -1221,7 +1060,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
delimiter = "/"
}
opts := &container.ListBlobsHierarchyOptions{
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &container.ListBlobsHierarchyOptions{
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
Include: container.ListBlobsInclude{
Copy: false,
@@ -1232,53 +1071,8 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
},
Prefix: &directory,
MaxResults: &maxResults,
}
// Request the Apache Arrow listing format. The SDK pager requests an
// Arrow IPC stream and decodes it, falling back to XML if the account
// doesn't have Arrow listing enabled. Skip the maxResults==1 probe
// (isEmpty) which doesn't benefit.
useArrow := f.opt.UseArrowList && maxResults != 1
if useArrow {
opts.ResponseFormat = container.StorageResponseFormatArrow
}
var foundItems int
var err error
if useArrow && f.opt.ListParallelism > 1 && !f.arrowXMLFallback.Load() {
// Arrow exposes server-side startFrom/endBefore name ranges, so the
// keyspace can be sharded and listed concurrently.
foundItems, err = f.listArrowParallel(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
} else {
foundItems, err = f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
if err != nil {
return err
}
if f.dirMarkers() && foundItems == 0 && directory != "" {
// Determine whether the directory exists or not by whether it has a marker
props, err := f.readMetaData(ctx, containerName, directory)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
if f.hns && !isHNSDirectory(props) {
return fs.ErrorDirNotFound
}
}
return nil
}
// listBlobsPager runs the hierarchy listing described by opts, calling fn for
// each blob and subdirectory, and returns the number of raw items seen.
// delimiter selects flat (recurse) vs hierarchical listing. If opts requests
// the Apache Arrow format and the service answers with XML (Arrow listing
// not enabled) a debug message is logged.
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
checkedArrow := false
})
foundItems := 0
for pager.More() {
var response container.ListBlobsHierarchyResponse
err := f.pacer.Call(func() (bool, error) {
@@ -1288,22 +1082,12 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
return f.shouldRetry(ctx, err)
})
// If Arrow was requested but the service answered with XML, Arrow
// listing is not enabled on this account; the listing is still correct
// but not accelerated.
if useArrow && !checkedArrow && err == nil {
checkedArrow = true
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
}
}
if err != nil {
// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
if storageErr, ok := err.(*azcore.ResponseError); ok && (storageErr.ErrorCode == string(bloberror.ContainerNotFound) || storageErr.StatusCode == http.StatusNotFound) {
return foundItems, fs.ErrorDirNotFound
return fs.ErrorDirNotFound
}
return foundItems, err
return err
}
// Advance marker to next
// marker = response.NextMarker
@@ -1318,24 +1102,12 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
fs.Debugf(f, "Nil name received")
continue
}
// Storage accounts with a hierarchical namespace return
// the file "dir" when listing "dir/"
if !strings.HasPrefix(*file.Name, directory) {
fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
foundItems--
continue
}
remote := f.opt.Enc.ToStandardPath(*file.Name)
if !strings.HasPrefix(remote, prefix) {
fs.Debugf(f, "Odd name received %q", remote)
continue
}
// ContentLength is documented as nullable in the Arrow listing schema
size := int64(-1)
if file.Properties.ContentLength != nil {
size = *file.Properties.ContentLength
}
isDirectory := isDirectoryMarker(size, file.Metadata, remote)
isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
if isDirectory {
// Don't insert the root directory
if remote == f.opt.Enc.ToStandardPath(directory) {
@@ -1351,7 +1123,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
// Send object
err = fn(remote, file, isDirectory)
if err != nil {
return foundItems, err
return err
}
}
// Send the subdirectories
@@ -1366,13 +1138,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
fs.Debugf(f, "Odd directory name received %q", remote)
continue
}
// Don't insert the root directory. The Arrow listing returns a
// directory marker blob with the same name as the listed directory
// as a BlobPrefix, whereas XML returns it as a blob (handled
// above).
if remote == f.opt.Enc.ToStandardPath(directory) {
continue
}
remote = remote[len(prefix):]
// Trim one slash off the remote name
remote, _ = strings.CutSuffix(remote, "/")
@@ -1385,11 +1150,21 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
// Send object
err = fn(remote, nil, true)
if err != nil {
return foundItems, err
return err
}
}
}
return foundItems, nil
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
// Determine whether the directory exists or not by whether it has a marker
_, err := f.readMetaData(ctx, containerName, directory)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
}
return nil
}
// Convert a list item into a DirEntry
@@ -1586,7 +1361,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
func (f *Fs) listContainersToFn(fn listContainerFn) error {
max := int32(f.opt.ListChunkSize)
pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
Include: service.ListContainersInclude{Metadata: true},
Include: service.ListContainersInclude{Metadata: true, Deleted: true},
MaxResults: &max,
})
ctx := context.Background()
@@ -1636,7 +1411,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
// Create directory marker file and parents
func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
if !f.dirMarkers() || container == "" {
if !f.opt.DirectoryMarkers || container == "" {
return nil
}
@@ -1693,11 +1468,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// mkdirParent creates the parent bucket/directory if it doesn't exist
func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
if f.hns {
// Parent directories are created by the upload
container, _ := f.split(remote)
return f.makeContainer(ctx, container)
}
remote, _ = strings.CutSuffix(remote, "/")
dir := path.Dir(remote)
if dir == "/" || dir == "." {
@@ -1803,9 +1573,6 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
container, directory := f.split(dir)
if f.hns && container != "" && directory != "" {
return f.rmdirHNS(ctx, container, directory)
}
// Remove directory marker file
if f.opt.DirectoryMarkers && container != "" && directory != "" {
o := &Object{
@@ -1828,39 +1595,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return f.deleteContainer(ctx, container)
}
// rmdirHNS removes the directory on a storage account with a
// hierarchical namespace
//
// The directory must be deleted without a trailing "/" and the server
// refuses to delete it if it isn't empty.
func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
directory, _ = strings.CutSuffix(directory, "/")
// Check it is a directory as deleting a file would succeed
props, err := f.readMetaData(ctx, container, directory)
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
} else if err != nil {
return err
}
if !isHNSDirectory(props) {
return fs.ErrorIsFile
}
blb := f.getBlobSVC(container, directory)
err = f.pacer.Call(func() (bool, error) {
_, err := blb.Delete(ctx, nil)
return f.shouldRetry(ctx, err)
})
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
return fs.ErrorDirectoryNotEmpty
}
if storageErr.StatusCode == http.StatusNotFound {
return fs.ErrorDirNotFound
}
}
return err
}
// Precision of the remote
func (f *Fs) Precision() time.Duration {
return time.Nanosecond
@@ -1896,7 +1630,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
}
// Validity window
start := time.Now().UTC().Add(-sasCopyStartSkew)
start := time.Now().UTC()
expiry := start.Add(2 * sasCopyValidity)
startStr := start.Format(time.RFC3339)
expiryStr := expiry.Format(time.RFC3339)
@@ -1945,7 +1679,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
// Build the SAS values
perms := sas.BlobPermissions{Read: true}
container, containerPath := o.split()
start := time.Now().UTC().Add(-sasCopyStartSkew)
start := time.Now().UTC()
expiry := start.Add(sasCopyValidity)
vals := sas.BlobSignatureValues{
StartTime: start,
@@ -2267,10 +2001,6 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.MD5 {
return "", hash.ErrUnsupported
}
// If decompressing, erase the hash
if o.size < 0 {
return "", nil
}
// Convert base64 encoded md5 into lower case hex
if o.md5 == "" {
return "", nil
@@ -2396,13 +2126,6 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *blob.GetPropertiesRe
o.accessTier = blob.AccessTier(*info.AccessTier)
}
o.setMetadata(metadata)
o.contentEncoding = info.ContentEncoding
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
o.size = -1
o.md5 = ""
}
return nil
}
@@ -2415,31 +2138,12 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
} else {
size = *info.ContentLength
}
// On a range request Content-Length is the length of the range, not the object, so take the
// object's size from the total in Content-Range instead.
if info.ContentRange != nil {
contentRange, err := rest.ParseContentRange(*info.ContentRange)
if err != nil {
fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
} else if contentRange.Size >= 0 {
size = contentRange.Size
}
}
if isDirectoryMarker(size, metadata, o.remote) {
return fs.ErrorNotAFile
}
// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
// this as base64 encoded string.
//
// On a range request the service returns the whole-blob MD5 in the
// x-ms-blob-content-md5 header (BlobContentMD5) and leaves Content-MD5
// (ContentMD5) empty, so prefer BlobContentMD5. Never overwrite a known
// hash with an empty one, or a ranged read would drop the object's MD5.
if md5 := info.BlobContentMD5; len(md5) > 0 {
o.md5 = base64.StdEncoding.EncodeToString(md5)
} else if len(info.ContentMD5) > 0 {
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
}
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
if info.ContentType == nil {
o.mimeType = ""
} else {
@@ -2458,12 +2162,21 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
// o.accessTier = blob.AccessTier(*info.AccessTier)
// }
o.setMetadata(metadata)
o.contentEncoding = info.ContentEncoding
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
o.size = -1
o.md5 = ""
// If it was a Range request, the size is wrong, so correct it
if info.ContentRange != nil {
contentRange := *info.ContentRange
slash := strings.IndexRune(contentRange, '/')
if slash >= 0 {
i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
if err == nil {
o.size = i
} else {
fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
}
} else {
fs.Debugf(o, "Failed to find length in %q", contentRange)
}
}
return nil
@@ -2503,13 +2216,6 @@ func (o *Object) decodeMetaDataFromBlob(info *container.BlobItem) (err error) {
o.accessTier = *info.Properties.AccessTier
}
o.setMetadata(metadata)
o.contentEncoding = info.Properties.ContentEncoding
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
o.size = -1
o.md5 = ""
}
return nil
}
@@ -2685,20 +2391,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
if err != nil {
return nil, fmt.Errorf("failed to decode metadata for download: %w", err)
}
// Decompress body if necessary
if downloadResponse.ContentEncoding != nil && *downloadResponse.ContentEncoding == "gzip" {
if o.fs.opt.Decompress {
return readers.NewGzipReader(downloadResponse.Body)
}
o.fs.warnCompressed.Do(func() {
fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --azureblob-decompress to override")
})
}
return downloadResponse.Body, nil
}
// Converts a string into a pointer to a string
func pString(s string) *string {
return &s
}
// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
type readSeekCloser struct {
io.Reader
@@ -2853,7 +2553,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
// isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
// returning true if it is found
func isInvalidBlockOrBlob(err error) bool {
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
var storageErr *azcore.ResponseError
if errors.As(err, &storageErr) {
return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
}
return false
@@ -3225,7 +2926,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
// Start with default content-type based on source
ui.httpHeaders = blob.HTTPHeaders{
BlobContentType: new(fs.MimeType(ctx, src)),
BlobContentType: pString(fs.MimeType(ctx, src)),
}
// Apply mapped metadata/headers/tags if requested
@@ -3271,15 +2972,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
o.tags[parts[0]] = parts[1]
}
case "cache-control":
ui.httpHeaders.BlobCacheControl = new(value)
ui.httpHeaders.BlobCacheControl = pString(value)
case "content-disposition":
ui.httpHeaders.BlobContentDisposition = new(value)
ui.httpHeaders.BlobContentDisposition = pString(value)
case "content-encoding":
ui.httpHeaders.BlobContentEncoding = new(value)
ui.httpHeaders.BlobContentEncoding = pString(value)
case "content-language":
ui.httpHeaders.BlobContentLanguage = new(value)
ui.httpHeaders.BlobContentLanguage = pString(value)
case "content-type":
ui.httpHeaders.BlobContentType = new(value)
ui.httpHeaders.BlobContentType = pString(value)
}
}
+1 -125
View File
@@ -1,12 +1,9 @@
//go:build !plan9 && !js
//go:build !plan9 && !solaris && !js
package azureblob
import (
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"encoding/base64"
"fmt"
"net/http"
@@ -17,7 +14,6 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
@@ -54,58 +50,6 @@ func TestBlockIDCreator(t *testing.T) {
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
}
func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
pInt64 := func(i int64) *int64 { return &i }
pString := func(s string) *string { return &s }
newTestObject := func() *Object {
return &Object{
fs: &Fs{},
remote: "test.bin",
size: -2, // sentinel to check it gets set
}
}
t.Run("WholeBlob", func(t *testing.T) {
o := newTestObject()
info := blob.DownloadStreamResponse{}
info.ContentLength = pInt64(12345)
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(12345), o.size)
})
t.Run("RangeRequest", func(t *testing.T) {
// On a ranged download Content-Length is the chunk length so the
// size must come from the total in Content-Range
o := newTestObject()
info := blob.DownloadStreamResponse{}
info.ContentLength = pInt64(67108864)
info.ContentRange = pString("bytes 67108864-134217727/169721004032")
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(169721004032), o.size)
})
t.Run("BadContentRange", func(t *testing.T) {
o := newTestObject()
info := blob.DownloadStreamResponse{}
info.ContentLength = pInt64(67108864)
info.ContentRange = pString("potato")
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(67108864), o.size)
})
t.Run("UnknownLength", func(t *testing.T) {
o := newTestObject()
info := blob.DownloadStreamResponse{}
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(-1), o.size)
})
}
func TestCopySASTimingConstants(t *testing.T) {
require.Greater(t, sasCopyStartSkew, time.Duration(0))
require.Greater(t, sasCopyValidity, sasCopyStartSkew)
}
func (f *Fs) testFeatures(t *testing.T) {
// Check first feature flags are set on this remote
enabled := f.Features().SetTier
@@ -206,66 +150,10 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
require.NoError(t, dst.Remove(ctx))
}
func gz(t *testing.T, s string) string {
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
_, err := zw.Write([]byte(s))
require.NoError(t, err)
err = zw.Close()
require.NoError(t, err)
return buf.String()
}
func md5sum(t *testing.T, s string) string {
hash := md5.Sum([]byte(s))
return fmt.Sprintf("%x", hash)
}
func (f *Fs) testGzipEncoding(t *testing.T) {
ctx := context.Background()
original := random.String(1000)
contents := gz(t, original)
item := fstest.NewItem("test-gzip", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
metadata := fs.Metadata{
"content-encoding": "gzip",
"content-type": "text/plain",
}
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", metadata)
defer func() {
assert.NoError(t, obj.Remove(ctx))
}()
o := obj.(*Object)
// Test that the gzipped file we uploaded can be
// downloaded with and without decompression
checkDownload := func(wantContents string, wantSize int64, wantHash string) {
gotContents := fstests.ReadObject(ctx, t, o, -1)
assert.Equal(t, wantContents, gotContents)
assert.Equal(t, wantSize, o.Size())
gotHash, err := o.Hash(ctx, hash.MD5)
require.NoError(t, err)
assert.Equal(t, wantHash, gotHash)
}
t.Run("NoDecompress", func(t *testing.T) {
checkDownload(contents, int64(len(contents)), md5sum(t, contents))
})
t.Run("Decompress", func(t *testing.T) {
f.opt.Decompress = true
defer func() {
f.opt.Decompress = false
}()
checkDownload(original, -1, "")
})
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("Features", f.testFeatures)
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
t.Run("Metadata", f.testMetadataPaths)
t.Run("GzipEncoding", f.testGzipEncoding)
t.Run("ArrowList", f.testArrowList)
}
// helper to read blob properties for an object
@@ -325,14 +213,6 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
}
// helper to read blob tags for an object
// skipIfHNS skips the test if the storage account has a hierarchical
// namespace as these don't support blob tags
func skipIfHNS(t *testing.T, f *Fs) {
if f.hns {
t.Skip("blob tags not supported with hierarchical namespace")
}
}
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
ao := o.(*Object)
blb := ao.getBlobSVC()
@@ -418,7 +298,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Singlepart upload
t.Run("PutSinglepartTags", func(t *testing.T) {
skipIfHNS(t, f)
contents := random.String(int(f.opt.ChunkSize / 2))
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
meta := fs.Metadata{
@@ -434,7 +313,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Multipart upload
t.Run("PutMultipartTags", func(t *testing.T) {
skipIfHNS(t, f)
contents := random.String(int(f.opt.ChunkSize + 2048))
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
meta := fs.Metadata{
@@ -622,7 +500,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Singlepart copy
t.Run("CopySinglepartTags", func(t *testing.T) {
skipIfHNS(t, f)
// create small source
contents := random.String(int(f.opt.ChunkSize / 2))
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
@@ -648,7 +525,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
// Tags: Multipart copy
t.Run("CopyMultipartTags", func(t *testing.T) {
skipIfHNS(t, f)
// create large source to force multipart
contents := random.String(int(f.opt.CopyCutoff + 4096))
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+1 -1
View File
@@ -1,6 +1,6 @@
// Test AzureBlob filesystem interface
//go:build !plan9 && !js
//go:build !plan9 && !solaris && !js
package azureblob
+1 -1
View File
@@ -1,7 +1,7 @@
// Build for azureblob for unsupported platforms to stop go complaining
// about "no buildable Go source files "
//go:build plan9 || js
//go:build plan9 || solaris || js
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
package azureblob
+23 -43
View File
@@ -58,9 +58,6 @@ const (
maxFileSize = 4 * fs.Tebi
defaultChunkSize = 4 * fs.Mebi
storageDefaultBaseURL = "file.core.windows.net"
// smbTimePrecision is the precision the server stores
// modtimes with (100 ns FILETIME ticks)
smbTimePrecision = 100 * time.Nanosecond
)
func init() {
@@ -79,7 +76,7 @@ This is required and is the name of the share to access.
Help: `Upload chunk size.
Note that this is stored in memory and there may be up to
"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
in memory.`,
Default: defaultChunkSize,
Advanced: true,
@@ -95,7 +92,7 @@ links and these uploads do not fully utilize your bandwidth, then
increasing this may help to speed up the transfers.
Note that chunks are stored in memory and there may be up to
"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
in memory.`,
Default: 16,
Advanced: true,
@@ -176,10 +173,6 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
NewSharedKeyCredential: service.NewSharedKeyCredential,
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
options.ClientOptions = policyClientOptions
// FileRequestIntent is required for TokenCredential (OAuth)
// auth; the SDK omits it for shared-key / SAS.
backup := service.ShareTokenIntentBackup
options.FileRequestIntent = &backup
},
}
res, err := auth.NewClient(ctx, conf, &opt.Options)
@@ -253,10 +246,10 @@ func (f *Fs) Features() *fs.Features {
// Precision return the precision of this Fs
//
// The SMB last write time properties are ISO 8601 with 100 ns (FILETIME) precision
// One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers
func (f *Fs) Precision() time.Duration {
return smbTimePrecision
return time.Second
}
// Hashes returns the supported hash sets.
@@ -601,10 +594,7 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
if err != nil {
return fmt.Errorf("unable to set modTime: %w", err)
}
// Truncate to the precision the server stores the modtime with
// to keep the in-memory modtime identical to the one a fresh
// listing returns.
o.modTime = t.Truncate(smbTimePrecision)
o.modTime = t
return nil
}
@@ -650,6 +640,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
return resp.Body, nil
}
// Returns a pointer to t - useful for returning pointers to constants
func ptr[T any](t T) *T {
return &t
}
var warnStreamUpload sync.Once
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
@@ -691,9 +686,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
}
// Count the bytes read so short reads from the source can be detected
counter = readers.NewCountingReader(in)
in = counter
// Measure the size if it is unknown
if sizeUnknown {
counter = readers.NewCountingReader(in)
in = counter
}
// Check we have a source MD5 hash...
if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
@@ -728,21 +725,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if sizeUnknown {
// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
size = int64(counter.BytesRead())
} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
// The file was created at the declared size so a short source
// leaves the remainder padded with zeroes
if isNewlyCreated {
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
}
} else if bytesRead < size {
// Truncate the file to the bytes actually received so it
// is not left padded to the declared size with zeroes
if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
}
}
return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
}
if hashUnknown {
md5Hash = hasher.Sum(nil)
@@ -783,12 +765,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return fmt.Errorf("update: failed to set properties: %w", err)
}
// Make sure Object is in sync, truncating the modtime to the
// precision the server stores it with so it is identical to the
// one a fresh listing returns
// Make sure Object is in sync
o.size = size
o.md5 = md5Hash
o.modTime = modTime.Truncate(smbTimePrecision)
o.modTime = modTime
o.contentType = contentType
return nil
}
@@ -813,8 +793,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
}
opt := file.RenameOptions{
IgnoreReadOnly: new(true),
ReplaceIfExists: new(true),
IgnoreReadOnly: ptr(true),
ReplaceIfExists: ptr(true),
}
dstAbsPath := f.absPath(remote)
fc := srcObj.fileClient()
@@ -859,8 +839,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
opt := directory.RenameOptions{
IgnoreReadOnly: new(false),
ReplaceIfExists: new(false),
IgnoreReadOnly: ptr(false),
ReplaceIfExists: ptr(false),
}
dstAbsPath := dstFs.absPath(dstRemote)
dirClient := srcFs.dirClient(srcRemote)
@@ -899,8 +879,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
ChangeTime: file.SourceCopyFileChangeTime{},
CreationTime: file.SourceCopyFileCreationTime{},
LastWriteTime: file.SourceCopyFileLastWriteTime{},
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
IgnoreReadOnly: new(true),
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
IgnoreReadOnly: ptr(true),
},
}
srcURL := srcObj.fileClient().URL()
+3 -27
View File
@@ -191,11 +191,6 @@ If the custom endpoint rewrites the requests for authentication,
e.g., in Cloudflare Workers, this header needs to be handled properly.
Leave blank if you want to use the endpoint provided by Backblaze.
Rclone will not follow a redirect from this endpoint which downgrades
HTTPS to HTTP, and will not send the "Authorization" header or the
SSE-C key headers once a redirect has left the host and port of this
endpoint.
The URL provided here SHOULD have the protocol and SHOULD NOT have
a trailing slash or specify the /file/bucket subpath as rclone will
request files with "{download_url}/file/{bucket_name}/{path}".
@@ -277,7 +272,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
Name: "sse_customer_key",
Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
Alternatively you can provide --b2-sse-customer-key-base64.`,
Alternatively you can provide --sse-customer-key-base64.`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "",
@@ -288,7 +283,7 @@ Alternatively you can provide --b2-sse-customer-key-base64.`,
Name: "sse_customer_key_base64",
Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
Alternatively you can provide --b2-sse-customer-key.`,
Alternatively you can provide --sse-customer-key.`,
Advanced: true,
Examples: []fs.OptionExample{{
Value: "",
@@ -534,25 +529,6 @@ func (f *Fs) setRoot(root string) {
f.rootBucket, f.rootDirectory = bucket.Split(f.root)
}
// redirectSecretHeaders are the request headers carrying secrets that
// must not be forwarded when a redirect leaves the original host.
var redirectSecretHeaders = []string{
"Authorization",
sseAlgorithmHeader,
sseKeyHeader,
sseMd5Header,
}
// newClient makes the http client used for all B2 requests.
//
// The download_url option sends the requests to a host the user does
// not control so it strips headers on cross host redirects.
func newClient(ctx context.Context) *http.Client {
client := fshttp.NewClient(ctx)
client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
return client
}
// NewFs constructs an Fs from the path, bucket:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
// Parse config into Options struct
@@ -605,7 +581,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
name: name,
opt: *opt,
ci: ci,
srv: rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
cache: bucket.NewCache(),
_bucketID: make(map[string]string, 1),
_bucketType: make(map[string]string, 1),
-59
View File
@@ -4,8 +4,6 @@ import (
"context"
"crypto/sha1"
"fmt"
"net/http"
"net/http/httptest"
"path"
"sort"
"strings"
@@ -619,60 +617,3 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
// redirectTestHeaders is a deliberately literal copy of
// redirectSecretHeaders so that a header dropped from the production
// list fails TestRedirectSecretHeaders rather than silently losing
// test coverage.
var redirectTestHeaders = map[string]string{
"Authorization": "secret-token",
"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
"X-Bz-Server-Side-Encryption-Customer-Key": "secret-key",
"X-Bz-Server-Side-Encryption-Customer-Key-Md5": "secret-md5",
}
func TestRedirectSecretHeaders(t *testing.T) {
want := make([]string, 0, len(redirectTestHeaders))
for header := range redirectTestHeaders {
want = append(want, header)
}
sort.Strings(want)
got := append([]string(nil), redirectSecretHeaders...)
sort.Strings(got)
assert.Equal(t, want, got)
}
// redirectTestRequest makes a GET request to rawURL carrying the
// secret headers plus a harmless marker header.
func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
require.NoError(t, err)
for header, value := range redirectTestHeaders {
req.Header.Set(header, value)
}
req.Header.Set("X-Bz-Test-Mode", "marker")
return req
}
func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
client := newClient(context.Background())
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
for header := range redirectTestHeaders {
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
}
assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
w.WriteHeader(http.StatusOK)
}))
defer redirectServer.Close()
initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
}))
defer initialServer.Close()
resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
assert.NoError(t, resp.Body.Close())
}
+13 -21
View File
@@ -27,7 +27,7 @@ import (
"sync/atomic"
"time"
"github.com/golang-jwt/jwt/v5"
"github.com/golang-jwt/jwt/v4"
"github.com/rclone/rclone/backend/box/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
@@ -45,7 +45,6 @@ import (
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/youmark/pkcs8"
)
@@ -77,10 +76,7 @@ var (
)
type boxCustomClaims struct {
jwt.RegisteredClaims
// Audience overrides RegisteredClaims.Audience so it is sent as
// a single string rather than an array of strings
Audience string `json:"aud,omitempty"`
jwt.StandardClaims
BoxSubType string `json:"box_sub_type,omitempty"`
}
@@ -226,7 +222,7 @@ func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
if configFileCredentials == "" {
configFile, _ := m.Get("box_config_file")
configFileBytes, err = os.ReadFile(env.ShellExpand(configFile))
configFileBytes, err = os.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("box: failed to read Box config: %w", err)
}
@@ -246,13 +242,15 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
}
claims = &boxCustomClaims{
RegisteredClaims: jwt.RegisteredClaims{
ID: val,
//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
StandardClaims: jwt.StandardClaims{
Id: val,
Issuer: boxConfig.BoxAppSettings.ClientID,
Subject: boxConfig.EnterpriseID,
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
Audience: tokenURL,
ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
},
Audience: tokenURL,
BoxSubType: boxSubType,
}
return claims, nil
@@ -1462,7 +1460,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
// Item in the cache has the same or newer SequenceID than
// this event. Ignore this event, it must be old.
f.itemMetaCacheMu.Unlock()
fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
continue
}
@@ -1693,7 +1691,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// upload does a single non-multipart upload
//
// This is recommended for less than 50 MiB of content
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
upload := api.UploadFile{
Name: o.fs.opt.Enc.FromStandardName(leaf),
ContentModifiedAt: api.Time(modTime),
@@ -1705,10 +1703,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
var resp *http.Response
var result api.FolderItems
counter := readers.NewCountingReader(in)
opts := rest.Opts{
Method: "POST",
Body: counter,
Body: in,
MultipartMetadataName: "attributes",
MultipartContentName: "contents",
MultipartFileName: upload.Name,
@@ -1728,11 +1725,6 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
if err != nil {
return err
}
// Check the source supplied the number of bytes it declared
// otherwise a truncated file would be stored as a good upload.
if size >= 0 && int64(counter.BytesRead()) != size {
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
}
if result.TotalCount != 1 || len(result.Entries) != 1 {
return fmt.Errorf("failed to upload %v - not sure why", o)
}
@@ -1762,7 +1754,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Upload with simple or multipart
if size <= int64(o.fs.opt.UploadCutoff) {
err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
} else {
err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
}
-118
View File
@@ -1,118 +0,0 @@
package box
import (
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/rclone/rclone/backend/box/api"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/lib/jwtutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// decodeJWTPart decodes one base64url encoded part of a JWT into a map
func decodeJWTPart(t *testing.T, part string) map[string]any {
data, err := base64.RawURLEncoding.DecodeString(part)
require.NoError(t, err)
var result map[string]any
require.NoError(t, json.Unmarshal(data, &result))
return result
}
// Test the JWT assertion sent to the token endpoint is in the format Box expects
func TestJWTAssertion(t *testing.T) {
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
boxConfig := &api.ConfigJSON{
BoxAppSettings: api.AppSettings{
ClientID: "client-id",
ClientSecret: "client-secret",
AppAuth: api.AppAuth{
PublicKeyID: "public-key-id",
},
},
EnterpriseID: "enterprise-id",
}
var form map[string][]string
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "POST", r.Method)
assert.NoError(t, r.ParseForm())
form = r.PostForm
_, _ = w.Write([]byte(`{"access_token":"access-token","token_type":"bearer","expires_in":3600}`))
}))
defer ts.Close()
claims, err := getClaims(boxConfig, "enterprise")
require.NoError(t, err)
m := configmap.Simple{}
start := time.Now()
err = jwtutil.Config("box", "test", ts.URL, *claims, getSigningHeaders(boxConfig), getQueryParams(boxConfig), privateKey, m, ts.Client())
require.NoError(t, err)
// Check the token was stored
assert.Contains(t, m["token"], `"access_token":"access-token"`)
// Check the form parameters
assert.Equal(t, []string{"urn:ietf:params:oauth:grant-type:jwt-bearer"}, form["grant_type"])
assert.Equal(t, []string{"client-id"}, form["client_id"])
assert.Equal(t, []string{"client-secret"}, form["client_secret"])
require.Len(t, form["assertion"], 1)
parts := strings.Split(form["assertion"][0], ".")
require.Len(t, parts, 3)
// Check the signature independently of the JWT library
signature, err := base64.RawURLEncoding.DecodeString(parts[2])
require.NoError(t, err)
hash := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
require.NoError(t, rsa.VerifyPKCS1v15(&privateKey.PublicKey, crypto.SHA256, hash[:], signature))
// Check the header
assert.Equal(t, map[string]any{
"alg": "RS256",
"typ": "JWT",
"kid": "public-key-id",
}, decodeJWTPart(t, parts[0]))
// Check the claims - aud must be a string, not an array
payload := decodeJWTPart(t, parts[1])
exp, ok := payload["exp"].(float64)
require.True(t, ok, "exp should be a number")
assert.InDelta(t, float64(start.Add(45*time.Second).Unix()), exp, 5)
jti, ok := payload["jti"].(string)
require.True(t, ok, "jti should be a string")
assert.Len(t, jti, 40)
delete(payload, "exp")
delete(payload, "jti")
assert.Equal(t, map[string]any{
"aud": tokenURL,
"iss": "client-id",
"sub": "enterprise-id",
"box_sub_type": "enterprise",
}, payload)
}
// Test box_config_file has ~ and environment variables expanded
func TestGetBoxConfigShellExpand(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"enterpriseID":"enterprise-id"}`), 0600))
t.Setenv("RCLONE_TEST_BOX_CONFIG_DIR", dir)
m := configmap.Simple{
"box_config_file": "$RCLONE_TEST_BOX_CONFIG_DIR/config.json",
}
boxConfig, err := getBoxConfig(m)
require.NoError(t, err)
assert.Equal(t, "enterprise-id", boxConfig.EnterpriseID)
}
+15 -23
View File
@@ -3,6 +3,7 @@
package box
import (
"bytes"
"context"
"crypto/sha1"
"encoding/base64"
@@ -19,7 +20,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/rest"
)
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
}
// uploadPart uploads a part in an upload session
//
// chunk must contain exactly the part's data and is read from the start
// on each attempt.
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
hasher := sha1.New()
chunkSize, err := io.Copy(hasher, chunk)
if err != nil {
return nil, err
}
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
chunkSize := int64(len(chunk))
sha1sum := sha1.Sum(chunk)
opts := rest.Opts{
Method: "PUT",
Path: "/files/upload_sessions/" + SessionID,
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
Options: options,
ExtraHeaders: map[string]string{
"Digest": sha1Digest(hasher.Sum(nil)),
"Digest": sha1Digest(sha1sum[:]),
},
}
var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) {
_, err = chunk.Seek(0, io.SeekStart)
if err != nil {
return false, err
}
opts.Body = wrap(chunk)
opts.Body = wrap(bytes.NewReader(chunk))
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(ctx, resp, err)
})
@@ -223,25 +213,27 @@ outer:
reqSize := min(remaining, chunkSize)
// Read the chunk into memory from the global pool, hashing
// it for the whole file hash (must be done sequentially)
rw := multipart.NewRW().Reserve(reqSize)
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
// Make a block of memory
buf := make([]byte, reqSize)
// Read the chunk
_, err = io.ReadFull(in, buf)
if err != nil {
_ = rw.Close()
err = fmt.Errorf("multipart upload failed to read source: %w", err)
break outer
}
// Make the global hash (must be done sequentially)
_, _ = hash.Write(buf)
// Transfer the chunk
wg.Add(1)
o.fs.uploadToken.Get()
go func(part int, position int64) {
defer wg.Done()
defer o.fs.uploadToken.Put()
defer func() { _ = rw.Close() }()
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
if err != nil {
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
select {
+5 -10
View File
@@ -503,19 +503,15 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
}
go func() {
// Wait on the timer and the stop channel together so that a stop
// signalled by StopBackgroundRunners is honoured immediately.
timer := time.NewTimer(time.Duration(f.opt.ChunkCleanInterval))
defer timer.Stop()
for {
time.Sleep(time.Duration(f.opt.ChunkCleanInterval))
select {
case <-f.cleanupChan:
fs.Infof(f, "stopping cleanup")
return
case <-timer.C:
default:
fs.Debugf(f, "starting cleanup")
f.CleanUpCache(false)
timer.Reset(time.Duration(f.opt.ChunkCleanInterval))
}
}
}()
@@ -567,10 +563,9 @@ Eg
})
rc.Add(rc.Call{
Path: "cache/stats",
NoAuth: true,
Fn: f.httpStats,
Title: "Get cache stats",
Path: "cache/stats",
Fn: f.httpStats,
Title: "Get cache stats",
Help: `
Show statistics for the cache remote.
`,
+6 -30
View File
@@ -801,12 +801,9 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
fs.Debugf(f, "skip orphan data chunk %q", remote)
break
}
if mainObject == nil && !f.useMeta && !isSpecial {
if mainObject == nil && !f.useMeta {
// this is the "nometa" case
// create dummy chunked object without metadata
//
// not for temporary or control chunks as they may be all
// that is left of an interrupted upload
mainObject = f.newObject(mainRemote, nil, nil)
byRemote[mainRemote] = mainObject
if !badEntry[mainRemote] {
@@ -1502,21 +1499,11 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {
func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
oldFsObject, err := f.NewObject(ctx, remote)
if err == nil {
removeStaleChunks(ctx, oldFsObject.(*Object), nil)
}
}
func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {
currentRemotes := make(map[string]struct{}, len(currentChunks))
for _, chunk := range currentChunks {
currentRemotes[chunk.Remote()] = struct{}{}
}
for _, chunk := range oldObject.chunks {
if _, found := currentRemotes[chunk.Remote()]; found {
continue
}
if err := chunk.Remove(ctx); err != nil {
fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
oldObject := oldFsObject.(*Object)
for _, chunk := range oldObject.chunks {
if err := chunk.Remove(ctx); err != nil {
fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
}
}
}
}
@@ -1697,20 +1684,12 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
// metadata format which might involve unsupported chunk types.
return nil, fmt.Errorf("can't %s this file: %w", opName, err)
}
// Snapshot destination chunks before its metadata is replaced.
var oldObject *Object
if oldFsObject, err := f.NewObject(ctx, remote); err == nil {
oldObject = oldFsObject.(*Object)
}
if !o.isComposite() {
fs.Debugf(o, "%s non-chunked object...", opName)
oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
if err != nil {
return nil, err
}
if oldObject != nil {
removeStaleChunks(ctx, oldObject, nil)
}
return f.newObject("", oResult, nil), nil
}
@@ -1776,9 +1755,6 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
silentlyRemove(ctx, newObj)
return nil, err
}
if oldObject != nil {
removeStaleChunks(ctx, oldObject, newChunks)
}
return newObj, nil
}
+1 -122
View File
@@ -11,7 +11,6 @@ import (
"strings"
"testing"
_ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/fspath"
@@ -44,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
type settings map[string]any
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
configMap := configmap.Simple{}
for key, val := range opts {
configMap[key] = fmt.Sprintf("%v", val)
@@ -628,22 +627,6 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
_ = file.Remove(ctx)
}
// testOrphanTempChunks checks that temporary chunks left behind by an
// interrupted upload are not listed and don't break the listing
func testOrphanTempChunks(t *testing.T, f *Fs) {
const dir = "orphan"
ctx := context.Background()
defer func() {
_ = operations.Purge(ctx, f.base, dir)
}()
modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
}
func testMetadataInput(t *testing.T, f *Fs) {
const minChunkForTest = 50
if f.opt.ChunkSize < minChunkForTest {
@@ -878,104 +861,6 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
_ = operations.Purge(ctx, f.base, dir)
}
func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {
ctx := context.Background()
const dir = "serveroverwrite"
remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)
fsResult, err := fs.NewFs(ctx, remote)
require.NoError(t, err)
copyFs, ok := fsResult.(*Fs)
require.True(t, ok)
oldMoveChunkSize := moveFs.opt.ChunkSize
moveFs.opt.ChunkSize = fs.SizeSuffix(3)
defer func() {
moveFs.opt.ChunkSize = oldMoveChunkSize
_ = operations.Purge(ctx, copyFs.base, "")
_ = operations.Purge(ctx, moveFs.base, dir)
}()
type operation struct {
name string
fs *Fs
enabled bool
do copyMoveFn
}
operations := []operation{
{name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},
{name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},
}
testCases := []struct {
name string
sourceContents string
targetContents string
}{
{name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},
{name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},
}
transactionModes := []struct {
name string
useNoRename bool
}{
{name: "Rename"},
{name: "NoRename", useNoRename: true},
}
for _, operation := range operations {
t.Run(operation.name, func(t *testing.T) {
if !operation.enabled {
t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))
}
for _, transactionMode := range transactionModes {
t.Run(transactionMode.name, func(t *testing.T) {
f := operation.fs
if transactionMode.useNoRename && !f.useMeta {
t.Skip("Can't test norename transactions without metadata")
}
oldUseNoRename := f.useNoRename
f.useNoRename = transactionMode.useNoRename
defer func() { f.useNoRename = oldUseNoRename }()
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))
source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)
target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)
targetObject := target.(*Object)
require.NotEmpty(t, targetObject.chunks)
oldChunks := append([]fs.Object(nil), targetObject.chunks...)
result, err := operation.do(ctx, source, target.Remote())
require.NoError(t, err)
reader, err := result.Open(ctx)
require.NoError(t, err)
contents, err := io.ReadAll(reader)
require.NoError(t, err)
require.NoError(t, reader.Close())
assert.Equal(t, testCase.sourceContents, string(contents))
resultObject := result.(*Object)
currentChunks := make(map[string]struct{}, len(resultObject.chunks))
for _, chunk := range resultObject.chunks {
currentChunks[chunk.Remote()] = struct{}{}
}
removed := 0
for _, chunk := range oldChunks {
if _, found := currentChunks[chunk.Remote()]; found {
continue
}
_, err := f.base.NewObject(ctx, chunk.Remote())
assert.Equal(t, fs.ErrorObjectNotFound, err)
removed++
}
assert.Positive(t, removed)
})
}
})
}
})
}
}
// Test that md5all creates metadata even for small files
func testMD5AllSlow(t *testing.T, f *Fs) {
ctx := context.Background()
@@ -1043,15 +928,9 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("BackwardsCompatibility", func(t *testing.T) {
testBackwardsCompatibility(t, f)
})
t.Run("OrphanTempChunks", func(t *testing.T) {
testOrphanTempChunks(t, f)
})
t.Run("ChunkerServerSideMove", func(t *testing.T) {
testChunkerServerSideMove(t, f)
})
t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {
testChunkerServerSideOverwrite(t, f)
})
t.Run("MD5AllSlow", func(t *testing.T) {
testMD5AllSlow(t, f)
})
+9 -13
View File
@@ -34,13 +34,6 @@ import (
"github.com/zeebo/blake3"
)
// Pacer settings - tuned to match other rclone backends
const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
)
// Cloudinary shouldn't have a trailing dot if there is no path
func cldPathDir(somePath string) string {
if somePath == "" || somePath == "." {
@@ -196,7 +189,7 @@ func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (f
root: root,
opt: *opt,
cld: cld,
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
srv: rest.NewClient(client),
}
@@ -472,8 +465,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
if updateOptions, ok := option.(*api.UpdateOptions); ok {
if updateOptions.PublicID != "" {
updateObject = true
params.Overwrite = new(true)
params.Invalidate = new(true)
params.Overwrite = SDKApi.Bool(true)
params.Invalidate = SDKApi.Bool(true)
params.PublicID = updateOptions.PublicID
params.ResourceType = updateOptions.ResourceType
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
@@ -593,18 +586,21 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
if fserrors.ContextError(ctx, &err) {
return false, err
}
// Cloudinary returns rate-limit windows as "Try again on YYYY-MM-DD HH:MM:SS UTC"
// embedded in the error message - honour it as a retry-after.
if err != nil {
tryAgain := "Try again on "
if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
layout := "2006-01-02 15:04:05 UTC"
dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
if timestamp, err2 := time.Parse(layout, dateStr); err2 == nil {
timestamp, err2 := time.Parse(layout, dateStr)
if err2 == nil {
return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
}
}
fs.Debugf(nil, "Retrying API error %v", err)
return true, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
+1 -1
View File
@@ -1104,7 +1104,7 @@ func (o *Object) Remote() string {
newPath, err := o.u.pathAdjustment.do(o.Object.String())
if err != nil {
fs.Errorf(o.Object, "Bad object: %v", err)
return ""
return err.Error()
}
return newPath
}
+5 -61
View File
@@ -31,7 +31,6 @@ import (
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/multipart"
)
// Globals
@@ -564,19 +563,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
// cache small files in memory and do normal upload
rw := multipart.NewRW()
defer fs.CheckClose(rw, &err)
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
if err == io.EOF {
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
return f.Fs.Put(ctx, rw, src, options...)
} else if err != nil {
return nil, err
buf := make([]byte, f.opt.RAMCacheLimit)
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
}
// Need to include what we already read
in = &ReadCloserWrapper{
Reader: io.MultiReader(rw, in),
Reader: io.MultiReader(bytes.NewReader(buf), in),
Closer: in,
}
@@ -703,16 +698,6 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
return nil, err
}
// Check the source supplied the number of bytes it declared -
// compressed data is stored under a name containing the
// original size so a mismatch makes the object unreadable.
if size := src.Size(); size >= 0 && meta.Size != size {
if removeErr := dataObject.Remove(ctx); removeErr != nil {
fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
}
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
}
mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
// meta data upload may fail. in this case we try to remove the original object
@@ -1085,35 +1070,6 @@ type ObjectMetadata struct {
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
}
// validate checks metadata read from the wrapped remote before it is used to
// construct a compressed reader.
func (meta *ObjectMetadata) validate() error {
if meta.Mode != Gzip {
return nil
}
if meta.CompressionMetadataGzip == nil {
return errors.New("missing gzip metadata")
}
if meta.CompressionMetadataGzip.BlockSize <= 0 {
return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
}
if meta.Size != meta.CompressionMetadataGzip.Size {
return errors.New("gzip metadata size does not match object size")
}
if meta.Size < 0 {
return errors.New("invalid gzip object size")
}
blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
blocks := meta.Size / blockSize
if meta.Size%blockSize != 0 {
blocks++
}
if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
return errors.New("gzip block data is incomplete")
}
return nil
}
// Object with external metadata
type Object struct {
fs.Object // Wraps around data object for this object
@@ -1137,9 +1093,6 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
if err = jr.Decode(meta); err != nil {
return nil, err
}
if err = meta.validate(); err != nil {
return nil, err
}
return meta, nil
}
@@ -1157,15 +1110,6 @@ func (o *Object) Remove(ctx context.Context) error {
return objErr
}
// countingDiscard is an io.Writer which discards its input and counts
// the bytes written
type countingDiscard int64
func (c *countingDiscard) Write(p []byte) (int, error) {
*c += countingDiscard(len(p))
return len(p), nil
}
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
type ReadCloserWrapper struct {
io.Reader
-89
View File
@@ -6,8 +6,6 @@ import (
"path/filepath"
"testing"
"github.com/buengese/sgzip"
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/local"
_ "github.com/rclone/rclone/backend/s3"
@@ -16,93 +14,6 @@ import (
"github.com/rclone/rclone/fstest/fstests"
)
func TestObjectMetadataValidate(t *testing.T) {
tests := []struct {
name string
meta ObjectMetadata
want bool
}{
{
name: "nil gzip metadata",
meta: ObjectMetadata{Mode: Gzip},
want: true,
},
{
name: "zero block size",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 0,
Size: 1024,
BlockData: []uint32{0},
},
},
want: true,
},
{
name: "negative block size",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: -1,
Size: 1024,
BlockData: []uint32{0},
},
},
want: true,
},
{
name: "mismatched sizes",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 2048,
BlockData: []uint32{0, 1},
},
},
want: true,
},
{
name: "short block data",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1025,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 1025,
BlockData: []uint32{0, 1},
},
},
want: true,
},
{
name: "valid",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1025,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 1025,
BlockData: []uint32{0, 1, 2},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := test.meta.validate()
if (err != nil) != test.want {
t.Fatalf("validate() error = %v, want error: %v", err, test.want)
}
})
}
}
var defaultOpt = fstests.Opt{
RemoteName: "TestCompress:",
NilObject: (*Object)(nil),
+3 -2
View File
@@ -2,6 +2,7 @@ package compress
import (
"bufio"
"bytes"
"context"
"crypto/md5"
"encoding/hex"
@@ -22,7 +23,7 @@ type gzipModeHandler struct{}
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
// the configured threshold
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
var b countingDiscard
var b bytes.Buffer
var n int64
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
if err != nil {
@@ -36,7 +37,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
if err != nil {
return false, err
}
ratio := float64(n) / float64(b)
ratio := float64(n) / float64(b.Len())
return ratio > minCompressionRatio, nil
}
-137
View File
@@ -1,137 +0,0 @@
package compress
import (
"bytes"
"context"
"crypto/rand"
"io"
"testing"
"time"
_ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/random"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
// PutStream. It reads each body twice when it is seekable to model a
// backend retrying an upload.
type putRecorder struct {
fs.Fs
t *testing.T
stream bool // expose PutStream from the wrapped Fs
size int64
seekable bool
body []byte
streamed bool
}
func (r *putRecorder) Features() *fs.Features {
features := *r.Fs.Features()
features.PutStream = nil
if r.stream {
features.PutStream = r.PutStream
}
return &features
}
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
r.size = src.Size()
body, err := io.ReadAll(in)
require.NoError(r.t, err)
if seeker, ok := in.(io.Seeker); ok {
r.seekable = true
_, err = seeker.Seek(0, io.SeekStart)
require.NoError(r.t, err)
again, err := io.ReadAll(in)
require.NoError(r.t, err)
assert.Equal(r.t, body, again, "retried body should be identical")
}
r.body = body
return body
}
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
body := r.record(in, src)
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
}
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
r.streamed = true
return r.Put(ctx, in, src, options...)
}
// TestRcat checks the three upload paths rcat takes: small files buffered
// in memory and uploaded with a known size and a seekable body, large
// files streamed when the wrapped remote supports it, and large files
// spooled to a temporary file when it doesn't. Pool pages taken for the
// buffer must be returned on every path.
func TestRcat(t *testing.T) {
ctx := context.Background()
const limit = 1024
for _, test := range []struct {
name string
size int
stream bool
wantSize int64
}{
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
{name: "Empty", size: 0, stream: true, wantSize: 0},
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
} {
t.Run(test.name, func(t *testing.T) {
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
require.NoError(t, err)
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
f := &Fs{Fs: rec}
f.opt.RAMCacheLimit = limit
content := []byte(random.String(test.size))
inUse := pool.Global().InUse()
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
require.NoError(t, err)
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
assert.Equal(t, test.wantSize, rec.size)
assert.Equal(t, content, rec.body)
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
if test.size < limit {
assert.True(t, rec.seekable, "small file body should be seekable")
}
rc, err := o.Open(ctx)
require.NoError(t, err)
got, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, content, got)
})
}
}
// TestIsCompressible checks the compressibility heuristic on data at
// both ends of the scale for every compression mode which has one.
func TestIsCompressible(t *testing.T) {
compressible := bytes.Repeat([]byte("compress me "), 4096)
incompressible := make([]byte, len(compressible))
_, err := io.ReadFull(rand.Reader, incompressible)
require.NoError(t, err)
for name, handler := range map[string]compressionModeHandler{
"gzip": &gzipModeHandler{},
"zstd": &zstdModeHandler{},
} {
t.Run(name, func(t *testing.T) {
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
require.NoError(t, err)
assert.True(t, ok)
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
require.NoError(t, err)
assert.False(t, ok)
})
}
}
+3 -2
View File
@@ -2,6 +2,7 @@ package compress
import (
"bufio"
"bytes"
"context"
"crypto/md5"
"encoding/hex"
@@ -22,7 +23,7 @@ type zstdModeHandler struct{}
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
// the configured threshold
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
var b countingDiscard
var b bytes.Buffer
var n int64
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
if err != nil {
@@ -36,7 +37,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
if err != nil {
return false, err
}
ratio := float64(n) / float64(b)
ratio := float64(n) / float64(b.Len())
return ratio > minCompressionRatio, nil
}
+13 -40
View File
@@ -11,7 +11,6 @@ import (
"errors"
"fmt"
"io"
"path"
"strconv"
"strings"
"sync"
@@ -118,8 +117,8 @@ func (mode NameEncryptionMode) String() (out string) {
return out
}
// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
type FileNameEncoding interface {
// fileNameEncoding are the encoding methods dealing with encrypted file names
type fileNameEncoding interface {
EncodeToString(src []byte) string
DecodeString(s string) ([]byte, error)
}
@@ -154,7 +153,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
}
// NewNameEncoding creates a NameEncoding from a string
func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
s = strings.ToLower(s)
switch s {
case "base32":
@@ -176,7 +175,7 @@ type Cipher struct {
nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto
block gocipher.Block
mode NameEncryptionMode
fileNameEnc FileNameEncoding
fileNameEnc fileNameEncoding
buffers sync.Pool // encrypt/decrypt buffers
cryptoRand io.Reader // read crypto random numbers from here
dirNameEncrypt bool
@@ -185,7 +184,7 @@ type Cipher struct {
}
// newCipher initialises the cipher. If salt is "" then it uses a built in salt val
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
c := &Cipher{
mode: mode,
fileNameEnc: enc,
@@ -487,12 +486,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
}
// encryptFileName encrypts a file path
//
// If stripVersion is set, a version string (as used by --b2-versions)
// on the last segment is kept in plain text. Set it for file names
// only: directory names must encrypt identically whether passed alone
// or as the parent of a file name.
func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
func (c *Cipher) encryptFileName(in string) string {
segments := strings.Split(in, "/")
for i := range segments {
// Skip directory name encryption if the user chose to
@@ -505,7 +499,7 @@ func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
// of the file name gets encrypted/obfuscated
hasVersion := false
var t time.Time
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
if i == (len(segments)-1) && version.Match(segments[i]) {
var s string
t, s = version.Remove(segments[i])
// version.Remove can fail, in which case it returns segments[i]
@@ -535,24 +529,19 @@ func (c *Cipher) EncryptFileName(in string) string {
if c.mode == NameEncryptionOff {
return in + c.encryptedSuffix
}
return c.encryptFileName(in, true)
return c.encryptFileName(in)
}
// EncryptDirName encrypts a directory path
//
// Unlike EncryptFileName, a version string on the last segment is
// encrypted along with the rest of the name.
func (c *Cipher) EncryptDirName(in string) string {
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
return in
}
return c.encryptFileName(in, false)
return c.encryptFileName(in)
}
// decryptFileName decrypts a file path
//
// stripVersion is as for encryptFileName.
func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
func (c *Cipher) decryptFileName(in string) (string, error) {
segments := strings.Split(in, "/")
for i := range segments {
var err error
@@ -566,7 +555,7 @@ func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
// of the file name gets decrypted/deobfuscated
hasVersion := false
var t time.Time
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
if i == (len(segments)-1) && version.Match(segments[i]) {
var s string
t, s = version.Remove(segments[i])
// version.Remove can fail, in which case it returns segments[i]
@@ -612,31 +601,15 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
// Leave the version string on, if it was there
return decrypted, nil
}
return c.decryptFileName(in, true)
return c.decryptFileName(in)
}
// DecryptDirName decrypts a directory path
//
// Unlike DecryptFileName, a version string on the last segment is
// expected to be part of the encrypted name. Directory names created
// by rclone before v1.76 had a version-like suffix left in plain
// text; these are still decrypted so that they appear in listings,
// but they can't be opened or removed until renamed on the
// underlying remote.
func (c *Cipher) DecryptDirName(in string) (string, error) {
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
return in, nil
}
out, err := c.decryptFileName(in, false)
if err != nil && version.Match(path.Base(in)) {
var legacyErr error
out, legacyErr = c.decryptFileName(in, true)
if legacyErr == nil {
fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
return out, nil
}
}
return out, err
return c.decryptFileName(in)
}
// NameEncryptionMode returns the encryption mode in use for names
-38
View File
@@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"io"
"path"
"strings"
"testing"
@@ -683,43 +682,6 @@ func TestNonStandardDecryptDirName(t *testing.T) {
}
}
// Test directories whose name looks like it has a version string -
// these encrypt verbatim, so that EncryptDirName agrees with the same
// directory encrypted as the parent of a file name, whereas file names
// keep the version string in plain text
func TestVersionedDirName(t *testing.T) {
const dir = "dir-v2001-02-03-040506-123"
enc, err := NewNameEncoding("base32")
require.NoError(t, err)
for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
c, err := newCipher(mode, "", "", true, enc)
require.NoError(t, err)
what := fmt.Sprintf("Testing mode=%v", mode)
// Check EncryptDirName matches the parent of an encrypted file name
encryptedDir := c.EncryptDirName(dir)
encryptedFile := c.EncryptFileName(dir + "/file.txt")
assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
// Check the encrypted directory name round trips OK
decryptedDir, err := c.DecryptDirName(encryptedDir)
assert.NoError(t, err, what)
assert.Equal(t, dir, decryptedDir, what)
// Check a file with the same name keeps its version string in plain text
encryptedFile = c.EncryptFileName(dir)
assert.NotEqual(t, encryptedDir, encryptedFile, what)
assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
decryptedFile, err := c.DecryptFileName(encryptedFile)
assert.NoError(t, err, what)
assert.Equal(t, dir, decryptedFile, what)
// Check a directory with a legacy encrypted name (the file form) still decrypts
decryptedDir, err = c.DecryptDirName(encryptedFile)
assert.NoError(t, err, what)
assert.Equal(t, dir, decryptedDir, what)
// but a name which is bad in both forms is still an error
_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
assert.Error(t, err, what)
}
}
func TestEncryptedSize(t *testing.T) {
c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
for _, test := range []struct {
+1 -6
View File
@@ -1179,11 +1179,6 @@ func (o *ObjectInfo) Size() int64 {
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
// If the data is unchanged then the hash of the source is the
// hash of the object which will be uploaded
if o.f.opt.NoDataEncryption {
return o.ObjectInfo.Hash(ctx, hash)
}
var srcObj fs.Object
var ok bool
// Get the underlying object if there is one
@@ -1197,7 +1192,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
return "", nil
}
// if this is wrapping a local object then we work out the hash
if srcObj != nil && srcObj.Fs().Features().IsLocal {
if srcObj.Fs().Features().IsLocal {
// Read the data and encrypt it to calculate the hash
fs.Debugf(o, "Computing %v hash of encrypted source", hash)
return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)
-14
View File
@@ -84,25 +84,11 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
// Test ObjectInfo.Hash
wantHash := md5.Sum(outBuf.Bytes())
if f.opt.NoDataEncryption {
// If the data isn't encrypted, the hash should be that of the plaintext
wantHash = md5.Sum([]byte(contents))
}
gotHash, err := src.Hash(ctx, hash.MD5)
require.NoError(t, err)
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
}
// Test ObjectInfo.Hash doesn't panic when the source is an
// fs.OverrideRemote which wraps an fs.ObjectInfo that isn't an fs.Object
func TestObjectInfoHashOverrideRemoteNotObject(t *testing.T) {
src := object.NewStaticObjectInfo("file", time.Now(), 3, true, nil, nil)
oi := (&Fs{}).newObjectInfo(fs.NewOverrideRemote(src, "new_remote"), nonce{})
gotHash, err := oi.Hash(context.Background(), hash.MD5)
require.NoError(t, err)
assert.Equal(t, "", gotHash)
}
func testComputeHash(t *testing.T, f *Fs) {
var (
contents = random.String(100)
-259
View File
@@ -1,259 +0,0 @@
// Package api has type definitions for dosya.dev
package api
import "fmt"
// Response is the base response wrapper
type Response struct {
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
}
// Error is an error response from the API.
//
// StatusCode and Status are filled in from the HTTP response. The API
// sets ErrorCode on the errors a client is expected to act on rather
// than just report, e.g. "concurrent_upload_limit".
type Error struct {
StatusCode int `json:"-"`
Status string `json:"-"`
OK bool `json:"ok"`
Message string `json:"error"`
ErrorCode string `json:"error_code,omitempty"`
MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
}
// Error satisfies the error interface
func (e *Error) Error() string {
out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
if e.Message != "" {
out += ": " + e.Message
}
if e.ErrorCode != "" {
out += " (" + e.ErrorCode + ")"
}
return out
}
// FileItem represents a file from the dosya API
type FileItem struct {
ID string `json:"id"`
Name string `json:"name"`
SizeBytes int64 `json:"size_bytes"`
MimeType string `json:"mime_type"`
Extension string `json:"extension"`
Region string `json:"region"`
CreatedAt float64 `json:"created_at"`
UpdatedAt float64 `json:"updated_at"`
FolderID *string `json:"folder_id"`
IsSynced int `json:"is_synced"`
// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
// are never re-read to hash the whole object - the same limitation S3 has).
ContentHash string `json:"content_hash"`
}
// FolderItem represents a folder from the dosya API
type FolderItem struct {
ID string `json:"id"`
Name string `json:"name"`
ParentID *string `json:"parent_id"`
CreatedAt float64 `json:"created_at"`
FileCount int `json:"file_count"`
IsSynced int `json:"is_synced"`
}
// ListResponse is the response from GET /api/files
type ListResponse struct {
Response
Files []FileItem `json:"files"`
Folders []FolderItem `json:"folders"`
Pagination Pagination `json:"pagination"`
}
// Pagination describes which page of files a ListResponse holds
type Pagination struct {
Page int `json:"page"`
PerPage int `json:"per_page"`
TotalFiles int `json:"total_files"`
TotalPages int `json:"total_pages"`
}
// CreateFolderRequest is the request body for POST /api/folders
type CreateFolderRequest struct {
WorkspaceID string `json:"workspace_id"`
ParentID *string `json:"parent_id"`
Name string `json:"name"`
}
// CreateFolderResponse is the response from POST /api/folders
type CreateFolderResponse struct {
Response
Folder struct {
ID string `json:"id"`
Name string `json:"name"`
ParentID *string `json:"parent_id"`
Workspace string `json:"workspace_id"`
} `json:"folder"`
}
// DeleteFolderResponse is the response from DELETE /api/folders/:id
type DeleteFolderResponse struct {
Response
Permanent bool `json:"permanent"`
Complete bool `json:"complete"`
Remaining int `json:"remaining"`
FilesAffected int `json:"files_affected"`
FoldersRemoved int `json:"folders_removed"`
}
// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
type RenameFolderRequest struct {
Name string `json:"name"`
}
// MoveFolderRequest is the request body for PUT /api/folders/:id/move
type MoveFolderRequest struct {
ParentID *string `json:"parent_id"`
Name string `json:"name,omitempty"`
}
// MoveFolderResponse is the response from PUT /api/folders/:id/move
type MoveFolderResponse struct {
Response
Name string `json:"name"`
}
// UploadInitRequest is the request body for POST /api/upload/init
type UploadInitRequest struct {
WorkspaceID string `json:"workspace_id"`
FileName string `json:"file_name"`
FileSize int64 `json:"file_size"`
MimeType string `json:"mime_type"`
Region string `json:"region,omitempty"`
FolderID *string `json:"folder_id"`
FileID *string `json:"file_id,omitempty"`
}
// UploadInitResponse is the response from POST /api/upload/init
type UploadInitResponse struct {
Response
SessionID string `json:"session_id"`
UploadURL string `json:"upload_url"`
WorkspaceID string `json:"workspace_id"`
FileName string `json:"file_name"`
FileSize int64 `json:"file_size"`
MimeType string `json:"mime_type"`
Extension string `json:"extension"`
Region string `json:"region"`
Resumable *struct {
PartSize int64 `json:"part_size"`
TotalParts int `json:"total_parts"`
PartUploadURL string `json:"part_upload_url"`
CompleteURL string `json:"complete_url"`
StatusURL string `json:"status_url"`
} `json:"resumable"`
}
// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
type UploadCompleteResponse struct {
Response
File struct {
ID string `json:"id"`
Name string `json:"name"`
SizeBytes int64 `json:"size_bytes"`
MimeType string `json:"mime_type"`
Extension string `json:"extension"`
Region string `json:"region"`
Version int `json:"version"`
CreatedAt float64 `json:"created_at"`
// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
// "" for a multipart upload, which the server does not hash.
ContentHash string `json:"content_hash"`
} `json:"file"`
}
// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
type UploadPartResponse struct {
Response
PartNumber int `json:"part_number"`
ETag string `json:"etag"`
BytesUploaded int64 `json:"bytes_uploaded"`
PartsCompleted int `json:"parts_completed"`
TotalParts int `json:"total_parts"`
}
// DeleteFileResponse is the response from DELETE /api/files/:id
type DeleteFileResponse struct {
Response
Permanent bool `json:"permanent"`
}
// RenameFileRequest is the request body for PUT /api/files/:id/rename
type RenameFileRequest struct {
Name string `json:"name"`
}
// RenameFileResponse is the response from PUT /api/files/:id/rename
type RenameFileResponse struct {
Response
Name string `json:"name"`
}
// MoveFileRequest is the request body for PUT /api/files/:id/move
type MoveFileRequest struct {
FolderID *string `json:"folder_id"`
Name *string `json:"name,omitempty"`
}
// CopyFileRequest is the request body for POST /api/files/:id/copy
type CopyFileRequest struct {
FolderID *string `json:"folder_id"`
Name string `json:"name,omitempty"`
}
// CopyFileResponse is the response from POST /api/files/:id/copy
type CopyFileResponse struct {
Response
FileID string `json:"file_id"`
Name string `json:"name"`
}
// ShareLinkRequest is the request body for POST /api/files/:id/share
type ShareLinkRequest struct {
ExpiresInDays *int `json:"expires_in_days,omitempty"`
}
// ShareLinkResponse is the response from POST /api/files/:id/share
type ShareLinkResponse struct {
Response
Link struct {
ID string `json:"id"`
Token string `json:"token"`
URL string `json:"url"`
ExpiresAt *int64 `json:"expires_at"`
CreatedAt float64 `json:"created_at"`
} `json:"link"`
}
// Workspace is one entry of the workspace list
type Workspace struct {
ID string `json:"id"`
Name string `json:"name"`
}
// WorkspaceListResponse is the response from GET /api/workspaces
type WorkspaceListResponse struct {
Response
Workspaces []Workspace `json:"workspaces"`
}
// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
type WorkspaceInfoResponse struct {
Response
Storage struct {
Used int64 `json:"used"`
Total int64 `json:"total"`
Free int64 `json:"free"`
} `json:"storage"`
}
-772
View File
@@ -1,772 +0,0 @@
// Package dosya provides an interface to the dosya.dev cloud storage system.
package dosya
import (
"context"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/rest"
)
const (
rootID = "root"
defaultBaseURL = "https://api.dosya.dev"
minSleep = 100 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2
attackConstant = 0
)
func init() {
fs.Register(&fs.RegInfo{
Name: "dosya",
Description: "dosya.dev",
NewFs: NewFs,
Options: []fs.Option{{
Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
Name: "api_key",
Sensitive: true,
Required: true,
}, {
Name: "workspace_id",
Help: `Your workspace ID.
Leave this blank to use the one workspace the API key can reach, which
is what a key restricted to a single workspace does. If the key can
reach several, rclone will refuse and list their IDs for you to choose
from.`,
}, {
Help: "API base URL.",
Name: "api_url",
Default: defaultBaseURL,
Advanced: true,
}, {
Name: "upload_concurrency",
Help: `Concurrency for multipart uploads.
This is the number of parts of the same file that are uploaded
concurrently. The first part is always uploaded on its own, because
the server creates the upload when it receives it.
Each part in flight is held in memory, so memory use grows with
this value times the part size the server chooses.
If you are uploading large files over high-speed links and these
uploads do not fully utilize your bandwidth, then increasing this
may help to speed up the transfers.`,
Default: 4,
Advanced: true,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
Advanced: true,
Default: (encoder.Display |
encoder.EncodeBackSlash |
encoder.EncodeSingleQuote |
encoder.EncodeDoubleQuote |
encoder.EncodeLtGt |
encoder.EncodeCtl |
encoder.EncodeLeftSpace |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8),
}},
})
}
// Options defines the configuration for this backend
type Options struct {
APIKey string `config:"api_key"`
WorkspaceID string `config:"workspace_id"`
APIURL string `config:"api_url"`
UploadConcurrency int `config:"upload_concurrency"`
Enc encoder.MultiEncoder `config:"encoding"`
}
// Fs represents a remote dosya.dev filesystem
type Fs struct {
name string
root string
opt Options
features *fs.Features
rest *rest.Client
pacer *fs.Pacer
dirCache *dircache.DirCache
}
// Object represents a dosya.dev file
type Object struct {
fs *Fs
remote string
file api.FileItem
}
// findWorkspace fills in the workspace ID when the config leaves it blank.
//
// The ID is not shown anywhere in the dosya.dev web interface, so requiring
// it meant asking people to find it in a developer console. A key restricted
// to one workspace can only mean that one, and an account with a single
// workspace has only one possible answer; anything else genuinely has to be
// chosen, so list the choices in the error.
func (f *Fs) findWorkspace(ctx context.Context) error {
workspaces, err := f.listWorkspaces(ctx)
if err != nil {
return err
}
switch len(workspaces) {
case 0:
return errors.New("no workspaces found for this API key")
case 1:
f.opt.WorkspaceID = workspaces[0].ID
fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
return nil
}
var choices strings.Builder
for _, w := range workspaces {
_, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
}
return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
len(workspaces), choices.String())
}
// FindLeaf finds a directory of name leaf in the folder with ID pathID
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
listing, err := f.listFilesAndFolders(ctx, pathID)
if err != nil {
return "", false, err
}
for _, folder := range listing.Folders {
if f.opt.Enc.ToStandardName(folder.Name) == leaf {
return folder.ID, true, nil
}
}
return "", false, nil
}
// CreateDir makes a directory with pathID as parent and name leaf
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
name := f.opt.Enc.FromStandardName(leaf)
resp, err := f.createFolder(ctx, name, pathID)
if err != nil {
return "", err
}
return resp.Folder.ID, nil
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
return f.name
}
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
return f.root
}
// String returns a description of the FS
func (f *Fs) String() string {
return fmt.Sprintf("dosya.dev root '%s'", f.root)
}
// Precision of the ModTimes in this Fs
func (f *Fs) Precision() time.Duration {
// Modification times are stored and returned to the second (the API
// keeps them as Unix seconds, sent on upload via the source-mtime header).
return time.Second
}
// Hashes returns the supported hash types of the filesystem
func (f *Fs) Hashes() hash.Set {
// The server computes and stores the SHA-256 of a single-shot upload's
// bytes; multipart uploads have none (see Object.Hash).
return hash.NewHashSet(hash.SHA256)
}
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// NewFs makes a new Fs object from the path
func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
opt := new(Options)
err := configstruct.Set(config, opt)
if err != nil {
return nil, err
}
root = strings.Trim(root, "/")
f := &Fs{
name: name,
root: root,
opt: *opt,
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
}
// No DuplicateFiles: an upload that names no existing file but whose
// (workspace, folder, name) is already taken becomes a new VERSION of
// that file rather than a second row, so a name is unique within a
// folder and the same upload twice returns the same file id.
f.features = (&fs.Features{
CaseInsensitive: false,
CanHaveEmptyDirectories: true,
ReadMimeType: true,
}).Fill(ctx, f)
client := fshttp.NewClient(ctx)
f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
if f.opt.WorkspaceID == "" {
err = f.findWorkspace(ctx)
if err != nil {
return nil, err
}
}
f.dirCache = dircache.New(root, rootID, f)
// Find the current root
err = f.dirCache.FindRoot(ctx, false)
if err != nil {
// Assume it is a file
newRoot, remote := dircache.SplitPath(root)
tempF := *f
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
tempF.root = newRoot
// Make new Fs which is the parent
err = tempF.dirCache.FindRoot(ctx, false)
if err != nil {
// No root so return old f
return f, nil
}
_, err := tempF.NewObject(ctx, remote)
if err != nil {
if err == fs.ErrorObjectNotFound {
// File doesn't exist so return old f
return f, nil
}
return nil, err
}
f.features.Fill(ctx, &tempF)
f.dirCache = tempF.dirCache
f.root = tempF.root
return f, fs.ErrorIsFile
}
return f, nil
}
// List the objects and directories in dir into entries
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return nil, err
}
listing, err := f.listFilesAndFolders(ctx, directoryID)
if err != nil {
return nil, err
}
for _, folder := range listing.Folders {
folderName := f.opt.Enc.ToStandardName(folder.Name)
fullPath := getRemote(dir, folderName)
modTime := time.Unix(int64(folder.CreatedAt), 0)
d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
entries = append(entries, d)
f.dirCache.Put(fullPath, folder.ID)
}
for _, file := range listing.Files {
fileName := f.opt.Enc.ToStandardName(file.Name)
remote := getRemote(dir, fileName)
o := &Object{
fs: f,
remote: remote,
file: file,
}
entries = append(entries, o)
}
return entries, nil
}
// NewObject finds the Object at remote
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
return nil, fs.ErrorObjectNotFound
}
return nil, err
}
listing, err := f.listFilesAndFolders(ctx, directoryID)
if err != nil {
return nil, err
}
for _, file := range listing.Files {
if f.opt.Enc.ToStandardName(file.Name) == leaf {
return &Object{
fs: f,
remote: remote,
file: file,
}, nil
}
}
return nil, fs.ErrorObjectNotFound
}
// Put uploads a new file
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
existingObj, err := f.NewObject(ctx, src.Remote())
switch err {
case nil:
return existingObj, existingObj.Update(ctx, in, src, options...)
case fs.ErrorObjectNotFound:
return f.put(ctx, in, src, options...)
default:
return nil, err
}
}
// put uploads src as a new file
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
remote := src.Remote()
_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}
modTime := src.ModTime(ctx)
resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
if err != nil {
return nil, err
}
return &Object{
fs: f,
remote: remote,
file: fileFromUpload(resp, modTime),
}, nil
}
// fileFromUpload builds the FileItem for the object an upload just created.
// The single-shot upload response omits updated_at, so the modification time
// is taken from what was sent (modTime) rather than defaulting to the create
// time - it is the time the server stored via the source-mtime header. The
// SHA-256 comes straight from the response (empty for a multipart upload).
func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
updatedAt := resp.File.CreatedAt
if !modTime.IsZero() {
updatedAt = float64(modTime.Unix())
}
return api.FileItem{
ID: resp.File.ID,
Name: resp.File.Name,
SizeBytes: resp.File.SizeBytes,
MimeType: resp.File.MimeType,
Extension: resp.File.Extension,
Region: resp.File.Region,
CreatedAt: resp.File.CreatedAt,
UpdatedAt: updatedAt,
ContentHash: resp.File.ContentHash,
}
}
// Mkdir makes the directory (container, bucket)
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
_, err := f.dirCache.FindDir(ctx, dir, true)
return err
}
// Rmdir removes the directory (container, bucket) if empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return err
}
// The server checks emptiness itself (?if_empty=1) and answers
// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
err = f.removeFolder(ctx, directoryID, true)
if err != nil {
return err
}
f.dirCache.FlushDir(dir)
return nil
}
// Move src to this remote using server-side move operations
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
if err != nil {
return nil, err
}
// Find destination directory
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}
needsMove := srcDirID != dstDirID
needsRename := srcLeaf != dstLeaf
// Moving and renaming in the same request (rather than a moveFile call
// followed by a separate renameFile call) matters when both are needed:
// a two-step move-then-rename can transiently place the file's CURRENT
// name into a destination folder that already holds an unrelated file
// with that name, even though the FINAL name is free - the server only
// allows one active name per folder, so that intermediate state 409s.
if needsMove || needsRename {
newName := ""
if needsRename {
newName = f.opt.Enc.FromStandardName(dstLeaf)
}
err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
if err != nil {
return nil, err
}
}
// Return new object. file.Name must reflect the destination leaf, not
// the source's - a caller that reuses this object as the src of a
// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
// recomputes srcLeaf from file.Name; a stale name there makes that
// second move see srcLeaf == dstLeaf and skip the rename entirely,
// silently leaving the object under the wrong name.
newFile := srcObj.file
newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
return &Object{
fs: f,
remote: remote,
file: newFile,
}, nil
}
// DirMove moves src, srcRemote to this remote at dstRemote
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
srcFs, ok := src.(*Fs)
if !ok {
fs.Debugf(srcFs, "Can't move directory - not same remote type")
return fs.ErrorCantDirMove
}
srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
if err != nil {
return err
}
srcLeaf := srcRemote
if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
srcLeaf = srcRemote[idx+1:]
}
needsRename := srcLeaf != dstLeaf
newName := ""
if needsRename {
newName = f.opt.Enc.FromStandardName(dstLeaf)
}
// Move folder to new parent only if parent changed. The new name goes
// in the same request, as for files in Move: a move followed by a
// separate rename can collide with an unrelated folder sharing the
// CURRENT name in the destination even though the final name is free.
if srcDirectoryID != dstDirectoryID {
renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
if err != nil {
return fmt.Errorf("couldn't move directory: %w", err)
}
if renamed {
needsRename = false
}
}
if needsRename {
err = f.renameFolder(ctx, srcID, newName)
if err != nil {
return fmt.Errorf("couldn't rename directory: %w", err)
}
}
srcFs.dirCache.FlushDir(srcRemote)
return nil
}
// Copy src to this remote using server-side copy operations
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}
// The copy API creates a new row named after the source file when
// copying across folders, which collides with one already at the
// destination. Copy under a temporary name, then delete the old
// destination, then move the copy onto the name - so a copy that fails
// leaves the destination as it was. Deleting it up front lost it
// (prior art: backend/box).
existingObj, err := f.NewObject(ctx, remote)
switch err {
case nil:
tempRemote := remote + "-rclone-copy-" + random.String(8)
fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
tempObj, err := f.Copy(ctx, src, tempRemote)
if err != nil {
return nil, err
}
fs.Debugf(tempRemote, "moving to real name %v", remote)
if err := existingObj.Remove(ctx); err != nil {
return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
}
return f.Move(ctx, tempObj, remote)
case fs.ErrorObjectNotFound:
// nothing at the destination, so copy straight onto the name
default:
return nil, err
}
dstName := f.opt.Enc.FromStandardName(dstLeaf)
resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
if err != nil {
return nil, fmt.Errorf("couldn't copy file: %w", err)
}
// Servers that predate the name field ignore it and name the copy after
// the source, or "Copy of <original>" in the same folder. Rename to the
// requested destination name if it differs.
if resp.Name != dstName {
err = f.renameFile(ctx, resp.FileID, dstName)
if err != nil {
return nil, fmt.Errorf("couldn't rename copied file: %w", err)
}
resp.Name = dstName
}
// A server-side copy is byte-identical to the source and the API
// preserves the source's modification time and content hash onto the
// copy, so the returned object must report the SOURCE's modtime and hash,
// not the copy's wall-clock time. Reading "now" here made the object
// rclone holds in memory disagree with the one read back from the remote
// (its Fingerprint check compares the two exactly).
return &Object{
fs: f,
remote: remote,
file: api.FileItem{
ID: resp.FileID,
Name: resp.Name,
SizeBytes: srcObj.file.SizeBytes,
MimeType: srcObj.file.MimeType,
Extension: srcObj.file.Extension,
Region: srcObj.file.Region,
CreatedAt: srcObj.file.CreatedAt,
UpdatedAt: srcObj.file.UpdatedAt,
ContentHash: srcObj.file.ContentHash,
},
}, nil
}
// Purge deletes all the files and the container
func (f *Fs) Purge(ctx context.Context, dir string) error {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return err
}
// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
err = f.removeFolder(ctx, directoryID, false)
if err != nil {
return err
}
f.dirCache.FlushDir(dir)
return nil
}
// PublicLink generates a public link to the remote path
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
o, err := f.NewObject(ctx, remote)
if err != nil {
return "", err
}
obj := o.(*Object)
resp, err := f.createShareLink(ctx, obj.file.ID, expire)
if err != nil {
return "", err
}
return resp.Link.URL, nil
}
// DirCacheFlush resets the directory cache - used in testing as an
// optional interface
func (f *Fs) DirCacheFlush() {
f.dirCache.ResetRoot()
}
// About gets quota information from the Fs
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
info, err := f.getWorkspaceInfo(ctx)
if err != nil {
return nil, err
}
usage := &fs.Usage{
Used: fs.NewUsageValue(info.Storage.Used),
Total: fs.NewUsageValue(info.Storage.Total),
Free: fs.NewUsageValue(info.Storage.Free),
}
return usage, nil
}
// ------------------------------------------------------------
// Object methods
// ------------------------------------------------------------
// String returns a description of the Object
func (o *Object) String() string {
return o.remote
}
// Remote returns the remote path
func (o *Object) Remote() string {
return o.remote
}
// ModTime returns the modification date of the file
func (o *Object) ModTime(ctx context.Context) time.Time {
if o.file.UpdatedAt > 0 {
return time.Unix(int64(o.file.UpdatedAt), 0)
}
return time.Unix(int64(o.file.CreatedAt), 0)
}
// Size returns the size of the file
func (o *Object) Size() int64 {
return o.file.SizeBytes
}
// Fs returns read only access to the Fs that this object is part of
func (o *Object) Fs() fs.Info {
return o.fs
}
// Hash returns the selected checksum of the file
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.SHA256 {
return "", hash.ErrUnsupported
}
// May be "" - a multipart-uploaded object carries no server-side hash,
// which rclone treats as "hash unavailable for this object" (as it does
// for S3 multipart objects).
return o.file.ContentHash, nil
}
// Storable says whether this object can be stored
func (o *Object) Storable() bool {
return true
}
// SetModTime sets the metadata on the object to set the modification date
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
return fs.ErrorCantSetModTime
}
// Open opens the file for read
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
fs.FixRangeOption(options, o.file.SizeBytes)
return o.fs.downloadFile(ctx, o.file.ID, options)
}
// Update replaces the file content
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
// Find the directory ID for this file
_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
if err != nil {
return err
}
// Upload as new version of existing file
fileID := o.file.ID
modTime := src.ModTime(ctx)
resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
if err != nil {
return err
}
// Update object metadata
o.file = fileFromUpload(resp, modTime)
return nil
}
// Remove removes this object
func (o *Object) Remove(ctx context.Context) error {
return o.fs.deleteFile(ctx, o.file.ID)
}
// MimeType of an Object if known, "" otherwise
func (o *Object) MimeType(ctx context.Context) string {
return o.file.MimeType
}
// ID returns the ID of the Object if known, or "" if not
func (o *Object) ID() string {
return o.file.ID
}
func getRemote(dir, fileName string) string {
if dir == "" {
return fileName
}
return dir + "/" + fileName
}
// Check the interfaces are satisfied
var (
_ fs.Fs = (*Fs)(nil)
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.Copier = (*Fs)(nil)
_ fs.Purger = (*Fs)(nil)
_ fs.PublicLinker = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ dircache.DirCacher = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = (*Object)(nil)
_ fs.IDer = (*Object)(nil)
)
-880
View File
@@ -1,880 +0,0 @@
package dosya
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"path"
"strconv"
"strings"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"golang.org/x/sync/errgroup"
)
// sourceMTimeHeader is the request header the upload endpoints read to set a
// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
const sourceMTimeHeader = "x-dosya-source-mtime"
// modTimeHeaders returns the ExtraHeaders that carry a source modification time
// on an upload, or nil when there is nothing plausible to send. The server
// clamps implausible values itself, but a zero time is never worth a header.
func modTimeHeaders(modTime time.Time) map[string]string {
if modTime.IsZero() {
return nil
}
return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
}
// workspaceHintHeader names the workspace a by-id request is about. The API
// maps an id to its workspace through an index the workspace fills about a
// second after a write, so an id this client was handed moments ago can still
// miss; the hint lets the server look for it in the named workspace instead.
// It only locates: every route still checks membership against the workspace
// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
const workspaceHintHeader = "X-Dosya-Workspace"
// byIDHeaders returns the ExtraHeaders every /api/files/:id and
// /api/folders/:id call carries, or nil when no workspace is configured.
func (f *Fs) byIDHeaders() map[string]string {
if f.opt.WorkspaceID == "" {
return nil
}
return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
}
// retryErrorCodes is a slice of error codes that we will retry
var retryErrorCodes = []int{
429, // Too Many Requests
500, // Internal Server Error
502, // Bad Gateway
503, // Service Unavailable
504, // Gateway Timeout
}
// shouldRetry returns a boolean as to whether this resp and err
// deserve to be retried
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
if retry {
if d, ok := retryAfter(resp); ok {
// The API answers 503 + Retry-After while a workspace is being
// moved between storage homes, and Cloudflare sends it with 429.
// The pacer's own backoff tops out at maxSleep, far short of that.
fs.Debugf(nil, "Server asked to retry after %v", d)
err = pacer.RetryAfterError(err, d)
}
}
return retry, err
}
// maxRetryAfter caps how long a single Retry-After is honoured for
const maxRetryAfter = 5 * time.Minute
// retryAfter reads a 429 or 503 response's Retry-After header, which the API
// sends as whole seconds
func retryAfter(resp *http.Response) (time.Duration, bool) {
if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
return 0, false
}
seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
if err != nil || seconds <= 0 {
return 0, false
}
return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
}
// errorHandler decodes a non-2xx response into an *api.Error so callers can
// act on the machine-readable error_code rather than the message text.
func errorHandler(resp *http.Response) error {
body, err := rest.ReadBody(resp)
if err != nil {
return fmt.Errorf("error reading error out of body: %w", err)
}
apiErr := &api.Error{
StatusCode: resp.StatusCode,
Status: resp.Status,
}
if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
// Not the API's JSON shape (e.g. an edge response that never
// reached the API) - keep the body text so it isn't lost.
apiErr.ErrorCode = ""
apiErr.Message = strings.TrimSpace(string(body))
}
return apiErr
}
// The workspace caps the number of in-flight uploads per user. The API
// refuses an upload/init past that cap with a 400 carrying
// error_code "concurrent_upload_limit", and documents it as an error to
// wait on rather than give up on: slots free as uploads finish, and the
// server marks sessions idle for 10 minutes as failed, so waiting up to
// that long guarantees progress unless the workspace really is saturated.
// A 400 is deliberately outside the pacer's retry codes, so the wait
// lives in initUpload instead of shouldRetry.
var (
concurrentUploadRetryDelay = 5 * time.Second
concurrentUploadWaitMax = 10 * time.Minute
)
// abortUploadTimeout bounds the abort of a session the client is giving up on,
// which runs outside the transfer's context so a cancelled transfer still
// sends it
const abortUploadTimeout = 10 * time.Second
// isConcurrentUploadLimit reports whether err is the API refusing an
// upload because the workspace's concurrent upload limit is reached
func isConcurrentUploadLimit(err error) bool {
var apiErr *api.Error
return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
}
// listPageSize is the largest page GET /api/files will serve
const listPageSize = 500
// listFilesAndFolders lists files and folders in a directory
//
// The API pages files (at most listPageSize per page) but returns every
// folder on each page, so the files from all pages are gathered and the
// folders are taken from the first page only. Pages are ordered by name,
// which the API tie-breaks on id, so offsets are stable between requests.
func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
var listing *api.ListResponse
for page := 1; ; page++ {
opts := rest.Opts{
Method: "GET",
Path: "/api/files",
Parameters: map[string][]string{
"workspace_id": {f.opt.WorkspaceID},
"per_page": {strconv.Itoa(listPageSize)},
"page": {strconv.Itoa(page)},
"sort": {"name"},
"dir": {"asc"},
},
}
if folderID != "" && folderID != rootID {
opts.Parameters["folder_id"] = []string{folderID}
}
var result api.ListResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't list files: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
if listing == nil {
listing = &result
} else {
listing.Files = append(listing.Files, result.Files...)
}
if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
return listing, nil
}
}
}
// createFolder creates a folder
func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
request := api.CreateFolderRequest{
WorkspaceID: f.opt.WorkspaceID,
Name: name,
}
if parentID != "" && parentID != rootID {
request.ParentID = &parentID
}
opts := rest.Opts{
Method: "POST",
Path: "/api/folders",
}
var result api.CreateFolderResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't create folder: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// removeFolder deletes a folder
//
// ?permanent=1 trashes and purges a live folder in one request on today's
// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
// HTTP 409 "Folder is not empty" instead of purging a folder that still
// holds a live file or a live child folder; that 409 is turned into
// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
// error, and it is never retried (409 is not one of retryErrorCodes). A big
// subtree is purged in bounded passes (202, complete=false), so keep
// calling until the purge is complete. An older server that ignores
// permanent=1 only trashes on the first call (answering permanent=false),
// so the loop's second pass sends the plain DELETE that purges it.
func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
parameters := map[string][]string{"permanent": {"1"}}
if ifEmpty {
parameters["if_empty"] = []string{"1"}
}
opts := rest.Opts{
Method: "DELETE",
Path: "/api/folders/" + folderID,
Parameters: parameters,
ExtraHeaders: f.byIDHeaders(),
}
for pass := 0; ; pass++ {
var result api.DeleteFolderResponse
var statusCode int
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
if resp != nil {
statusCode = resp.StatusCode
}
return shouldRetry(ctx, resp, err)
})
if err != nil {
if ifEmpty && statusCode == http.StatusConflict {
return fs.ErrorDirectoryNotEmpty
}
if pass == 0 {
return fmt.Errorf("couldn't remove folder: %w", err)
}
return fmt.Errorf("couldn't permanently remove folder: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
if result.Permanent && result.Complete {
return nil
}
if pass >= maxFolderPurgePasses {
return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
}
}
}
// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
const maxFolderPurgePasses = 1000
// renameFolder renames a folder
func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
request := api.RenameFolderRequest{Name: newName}
opts := rest.Opts{
Method: "PUT",
Path: "/api/folders/" + folderID + "/rename",
ExtraHeaders: f.byIDHeaders(),
}
var result api.Response
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't rename folder: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
return nil
}
// moveFolder moves a folder to a new parent
func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
request := api.MoveFolderRequest{Name: newName}
if newParentID != "" && newParentID != rootID {
request.ParentID = &newParentID
}
opts := rest.Opts{
Method: "PUT",
Path: "/api/folders/" + folderID + "/move",
ExtraHeaders: f.byIDHeaders(),
}
var result api.MoveFolderResponse
err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return false, fmt.Errorf("couldn't move folder: %w", err)
}
if !result.OK {
return false, fmt.Errorf("API error: %s", result.Error)
}
// Servers that predate the name field ignore it and answer without a
// name, leaving the caller to rename separately
return newName != "" && result.Name == newName, nil
}
// deleteFile deletes a file
//
// ?permanent=1 trashes and purges it in one request on today's API. An
// older server that ignores the flag only soft-deletes it, answering
// permanent=false, so a second DELETE on the now soft-deleted file purges
// it from the database and R2, as it always has.
func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
opts := rest.Opts{
Method: "DELETE",
Path: "/api/files/" + fileID,
Parameters: map[string][]string{"permanent": {"1"}},
ExtraHeaders: f.byIDHeaders(),
}
var result api.DeleteFileResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't delete file: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
// Answered into its own value so a field the second response omits
// can't be read off the first.
if !result.Permanent {
var permanent api.DeleteFileResponse
err = f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't permanently delete file: %w", err)
}
if !permanent.OK {
return fmt.Errorf("API error: %s", permanent.Error)
}
}
return nil
}
// renameFile renames a file
func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
request := api.RenameFileRequest{Name: newName}
opts := rest.Opts{
Method: "PUT",
Path: "/api/files/" + fileID + "/rename",
ExtraHeaders: f.byIDHeaders(),
}
var result api.RenameFileResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't rename file: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
return nil
}
// moveFile moves a file to a new folder and, if newName is non-empty,
// renames it in the same request
func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
request := api.MoveFileRequest{}
if folderID != "" && folderID != rootID {
request.FolderID = &folderID
}
if newName != "" {
request.Name = &newName
}
opts := rest.Opts{
Method: "PUT",
Path: "/api/files/" + fileID + "/move",
ExtraHeaders: f.byIDHeaders(),
}
var result api.Response
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return fmt.Errorf("couldn't move file: %w", err)
}
if !result.OK {
return fmt.Errorf("API error: %s", result.Error)
}
return nil
}
// copyFile copies a file to a folder
func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
request := api.CopyFileRequest{Name: name}
if folderID != "" && folderID != rootID {
request.FolderID = &folderID
}
opts := rest.Opts{
Method: "POST",
Path: "/api/files/" + fileID + "/copy",
ExtraHeaders: f.byIDHeaders(),
}
var result api.CopyFileResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't copy file: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// downloadFile downloads a file by its ID
//
// The download endpoint answers 302 with a presigned R2 URL in Location, so
// this is two calls. Both go through the rest client, which is built on
// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
// --user-agent, --bind, --dump and the proxy settings apply to downloads as
// they do to every other call, and errorHandler and shouldRetry decide the
// errors and the retries uniformly.
func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
// NoRedirect stops the client following the 302 with the API's
// Authorization header attached. A 302 is not a 2xx, so Call reports it
// as an error (having read and closed the body); the Location header
// is what actually matters, and finding it clears the error, as in
// backend/linkbox.
opts := rest.Opts{
Method: "GET",
Path: "/api/files/" + fileID + "/download",
ExtraHeaders: f.byIDHeaders(),
NoRedirect: true,
}
var downloadURL string
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.Call(ctx, &opts)
if resp != nil {
if location := resp.Header.Get("Location"); location != "" {
downloadURL, err = location, nil
}
}
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't get download URL: %w", err)
}
if downloadURL == "" {
return nil, errors.New("no download URL in redirect response")
}
// Fetch the object from the presigned URL. It carries its own
// credentials and R2 refuses a request that also has an Authorization
// header, so the client-level bearer token is removed by setting the
// header empty: the rest client skips a header with an empty value.
opts = rest.Opts{
Method: "GET",
RootURL: downloadURL,
Options: options,
ExtraHeaders: map[string]string{"Authorization": ""},
}
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
var err error
resp, err = f.rest.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't download file: %w", err)
}
return resp.Body, nil
}
// uploadSmallFile uploads a file smaller than 10MB in a single request
func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
opts := rest.Opts{
Method: "PUT",
Path: "/api/upload/" + sessionID,
Body: in,
ContentLength: &size,
ContentType: "application/octet-stream",
ExtraHeaders: modTimeHeaders(modTime),
}
// The body is a stream which can't be rewound, so a low level retry
// would resend whatever is left of it - nothing - and the server
// would store that as an empty file. Don't retry here; rclone's high
// level retries redo the whole transfer instead.
var result api.UploadCompleteResponse
err := f.pacer.CallNoRetry(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't upload file: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// uploadPart uploads a single part of a multipart upload
func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
size := int64(len(part))
opts := rest.Opts{
Method: "PUT",
Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
ContentLength: &size,
ContentType: "application/octet-stream",
}
var result api.UploadPartResponse
err := f.pacer.Call(func() (bool, error) {
// A fresh reader per attempt so a retry resends the whole part
opts.Body = bytes.NewReader(part)
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// completeUpload finalizes a multipart upload
func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
opts := rest.Opts{
Method: "POST",
Path: "/api/upload/" + sessionID + "/complete",
ExtraHeaders: modTimeHeaders(modTime),
}
var result api.UploadCompleteResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't complete upload: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// initUpload initializes a new upload session
func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
request := api.UploadInitRequest{
WorkspaceID: f.opt.WorkspaceID,
FileName: fileName,
FileSize: fileSize,
MimeType: mimeType,
FileID: fileID,
}
if folderID != "" && folderID != rootID {
request.FolderID = &folderID
}
opts := rest.Opts{
Method: "POST",
Path: "/api/upload/init",
}
var result api.UploadInitResponse
deadline := time.Now().Add(concurrentUploadWaitMax)
for {
result = api.UploadInitResponse{}
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err == nil {
break
}
if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
return nil, fmt.Errorf("couldn't init upload: %w", err)
}
fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
select {
case <-ctx.Done():
return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
case <-time.After(concurrentUploadRetryDelay):
}
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// uploadParts reads the source part by part and uploads the parts.
//
// Part 1 goes on its own: the server creates the R2 multipart upload (and a
// new file's id and key) when it receives the first part, so parts sent
// before that has finished would each start an upload of their own. The
// rest go out opt.UploadConcurrency at a time. A part is only read from the
// source once a buffer is free, so at most that many parts are held in
// memory, and a short source is still found by the reader before the part
// it could not fill is sent.
func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
readPart := func(buf []byte, partNum int) ([]byte, error) {
currentPartSize := partSize
if partNum == totalParts {
currentPartSize = size - partSize*int64(totalParts-1)
}
part := buf[:currentPartSize]
if _, err := io.ReadFull(in, part); err != nil {
if err == io.EOF || err == io.ErrUnexpectedEOF {
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
}
return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
}
return part, nil
}
concurrency := max(f.opt.UploadConcurrency, 1)
buffers := make(chan []byte, concurrency)
buffers <- make([]byte, partSize)
allocated := 1
first := <-buffers
part, err := readPart(first, 1)
if err != nil {
return err
}
if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
return err
}
buffers <- first
// A failed read has to take the parts already in flight down with it,
// or they upload to completion while the error waits on Wait.
ctx, cancel := context.WithCancel(ctx)
defer cancel()
g, gCtx := errgroup.WithContext(ctx)
var readErr error
for partNum := 2; partNum <= totalParts; partNum++ {
// Take a free buffer, allocating one while under the limit.
var buf []byte
select {
case buf = <-buffers:
default:
if allocated < concurrency {
buf = make([]byte, partSize)
allocated++
} else {
select {
case buf = <-buffers:
case <-gCtx.Done():
}
}
}
if buf == nil {
break // a part failed; its error comes back from Wait
}
part, err := readPart(buf, partNum)
if err != nil {
readErr = err
cancel()
break
}
g.Go(func() error {
defer func() { buffers <- buf }()
_, err := f.uploadPart(gCtx, part, sessionID, partNum)
return err
})
}
err = g.Wait()
if readErr != nil {
return readErr
}
return err
}
// abortUpload retires an upload session the client is giving up on.
//
// Best effort: the server reclaims an abandoned session itself, so a failure
// here - including an older server with no such route - costs the slot the
// wait it would have cost anyway and is not worth failing an upload over.
// Sent on a context of its own so a cancelled transfer still gives its slot
// back rather than leaving it to the sweep.
func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
defer cancel()
opts := rest.Opts{
Method: "DELETE",
Path: "/api/upload/" + sessionID,
NoResponse: true,
}
err := f.pacer.CallNoRetry(func() (bool, error) {
resp, err := f.rest.Call(ctx, &opts)
return shouldRetry(ctx, resp, err)
})
if err != nil {
fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
}
}
// uploadFile handles the full upload flow (small or multipart)
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
// Both refusals live here so a new file and a new version of an existing
// one answer the same way: upload/init needs a byte count up front, and
// the server stores no zero-length object.
if size < 0 {
return nil, errors.New("can't upload files with unknown size")
}
if size == 0 {
return nil, fs.ErrorCantUploadEmptyFiles
}
leaf := f.opt.Enc.FromStandardName(path.Base(remote))
initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
if err != nil {
return nil, err
}
// A session abandoned here holds one of the workspace's upload slots
// until the server's sweep retires it ten minutes later, and rclone's
// high-level retry opens a fresh session rather than resuming this one -
// so without this, a few failed attempts use up every slot.
defer func() {
if err != nil {
f.abortUpload(ctx, initResp.SessionID)
}
}()
// Count what the source actually supplies, so a source which ends
// early is reported as an error rather than stored as a truncated
// file.
counter := readers.NewCountingReader(in)
// Small file upload (single PUT, no resumable)
if initResp.Resumable == nil {
resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
if err != nil {
return nil, err
}
if got := int64(counter.BytesRead()); got != size {
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
}
return resp, nil
}
// Multipart upload. Each part is read fully into memory before it is
// sent, so a short source is caught before the part goes out and a
// part which has to be retried is resent with all of its bytes.
partSize := initResp.Resumable.PartSize
totalParts := initResp.Resumable.TotalParts
// Both bounds matter. partSize*(totalParts-1) >= size means a part would
// have nothing to carry; size > partSize*totalParts means the last part
// is bigger than partSize, and readPart would slice past its buffer and
// panic.
if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
}
if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
return nil, err
}
return f.completeUpload(ctx, initResp.SessionID, modTime)
}
// createShareLink creates a public share link for a file
func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
request := api.ShareLinkRequest{}
if expire < fs.DurationOff {
days := int(time.Duration(expire).Hours() / 24)
if days < 1 {
days = 1
}
request.ExpiresInDays = &days
}
opts := rest.Opts{
Method: "POST",
Path: "/api/files/" + fileID + "/share",
ExtraHeaders: f.byIDHeaders(),
}
var result api.ShareLinkResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't create share link: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
// getWorkspaceInfo fetches workspace storage info
// listWorkspaces lists the workspaces this credential can reach. An API key
// pinned to a workspace sees exactly one: its own.
func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
opts := rest.Opts{
Method: "GET",
Path: "/api/workspaces",
}
var result api.WorkspaceListResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't list workspaces: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return result.Workspaces, nil
}
func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
opts := rest.Opts{
Method: "GET",
Path: "/api/workspaces/" + f.opt.WorkspaceID,
}
var result api.WorkspaceInfoResponse
err := f.pacer.Call(func() (bool, error) {
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("couldn't get workspace info: %w", err)
}
if !result.OK {
return nil, fmt.Errorf("API error: %s", result.Error)
}
return &result, nil
}
-235
View File
@@ -1,235 +0,0 @@
package dosya
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
status int
header string
want time.Duration
ok bool
}{
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
{"missing header", http.StatusServiceUnavailable, "", 0, false},
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
} {
t.Run(tc.name, func(t *testing.T) {
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
if tc.header != "" {
resp.Header.Set("Retry-After", tc.header)
}
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
assert.True(t, retry)
d, ok := pacer.IsRetryAfter(err)
assert.Equal(t, tc.ok, ok)
if tc.ok {
assert.Equal(t, tc.want, d)
}
})
}
}
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
resp.Header.Set("Retry-After", "30")
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
assert.False(t, retry)
_, ok := pacer.IsRetryAfter(err)
assert.False(t, ok)
}
func TestRemoveFolderPurgesPermanently(t *testing.T) {
// a big subtree purges in bounded passes: 202 complete=false repeats
// the same request until complete=true
answers := []struct {
status int
body string
}{
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
}
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
require.Equal(t, "DELETE", r.Method)
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
a := answers[calls]
calls++
return jsonResp(a.status, a.body), nil
}))
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
assert.Equal(t, len(answers), calls)
}
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
}))
err := f.removeFolder(context.Background(), "fld_1", false)
require.Error(t, err)
assert.Contains(t, err.Error(), "locked file")
assert.Equal(t, 1, calls)
}
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
}))
err := f.removeFolder(context.Background(), "fld_1", true)
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
}
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
}))
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
assert.Equal(t, 1, calls)
}
func TestRmdirMakesOneRequest(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "DELETE", r.Method)
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
}))
f.dirCache = dircache.New("", rootID, f)
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
f.dirCache.Put("sub", "fld_1")
require.NoError(t, f.Rmdir(context.Background(), "sub"))
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
}
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
}))
f.dirCache = dircache.New("", rootID, f)
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
f.dirCache.Put("sub", "fld_1")
err := f.Rmdir(context.Background(), "sub")
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
}
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
for _, tc := range []struct {
name string
permanent bool
wantCalls int
}{
{"today's API purges in one call", true, 1},
{"an older server ignores the flag and still gets a second DELETE", false, 2},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
calls++
assert.Equal(t, "DELETE", r.Method)
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
}))
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
assert.Equal(t, tc.wantCalls, calls)
})
}
}
// decodeBody reads a JSON request body into a map
func decodeBody(t *testing.T, r *http.Request) map[string]any {
b, err := io.ReadAll(r.Body)
require.NoError(t, err)
var m map[string]any
require.NoError(t, json.Unmarshal(b, &m))
return m
}
func TestMoveFolderSendsName(t *testing.T) {
for _, tc := range []struct {
name string
newName string
respBody string
wantRenamed bool
}{
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
{"older server ignores name", "dst", `{"ok":true}`, false},
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
} {
t.Run(tc.name, func(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
body := decodeBody(t, r)
assert.Equal(t, "fld_parent", body["parent_id"])
if tc.newName == "" {
assert.NotContains(t, body, "name")
} else {
assert.Equal(t, tc.newName, body["name"])
}
return jsonResp(http.StatusOK, tc.respBody), nil
}))
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
require.NoError(t, err)
assert.Equal(t, tc.wantRenamed, renamed)
})
}
}
func TestCopyFileSendsName(t *testing.T) {
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
body := decodeBody(t, r)
assert.Equal(t, "fld_dst", body["folder_id"])
assert.Equal(t, "final.txt", body["name"])
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
}))
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
require.NoError(t, err)
assert.Equal(t, "fil_2", resp.FileID)
assert.Equal(t, "final.txt", resp.Name)
}
-171
View File
@@ -1,171 +0,0 @@
package dosya
import (
"context"
"io"
"net/http"
"strconv"
"strings"
"testing"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
f := &Fs{}
assert.Equal(t, time.Second, f.Precision())
assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
// Exactly SHA-256, nothing else.
assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
}
func TestObjectHashReturnsContentHash(t *testing.T) {
o := &Object{file: api.FileItem{ContentHash: "abc123"}}
got, err := o.Hash(context.Background(), hash.SHA256)
require.NoError(t, err)
assert.Equal(t, "abc123", got)
// A multipart object with no server-side hash yields "" and no error,
// exactly as an S3 multipart object does.
empty := &Object{file: api.FileItem{ContentHash: ""}}
got, err = empty.Hash(context.Background(), hash.SHA256)
require.NoError(t, err)
assert.Equal(t, "", got)
// Any other algorithm is unsupported.
_, err = o.Hash(context.Background(), hash.MD5)
assert.ErrorIs(t, err, hash.ErrUnsupported)
}
func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
assert.True(t, mt.Equal(o.ModTime(context.Background())))
}
func TestModTimeHeaders(t *testing.T) {
assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
h := modTimeHeaders(mt)
require.NotNil(t, h)
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
}
func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
resp := &api.UploadCompleteResponse{}
resp.File.ID = "fil_1"
resp.File.Name = "x.txt"
resp.File.SizeBytes = 10
resp.File.CreatedAt = 1_000
resp.File.ContentHash = "deadbeef"
mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
item := fileFromUpload(resp, mt)
assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
assert.Equal(t, "deadbeef", item.ContentHash)
assert.Equal(t, float64(1_000), item.CreatedAt)
// A zero modtime falls back to the create time.
item = fileFromUpload(resp, time.Time{})
assert.Equal(t, float64(1_000), item.UpdatedAt)
}
// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
// fake transport and asserts the source-mtime header actually rides on the PUT.
func TestUploadSendsSourceMTimeHeader(t *testing.T) {
var putMTime string
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
}
switch {
case r.Method == "POST" && r.URL.Path == "/api/upload/init":
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
putMTime = r.Header.Get(sourceMTimeHeader)
return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
}
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
return nil, nil
})
f := newFakeFs(rt)
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, "h", resp.File.ContentHash)
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
}
// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
// the completion request for a multipart upload (the server stores the modtime
// there, not on the individual parts).
func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
var completeMTime string
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
if r.Body != nil {
_, _ = io.ReadAll(r.Body)
}
switch {
case r.URL.Path == "/api/upload/init":
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
return jsonResp(http.StatusOK, `{"ok":true}`), nil
case r.URL.Path == "/api/upload/upl_1/complete":
completeMTime = r.Header.Get(sourceMTimeHeader)
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
}
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
return nil, nil
})
f := newFakeFs(rt)
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
}
// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
// server-side Copy returns: it must report the SOURCE's modtime and hash (the
// copy is byte-identical and the API preserves both), not the copy's wall
// clock. Reading "now" here made rclone's in-memory object disagree with the
// one read back from the remote and failed FsCopy's fingerprint check.
func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
switch {
case r.Method == "GET" && r.URL.Path == "/api/files":
// NewObject's existence probe on the destination: nothing there.
return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
}
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
return nil, nil
})
f := newFakeFs(rt)
f.dirCache = dircache.New("", rootID, f)
mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
}}
dst, err := f.Copy(context.Background(), src, "f-copy.txt")
require.NoError(t, err)
assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
assert.Equal(t, int64(100), dst.Size())
got, err := dst.Hash(context.Background(), hash.SHA256)
require.NoError(t, err)
assert.Equal(t, "srchash", got, "copy keeps the source hash")
}
-156
View File
@@ -1,156 +0,0 @@
package dosya
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/backend/dosya/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
func TestErrorHandlerDecodesAPIError(t *testing.T) {
resp := &http.Response{
StatusCode: http.StatusBadRequest,
Status: "400 Bad Request",
Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),
}
err := errorHandler(resp)
var apiErr *api.Error
require.ErrorAs(t, err, &apiErr)
assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
assert.Contains(t, err.Error(), "uploads in progress")
assert.Contains(t, err.Error(), "concurrent_upload_limit")
}
func TestErrorHandlerNonJSONBody(t *testing.T) {
// e.g. an edge (Cloudflare) response that never reached the API
resp := &http.Response{
StatusCode: http.StatusTooManyRequests,
Status: "429 Too Many Requests",
Body: io.NopCloser(strings.NewReader("error code: 1015\n")),
}
err := errorHandler(resp)
var apiErr *api.Error
require.ErrorAs(t, err, &apiErr)
assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
assert.Equal(t, "", apiErr.ErrorCode)
assert.Contains(t, err.Error(), "429")
assert.Contains(t, err.Error(), "error code: 1015")
}
func TestShouldRetryAPIErrors(t *testing.T) {
ctx := context.Background()
// The concurrent upload limit is handled by initUpload's slot wait,
// not by the pacer's low level retries.
limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
assert.False(t, retry)
assert.Equal(t, limitErr, err)
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
assert.True(t, retry)
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
assert.False(t, retry)
}
// swapWaitTimings shortens the slot wait timings for a test and returns a
// function which restores them
func swapWaitTimings(delay, max time.Duration) func() {
oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
}
// newTestFs makes an Fs which talks to srv
func newTestFs(srv *httptest.Server) *Fs {
ctx := context.Background()
f := &Fs{
name: "test",
opt: Options{WorkspaceID: "ws_test"},
pacer: fs.NewPacer(ctx, pacer.NewDefault(
pacer.MinSleep(time.Millisecond),
pacer.MaxSleep(time.Millisecond),
pacer.DecayConstant(decayConstant),
pacer.AttackConstant(attackConstant),
)),
}
f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
return f
}
// limitServer answers the first n init calls with the concurrent upload
// limit error and the rest with success
func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/api/upload/init", r.URL.Path)
w.Header().Set("Content-Type", "application/json")
if calls.Add(1) <= n {
w.WriteHeader(http.StatusBadRequest)
_, _ = w.Write([]byte(concurrentLimitBody))
return
}
w.WriteHeader(http.StatusCreated)
_, _ = w.Write([]byte(`{"ok":true}`))
}))
}
func TestInitUploadWaitsForFreeSlot(t *testing.T) {
defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
var calls atomic.Int32
srv := limitServer(t, 2, &calls)
defer srv.Close()
f := newTestFs(srv)
resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
require.NoError(t, err)
assert.True(t, resp.OK)
assert.Equal(t, int32(3), calls.Load())
}
func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
var calls atomic.Int32
srv := limitServer(t, 1<<30, &calls)
defer srv.Close()
f := newTestFs(srv)
_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
require.Error(t, err)
var apiErr *api.Error
require.ErrorAs(t, err, &apiErr)
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
assert.Contains(t, err.Error(), "couldn't init upload")
assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
}
func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
defer swapWaitTimings(time.Minute, time.Hour)()
var calls atomic.Int32
srv := limitServer(t, 1<<30, &calls)
defer srv.Close()
f := newTestFs(srv)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
start := time.Now()
_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, context.DeadlineExceeded)
assert.Less(t, time.Since(start), 5*time.Second)
}
-68
View File
@@ -1,68 +0,0 @@
package dosya
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeListServer serves GET /api/files the way the API does: files paged at
// per_page, every folder repeated on each page
func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
return func(r *http.Request) (*http.Response, error) {
require.Equal(t, "/api/files", r.URL.Path)
q := r.URL.Query()
assert.Equal(t, "name", q.Get("sort"))
assert.Equal(t, "asc", q.Get("dir"))
page, _ := strconv.Atoi(q.Get("page"))
perPage, _ := strconv.Atoi(q.Get("per_page"))
require.Positive(t, page)
require.Positive(t, perPage)
*pages = append(*pages, page)
var files []string
for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
}
totalPages := max(1, (totalFiles+perPage-1)/perPage)
body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
strings.Join(files, ","), page, perPage, totalFiles, totalPages)
return jsonResp(http.StatusOK, body), nil
}
}
func TestListFilesAndFoldersFollowsPages(t *testing.T) {
var pages []int
f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
require.NoError(t, err)
assert.Equal(t, []int{1, 2, 3}, pages)
require.Len(t, listing.Files, 2*listPageSize+7)
assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
// folders repeat on every page and must not be duplicated
assert.Len(t, listing.Folders, 1)
}
func TestListFilesAndFoldersSinglePage(t *testing.T) {
for _, total := range []int{0, 1, listPageSize} {
t.Run(strconv.Itoa(total), func(t *testing.T) {
var pages []int
f := newFakeFs(fakeListServer(t, total, &pages))
listing, err := f.listFilesAndFolders(context.Background(), rootID)
require.NoError(t, err)
assert.Equal(t, []int{1}, pages)
assert.Len(t, listing.Files, total)
})
}
}
-153
View File
@@ -1,153 +0,0 @@
package dosya
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// parallelUploadServer answers the multipart endpoints without holding a
// lock across a request, so parts really do overlap, and records what the
// ordering rules below are about: how many parts were in flight at once, and
// whether any part started before part 1 had finished.
//
// Part 1 matters because the API creates the R2 multipart upload (and, for a
// new file, its id and key) on the first part it sees. Parts that race ahead
// of part 1 would each create their own upload, and only one of those would
// be completed.
type parallelUploadServer struct {
t *testing.T
resumable string
partDelay time.Duration
failPart int // answer this part with a 400, 0 for none
mu sync.Mutex
inFlight int
maxFlight int
part1Done bool
earlyParts []int // parts that started before part 1 finished
parts map[int]string
completes int
aborts int
}
func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
var body []byte
if r.Body != nil {
var err error
body, err = io.ReadAll(r.Body)
require.NoError(s.t, err)
}
path := r.URL.Path
switch {
case r.Method == "POST" && path == "/api/upload/init":
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
var n int
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
require.NoError(s.t, err)
s.mu.Lock()
if n != 1 && !s.part1Done {
s.earlyParts = append(s.earlyParts, n)
}
s.inFlight++
s.maxFlight = max(s.maxFlight, s.inFlight)
s.mu.Unlock()
time.Sleep(s.partDelay)
s.mu.Lock()
s.inFlight--
if n == 1 {
s.part1Done = true
}
s.parts[n] = string(body)
fail := n == s.failPart
s.mu.Unlock()
if fail {
return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
}
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
s.mu.Lock()
s.completes++
s.mu.Unlock()
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
case r.Method == "DELETE" && path == "/api/upload/upl_1":
s.mu.Lock()
s.aborts++
s.mu.Unlock()
return jsonResp(http.StatusOK, `{"ok":true}`), nil
}
s.t.Fatalf("unexpected request %s %s", r.Method, path)
return nil, nil
}
func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
return &parallelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
}
const tenParts = `{"part_size":2,"total_parts":10}`
func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
srv := newParallelServer(t, tenParts)
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 4
content := "aabbccddeeffgghhiijj"
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
assert.Equal(t, 1, srv.completes)
for i := 1; i <= 10; i++ {
assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
}
}
func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
srv := newParallelServer(t, tenParts)
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 1
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, 1, srv.maxFlight)
assert.Equal(t, 1, srv.completes)
}
func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
srv := newParallelServer(t, tenParts)
srv.failPart = 6
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 4
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.Contains(t, err.Error(), "part 6")
assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
}
func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
// 20 bytes declared, 15 supplied: the short read is found by the
// reader, whichever parts are already in flight, and nothing completes.
srv := newParallelServer(t, tenParts)
f := newFakeFs(rtFunc(srv.roundTrip))
f.opt.UploadConcurrency = 4
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.Equal(t, 0, srv.completes)
assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
}
-15
View File
@@ -1,15 +0,0 @@
// Test Dosya filesystem interface
package dosya
import (
"testing"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestDosya:",
})
}
-247
View File
@@ -1,247 +0,0 @@
package dosya
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"sync"
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// rtFunc adapts a function into an http.RoundTripper so the upload path
// can be driven without a network. Unlike a real server it always reads
// the whole request body and answers, which is exactly the misbehaviour
// (accepting a short body as a complete upload) these tests guard against.
type rtFunc func(*http.Request) (*http.Response, error)
func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func jsonResp(status int, body string) *http.Response {
return &http.Response{
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
Header: http.Header{"Content-Type": {"application/json"}},
Body: io.NopCloser(strings.NewReader(body)),
ContentLength: int64(len(body)),
}
}
func newFakeFs(rt http.RoundTripper) *Fs {
ctx := context.Background()
f := &Fs{
name: "test",
opt: Options{WorkspaceID: "ws_test"},
pacer: fs.NewPacer(ctx, pacer.NewDefault(
pacer.MinSleep(time.Millisecond),
pacer.MaxSleep(time.Millisecond),
pacer.DecayConstant(decayConstant),
pacer.AttackConstant(attackConstant),
)),
}
f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
return f
}
// fakeUploadServer records what an upload sends and lets a test inject
// failures per request
type fakeUploadServer struct {
t *testing.T
mu sync.Mutex
resumable string // JSON for the init "resumable" field, "" for a single PUT
puts [][]byte // bodies of PUT /api/upload/<id>
parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
completes int
aborts int // DELETE /api/upload/<id>
failFirst map[string]bool // path -> answer the first request with a 502
seenBefore map[string]bool
}
func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
return &fakeUploadServer{
t: t,
resumable: resumable,
parts: map[int][][]byte{},
failFirst: map[string]bool{},
seenBefore: map[string]bool{},
}
}
func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
s.mu.Lock()
defer s.mu.Unlock()
var body []byte
if r.Body != nil {
var err error
body, err = io.ReadAll(r.Body)
require.NoError(s.t, err)
}
path := r.URL.Path
// Bodies are recorded for every attempt, including the ones answered
// with a 502, so tests can see exactly what each attempt carried.
fail := s.failFirst[path] && !s.seenBefore[path]
s.seenBefore[path] = true
bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
switch {
case r.Method == "POST" && path == "/api/upload/init":
if s.resumable != "" {
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
}
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
case r.Method == "PUT" && path == "/api/upload/upl_1":
s.puts = append(s.puts, body)
if fail {
return bad, nil
}
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
var n int
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
require.NoError(s.t, err)
s.parts[n] = append(s.parts[n], body)
if fail {
return bad, nil
}
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
s.completes++
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
case r.Method == "DELETE" && path == "/api/upload/upl_1":
s.aborts++
return jsonResp(http.StatusOK, `{"ok":true}`), nil
}
s.t.Fatalf("unexpected request %s %s", r.Method, path)
return nil, nil
}
func TestUploadSmallSendsWholeBody(t *testing.T) {
srv := newFakeUploadServer(t, "")
f := newFakeFs(rtFunc(srv.roundTrip))
content := strings.Repeat("a", 100)
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, "fil_1", resp.File.ID)
require.Len(t, srv.puts, 1)
assert.Equal(t, content, string(srv.puts[0]))
}
func TestUploadSmallRejectsShortSource(t *testing.T) {
// The source declares 100 bytes but only supplies 60. A server which
// accepts that as a complete upload must not turn into a success.
srv := newFakeUploadServer(t, "")
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
}
func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
// A retried PUT would resend an already-consumed reader, i.e. an
// empty body, which the server would store as an empty file. The
// single PUT therefore must not be retried at the low level.
srv := newFakeUploadServer(t, "")
srv.failFirst["/api/upload/upl_1"] = true
f := newFakeFs(rtFunc(srv.roundTrip))
content := strings.Repeat("a", 100)
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
assert.Equal(t, content, string(srv.puts[0]))
}
const threeParts = `{"part_size":4,"total_parts":3}`
func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
srv := newFakeUploadServer(t, threeParts)
f := newFakeFs(rtFunc(srv.roundTrip))
content := "0123456789"
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, "fil_1", resp.File.ID)
assert.Equal(t, 1, srv.completes)
require.Len(t, srv.parts, 3)
assert.Equal(t, "0123", string(srv.parts[1][0]))
assert.Equal(t, "4567", string(srv.parts[2][0]))
assert.Equal(t, "89", string(srv.parts[3][0]))
}
func TestUploadMultipartRejectsShortSource(t *testing.T) {
// 10 bytes declared, 8 supplied: the last part must never be sent
// and the upload must not be completed.
srv := newFakeUploadServer(t, threeParts)
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
assert.Equal(t, 0, srv.completes, "upload must not be completed")
}
func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
// A part which fails transiently must be resent with all its bytes,
// not with whatever is left of a consumed reader.
srv := newFakeUploadServer(t, threeParts)
srv.failFirst["/api/upload/upl_1/part/2"] = true
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
assert.Equal(t, "4567", string(srv.parts[2][0]))
assert.Equal(t, "4567", string(srv.parts[2][1]))
assert.Equal(t, "89", string(srv.parts[3][0]))
assert.Equal(t, 1, srv.completes)
}
// An upload that fails after init has to hand its session back: the session
// holds one of the workspace's upload slots for ten minutes otherwise, and
// rclone's high-level retry opens a new one rather than resuming it.
func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
for _, tc := range []struct {
name string
resumable string
content string
size int64
}{
{"small file short source", "", "0123", 10},
{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
} {
t.Run(tc.name, func(t *testing.T) {
srv := newFakeUploadServer(t, tc.resumable)
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.Error(t, err)
assert.Equal(t, 1, srv.aborts)
})
}
}
// A successful upload must not abort anything
func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
srv := newFakeUploadServer(t, "")
f := newFakeFs(rtFunc(srv.roundTrip))
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
require.NoError(t, err)
assert.Equal(t, 0, srv.aborts)
}
@@ -1,73 +0,0 @@
package dosya
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Every by-id call names the workspace, so the API can find an id it minted
// moments ago before its file_home index row lands.
func TestByIDCallsSendWorkspaceHint(t *testing.T) {
seen := map[string]string{}
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
switch r.URL.Path {
case "/api/folders/fld_1":
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
case "/api/files/fil_1":
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
case "/api/folders/fld_1/move":
return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
default:
return jsonResp(http.StatusOK, `{"ok":true}`), nil
}
}))
ctx := context.Background()
require.NoError(t, f.removeFolder(ctx, "fld_1", false))
require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
require.NoError(t, err)
require.NoError(t, f.deleteFile(ctx, "fil_1"))
require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
require.NoError(t, err)
_, err = f.createShareLink(ctx, "fil_1", 0)
require.NoError(t, err)
for call, header := range seen {
assert.Equal(t, "ws_test", header, call)
}
assert.Len(t, seen, 8)
}
// The download's API request carries the hint; the presigned R2 fetch must not.
func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
var apiHint, blobHint string
var srv *httptest.Server
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/files/fil_1/download" {
apiHint = r.Header.Get("X-Dosya-Workspace")
http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
return
}
blobHint = r.Header.Get("X-Dosya-Workspace")
_, _ = w.Write([]byte("bytes"))
}))
defer srv.Close()
f := newTestFs(srv)
f.opt.APIURL = srv.URL
rc, err := f.downloadFile(context.Background(), "fil_1", nil)
require.NoError(t, err)
_ = rc.Close()
assert.Equal(t, "ws_test", apiHint)
assert.Equal(t, "", blobHint)
}
-52
View File
@@ -1,52 +0,0 @@
package dosya
import (
"context"
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// fakeWorkspaceServer answers GET /api/workspaces with the given body
func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
return func(r *http.Request) (*http.Response, error) {
require.Equal(t, "/api/workspaces", r.URL.Path)
*calls++
return jsonResp(http.StatusOK, body), nil
}
}
func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
calls := 0
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
f.opt.WorkspaceID = ""
require.NoError(t, f.findWorkspace(context.Background()))
assert.Equal(t, "ws_only", f.opt.WorkspaceID)
assert.Equal(t, 1, calls)
}
func TestFindWorkspaceListsTheChoices(t *testing.T) {
calls := 0
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
f.opt.WorkspaceID = ""
err := f.findWorkspace(context.Background())
require.Error(t, err)
// the ids have to be in the message: they are the only place the user
// can read them, which is the whole point of listing them
assert.Contains(t, err.Error(), "ws_one (One)")
assert.Contains(t, err.Error(), "ws_two (Two)")
assert.Empty(t, f.opt.WorkspaceID)
}
func TestFindWorkspaceWithNone(t *testing.T) {
calls := 0
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
f.opt.WorkspaceID = ""
require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
}
+6 -5
View File
@@ -21,7 +21,7 @@ type User struct {
Avatar string `json:"avatar"`
ModelType string `json:"model_type"`
OwnsEntry bool `json:"owns_entry"`
EntryPermissions any `json:"entry_permissions"`
EntryPermissions []any `json:"entry_permissions"`
DisplayName string `json:"display_name"`
}
@@ -57,6 +57,7 @@ type Item struct {
FileHash any `json:"file_hash"`
Public bool `json:"public"`
Thumbnail bool `json:"thumbnail"`
MuxStatus any `json:"mux_status"`
ThumbnailURL any `json:"thumbnail_url"`
WorkspaceID int `json:"workspace_id"`
IsEncrypted int `json:"is_encrypted"`
@@ -76,9 +77,9 @@ type Listing struct {
Data []Item `json:"data"`
From int `json:"from"`
LastPage int `json:"last_page"`
NextPageURL string `json:"next_page_url"`
NextPage int `json:"next_page"`
PerPage int `json:"per_page"`
PrevPageURL string `json:"prev_page_url"`
PrevPage int `json:"prev_page"`
To int `json:"to"`
Total int `json:"total"`
}
@@ -170,7 +171,7 @@ type MultiPartCreateRequest struct {
Mime string `json:"mime"`
Size int64 `json:"size"`
Extension string `json:"extension"`
ParentID json.Number `json:"parentId"`
ParentID json.Number `json:"parent_id"`
RelativePath string `json:"relativePath"`
WorkspaceID string `json:"workspaceId,omitempty"`
}
@@ -221,7 +222,7 @@ type MultiPartEntriesRequest struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
ClientExtension string `json:"clientExtension"`
ParentID json.Number `json:"parentId"`
ParentID json.Number `json:"parent_id"`
RelativePath string `json:"relativePath"`
WorkspaceID string `json:"workspaceId,omitempty"`
}
+37 -132
View File
@@ -54,7 +54,7 @@ const (
rootURL = baseURL + "api/v1"
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
minChunkSize = fs.SizeSuffix(1024 * 1024 * 5)
defaultUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024) // as per https://docs.drime.cloud/uploads-guide
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
)
// Register with Fs
@@ -146,6 +146,14 @@ and these uploads do not fully utilize your bandwidth, then increasing
this may help to speed up the transfers.`,
Default: 4,
Advanced: true,
}, {
Name: "upload_cutoff",
Help: `Cutoff for switching to chunked upload.
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5 GiB.`,
Default: defaultUploadCutoff,
Advanced: true,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
@@ -270,11 +278,6 @@ var retryErrorCodes = []int{
503, // Service Unavailable
504, // Gateway Timeout
509, // Bandwidth Limit Exceeded
520, // Cloudflare: Web server returns an unknown error
521, // Cloudflare: Web server is down
522, // Cloudflare: Connection timed out
523, // Cloudflare: Origin is unreachable
524, // Cloudflare: A timeout occurred
}
// shouldRetry returns a boolean as to whether this resp and err
@@ -321,13 +324,10 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
}
return false
})
if err != nil {
return nil, err
}
if !found {
return nil, fs.ErrorObjectNotFound
}
return info, nil
return info, err
}
// errorHandler parses a non 2xx error response into an error
@@ -527,7 +527,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
Parameters: url.Values{},
}
if dirID != "" {
opts.Parameters.Add("folderId", dirID)
opts.Parameters.Add("parentIds", dirID)
}
if directoriesOnly {
opts.Parameters.Add("type", api.ItemTypeFolder)
@@ -549,11 +549,6 @@ OUTER:
if err != nil {
return found, fmt.Errorf("couldn't list files: %w", err)
}
// Drime won't page past a fixed number of entries (20,000 for
// rclone) and serves the last page it allows again instead
if result.CurrentPage != page {
return found, errors.New("couldn't list files: directory has more entries than drime will list")
}
for _, item := range result.Data {
if item.Type == api.ItemTypeFolder {
if filesOnly {
@@ -570,10 +565,10 @@ OUTER:
break OUTER
}
}
if result.CurrentPage == result.LastPage {
if result.NextPage == 0 {
break
}
page = result.CurrentPage + 1
page = result.NextPage
}
return found, err
}
@@ -700,27 +695,13 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// deleteObject removes an object by ID
func (f *Fs) deleteObject(ctx context.Context, id string) error {
if f.opt.HardDelete {
// Deleting forever doesn't invalidate the server's cache of
// the parent folder listing, so the entry keeps being listed
// for about a second. Moving to the trash first does.
err := f.deleteEntry(ctx, id, false)
if err != nil {
return err
}
}
return f.deleteEntry(ctx, id, f.opt.HardDelete)
}
// deleteEntry moves the entry with ID to the trash or deletes it forever
func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) error {
opts := rest.Opts{
Method: "POST",
Path: "/file-entries/delete",
}
request := api.DeleteRequest{
EntryIDs: []string{id},
DeleteForever: deleteForever,
DeleteForever: f.opt.HardDelete,
}
var result api.DeleteResponse
err := f.pacer.Call(func() (bool, error) {
@@ -728,11 +709,6 @@ func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) err
return shouldRetry(ctx, resp, err)
})
if err != nil {
// If Drime returns 422 with invalid entry ID, the object is already gone.
// Map this to a standard not-found error
if strings.Contains(err.Error(), "entry ids is invalid") {
return fs.ErrorObjectNotFound
}
return fmt.Errorf("failed to delete item: %w", err)
}
// Check the individual result codes also
@@ -806,16 +782,10 @@ func (f *Fs) patch(ctx context.Context, id, attribute string, value string) (ite
Name: value,
}
var result api.UpdateItemResponse
// The drime origin returns a malformed response (seen by Cloudflare as
// a 520) for a literal PUT to this endpoint, so use a POST with Laravel
// method spoofing instead - the API routes it to the same handler.
opts := rest.Opts{
Method: "POST",
Method: "PUT",
Path: "/file-entries/" + id,
Parameters: url.Values{},
ExtraHeaders: map[string]string{
"X-HTTP-Method-Override": "PUT",
},
}
if f.opt.WorkspaceID != "" {
opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
@@ -949,48 +919,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return dstObj, nil
}
// countDirChildren returns the number of direct children of dirID.
func (f *Fs) countDirChildren(ctx context.Context, dirID string) (int, error) {
n := 0
_, err := f.listAll(ctx, dirID, false, false, "", func(*api.Item) bool {
n++
return false
})
return n, err
}
// waitForDirChildren polls dirID until it lists at least want direct
// children, or the timeout elapses. Drime's folder rename has visible
// eventual-consistency on the children listing - the rename returns
// success immediately but the renamed folder lists as empty for a
// short window - so callers that rename a folder use this to wait for
// the listing to settle before returning (see issue #9450).
func (f *Fs) waitForDirChildren(ctx context.Context, dirID string, want int) error {
const timeout = 30 * time.Second
deadline := time.Now().Add(timeout)
backoff := 100 * time.Millisecond
for {
got, err := f.countDirChildren(ctx, dirID)
if err != nil {
return err
}
if got >= want {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("timed out waiting for directory listing to settle: got %d, want %d", got, want)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(backoff):
}
if backoff < 2*time.Second {
backoff *= 2
}
}
}
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
@@ -1011,25 +939,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return err
}
// Count children before the move so we can wait for the server's
// children listing to reflect the rename - see waitForDirChildren.
preCount, err := f.countDirChildren(ctx, srcID)
if err != nil {
return fmt.Errorf("DirMove: count source children: %w", err)
}
// Do the move
info, err := f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
_, err = f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
if err != nil {
return err
}
if preCount > 0 {
if err := f.waitForDirChildren(ctx, info.ID.String(), preCount); err != nil {
fs.Logf(f, "DirMove: %v (continuing)", err)
}
}
srcFs.dirCache.FlushDir(srcRemote)
return nil
}
@@ -1062,28 +976,17 @@ func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err
}
// copy and rename a file or folder to directoryID with leaf
//
// If existing is set it is removed once the copy has succeeded.
func (f *Fs) copyTo(ctx context.Context, srcID, dstLeaf, dstDirectoryID string, existing fs.Object) (info *api.Item, err error) {
func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
// Can have duplicates so don't have to be careful here
// Copy to dstDirectoryID first
info, err = f.copy(ctx, srcID, dstDirectoryID)
if err != nil {
return nil, err
}
// The server names the copy "name (1)" if the destination
// directory already has an entry called "name" and refuses to
// rename it to "name" until that entry is removed.
if existing != nil {
fs.Debugf(existing, "Server side copy: removing existing object after successful copy")
err = existing.Remove(ctx)
if err != nil {
return nil, err
}
}
// Rename if required
if info.Name != f.opt.Enc.FromStandardName(dstLeaf) {
if srcLeaf != dstLeaf {
info, err = f.rename(ctx, info.ID.String(), dstLeaf)
if err != nil {
return nil, err
@@ -1107,6 +1010,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
fs.Debugf(src, "Can't copy - not same remote type")
return nil, fs.ErrorCantCopy
}
srcLeaf := path.Base(srcObj.remote)
srcPath := srcObj.fs.rootSlash() + srcObj.remote
dstPath := f.rootSlash() + remote
if srcPath == dstPath {
@@ -1115,8 +1020,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
// Find existing object
existingObj, err := f.NewObject(ctx, remote)
if err != nil {
existingObj = nil
if err == nil {
defer func() {
// Don't remove existing object if returning an error
if err != nil {
return
}
fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
err = existingObj.Remove(ctx)
}()
}
// Create temporary object
@@ -1126,7 +1038,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
}
// Copy the object
info, err := f.copyTo(ctx, srcObj.id, dstLeaf, dstDirectoryID, existingObj)
info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
if err != nil {
return nil, err
}
@@ -1184,11 +1096,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
return info, nil, err
}
// Send just the leaf as relativePath, matching the single-part /uploads
// convention. The file is placed by parentId; sending an absolute path
// here makes the server build folders from it and drop "0" path segments.
relPath := f.opt.Enc.FromStandardName(leaf)
// Temporary Object under construction
o := &Object{
fs: f,
@@ -1222,7 +1129,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
Size: createSize,
Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
ParentID: json.Number(directoryID),
RelativePath: relPath,
RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
WorkspaceID: f.opt.WorkspaceID,
}
@@ -1249,6 +1156,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
if ext == "" {
ext = "bin"
}
rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
chunkWriter := &drimeChunkWriter{
uploadID: resp.UploadID,
key: resp.Key,
@@ -1261,7 +1170,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
mime: mime,
extension: ext,
parentID: json.Number(directoryID),
relativePath: relPath,
relativePath: rel,
}
info = fs.ChunkWriterInfo{
ChunkSize: int64(chunkSize),
@@ -1621,11 +1530,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
fs.Debugf(o, "Removing old object on successful upload")
deleteErr := o.fs.deleteObject(ctx, id)
if deleteErr != nil {
if errors.Is(deleteErr, fs.ErrorObjectNotFound) {
fs.Debugf(o, "Old object already deleted, safely ignoring")
} else {
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
}
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
}
}()
}
-158
View File
@@ -1,158 +0,0 @@
package drime
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/rclone/rclone/backend/drime/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
testPerPage = 5 // entries per page the test server returns
testLimit = 20 // entries the test server will page through
)
// listingServer serves /drive/file-entries from entries the way Drime
// does.
//
// It returns testPerPage entries per page and won't page past testLimit
// entries. Asking for a later page returns the last page it allows
// again, with last_page one beyond it.
//
// It counts the requests made in requests.
func listingServer(t *testing.T, entries []api.Item, requests *atomic.Int64) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
assert.Equal(t, "/drive/file-entries", r.URL.Path)
params := r.URL.Query()
page, err := strconv.Atoi(params.Get("page"))
require.NoError(t, err)
maxPage := testLimit / testPerPage
currentPage := min(page, maxPage)
start := min((currentPage-1)*testPerPage, len(entries))
end := min(start+testPerPage, len(entries))
lastPage := currentPage
if page > maxPage || end < len(entries) {
lastPage = currentPage + 1
}
w.Header().Set("Content-Type", "application/json")
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
"current_page": currentPage,
"last_page": lastPage,
"per_page": testPerPage,
"total": (currentPage + 1) * testPerPage,
"data": entries[start:end],
}))
}))
}
// makeEntries makes n file entries
func makeEntries(n int) []api.Item {
entries := make([]api.Item, n)
for i := range entries {
entries[i] = api.Item{
ID: json.Number(strconv.Itoa(1000 + i)),
Name: fmt.Sprintf("file%05d", i),
Type: "text",
}
}
return entries
}
// testFs makes an Fs which uses server as its API
func testFs(t *testing.T, server *httptest.Server) *Fs {
t.Helper()
f := &Fs{
opt: Options{ListChunk: 1000},
srv: rest.NewClient(server.Client()).SetRoot(server.URL),
}
f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
f.srv.SetErrorHandler(errorHandler)
return f
}
// listNames lists a directory with listAll and returns the names found
func listNames(ctx context.Context, f *Fs) (names []string, err error) {
_, err = f.listAll(ctx, "1", false, false, "", func(item *api.Item) bool {
names = append(names, item.Name)
return false
})
return names, err
}
// TestListAll checks that listAll returns every entry exactly once
func TestListAll(t *testing.T) {
for _, test := range []struct {
name string
entries int
}{
{name: "Empty", entries: 0},
{name: "OnePage", entries: 3},
{name: "ExactPages", entries: 10},
{name: "PartialLastPage", entries: 13},
{name: "AtLimit", entries: testLimit},
} {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var requests atomic.Int64
entries := makeEntries(test.entries)
server := listingServer(t, entries, &requests)
defer server.Close()
f := testFs(t, server)
names, err := listNames(ctx, f)
require.NoError(t, err)
var want []string
for _, entry := range entries {
want = append(want, entry.Name)
}
assert.ElementsMatch(t, want, names)
})
}
}
// TestListAllTooManyEntries checks that listAll returns an error for a
// directory with more entries than the server will list, rather than
// looping forever.
func TestListAllTooManyEntries(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var requests atomic.Int64
server := listingServer(t, makeEntries(2*testLimit), &requests)
defer server.Close()
f := testFs(t, server)
_, err := listNames(ctx, f)
require.Error(t, err)
assert.NoError(t, ctx.Err(), "listing didn't finish")
assert.LessOrEqual(t, requests.Load(), int64(testLimit/testPerPage+1))
}
// TestGetItemListError checks that getItem returns listing errors
// rather than fs.ErrorObjectNotFound.
func TestGetItemListError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"message":"access denied"}`, http.StatusForbidden)
}))
defer server.Close()
f := testFs(t, server)
_, err := f.getItem(context.Background(), "1000", "1", "file00000")
require.Error(t, err)
assert.NotErrorIs(t, err, fs.ErrorObjectNotFound)
assert.Contains(t, err.Error(), "access denied")
}
+24 -104
View File
@@ -144,7 +144,6 @@ var (
_importFormats map[string][]string // allowed import MIME type conversions
templatesOnce sync.Once // parse link templates only once
_linkTemplates map[string]*template.Template // available link types
gdocsWarnOnce sync.Once // warn once about skipped non-exportable Google documents
)
// rwChoices type for fs.Bits
@@ -208,10 +207,7 @@ func driveOAuthOptions() []fs.Option {
opts := []fs.Option{}
for _, opt := range oauthutil.SharedOptions {
if opt.Name == config.ConfigClientID {
opt.Help = "Google Application Client Id\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nThe shared client_id is being retired and will stop working during 2026, so creating your own is now strongly recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own."
}
if opt.Name == config.ConfigClientSecret {
opt.Help = "Google Application Client Secret\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nIf you created your own client_id then enter its client secret here."
opt.Help = "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance."
}
opts = append(opts, opt)
}
@@ -225,7 +221,7 @@ func init() {
Description: "Google Drive",
NewFs: NewFs,
CommandHelp: commandHelp,
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
@@ -233,7 +229,7 @@ func init() {
return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
}
switch configIn.State {
switch config.State {
case "":
// Fill in the scopes
driveConfig.Scopes = driveScopes(opt.Scope)
@@ -244,43 +240,24 @@ func init() {
}
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
return fs.ConfigGoto("client_id")
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
OAuth2Config: driveConfig,
})
}
return fs.ConfigGoto("teamdrive")
case "client_id":
if clientID, _ := m.Get(config.ConfigClientID); clientID != "" {
return fs.ConfigGoto("oauth")
}
return oauthutil.SharedClientIDConfigConfirm("client_id_warning", "Google Drive", "https://rclone.org/drive/#making-your-own-client-id")
case "client_id_warning":
if configIn.Result == "true" {
// Continue using the shared client_id
return fs.ConfigGoto("oauth")
}
return fs.ConfigInput("client_id_set", config.ConfigClientID, "Google Application Client Id")
case "client_id_set":
m.Set(config.ConfigClientID, configIn.Result)
return fs.ConfigInput("client_secret_set", config.ConfigClientSecret, "Google Application Client Secret")
case "client_secret_set":
m.Set(config.ConfigClientSecret, configIn.Result)
return fs.ConfigGoto("oauth")
case "oauth":
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
OAuth2Config: driveConfig,
})
case "teamdrive":
if opt.TeamDriveID == "" {
return fs.ConfigConfirm("teamdrive_ok", false, "config_change_team_drive", "Configure this as a Shared Drive (Team Drive)?\n")
}
return fs.ConfigConfirm("teamdrive_change", false, "config_change_team_drive", fmt.Sprintf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID))
case "teamdrive_ok":
if configIn.Result == "false" {
if config.Result == "false" {
m.Set("team_drive", "")
return nil, nil
}
return fs.ConfigGoto("teamdrive_config")
case "teamdrive_change":
if configIn.Result == "false" {
if config.Result == "false" {
return nil, nil
}
return fs.ConfigGoto("teamdrive_config")
@@ -301,14 +278,14 @@ func init() {
return teamDrive.Id, teamDrive.Name
})
case "teamdrive_final":
driveID := configIn.Result
driveID := config.Result
m.Set("team_drive", driveID)
m.Set("root_folder_id", "")
opt.TeamDriveID = driveID
opt.RootFolderID = ""
return nil, nil
}
return nil, fmt.Errorf("unknown state %q", configIn.State)
return nil, fmt.Errorf("unknown state %q", config.State)
},
MetadataInfo: &fs.MetadataInfo{
System: systemMetadataInfo,
@@ -568,7 +545,7 @@ recommended usage is using the flag form --drive-size-as-quota when
doing rclone ls/lsl/lsf/lsjson/etc only.
If you do use this flag for syncing (not recommended) then you will
need to use --ignore-size also.`,
need to use --ignore size also.`,
Advanced: true,
Hide: fs.OptionHideConfigurator,
}, {
@@ -1325,7 +1302,6 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
return nil, fmt.Errorf("failed to create client from environment: %w", err)
}
} else {
oauthutil.SharedClientIDWarning(name, "Google Drive", "https://rclone.org/drive/#making-your-own-client-id", m)
oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
if err != nil {
return nil, fmt.Errorf("failed to create oauth client: %w", err)
@@ -1710,9 +1686,6 @@ func (f *Fs) newObjectWithExportInfo(
// If item MimeType is in the ExportFormats then it is a google doc
if !isDocument {
fs.Debugf(remote, "Ignoring unknown document type %q", info.MimeType)
gdocsWarnOnce.Do(func() {
fs.Logf(remote, "Skipping unexportable google document %q. Use --drive-show-all-gdocs to include them in server side copy and move", info.MimeType)
})
return nil, fs.ErrorObjectNotFound
}
if extension == "" {
@@ -2298,9 +2271,15 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
wg.Wait()
// if the input channel overflowed add the collected entries to the channel now
for len(overflow) > 0 {
if !refillListRInput(&mu, &in, &overflow, &wg) {
return
mu.Lock()
// only fill half of the channel to prevent entries being put into overflow again
l := min(len(overflow), listRInputBuffer/2)
wg.Add(l)
for _, d := range overflow[:l] {
in <- d
}
overflow = overflow[l:]
mu.Unlock()
// wait again for the completion of all directories
wg.Wait()
@@ -2348,25 +2327,6 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return nil
}
// refillListRInput moves queued directory jobs back onto in.
// It returns false when a listing error has already closed in.
// Sending on that nil channel would block forever while holding mu.
func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {
mu.Lock()
defer mu.Unlock()
if *in == nil {
return false
}
// only fill half of the channel to prevent entries being put into overflow again
l := min(len(*overflow), listRInputBuffer/2)
wg.Add(l)
for _, d := range (*overflow)[:l] {
*in <- d
}
*overflow = (*overflow)[l:]
return true
}
const shortcutSeparator = '\t'
// joinID adds an actual drive ID to the shortcut ID it came from
@@ -2452,40 +2412,12 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
return newItem, nil
}
// shortcutLoop reports whether a folder shortcut at remote pointing to
// the directory targetID would create a recursive loop.
//
// This is the case when targetID is the ID of remote's parent or any
// other ancestor directory, as descending into it would list an
// ancestor again forever. The ancestor IDs are looked up in the
// directory cache which is always populated for the parents of remote
// as they are listed before their children.
func (f *Fs) shortcutLoop(remote, targetID string) bool {
for dir := path.Dir(remote); ; dir = path.Dir(dir) {
if dir == "." {
dir = ""
}
if id, ok := f.dirCache.Get(dir); ok && actualID(id) == targetID {
return true
}
if dir == "" {
return false
}
}
}
// itemToDirEntry converts a drive.File to an fs.DirEntry.
// When the drive.File cannot be represented as an fs.DirEntry
// (nil, nil) is returned.
func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
switch {
case item.MimeType == driveFolderType:
// A folder shortcut pointing at one of its own ancestors
// would make rclone recurse forever, so drop it.
if isShortcutID(item.Id) && f.shortcutLoop(remote, actualID(item.Id)) {
fs.Errorf(remote, "Ignoring folder shortcut as it points to an ancestor directory creating a loop")
return nil, nil
}
// cache the directory ID for later lookups
f.dirCache.Put(remote, item.Id)
// cache the resource key for later lookups
@@ -2779,12 +2711,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
}
trashedFiles := false
if check {
// trashedFiles is only consulted to decide between trashing and hard deleting
// below, so when the directory is going to be trashed anyway there is no reason
// to enumerate trashed children: let the server filter them out instead of
// paging through them. Hard deletes still need to see them (#1040).
includeAll := !f.opt.UseTrash || f.opt.TrashedOnly
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, includeAll, func(item *drive.File) bool {
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
if !item.Trashed {
fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
return true
@@ -3107,18 +3034,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
remote = remote[:len(remote)-len(ext)]
}
// Find the ID of the parent to remove the file from.
var srcParentID string
if len(srcObj.parents) == 1 {
srcParentID = srcObj.parents[0]
} else {
var err error
_, srcParentID, err = srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
if err != nil {
return nil, err
}
srcParentID = actualID(srcParentID)
_, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
if err != nil {
return nil, err
}
srcParentID = actualID(srcParentID)
// Temporary Object under construction
dstInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
-259
View File
@@ -8,14 +8,10 @@ import (
"fmt"
"io"
"mime"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strings"
stdsync "sync"
"sync/atomic"
"testing"
"time"
@@ -28,114 +24,13 @@ import (
"github.com/rclone/rclone/fs/sync"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"google.golang.org/api/option"
)
func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
t.Helper()
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
service, err := drive.NewService(
context.Background(),
option.WithHTTPClient(server.Client()),
option.WithEndpoint(server.URL+"/"),
)
require.NoError(t, err)
return &Fs{
svc: service,
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
permissionsMu: new(stdsync.Mutex),
permissions: make(map[string]*drive.Permission),
}
}
func TestInternalGetPermissionConcurrent(t *testing.T) {
const requestCount = 2
arrived := make(chan string, requestCount)
release := make(chan struct{})
var releaseOnce stdsync.Once
releaseRequests := func() {
releaseOnce.Do(func() { close(release) })
}
defer releaseRequests()
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
permissionID := path.Base(r.URL.Path)
arrived <- permissionID
<-release
w.Header().Set("Content-Type", "application/json")
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
assert.NoError(t, err)
}))
type permissionResult struct {
permission *drive.Permission
inherited bool
err error
}
results := make(chan permissionResult, requestCount)
permissionIDs := []string{"permission-1", "permission-2"}
for _, permissionID := range permissionIDs {
go func() {
permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
results <- permissionResult{permission: permission, inherited: inherited, err: err}
}()
}
arrivedIDs := make(map[string]bool, requestCount)
timer := time.NewTimer(time.Second)
defer timer.Stop()
for len(arrivedIDs) < requestCount {
select {
case permissionID := <-arrived:
arrivedIDs[permissionID] = true
case <-timer.C:
releaseRequests()
t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
}
}
releaseRequests()
assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
returnedIDs := make(map[string]bool, requestCount)
for range requestCount {
result := <-results
require.NoError(t, result.err)
require.NotNil(t, result.permission)
assert.False(t, result.inherited)
returnedIDs[result.permission.Id] = true
}
assert.Equal(t, arrivedIDs, returnedIDs)
}
func TestInternalGetPermissionCacheHit(t *testing.T) {
var requests atomic.Int32
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
permissionID := path.Base(r.URL.Path)
w.Header().Set("Content-Type", "application/json")
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
assert.NoError(t, err)
}))
first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
require.NoError(t, err)
assert.False(t, inherited)
second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
require.NoError(t, err)
assert.False(t, inherited)
assert.Same(t, first, second)
assert.Equal(t, int32(1), requests.Load())
}
func TestDriveScopes(t *testing.T) {
for _, test := range []struct {
in string
@@ -250,34 +145,6 @@ func TestInternalFindExportFormat(t *testing.T) {
}
}
func TestShortcutLoop(t *testing.T) {
f := &Fs{}
f.dirCache = dircache.New("", "root", f)
// root is FolderX with ID "X"
f.dirCache.Put("", "X")
// a real subfolder
f.dirCache.Put("sub", "sub-id")
// a nested subfolder reached via a shortcut, composite ID
f.dirCache.Put("sub/nested", joinID("nested-id", "shortcut-id"))
for _, test := range []struct {
name string
remote string
targetID string
want bool
}{
{"shortcut to the root itself", "loop", "X", true},
{"shortcut to the parent", "sub/loop", "sub-id", true},
{"shortcut to a grandparent", "sub/nested/loop", "X", true},
{"shortcut to a composite ancestor", "sub/nested/loop", "nested-id", true},
{"shortcut to an unrelated folder", "sub/loop", "other-id", false},
{"shortcut to a sibling", "sub/loop", "sub-id2", false},
} {
got := f.shortcutLoop(test.remote, test.targetID)
assert.Equal(t, test.want, got, test.name)
}
}
func TestMimeTypesToExtension(t *testing.T) {
for mimeType, extension := range _mimeTypeToExtension {
extensions, err := mime.ExtensionsByType(mimeType)
@@ -814,83 +681,6 @@ func (f *Fs) InternalTestSingleQuoteFolder(t *testing.T) {
assert.NoError(t, err)
}
// TestIntegration/FsMkdir/FsPutFiles/Internal/MoveDuplicateParent
//
// Check that moving a file removes it from its real parent even when there
// are two directories with the same name and the directory cache resolves the
// path to the wrong duplicate. Otherwise the file ends up with two parents
// which fails on shared drives with a teamDrivesParentLimit error (#9472).
func (f *Fs) InternalTestMoveDuplicateParent(t *testing.T) {
ctx := context.Background()
if f.Features().Move == nil {
t.Skip("Move not supported")
}
rootID, err := f.dirCache.RootID(ctx, false)
require.NoError(t, err)
const dupName = "dup9472"
const dstName = "dst9472"
const fileName = "michael.txt"
// Create the first "dup9472" directory - this is the one the directory
// cache will resolve the path to.
dirA, err := f.createDir(ctx, rootID, dupName, nil)
require.NoError(t, err)
f.dirCache.Put(dupName, dirA.Id)
// Create a second directory with the SAME name and parent: a duplicate
// the directory cache does not know about.
dirB, err := f.createDir(ctx, rootID, dupName, nil)
require.NoError(t, err)
// Destination directory
dirDst, err := f.createDir(ctx, rootID, dstName, nil)
require.NoError(t, err)
defer func() {
_ = f.delete(ctx, dirA.Id, false)
_ = f.delete(ctx, dirB.Id, false)
_ = f.delete(ctx, dirDst.Id, false)
f.dirCache.Flush()
}()
// Upload a file directly into dirB (the duplicate the cache doesn't know).
createInfo := &drive.File{
Name: fileName,
Parents: []string{dirB.Id},
}
var fileInfo *drive.File
err = f.pacer.Call(func() (bool, error) {
fileInfo, err = f.svc.Files.Create(createInfo).
Media(strings.NewReader("9472 duplicate parent test")).
Fields(f.getFileFields(ctx)).
SupportsAllDrives(true).
Context(ctx).Do()
return f.shouldRetry(ctx, err)
})
require.NoError(t, err)
require.Equal(t, []string{dirB.Id}, fileInfo.Parents)
// Sanity check: the directory cache resolves the path to dirA, not the
// dirB the file actually lives in.
gotDirID, err := f.dirCache.FindDir(ctx, dupName, false)
require.NoError(t, err)
require.Equal(t, dirA.Id, gotDirID, "directory cache should resolve to the wrong duplicate")
// Build the source object pointing at the file in dirB and move it.
srcObj, err := f.newObjectWithInfo(ctx, dupName+"/"+fileName, fileInfo)
require.NoError(t, err)
dstObj, err := f.Move(ctx, srcObj, dstName+"/"+fileName)
require.NoError(t, err)
// The moved file must have exactly one parent: the destination.
movedInfo, err := f.getFile(ctx, dstObj.(fs.IDer).ID(), "id,parents")
require.NoError(t, err)
assert.Equal(t, []string{dirDst.Id}, movedInfo.Parents)
}
func (f *Fs) InternalTest(t *testing.T) {
// These tests all depend on each other so run them as nested tests
t.Run("DocumentImport", func(t *testing.T) {
@@ -911,56 +701,7 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("Query", f.InternalTestQuery)
t.Run("AgeQuery", f.InternalTestAgeQuery)
t.Run("SingleQuoteFolder", f.InternalTestSingleQuoteFolder)
t.Run("MoveDuplicateParent", f.InternalTestMoveDuplicateParent)
t.Run("ShouldRetry", f.InternalTestShouldRetry)
}
var _ fstests.InternalTester = (*Fs)(nil)
func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {
mu := stdsync.Mutex{}
wg := stdsync.WaitGroup{}
in := make(chan listREntry, listRInputBuffer)
overflow := []listREntry{{id: "folder", path: "folder"}}
mu.Lock()
close(in)
in = nil
mu.Unlock()
done := make(chan bool, 1)
go func() {
done <- refillListRInput(&mu, &in, &overflow, &wg)
}()
select {
case ok := <-done:
require.False(t, ok, "refill continued after the input channel was closed")
case <-time.After(time.Second):
t.Fatal("refill blocked after the input channel was closed")
}
require.Len(t, overflow, 1)
finished := make(chan struct{})
go func() {
wg.Wait()
close(finished)
}()
select {
case <-finished:
case <-time.After(time.Second):
t.Fatal("refill incremented the wait group after the input channel was closed")
}
}
func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {
mu := stdsync.Mutex{}
wg := stdsync.WaitGroup{}
in := make(chan listREntry, listRInputBuffer)
overflow := []listREntry{{id: "folder", path: "folder"}}
require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")
require.Empty(t, overflow)
got := <-in
require.Equal(t, listREntry{id: "folder", path: "folder"}, got)
wg.Done()
}
+2 -4
View File
@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
// getPermission returns permissions for the fileID and permissionID passed in
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
f.permissionsMu.Lock()
defer f.permissionsMu.Unlock()
if useCache {
f.permissionsMu.Lock()
perm = f.permissions[permissionID]
f.permissionsMu.Unlock()
if perm != nil {
return perm, false, nil
}
@@ -131,9 +131,7 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
cleanPermission(perm)
// cache the permission
f.permissionsMu.Lock()
f.permissions[permissionID] = perm
f.permissionsMu.Unlock()
return perm, inherited, err
}
+18 -30
View File
@@ -11,6 +11,7 @@
package drive
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -21,7 +22,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/readers"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
@@ -112,13 +112,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
}
// Make an http.Request for the range passed in
//
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
// to an io.Closer and close it after each attempt — the chunk buffer is
// pool-backed, so an early close would free its pages and a retry would then
// read a dead buffer. The upload loop owns the buffer's lifetime.
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
req.ContentLength = reqSize
totalSize := "*"
if rx.ContentLength >= 0 {
@@ -173,40 +168,37 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
start := int64(0)
var StatusCode int
var err error
chunkSize := int64(rx.f.opt.ChunkSize)
buf := make([]byte, int(rx.f.opt.ChunkSize))
for finished := false; !finished; {
reqSize := chunkSize
var reqSize int64
var chunk io.ReadSeeker
if rx.ContentLength >= 0 {
// If size known use repeatable reader for smoother bwlimit
if start >= rx.ContentLength {
break
}
reqSize = min(rx.ContentLength-start, chunkSize)
}
// Buffer the chunk in memory from the global pool so reads are
// repeatable for retries
rw := multipart.NewRW()
var n int64
n, err = io.CopyN(rw, rx.Media, reqSize)
if rx.ContentLength < 0 {
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
} else {
// If size unknown read into buffer
var n int
n, err = readers.ReadFill(rx.Media, buf)
if err == io.EOF {
// Send the last chunk with the correct ContentLength
// otherwise Google doesn't know we've finished
rx.ContentLength = start + n
rx.ContentLength = start + int64(n)
finished = true
err = nil
} else if err != nil {
return nil, err
}
reqSize = n
}
if err != nil {
_ = rw.Close()
return nil, err
reqSize = int64(n)
chunk = bytes.NewReader(buf[:reqSize])
}
// Transfer the chunk
err = rx.f.pacer.Call(func() (bool, error) {
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
again, err := rx.f.shouldRetry(ctx, err)
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
again = false
@@ -214,13 +206,9 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
}
return again, err
})
closeErr := rw.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
start += reqSize
}
-57
View File
@@ -1,57 +0,0 @@
package drive
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
// retried successfully. The chunk buffer is pool-backed and implements
// io.Closer, so if it reaches the http transport unwrapped the transport
// closes it after the failed attempt, returning its pages to the pool, and
// the retry then reads a freed buffer.
func TestResumableUploadRetry(t *testing.T) {
content := bytes.Repeat([]byte("resumable"), 512)
var attempts atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.NoError(t, err)
if attempts.Add(1) == 1 {
w.WriteHeader(http.StatusInternalServerError)
return
}
assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
}))
defer server.Close()
f := &Fs{
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
client: server.Client(),
}
f.opt.ChunkSize = fs.SizeSuffix(len(content))
rx := &resumableUpload{
f: f,
remote: "remote",
URI: server.URL,
Media: bytes.NewReader(content),
MediaType: "application/octet-stream",
ContentLength: int64(len(content)),
}
info, err := rx.Upload(context.Background())
require.NoError(t, err)
require.NotNil(t, info)
assert.Equal(t, "fake-id", info.Id)
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
}
+1 -1
View File
@@ -19,7 +19,7 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
Entries: items,
}
err = f.pacer.Call(func() (bool, error) {
complete, err = f.srv.UploadSessionFinishBatchV2Context(ctx, arg)
complete, err = f.srv.UploadSessionFinishBatchV2(arg)
if retry, err := shouldRetryExclude(ctx, err); !retry {
return retry, err
}
+115 -275
View File
@@ -51,7 +51,6 @@ import (
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/batcher"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/readers"
@@ -151,20 +150,14 @@ var (
// Gets an oauth config with the right scopes
func getOauthConfig(m configmap.Mapper) *oauthutil.Config {
impersonate, _ := m.Get("impersonate")
impersonateAdmin, _ := m.Get("impersonate_admin")
// If not impersonating, use standard scopes
if impersonate == "" && impersonateAdmin == "" {
if impersonate, _ := m.Get("impersonate"); impersonate == "" {
return dropboxConfig
}
// Make a copy of the config
config := *dropboxConfig
// Make a copy of the scopes with extra scopes requires appended
config.Scopes = append(config.Scopes, "team_data.member")
if impersonate != "" {
config.Scopes = append(config.Scopes, "members.read")
}
config.Scopes = append(config.Scopes, "members.read", "team_data.member")
return &config
}
@@ -213,29 +206,6 @@ You will have to use your own App (setting your own client_id and
client_secret) to use this option as currently rclone's default set of
permissions doesn't include "members.read". This can be added once
v1.55 or later is in use everywhere.
`,
Default: "",
Advanced: true,
Sensitive: true,
}, {
Name: "impersonate_admin",
Help: `Team admin ID to use when performing actions as a team administrator.
This sets the Dropbox-API-Select-Admin header with the given team
member ID (for example "dbmid:...").
This takes a team member ID directly rather than an email address.
Note that if you want to use impersonate_admin, you should make sure this
flag is set when running "rclone config" as this will cause rclone to
request the "team_data.member" scope which it won't normally.
Using the "team_data.member" scope will require a Dropbox Team Admin
to approve during the OAuth flow.
You will have to use your own App (setting your own client_id and
client_secret) to use this option as currently rclone's default set of
permissions doesn't include "team_data.member".
`,
Default: "",
Advanced: true,
@@ -268,36 +238,6 @@ See also --dropbox-root-namespace for an alternative way to work with shared
folders.`,
Default: false,
Advanced: true,
}, {
Name: "skip_shared_folders",
Help: `Instructs rclone to skip all shared folders.
When set, any folder that is a shared folder mount point will be
excluded from directory listings, regardless of ownership.
This is useful if you prefer to back up shared folders separately
using a separate remote configured with the shared folder namespace.`,
Default: false,
Advanced: true,
}, {
Name: "skip_unowned_folders",
Help: `Instructs rclone to skip shared folders not owned by the current user.
When set, any folder that is a shared folder mount point and not
owned by the current user will be excluded from directory listings.
This is useful when backing up multiple Dropbox accounts that share
common folders, to avoid duplicating the shared data across accounts.
Note: In Dropbox Business, 'Team Folders' are owned by the Team.
For standard team members, these folders evaluate as 'unowned'
(editor/viewer access) and will be excluded by this flag. To back up
Team Folders, do not use this flag or run the backup using a Team Admin
account.
If --dropbox-skip-shared-folders is also enabled, this flag has no effect.
This makes an extra API call per shared folder mount point.`,
Default: false,
Advanced: true,
}, {
Name: "pacer_min_sleep",
Default: defaultMinSleep,
@@ -362,41 +302,38 @@ will fail to download them.
// Options defines the configuration for this backend
type Options struct {
ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"`
ImpersonateAdmin string `config:"impersonate_admin"`
SharedFiles bool `config:"shared_files"`
SharedFolders bool `config:"shared_folders"`
SkipSharedFolders bool `config:"skip_shared_folders"`
SkipUnownedFolders bool `config:"skip_unowned_folders"`
BatchMode string `config:"batch_mode"`
BatchSize int `config:"batch_size"`
BatchTimeout fs.Duration `config:"batch_timeout"`
AsyncBatch bool `config:"async_batch"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
Enc encoder.MultiEncoder `config:"encoding"`
RootNsid string `config:"root_namespace"`
ExportFormats fs.CommaSepList `config:"export_formats"`
SkipExports bool `config:"skip_exports"`
ShowAllExports bool `config:"show_all_exports"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"`
SharedFiles bool `config:"shared_files"`
SharedFolders bool `config:"shared_folders"`
BatchMode string `config:"batch_mode"`
BatchSize int `config:"batch_size"`
BatchTimeout fs.Duration `config:"batch_timeout"`
AsyncBatch bool `config:"async_batch"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
Enc encoder.MultiEncoder `config:"encoding"`
RootNsid string `config:"root_namespace"`
ExportFormats fs.CommaSepList `config:"export_formats"`
SkipExports bool `config:"skip_exports"`
ShowAllExports bool `config:"show_all_exports"`
}
// Fs represents a remote dropbox server
type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv files.ContextClient // the connection to the dropbox server
svc files.ContextClient // the connection to the dropbox server (unauthorized)
sharing sharing.ContextClient // as above, but for generating sharing links
users users.ContextClient // as above, but for accessing user information
team team.ContextClient // for the Teams API
slashRoot string // root with "/" prefix
slashRootSlash string // root with "/" prefix and postfix
pacer *fs.Pacer // To pace the API calls
ns string // The namespace we are using or "" for none
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv files.Client // the connection to the dropbox server
svc files.Client // the connection to the dropbox server (unauthorized)
sharing sharing.Client // as above, but for generating sharing links
users users.Client // as above, but for accessing user information
team team.Client // for the Teams API
slashRoot string // root with "/" prefix, lowercase
slashRootSlash string // root with "/" prefix and postfix, lowercase
pacer *fs.Pacer // To pace the API calls
ns string // The namespace we are using or "" for none
batcher *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
exportExts []exportExtension
}
@@ -578,7 +515,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
f.team = team.NewContext(cfg)
f.team = team.New(cfg)
if opt.Impersonate != "" {
user := team.UserSelectorArg{
@@ -589,7 +526,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
members := []*team.UserSelectorArg{&user}
args := team.NewMembersGetInfoArgs(members)
memberIDs, err := f.team.MembersGetInfoContext(ctx, args)
memberIDs, err := f.team.MembersGetInfo(args)
if err != nil {
return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
}
@@ -600,14 +537,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId
}
if opt.ImpersonateAdmin != "" {
cfg.AsAdminID = opt.ImpersonateAdmin
}
f.srv = files.NewContext(cfg)
f.svc = files.NewContext(ucfg)
f.sharing = sharing.NewContext(cfg)
f.users = users.NewContext(cfg)
f.srv = files.New(cfg)
f.svc = files.New(ucfg)
f.sharing = sharing.New(cfg)
f.users = users.New(cfg)
f.features = (&fs.Features{
CaseInsensitive: true,
ReadMimeType: false,
@@ -634,8 +567,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return f, nil // our root it empty so we probably want to list shared folders
}
dir := path.Dir(f.root)
if dir == "." {
dir = f.root
}
// root is not empty so we have find the right shared folder if it exists
id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
id, err := f.findSharedFolder(ctx, dir)
if err != nil {
// if we didn't find the specified shared folder we have to bail out here
return nil, err
@@ -669,7 +607,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// If root starts with / then use the actual root
var acc *users.FullAccount
err = f.pacer.Call(func() (bool, error) {
acc, err = f.users.GetCurrentAccountContext(ctx)
acc, err = f.users.GetCurrentAccount()
return shouldRetry(ctx, err)
})
if err != nil {
@@ -689,7 +627,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// See if the root is actually an object
if f.root != "" {
_, _, err = f.getFileMetadata(ctx, f.slashRoot)
_, err = f.getFileMetadata(ctx, f.slashRoot)
if err == nil {
newRoot := path.Dir(f.root)
if newRoot == "." {
@@ -724,16 +662,15 @@ func (f *Fs) setRoot(root string) {
}
type getMetadataResult struct {
entry files.IsMetadata
remoteIsExportPath bool
notFound bool
err error
entry files.IsMetadata
notFound bool
err error
}
// getMetadata gets the metadata for a file or directory
func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
res.err = f.pacer.Call(func() (bool, error) {
res.entry, res.err = f.srv.GetMetadataContext(ctx, &files.GetMetadataArg{
res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
Path: f.opt.Enc.FromStandardPath(objPath),
})
return shouldRetry(ctx, res.err)
@@ -776,7 +713,6 @@ func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportE
ch <- getMetadataResult{notFound: true}
return
}
res.remoteIsExportPath = true
}
// Return our real result or error
@@ -822,7 +758,7 @@ func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-ch
}
// getFileMetadata gets the metadata for a file
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, bool, error) {
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
var res getMetadataResult
// Try all possible metadatas
@@ -831,7 +767,7 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
res = <-ch
if res.err != nil {
return nil, false, res.err
return nil, res.err
}
if !res.notFound {
break
@@ -839,17 +775,17 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
}
if res.notFound {
return nil, false, fs.ErrorObjectNotFound
return nil, fs.ErrorObjectNotFound
}
fileInfo, ok := res.entry.(*files.FileMetadata)
if !ok {
if _, ok = res.entry.(*files.FolderMetadata); ok {
return nil, false, fs.ErrorIsDir
return nil, fs.ErrorIsDir
}
return nil, false, fs.ErrorNotAFile
return nil, fs.ErrorNotAFile
}
return fileInfo, res.remoteIsExportPath, nil
return fileInfo, nil
}
// getDirMetadata gets the metadata for a directory
@@ -908,7 +844,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
Limit: 100,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFoldersContext(ctx, &arg)
res, err = f.sharing.ListFolders(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -920,7 +856,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
Cursor: res.Cursor,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFoldersContinueContext(ctx, &arg)
res, err = f.sharing.ListFoldersContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -943,21 +879,13 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
return nil
}
// sharedFolderName returns the shared folder name in root, which is its first
// path component. root must be the trimmed root as produced by setRoot (no
// leading slash).
func sharedFolderName(root string) string {
name, _, _ := strings.Cut(root, "/")
return name
}
// findSharedFolder find the id for a given shared folder name
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
// so our only option is to iterate over all shared folders
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
errFoundFile := errors.New("found file")
err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
if entry.(*fs.Dir).Remote() == name {
id = entry.(*fs.Dir).ID()
return errFoundFile
}
@@ -977,7 +905,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
SharedFolderId: id,
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.sharing.MountFolderContext(ctx, &arg)
_, err := f.sharing.MountFolder(&arg)
return shouldRetry(ctx, err)
})
return err
@@ -994,7 +922,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
Limit: 100,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFilesContext(ctx, &arg)
res, err = f.sharing.ListReceivedFiles(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1006,7 +934,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
Cursor: res.Cursor,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFilesContinueContext(ctx, &arg)
res, err = f.sharing.ListReceivedFilesContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1014,12 +942,13 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
}
}
for _, entry := range res.Entries {
entryPath := f.opt.Enc.ToStandardName(entry.Name)
fmt.Printf("%+v\n", entry)
entryPath := entry.Name
o := &Object{
fs: f,
url: entry.PreviewUrl,
remote: entryPath,
modTime: time.Time(*entry.TimeInvited),
modTime: *entry.TimeInvited,
}
if err != nil {
return err
@@ -1039,7 +968,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
errFoundFile := errors.New("found file")
err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
if strings.EqualFold(entry.(*Object).remote, name) {
if entry.(*Object).remote == name {
o = entry.(*Object)
return errFoundFile
}
@@ -1113,7 +1042,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
arg.Path = "" // Specify root folder as empty string
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContext(ctx, arg)
res, err = f.srv.ListFolder(arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1131,7 +1060,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
Cursor: res.Cursor,
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
res, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1159,26 +1088,6 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
remote := path.Join(dir, leaf)
if folderInfo != nil {
if folderInfo.SharingInfo != nil && folderInfo.SharingInfo.SharedFolderId != "" {
if f.opt.SkipSharedFolders {
fs.Debugf(remote, "Skipping shared folder")
continue
}
if f.opt.SkipUnownedFolders {
var sfMeta *sharing.SharedFolderMetadata
err = f.pacer.Call(func() (bool, error) {
var apiErr error
sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
return shouldRetry(ctx, apiErr)
})
if err != nil {
fs.Errorf(remote, "Failed to get shared folder metadata (defaulting to include): %v", err)
} else if sfMeta != nil && sfMeta.AccessType != nil && sfMeta.AccessType.Tag != sharing.AccessLevelOwner {
fs.Debugf(remote, "Skipping unowned shared folder")
continue
}
}
}
d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)
err = list.Add(d)
if err != nil {
@@ -1255,7 +1164,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
return cErr
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.CreateFolderV2Context(ctx, &arg2)
_, err = f.srv.CreateFolderV2(&arg2)
return shouldRetry(ctx, err)
})
return err
@@ -1273,41 +1182,34 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
encRoot := f.opt.Enc.FromStandardPath(root)
if check {
// ListFolder reports a missing path and a path that is a file, so it
// checks that the directory exists and is empty in a single request.
// check directory exists
_, err = f.getDirMetadata(ctx, root)
if err != nil {
return fmt.Errorf("Rmdir: %w", err)
}
// check directory empty
arg := files.NewListFolderArg(encRoot)
arg.Recursive = false
arg.Limit = 1
if root == "/" {
arg.Path = "" // Specify root folder as empty string
}
var res *files.ListFolderResult
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContext(ctx, arg)
res, err = f.srv.ListFolder(arg)
return shouldRetry(ctx, err)
})
if err != nil {
switch e := err.(type) {
case files.ListFolderAPIError:
if e.EndpointError != nil && e.EndpointError.Path != nil {
switch e.EndpointError.Path.Tag {
case files.LookupErrorNotFound:
err = fs.ErrorDirNotFound
case files.LookupErrorNotFolder:
err = fs.ErrorIsFile
}
}
}
return fmt.Errorf("Rmdir: %w", err)
}
if len(res.Entries) != 0 || res.HasMore {
if len(res.Entries) != 0 {
return errors.New("directory not empty")
}
}
// remove it
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.DeleteV2Context(ctx, &files.DeleteArg{Path: encRoot})
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
return shouldRetry(ctx, err)
})
return err
@@ -1366,7 +1268,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
}
var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.CopyV2Context(ctx, &arg)
result, err = f.srv.CopyV2(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1427,7 +1329,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
var err error
var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.MoveV2Context(ctx, &arg)
result, err = f.srv.MoveV2(&arg)
switch e := err.(type) {
case files.MoveV2APIError:
// There seems to be a bit of eventual consistency here which causes this to
@@ -1475,13 +1377,13 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
},
}
if expire < fs.DurationOff {
expiryTime := dropbox.DBXTime(time.Now().Add(time.Duration(expire)).UTC().Round(time.Second))
expiryTime := time.Now().Add(time.Duration(expire)).UTC().Round(time.Second)
createArg.Settings.Expires = &expiryTime
}
var linkRes sharing.IsSharedLinkMetadata
err = f.pacer.Call(func() (bool, error) {
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
return shouldRetry(ctx, err)
})
@@ -1490,7 +1392,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
fs.Debugf(absPath, "can't create link with expiry, trying without")
createArg.Settings.Expires = nil
err = f.pacer.Call(func() (bool, error) {
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
return shouldRetry(ctx, err)
})
}
@@ -1504,7 +1406,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
}
var listRes *sharing.ListSharedLinksResult
err = f.pacer.Call(func() (bool, error) {
listRes, err = f.sharing.ListSharedLinksContext(ctx, &listArg)
listRes, err = f.sharing.ListSharedLinks(&listArg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1565,7 +1467,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
},
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.MoveV2Context(ctx, &arg)
_, err = f.srv.MoveV2(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1579,7 +1481,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
var q *users.SpaceUsage
err = f.pacer.Call(func() (bool, error) {
q, err = f.users.GetSpaceUsageContext(ctx)
q, err = f.users.GetSpaceUsage()
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1666,7 +1568,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
arg.Path = ""
}
startCursor, err = f.srv.ListFolderGetLatestCursorContext(ctx, arg)
startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
return shouldRetry(ctx, err)
})
@@ -1676,34 +1578,6 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
return startCursor.Cursor, nil
}
// trimPrefixFold returns s with the leading prefix removed, matching
// case insensitively rune by rune so folded runes of differing UTF-8
// length still match.
//
// If s is exactly the root that prefix names (prefix without its
// trailing "/") it returns "". If prefix doesn't match it returns s
// unchanged.
func trimPrefixFold(s, prefix string) string {
if strings.HasSuffix(prefix, "/") && strings.EqualFold(s, strings.TrimSuffix(prefix, "/")) {
return ""
}
offset := 0
for prefix != "" {
if offset >= len(s) {
return s
}
sRune, sSize := utf8.DecodeRuneInString(s[offset:])
prefixRune, prefixSize := utf8.DecodeRuneInString(prefix)
if !strings.EqualFold(string(sRune), string(prefixRune)) {
return s
}
offset += sSize
prefix = prefix[prefixSize:]
}
return s[offset:]
}
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
cursor := startCursor
var res *files.ListFolderLongpollResult
@@ -1727,7 +1601,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
Timeout: timeout,
}
res, err = f.svc.ListFolderLongpollContext(ctx, &args)
res, err = f.svc.ListFolderLongpoll(&args)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1750,7 +1624,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
Cursor: cursor,
}
err = f.pacer.Call(func() (bool, error) {
changeList, err = f.srv.ListFolderContinueContext(ctx, &arg)
changeList, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1763,18 +1637,17 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
switch info := entry.(type) {
case *files.FolderMetadata:
entryType = fs.EntryDirectory
entryPath = info.PathDisplay
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
case *files.FileMetadata:
entryType = fs.EntryObject
entryPath = info.PathDisplay
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
case *files.DeletedMetadata:
entryType = fs.EntryObject
entryPath = info.PathDisplay
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
default:
fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
continue
}
entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
if entryPath != "" {
notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
@@ -1886,7 +1759,7 @@ func (o *Object) Size() int64 {
return o.bytes
}
func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPath bool) {
func (o *Object) setMetadataForExport(info *files.FileMetadata) {
o.bytes = -1
o.hash = ""
@@ -1907,10 +1780,8 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
o.exportType = exportExportable
// get rid of any paper extension, if present
o.remote = strings.TrimSuffix(o.remote, paperExtension)
if !remoteIsExportPath {
// add the export extension
o.remote += "." + string(exportExt)
}
// add the export extension
o.remote += "." + string(exportExt)
}
}
@@ -1918,23 +1789,19 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
//
// This isn't a complete set of metadata and has an inaccurate date
func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
return o.setMetadataFromEntryWithExportPath(info, false)
}
func (o *Object) setMetadataFromEntryWithExportPath(info *files.FileMetadata, remoteIsExportPath bool) error {
o.id = info.Id
o.bytes = int64(info.Size)
o.modTime = time.Time(info.ClientModified)
o.modTime = info.ClientModified
o.hash = info.ContentHash
if !info.IsDownloadable {
o.setMetadataForExport(info, remoteIsExportPath)
o.setMetadataForExport(info)
}
return nil
}
// Reads the entry for a file from dropbox
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, bool, error) {
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
return o.fs.getFileMetadata(ctx, o.remotePath())
}
@@ -1944,11 +1811,11 @@ func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
if !o.modTime.IsZero() {
return nil
}
entry, remoteIsExportPath, err := o.readEntry(ctx)
entry, err := o.readEntry(ctx)
if err != nil {
return err
}
return o.setMetadataFromEntryWithExportPath(entry, remoteIsExportPath)
return o.setMetadataFromEntry(entry)
}
// Returns the remote path for the object
@@ -2002,7 +1869,7 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
var exportResult *files.ExportResult
err = o.fs.pacer.Call(func() (bool, error) {
exportResult, in, err = o.fs.srv.ExportContext(ctx, &arg)
exportResult, in, err = o.fs.srv.Export(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -2024,7 +1891,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Url: o.url,
}
err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.sharing.GetSharedLinkFileContext(ctx, &arg)
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -2044,7 +1911,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
ExtraHeaders: headers,
}
err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.srv.DownloadContext(ctx, &arg)
_, in, err = o.fs.srv.Download(&arg)
return shouldRetry(ctx, err)
})
@@ -2068,7 +1935,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
// start upload
var res *files.UploadSessionStartResult
err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.srv.UploadSessionStartContext(ctx, &files.UploadSessionStartArg{}, nil)
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, nil)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -2083,16 +1950,12 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
// write chunks
in := readers.NewCountingReader(in0)
buf := make([]byte, int(chunkSize))
cursor := files.UploadSessionCursor{
SessionId: res.SessionId,
Offset: 0,
}
appendArg := files.UploadSessionAppendArg{
Cursor: &cursor,
// A known-size upload which fits in a single chunk can close the
// session with its only append, saving an empty append request
Close: size >= 0 && size <= chunkSize,
}
appendArg := files.UploadSessionAppendArg{Cursor: &cursor}
for currentChunk := 1; ; currentChunk++ {
cursor.Offset = in.BytesRead()
@@ -2102,21 +1965,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
}
// Buffer the chunk in memory from the global pool for retries
rw := multipart.NewRW()
var n int64
n, err = io.CopyN(rw, in, chunkSize)
if err != nil && err != io.EOF {
_ = rw.Close()
return nil, err
}
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
skip := int64(0)
err = o.fs.pacer.Call(func() (bool, error) {
// seek to the start in case this is a retry
if _, err = rw.Seek(skip, io.SeekStart); err != nil {
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
return false, err
}
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
// after session is started, we retry everything
if err != nil {
// Check for incorrect offset error and retry with new offset
@@ -2128,10 +1984,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
if skip < 0 {
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
} else if skip == n {
} else if skip == chunkSize {
fs.Debugf(o, "%s: chunk received OK - continuing", what)
return false, nil
} else if skip > n {
} else if skip > chunkSize {
// This error should never happen
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
}
@@ -2141,27 +1997,11 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
}
}
}
// Don't waste the low level retries if the context has
// been cancelled
if fserrors.ContextError(ctx, &err) {
return false, err
}
return err != nil, err
})
closeErr := rw.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
if size >= 0 {
// Check for sources which truncate early
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
if in.BytesRead() < expected {
return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
}
}
if appendArg.Close {
break
}
@@ -2191,7 +2031,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
}
err = o.fs.pacer.Call(func() (bool, error) {
entry, err = o.fs.srv.UploadSessionFinishContext(ctx, args, nil)
entry, err = o.fs.srv.UploadSessionFinish(args, nil)
if retry, err := shouldRetryExclude(ctx, err); !retry {
return retry, err
}
@@ -2245,7 +2085,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
commitInfo := files.NewCommitInfo(o.fs.opt.Enc.FromStandardPath(o.remotePath()))
commitInfo.Mode.Tag = "overwrite"
// The Dropbox API only accepts timestamps in UTC with second precision.
clientModified := dropbox.DBXTime(src.ModTime(ctx).UTC().Round(time.Second))
clientModified := src.ModTime(ctx).UTC().Round(time.Second)
commitInfo.ClientModified = &clientModified
// Don't attempt to create filenames that are too long
if cErr := checkPathLength(commitInfo.Path); cErr != nil {
@@ -2259,7 +2099,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
} else {
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
entry, err = o.fs.srv.UploadContext(ctx, &files.UploadArg{CommitInfo: *commitInfo}, in)
entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
return shouldRetry(ctx, err)
})
}
@@ -2271,7 +2111,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// This will only happen if we are uploading async batches
if entry == nil {
o.bytes = size
o.modTime = time.Time(*commitInfo.ClientModified)
o.modTime = *commitInfo.ClientModified
o.hash = "" // we don't have this
return nil
}
@@ -2284,7 +2124,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
return errNotSupportedInSharedMode
}
err = o.fs.pacer.Call(func() (bool, error) {
_, err = o.fs.srv.DeleteV2Context(ctx, &files.DeleteArg{
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
})
return shouldRetry(ctx, err)
Loaded 100 of 1177 files, more files were not shown because too many files have changed in this diff. Show more