Compare commits

..
Author SHA1 Message Date
Nick Craig-Wood 13a1d23bb8 mega: fix nil pointer panic when login with session keys fails - fixes #9232
When LoginWithKeys failed (e.g. after clearing Mega trash or with a
stale session), the error was silently ignored and only logged at debug
level. This left the filesystem uninitialized so GetRoot() returned
nil, causing a nil pointer dereference panic in findDir.

This returns the login error instead of swallowing it, and adds a nil
check for GetRoot() in findRoot, and adds defensive nil node checks in
findDir and findObject.
2026-03-05 12:30:43 +00:00
753 changed files with 14405 additions and 83247 deletions

No files matched your search

-78
View File
@@ -1,78 +0,0 @@
# CODEOWNERS for rclone
#
# Maps paths to the maintainers auto-requested for review when a pull request
# touches matching files. Patterns are evaluated top to bottom and the LAST
# matching pattern wins, so the catch-all below is overridden by the more
# specific subsystem/backend lines that follow.
#
# Every owner listed here has write access to the repo (GitHub ignores owners
# without it). Assignments are derived from `git log` history per directory:
# each area lists its dominant contributor(s) alongside @ncw, the project lead
# and global fallback.
# Global fallback - everything not matched below
* @ncw
# -----------------------------------------------------------------------------
# Core subsystems
# -----------------------------------------------------------------------------
/fs/ @ncw
/vfs/ @ncw
/lib/ @ncw
/cmd/ @ncw
/librclone/ @ncw
/fstest/ @ncw
/lib/encoder/ @ncw @klauspost
/fs/sync/ @ncw @nielash
/fs/operations/ @ncw @nielash
/cmd/bisync/ @nielash @ivandeex @ncw
/cmd/serve/ @albertony @ncw
/cmd/selfupdate/ @ivandeex @ncw
/cmd/mount2/ @ncw @darthShadow
/cmd/gitannex/ @dmcardle @ncw
# -----------------------------------------------------------------------------
# Backends - keyed to the dominant / specialist maintainer per `git log`
# -----------------------------------------------------------------------------
# albertony
/backend/jottacloud/ @albertony @buengese @ncw
/backend/local/ @ncw @albertony @nielash
/backend/netstorage/ @albertony @ncw
/backend/hdfs/ @albertony @ncw
# buengese
/backend/yandex/ @ncw @buengese
/backend/zoho/ @buengese @ncw
/backend/fichier/ @ncw @buengese
/backend/compress/ @ncw @buengese
# ivandeex
/backend/chunker/ @ncw @ivandeex
/backend/mailru/ @ncw @ivandeex
/backend/ftp/ @ncw @ivandeex
/backend/sia/ @ncw @ivandeex
# Other maintainer-owned backends (see MAINTAINERS.md)
/backend/cache/ @ncw @remusb
/backend/union/ @ncw @Max-Sum @albertony
/backend/filescom/ @ncw @childish-sambino
# Microsoft Azure
/backend/azureblob/ @ncw @sandeepkru @darthShadow
# Single-maintainer specialist backends
/backend/onedrive/ @ncw @Cnly @nielash
/backend/iclouddrive/ @lostb1t @ncw
/backend/pikpak/ @wiserain @ncw
/backend/protondrive/ @henrybear327 @ncw
/backend/seafile/ @ncw @creativeprojects
/backend/filen/ @Enduriel @ncw
/backend/storj/ @ncw @calebcase
/backend/combine/ @ncw @nielash
# -----------------------------------------------------------------------------
# Repo plumbing
# -----------------------------------------------------------------------------
/.github/ @ncw
+31
View File
@@ -0,0 +1,31 @@
<!--
Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
https://forum.rclone.org/
instead of filing an issue for a quick response.
If you are reporting a bug or asking for a new feature then please use one of the templates here:
https://github.com/rclone/rclone/issues/new
otherwise fill in the form below.
Thank you
The Rclone Developers
-->
#### Output of `rclone version`
#### Describe the issue
+74
View File
@@ -0,0 +1,74 @@
---
name: Bug report
about: Report a problem with rclone
---
<!--
We understand you are having a problem with rclone; we want to help you with that!
**STOP and READ**
**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
Please show the effort you've put into solving the problem and please be specific.
People are volunteering their time to help! Low effort posts are not likely to get good answers!
If you think you might have found a bug, try to replicate it with the latest beta (or stable).
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
If you can still replicate it or just got a question then please use the rclone forum:
https://forum.rclone.org/
for a quick response instead of filing an issue on this repo.
If nothing else helps, then please fill in the info below which helps us help you.
**DO NOT REDACT** any information except passwords/keys/personal info.
You should use 3 backticks to begin and end your paste to make it readable.
Make sure to include a log obtained with '-vv'.
You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
Thank you
The Rclone Developers
-->
#### The associated forum post URL from `https://forum.rclone.org`
#### What is the problem you are having with rclone?
#### What is your rclone version (output from `rclone version`)
#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
#### Which cloud storage system are you using? (e.g. Google Drive)
#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
<!--- Please keep the note below for others who read your bug report. -->
#### How to use GitHub
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
* Subscribe to receive notifications on status change and new comments.
+51
View File
@@ -0,0 +1,51 @@
---
name: Feature request
about: Suggest a new feature or enhancement for rclone
---
<!--
Welcome :-)
So you've got an idea to improve rclone? We love that!
You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
Probably the latest beta (or stable) release has your feature, so try to update your rclone.
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
If it still isn't there, here is a checklist of things to do:
1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
2. Discuss on the forum: https://forum.rclone.org/
3. Make a feature request issue (this is the right place!).
4. Be prepared to get involved making the feature :-)
Looking forward to your great idea!
The Rclone Developers
-->
#### The associated forum post URL from `https://forum.rclone.org`
#### What is your current rclone version (output from `rclone version`)?
#### What problem are you are trying to solve?
#### How do you think rclone should be changed to solve that?
<!--- Please keep the note below for others who read your feature request. -->
#### How to use GitHub
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
* Subscribe to receive notifications on status change and new comments.
-124
View File
@@ -1,124 +0,0 @@
name: Bug report
description: Report a reproducible problem with rclone
labels: ["bug"]
body:
- type: markdown
attributes:
value: |
We want to help you with rclone - thank you for taking the time to fill this in!
**This tracker is for reproducible bugs only.** If you have a question, are
not sure whether something is a bug, or want help with a configuration, please
use the [rclone forum](https://forum.rclone.org/) instead - you will get a much
quicker answer there.
Before filing, please **test with the [latest beta or stable](https://rclone.org/commands/rclone_selfupdate/)**
(downloads at <https://beta.rclone.org/>) as your problem may already be fixed.
**Do not redact** any information except passwords, keys and personal info.
- type: checkboxes
id: acknowledgements
attributes:
label: Before you start
description: Please confirm the following. Issues that skip these steps may be closed.
options:
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues for this problem.
required: true
- label: I have tested with the latest beta or stable release and can still reproduce the problem.
required: true
- label: This is a reproducible bug, not a usage question (questions belong on the forum).
required: true
- type: input
id: forum-url
attributes:
label: Associated forum post URL
description: If you discussed this on the forum first (encouraged), paste the link here.
placeholder: https://forum.rclone.org/t/...
validations:
required: false
- type: textarea
id: problem
attributes:
label: What is the problem you are having with rclone?
description: Describe the problem and the exact steps to reproduce it.
validations:
required: true
- type: textarea
id: version
attributes:
label: rclone version
description: The full output of `rclone version`.
placeholder: |
rclone v1.75.0
- os/version: ubuntu 24.04 (64 bit)
- os/kernel: 6.8.0-111-generic (x86_64)
- os/type: linux
- os/arch: amd64
- go/version: go1.26.4
- go/linking: dynamic
- go/tags: none
validations:
required: true
- type: dropdown
id: os
attributes:
label: Operating system
description: Choose the operating system you are using with rclone.
options:
- Windows
- macOS
- Linux
- FreeBSD
- NetBSD
- OpenBSD
- Plan 9
- Solaris
- Android
- iOS
- Other
validations:
required: true
- type: input
id: backend
attributes:
label: Which cloud storage system are you using?
description: e.g. Google Drive, S3, Dropbox, local disk. List all involved.
validations:
required: true
- type: textarea
id: config
attributes:
label: Output of `rclone config redacted`
description: |
Run `rclone config redacted` (or `rclone config redacted <remote>` for a
single remote). This command automatically replaces passwords and tokens
with `XXX`, so it should be safe to paste but please double-check before posting.
render: ini
validations:
required: true
- type: textarea
id: command
attributes:
label: The command you were trying to run
description: e.g. `rclone copy /tmp remote:tmp`
render: shell
validations:
required: true
- type: textarea
id: log
attributes:
label: A log from the command with the `-vv` flag
description: |
e.g. output from `rclone -vv copy /tmp remote:tmp`. For long logs (> 100 lines) use
`-vv --log-file bug.log` and attach it here or send to a service such as
<https://gist.github.com/> or <https://pastebin.com/> and link it here.
render: shell
validations:
required: true
- type: textarea
id: extra
attributes:
label: Anything else?
description: Anything else that might help us, e.g. a minimal way to replicate the problem.
validations:
required: false
+2 -5
View File
@@ -1,8 +1,5 @@
blank_issues_enabled: false
contact_links:
- name: Rclone Forum - questions & support
- name: Rclone Forum Community Support
url: https://forum.rclone.org/
about: Have a question, need help, or not sure if it's a bug? Please ask here for a quick answer.
- name: Rclone Documentation
url: https://rclone.org/docs/
about: The docs answer many "how do I..." questions, including per-backend guides.
about: Please ask and answer questions here.
-68
View File
@@ -1,68 +0,0 @@
name: Feature request
description: Suggest a new feature or enhancement for rclone
labels: ["enhancement"]
body:
- type: markdown
attributes:
value: |
So you've got an idea to improve rclone? We love that - we've already
incorporated hundreds of ideas from contributors!
Please **search first**: your idea may already exist in the latest
[beta or stable](https://rclone.org/commands/rclone_selfupdate/), or already
be requested. It is often worth [discussing on the forum](https://forum.rclone.org/)
to refine the idea or checking to see if rclone can do it already before filing it here.
- type: checkboxes
id: acknowledgements
attributes:
label: Before you start
options:
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues and this hasn't already been requested.
required: true
- label: I have checked the latest beta and this feature doesn't already exist.
required: true
- type: input
id: forum-url
attributes:
label: Associated forum post URL
description: If you discussed this on the forum first (encouraged), paste the link here.
placeholder: https://forum.rclone.org/t/...
validations:
required: false
- type: textarea
id: version
attributes:
label: rclone version
description: The full output of `rclone version`, not just the version number.
placeholder: |
rclone v1.75.0
- os/version: ubuntu 24.04 (64 bit)
- os/kernel: 6.8.0-111-generic (x86_64)
- os/type: linux
- os/arch: amd64
- go/version: go1.26.4
- go/linking: dynamic
- go/tags: none
validations:
required: true
- type: textarea
id: problem
attributes:
label: What problem are you trying to solve?
description: Describe the use case and what is difficult or impossible today.
validations:
required: true
- type: textarea
id: solution
attributes:
label: How do you think rclone should be changed to solve that?
validations:
required: true
- type: checkboxes
id: involvement
attributes:
label: Getting involved
description: Features are much more likely to happen if someone helps make them.
options:
- label: I'm willing to help implement, test or fund this feature.
required: false
+11 -36
View File
@@ -1,54 +1,29 @@
<!--
Thank you for contributing to rclone!
Thank you very much for contributing code or documentation to rclone! Please
fill out the following questions to make it easier for us to review your
changes.
Please discuss anything more than a trivial fix in an issue FIRST - see the
"Linked issue" section below. Then fill in the questions to help us review.
You do not need to check all the boxes below all at once, feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box.
-->
#### What does this change do?
<!-- Describe the change here. -->
#### Linked issue
#### What is the purpose of this change?
<!--
Put `Fixes #1234` here if this closes an issue, or `#1234` to link without closing.
IMPORTANT: Anything beyond a trivial fix (typo, doc tweak, small obvious bug fix)
should be discussed and agreed in an issue BEFORE you open a pull request.
Pull requests for larger changes that have not been discussed in an issue first
may be closed. This saves everyone's time if the change needs a different approach
or isn't a good fit.
Describe the changes here
-->
Fixes #
#### For new or changed backends
#### Was the change discussed in an issue or in the forum before?
<!--
Backend pull requests are often sent without the integration tests having been run.
We cannot merge a backend change until it passes the integration tests.
To be merged, a backend change REQUIRES:
1. A clean run of `go run ./fstest/test_all -backends <remote>` (summarise the
result below or paste a screenshot of the test_all output in the web browser).
2. A test account so the maintainers can run the integration tests daily against
the backend on the integration test server.
If the tests aren't quite passing yet and you need help getting them
to pass, then submit the PR and we can help getting you over the line.
See https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#writing-a-new-backend
and https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests
Link issues and relevant forum posts here.
-->
#### Checklist
- [ ] This change is trivial **OR** it has been discussed and agreed in the linked issue.
- [ ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
- [ ] **(If I used AI tools to help write this code)** I have read and understood the [AI-assisted contributions guidance](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#ai-assisted-contributions), and I have tested and take ownership of this change myself.
- [ ] I have added tests for all changes in this PR if appropriate.
- [ ] I have added documentation for the changes if appropriate.
- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
- [ ] **(Backend changes only)** `test_all` passes for this backend and if submitting a new backend can provide a test account for the integration tester - see [CONTRIBUTING.md](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests).
- [ ] This Pull Request is ready for review.
- [ ] I'm done, this Pull Request is ready for review :-)
-2
View File
@@ -4,5 +4,3 @@ updates:
directory: "/"
schedule:
interval: "daily"
cooldown:
default-days: 7
+19 -139
View File
@@ -19,9 +19,6 @@ on:
type: boolean
default: true
permissions:
contents: read
jobs:
build:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
@@ -32,12 +29,12 @@ jobs:
strategy:
fail-fast: false
matrix:
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
include:
- job_name: linux
os: ubuntu-latest
go: '~1.27.0'
go: '~1.26.0'
gotags: cmount
build_flags: '-include "^linux/"'
check: true
@@ -48,14 +45,14 @@ jobs:
- job_name: linux_386
os: ubuntu-latest
go: '~1.27.0'
go: '~1.26.0'
goarch: 386
gotags: cmount
quicktest: true
- job_name: mac_amd64
os: macos-latest
go: '~1.27.0'
go: '~1.26.0'
gotags: 'cmount'
build_flags: '-include "^darwin/amd64" -cgo'
quicktest: true
@@ -64,14 +61,14 @@ jobs:
- job_name: mac_arm64
os: macos-latest
go: '~1.27.0'
go: '~1.26.0'
gotags: 'cmount'
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
deploy: true
- job_name: windows
os: windows-latest
go: '~1.27.0'
go: '~1.26.0'
gotags: cmount
cgo: '0'
build_flags: '-include "^windows/"'
@@ -81,14 +78,14 @@ jobs:
- job_name: other_os
os: ubuntu-latest
go: '~1.27.0'
go: '~1.26.0'
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
compile_all: true
deploy: true
- job_name: go1.26
- job_name: go1.25
os: ubuntu-latest
go: '~1.26.6'
go: '~1.25.7'
quicktest: true
racequicktest: true
@@ -97,58 +94,16 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: Get runner parameters
id: get-runner-parameters
run: |
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Go
id: setup-go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: ${{ matrix.go }}
check-latest: true
# Caching is handled explicitly below. setup-go's built-in cache
# keys only on go.sum, so all matrix jobs collide on one key and
# only the first to finish saves - which loses the cross-compile
# build cache.
cache: false
- name: Get Go paths
id: go-paths
run: |
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
# The module cache holds downloaded source and is identical across
# jobs (it depends only on go.sum, not on OS, arch or Go version), so
# it is shared via a single key to avoid storing the same ~600 MiB in
# every job's cache and overflowing the repo cache limit.
- name: Module cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.mod-cache }}
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
# The build cache holds compiled artifacts which are specific to the
# OS, arch and Go version, so it is kept per job.
- name: Build cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.build-cache }}
key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
- name: Set environment variables
run: |
@@ -165,11 +120,6 @@ jobs:
sudo chown root:$USER /etc/fuse.conf
sudo apt-get update
sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
# The podman static bundle in runner image ubuntu24/20260726.254 and
# later ships a non-setuid fusermount3 in /usr/local/bin which
# shadows the setuid one from the fuse3 package and breaks FUSE
# mounts with "Operation not permitted", so remove it.
sudo rm -f /usr/local/bin/fusermount /usr/local/bin/fusermount3
if: matrix.os == 'ubuntu-latest'
- name: Install Libraries on macOS
@@ -179,7 +129,6 @@ jobs:
unset HOMEBREW_NO_INSTALL_FROM_API
brew untap --force homebrew/core
brew untap --force homebrew/cask
brew untap --force aws/tap
brew update
brew install --cask macfuse
brew install git-annex git-annex-remote-rclone
@@ -253,15 +202,6 @@ jobs:
# Deploy binaries if enabled in config && not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
lint:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -276,31 +216,23 @@ jobs:
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Go
id: setup-go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: '~1.27.0'
go-version: '~1.26.0'
check-latest: true
cache: false
- name: Module cache
uses: actions/cache@v6
with:
path: ~/go/pkg/mod
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
- name: Cache
uses: actions/cache@v6
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
@@ -367,15 +299,6 @@ jobs:
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
if: github.event_name == 'pull_request'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
android:
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
timeout-minutes: 30
@@ -383,50 +306,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Get runner parameters
id: get-runner-parameters
run: |
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
# Upgrade together with NDK version
- name: Set up Go
id: setup-go
uses: actions/setup-go@v7
uses: actions/setup-go@v6
with:
go-version: '~1.27.0'
# Caching is handled explicitly below to share the module cache
# with the other jobs - see the build job for the rationale.
cache: false
- name: Get Go paths
id: go-paths
run: |
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
- name: Module cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.mod-cache }}
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
go-mod-
- name: Build cache
uses: actions/cache@v6
with:
path: ${{ steps.go-paths.outputs.build-cache }}
key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
restore-keys: |
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
go-version: '~1.26.0'
- name: Set global environment variables
run: |
@@ -502,12 +391,3 @@ jobs:
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
# Upload artifacts if not a PR && not a fork
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
- name: Trim module cache before saving
if: always()
run: |
# The downloaded module .zip archives are not needed to build from
# the already-extracted module cache, so drop them before the cache
# is saved - this roughly halves the module cache size.
modcache=$(go env GOMODCACHE | tr '\\' '/')
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
@@ -18,10 +18,6 @@ on:
type: boolean
default: true
permissions:
contents: read
packages: write
jobs:
build-image:
if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
@@ -56,7 +52,7 @@ jobs:
df -h .
- name: Checkout Repository
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
@@ -96,7 +92,7 @@ jobs:
# There's no way around this, because "ImageOS" is only available to
# processes, but the setup-go action uses it in its key.
id: imageos
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
result-encoding: string
script: |
@@ -104,7 +100,7 @@ jobs:
- name: Extract Metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
env:
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
with:
@@ -126,14 +122,14 @@ jobs:
type=raw,value=beta,enable={{is_default_branch}}
- name: Setup QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Load Go Build Cache for Docker
id: go-cache
uses: actions/cache@v6
uses: actions/cache@v5
with:
key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
@@ -152,7 +148,7 @@ jobs:
skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
# This is the user that triggered the Workflow. In this case, it will
@@ -163,7 +159,7 @@ jobs:
- name: Build and Publish Image Digest
id: build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
file: Dockerfile
context: .
@@ -214,7 +210,7 @@ jobs:
- name: Extract Metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@v5
env:
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
with:
@@ -265,16 +261,16 @@ jobs:
env.write(f"ANNOTATIONS={annotations_string}\n")
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
# This is the user that triggered the Workflow. In this case, it will
@@ -14,9 +14,6 @@ on:
type: boolean
default: true
permissions:
contents: read
jobs:
build_docker_volume_plugin:
if: inputs.manual || github.repository == 'rclone/rclone'
@@ -33,7 +30,7 @@ jobs:
sudo rm -rf /usr/share/dotnet || true
df -h .
- name: Checkout master
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Build and publish docker plugin
-3
View File
@@ -4,9 +4,6 @@ on:
issues:
types: [labeled]
permissions:
contents: read
jobs:
notify:
runs-on: ubuntu-latest
-3
View File
@@ -3,9 +3,6 @@ on:
release:
types: [released]
permissions:
contents: read
jobs:
publish:
runs-on: ubuntu-latest
-5
View File
@@ -46,11 +46,6 @@ linters:
- -ST1022
# Disable quickfix checks
- -QF*
# Disabled because its dataflow analysis (new in staticcheck
# 0.8.0) makes linting large packages more than 10x slower,
# which takes the CI lint job past its timeout, and it
# produces false positives (eg on operations.Delete).
- -SA4023
gocritic:
# With gocritic there are different settings, but since enabled-checks
# and disabled-checks cannot both be set, for full customization the
+3 -3
View File
@@ -1,6 +1,6 @@
default: true
# Use specific styles, to be consistent across all documents.
# Use specific styles, to be consistent accross all documents.
# Default is to accept any as long as it is consistent within the same document.
heading-style: # MD003
style: atx
@@ -48,12 +48,12 @@ link-fragments: false # MD051
# https://gohugo.io/content-management/syntax-highlighting/#languages
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
# In addition, we only want to allow identifiers (aliases) that correspond to
# the same language in Hugo and GitHub, and preferably also VSCode and other
# the same language in Hugo and GitHub, and preferrably also VSCode and other
# commonly used tools, to avoid confusion. An example of this is that "shell"
# by some are considered an identifier for shell scripts, i.e. an alias for
# "sh", while others consider it an identifier for shell sessions, i.e. an
# alias for "console". Although Hugo and GitHub in this case are consistent and
# have chosen the former, using "sh" instead, and not allowing use of "shell",
# have choosen the former, using "sh" instead, and not allowing use of "shell",
# avoids the confusion entirely.
fenced-code-language: # MD040
allowed_languages:
-134
View File
@@ -1,134 +0,0 @@
# AGENTS.md
This file provides guidance to AI coding agents (e.g. Claude Code, Codex, Cursor, Gemini CLI, and similar tools) when working with code in this repository.
Rclone welcomes AI-assisted contributions, but the expectation is that you, the human submitter, understand every line you propose and have compiled and tested it against real rclone code - not just generated it. See the "AI-assisted contributions" section of [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request.
## Project Overview
Rclone is a command-line program to sync files and directories to and from cloud storage providers. It's written in Go and supports 70+ backends (cloud storage systems). Think "rsync for cloud storage".
## General Notes
**We take backwards compatibility very seriously.** PRs should not change the observable behaviour of existing commands, flags or rc API without very good reason. Rclone does not try to preserve a stable Go API but try not to change it gratuitously.
Rclone operates with a lot of different backends, so **compatibility is key**. It is the backend integration tests which guarantee that compatibility. Changes should consider both known and unknown backends and should take care not to break functionality of existing installations.
The core parts of rclone under `fs` and `vfs` need to work with all backends and **backend specific hacks won't be merged**. Fixes likely need to go in the relevant backend or if new behaviour is really needed, a new Feature flag needs to be added.
**Changes should be kept to the minimum.** Work hard to make the most elegant, smallest change you can. Do not refactor or re-order code unless necessary as this makes review more challenging. Re-use existing test scaffolding, existing or library routines (e.g. `lib`) where possible.
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
## Build and Test Commands
```bash
# Build rclone (simple)
go build
# Build with version info (preferred)
make
# Run all unit tests (no cloud credentials needed)
make quicktest
# or equivalently:
RCLONE_CONFIG="/notfound" go test ./...
# Run tests for a specific package
cd backend/memory && go test -v
# or from root:
go test -v ./backend/memory/
# Run a single test
go test -v -run TestIntegration/FsCheckWrap ./backend/memory/
# Run tests with race detector
make racequicktest
# Lint (requires golangci-lint)
golangci-lint run ./...
# Run backend integration tests (requires configured TestRemote remote)
cd backend/drive && go test -v
# Run sync/operations integration tests against a remote
cd fs/sync && go test -v -remote TestDrive:
cd fs/operations && go test -v -remote TestDrive:
# Run integration tests via test framework
go run ./fstest/test_all -backends drive
```
## Architecture
### Entry Point and Plugin Registration
`rclone.go` is the main entry point. It imports `backend/all` and `cmd/all` which use Go's `init()` pattern to register all backends and commands. Each backend calls `fs.Register()` with a `fs.RegInfo` struct during init.
### Core Interfaces (`fs/`)
The `fs` package defines the core abstractions:
- **`fs.Fs`** (`fs/types.go`): The filesystem interface every backend must implement (List, NewObject, Put, Mkdir, Rmdir).
- **`fs.Object`** (`fs/types.go`): Interface for a file/object (Open, Update, Remove, SetModTime).
- **`fs.Features`** (`fs/features.go`): Optional capabilities a backend can declare (Purge, Copy, Move, DirMove, etc.). Backends set function pointers for operations they support; nil means not supported.
- **`fs.RegInfo`** (`fs/registry.go`): Registration metadata for a backend including its name, config options, and NewFs constructor.
### Backend Structure (`backend/`)
Each backend is a single Go package (e.g., `backend/s3/`, `backend/drive/`). Key conventions:
- Main implementation in a single file (e.g., `s3.go`) - **do not** split into `fs.go`/`object.go`.
- API types go in a separate `api/types.go` file.
- Test file (e.g., `s3_test.go`) uses `fstests.Run()` from `fstest/fstests` for standardized integration tests.
- Register in `backend/all/all.go` via blank import.
- HTTP-based backends should use `lib/rest` for HTTP calls and `fs/fshttp` for the HTTP client.
- Use `lib/dircache` for directory-ID-based remotes, `lib/oauthutil` for OAuth, `lib/pacer` for rate limiting.
### Command Structure (`cmd/`)
Each command is a package under `cmd/` registered in `cmd/all/all.go` via blank import. Commands use cobra via `cmd.Main()`.
### Key Subsystems
- **`fs/operations/`**: Core file operations (Copy, Move, Delete, etc.)
- **`fs/sync/`**: Directory sync logic
- **`fs/march/`**: Parallel directory tree walker used by sync
- **`fs/filter/`**: Include/exclude filtering
- **`fs/accounting/`**: Transfer statistics and bandwidth limiting
- **`fs/config/`**: Configuration file management
- **`vfs/`**: Virtual filesystem layer (used by mount, serve)
- **`librclone/`**: C-compatible library interface for embedding rclone
- **`fstest/`**: Integration test framework; `fstest/fstests/` has the generic backend test suite
## Commit Message Convention
Prefix with the directory of the change, then a colon: `drive: add team drive support - fixes #885`. For cross-cutting changes use a broader prefix like `fs` or `operations`.
Make the first line of your commit message a summary of the change that a user (not a developer) of rclone would like to read. So write `drive: fix server side copy of big files` instead of `drive: no longer set the MimeType in Move or Copy`. This is important because these lines go into the change log which is read by users.
## Code Commenting Style
Comments describe the code as it is now, for a future reader who has no knowledge of the change that introduced it.
Every exported type, function, field, and constant has a **godoc comment** that starts with its name and is phrased as a present-tense statement of what it is or does (`// Mkdir makes the directory (container, bucket)`)
**Document the contract** callers need - preconditions, what's returned, which sentinel errors are returned and when, and any "shouldn't return an error if it already exists" style caveats - rather than the implementation.
**Keep comments terse**: a single line for most things, with extra paragraphs (separated by blank `//` lines) reserved for genuine subtlety.
Inline comments inside function bodies should **explain why** - a non-obvious API quirk, a workaround, a gotcha, or an ordering constraint - and may cite an external reference (forum thread, vendor docs, RFC) when that's what makes the behaviour non-obvious; skip comments that merely restate what the code plainly does.
Use `FIXME` and `TODO` for known shortcomings.
**Do not write comments that narrate the change itself** or compare against the previous behaviour (no "now we also handle...", "changed to...", "previously this returned...", or references to bug/PR numbers in the code) - that context belongs in the commit message, not in source that will outlive the change.
Code comments should only refer to **current** behaviour - and shouldn't describe behaviour that is trivially deducible from reading the code.
## Linting Configuration
Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck, govet, ineffassign, staticcheck, unused, gocritic, misspell, revive, unconvert. The `goimports` formatter is also enabled.
## Documentation
- Backend option docs come from `Help:` fields in the Go source Options structs, not from markdown files.
- Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
- Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
- Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
-6
View File
@@ -1,6 +0,0 @@
# CLAUDE.md
Guidance for this repository lives in AGENTS.md, shared with all AI coding
agents. Claude Code reads it via the import below.
@AGENTS.md
+4 -69
View File
@@ -113,36 +113,6 @@ GitHub.
You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master)
or [squash your commits](#squashing-your-commits).
## AI-assisted contributions
You are welcome to use AI coding assistants (Claude Code, Codex, Cursor, Gemini
CLI, and similar) to help write your contribution. Rclone has an
[AGENTS.md](AGENTS.md) file at the top of the repository describing the project's
conventions; point your tool at it so the code it produces matches rclone's
style.
However, the same standard applies to every pull request whether or not a tool
was involved: **you are responsible for the code you submit.** Before you open a
pull request please make sure that:
- You understand every line of the change and can explain why it is correct. If
a reviewer asks a question about it, you should be able to answer without
going back to the tool.
- You have actually built and run it. At a minimum `go build` and
`make quicktest` must pass, and for a backend change you should run the
[backend tests](#backend-testing) against a real remote where you can.
- The change is a genuine fix or feature that you have verified solves the
problem, not a plausible-looking guess. Unverified, AI-generated pull requests
that do not compile, do not pass the tests, invent APIs that do not exist, or
do not actually do what the description claims waste maintainer time and are
likely to be closed.
- You have trimmed the comments. AI tools tend to add verbose comments that
restate what the code plainly does or narrate the change being made; please
cut these down to match the [code commenting style](AGENTS.md#code-commenting-style).
In short: an AI assistant is a tool to help *you* contribute, not a substitute
for understanding and testing your own work.
## Using Git and GitHub
### Committing your changes
@@ -630,37 +600,10 @@ remote or an fs.
- **Remember** we have >50 backends to maintain so keeping them as similar as
possible to each other is a high priority!
### Managing memory
Use rclone's memory pool for upload buffers rather than allocating them
yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
rclone's memory usage smaller.
- If the provider supports multipart or parallel chunk uploads, implement
[fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
and drive it with
[multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
This gives you pooled chunk buffers, concurrency, retries and
`--multi-thread-streams` support for free. See s3/b2 backends for examples.
- If you must buffer data yourself take the buffer from the global pool with
[multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
be rewound for a retry.
- When using a pooled buffer `Close()` it on all paths so its pages go back to
the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
attempt, set the request's `ContentLength` explicitly, and if the buffer
reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
so the transport can't close it and free its pages before the retry.
### Unit tests
- Create a config entry called `TestRemote` for the unit tests to use
- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
- If your backend does chunked uploads, set `ChunkedUpload` in the
`fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
`FsPutRetry` tests can run the chunked path with small chunks.
- Make sure all tests pass with `go test -v`
### Integration tests
@@ -680,22 +623,14 @@ Or if you want to run the integration tests manually:
- If your remote defines `ListR` check with this also
- `go test -v -remote TestRemote: -fast-list`
Before a new or changed backend can be merged we require:
- A clean run of `go run ./fstest/test_all -backends remote` (please include the
result in your pull request).
- A test account for the backend so the maintainers can add it to the
[integration test server](https://integration.rclone.org) and keep the backend
working as rclone evolves. A backend that we cannot test is likely to break and
may be removed.
See the [testing](#testing) section for more information on integration tests.
### Backend documentation
Add your backend to the docs. Keep lists of remotes in alphabetical
order of full name of remote (e.g. `drive` is ordered as `Google
Drive`) but with the local file system last.
Add your backend to the docs - you'll need to pick an icon for it from
[fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in
alphabetical order of full name of remote (e.g. `drive` is ordered as
`Google Drive`) but with the local file system last.
First add a data file about your backend in
`docs/data/backends/remote.yaml` - this is used to build the overview
+1 -4
View File
@@ -35,9 +35,6 @@ RUN echo "**** Print Version Binary ****" && \
# Begin final image
FROM alpine:latest
ARG RCLONE_UID=1009
ARG RCLONE_GID=1009
RUN echo "**** Install Dependencies ****" && \
apk add --no-cache \
ca-certificates \
@@ -48,7 +45,7 @@ RUN echo "**** Install Dependencies ****" && \
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
ENTRYPOINT [ "rclone" ]
+21 -28
View File
@@ -2,34 +2,27 @@
Current active maintainers of rclone are:
| Name | GitHub ID |
| :--------------- | :---------------- |
| Nick Craig-Wood | @ncw |
| Stefan Breunig | @breunigs |
| Ishuah Kariuki | @ishuah |
| Remus Bunduc | @remusb |
| Fabian Möller | @B4dM4n |
| Alex Chen | @Cnly |
| Sandeep Ummadi | @sandeepkru |
| Sebastian Bünger | @buengese |
| Ivan Andreev | @ivandeex |
| Max Sum | @Max-Sum |
| Fred | @creativeprojects |
| Caleb Case | @calebcase |
| wiserain | @wiserain |
| albertony | @albertony |
| Chun-Hung Tseng | @henrybear327 |
| Hideo Aoyama | @boukendesho |
| nielash | @nielash |
| Dan McArdle | @dmcardle |
| Sam Harrison | @childish-sambino |
| Enduriel | @Enduriel |
Per-area ownership of code (which backend or command each maintainer looks
after) is defined in [.github/CODEOWNERS](.github/CODEOWNERS), which GitHub uses
to auto-request reviews. Two responsibilities don't map to a code path: @ncw
looks after overall project health, and @boukendesho maintains the snap
packaging.
| Name | GitHub ID | Specific Responsibilities |
| :--------------- | :---------------- | :-------------------------- |
| Nick Craig-Wood | @ncw | overall project health |
| Stefan Breunig | @breunigs | |
| Ishuah Kariuki | @ishuah | |
| Remus Bunduc | @remusb | cache backend |
| Fabian Möller | @B4dM4n | |
| Alex Chen | @Cnly | onedrive backend |
| Sandeep Ummadi | @sandeepkru | azureblob backend |
| Sebastian Bünger | @buengese | jottacloud, yandex & compress backends |
| Ivan Andreev | @ivandeex | chunker & mailru backends |
| Max Sum | @Max-Sum | union backend |
| Fred | @creativeprojects | seafile backend |
| Caleb Case | @calebcase | storj backend |
| wiserain | @wiserain | pikpak backend |
| albertony | @albertony | |
| Chun-Hung Tseng | @henrybear327 | Proton Drive Backend |
| Hideo Aoyama | @boukendesho | snap packaging |
| nielash | @nielash | bisync |
| Dan McArdle | @dmcardle | gitannex |
| Sam Harrison | @childish-sambino | filescom |
## This is a work in progress draft
Generated
+2577 -7670
View File
File diff suppressed because it is too large. Load diff
Generated
+968 -5149
View File
File diff suppressed because it is too large. Load diff
Generated
+918 -5426
View File
File diff suppressed because it is too large. Load diff
+3 -33
View File
@@ -45,7 +45,7 @@ LINTTAGS=--build-tags "$(GOTAGS)"
endif
LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
.PHONY: rclone test_all vars version fetch-gui fetch-gui-and-commit
.PHONY: rclone test_all vars version
rclone:
ifeq ($(GO_OS),windows)
@@ -59,12 +59,6 @@ endif
cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
fetch-gui:
$(SHELL) ./bin/fetch-gui-dist.sh
fetch-gui-and-commit:
$(SHELL) ./bin/fetch-gui-dist.sh --commit
test_all:
go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
@@ -93,15 +87,11 @@ test: rclone test_all
@echo "Written logs in test_all.log"
# Quick test
#
# The timeout is raised above the go test default of 10m as the
# cmd/gitannex end to end tests can take longer than that on slow CI
# runners.
quicktest:
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
racequicktest:
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
compiletest:
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
@@ -131,25 +121,6 @@ updatedirect:
go get $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
go mod tidy
# Update direct dependencies only but won't update the `go` line in go.mod
updatedirectnoupgrade:
@GO_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
for mod in $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all); do \
cp go.mod go.mod.bak && \
cp go.sum go.sum.bak && \
go get $$mod && \
NEW_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
if [ "$$NEW_VERSION" != "$$GO_VERSION" ]; then \
echo "SKIPPING $$mod (requires go $$NEW_VERSION)"; \
cp go.mod.bak go.mod; \
cp go.sum.bak go.sum; \
else \
echo "updated $$mod"; \
fi; \
done && \
rm -f go.mod.bak go.sum.bak && \
go mod tidy
# Update direct and indirect dependencies and test dependencies
update:
go get -u -t ./...
@@ -175,7 +146,6 @@ MANUAL.txt: MANUAL.md
commanddocs: rclone
go generate ./lib/transform
go generate ./cmd/bisync
-@rmdir -p '$$HOME/.config/rclone'
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
-6
View File
@@ -59,17 +59,14 @@ directories to and from different cloud storage providers.
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp)
- HTTP [:page_facing_up:](https://rclone.org/http/)
- Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
- Huawei Drive [:page_facing_up:](https://rclone.org/huaweidrive/)
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
- ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
- Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
- Internxt [:page_facing_up:](https://rclone.org/internxt/)
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
@@ -112,7 +109,6 @@ directories to and from different cloud storage providers.
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
- Seafile [:page_facing_up:](https://rclone.org/seafile/)
- Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
- SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
@@ -127,12 +123,10 @@ directories to and from different cloud storage providers.
- Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
- Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
- Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
- US3 Object Storage [:page_facing_up:](https://rclone.org/s3/#us3)
- Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
- WebDAV [:page_facing_up:](https://rclone.org/webdav/)
- Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
- Zadara Object Storage [:page_facing_up:](https://rclone.org/s3/#zadara)
- Zero Services (ZERO-Z3) [:page_facing_up:](https://rclone.org/s3/#zero-z3)
- Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
- Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
- The local filesystem [:page_facing_up:](https://rclone.org/local/)
+27 -31
View File
@@ -11,12 +11,11 @@ This file describes how to make the various kinds of releases
- git checkout master # see below for stable branch
- git pull # IMPORTANT
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
- git status - make sure everything is checked in
- Check GitHub actions build for master is Green
- make test # see integration test server or run locally
- make tag
- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
- edit docs/content/changelog.md # make sure to remove duplicate logs from point
releases
- make tidy
- make doc
@@ -33,20 +32,16 @@ This file describes how to make the various kinds of releases
- make sign_upload
- make check_sign
- make upload
- make upload_test_website # check sponsors are correct/unchanged
- make upload_website
- make upload_github
- make startdev # make startstable for stable branch
- \# announce with forum post, twitter post, and post to annoucements list
- \# announce with forum post, twitter post, patreon post
## Update dependencies
Early in the next release cycle update the dependencies.
- Active Pins
- `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
- See: [#9539](https://github.com/rclone/rclone/issues/9539)
- Review Active Pins in go.mod and remove if possible
- Review any pinned packages in go.mod and remove if possible
- `make updatedirect`
- `make GOTAGS=cmount`
- `make compiletest`
@@ -59,18 +54,32 @@ If the `make updatedirect` upgrades the version of go in the `go.mod`
go 1.22.0
```
then so this instead `make updatedirectnoupgrade`. This will try each
upgrade individually. Consider making an upstream bug report for any
`SKIPPING` lines as these only support the latest Go version not the
previous version too.
then go to manual mode. `go1.22` here is the lowest supported version
in the `go.mod`.
If `make updatedirect` or `make updatedirectnoupgrade` added a
`toolchain` directive then remove it. We don't want to force a
toolchain on our users. Linux packagers are often using a version of
Go that is a few versions out of date.
If `make updatedirect` added a `toolchain` directive then remove it.
We don't want to force a toolchain on our users. Linux packagers are
often using a version of Go that is a few versions out of date.
```console
go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
go get -d $(cat /tmp/potential-upgrades)
go mod tidy -go=1.22 -compat=1.22
```
If the `go mod tidy` fails use the output from it to remove the
package which can't be upgraded from `/tmp/potential-upgrades` when
done
```console
git co go.mod go.sum
```
And try again.
Optionally upgrade the direct and indirect dependencies. This is very
likely to fail if `make updatedirectnoupgrade` was used.
likely to fail if the manual method was used abve - in that case
ignore it as it is too time consuming to fix.
- `make update`
- `make GOTAGS=cmount`
@@ -88,19 +97,6 @@ build.
Once it compiles locally, push it on a test branch and commit fixes
until the tests pass.
### Pseudo versions
Go makes pseudo versions for untagged repos and repos not at a tag.
The pseudo versions on repos that have been tagged before do not get
updated automatically so need manually checking. These can be found with
```console
grep -E '[0-9]{14}-[0-9a-f]{12}' go.mod | grep -v indirect | grep -v 'v0\.0\.0'
```
These will need to be updated manually using the `go get ...@branch`
syntax.
### Major versions
The above procedure will not upgrade major versions, so v2 to v3.
@@ -195,7 +191,7 @@ this will be done already.
Now
- git co ${BASE_TAG}-stable
- `git cherry-pick -x` any fixes
- git cherry-pick any fixes
- make startstable
- Do the steps as above
- git co master
+1 -1
View File
@@ -1 +1 @@
v1.76.0
v1.74.0
-1
View File
@@ -32,7 +32,6 @@ import (
_ "github.com/rclone/rclone/backend/hdfs"
_ "github.com/rclone/rclone/backend/hidrive"
_ "github.com/rclone/rclone/backend/http"
_ "github.com/rclone/rclone/backend/huaweidrive"
_ "github.com/rclone/rclone/backend/iclouddrive"
_ "github.com/rclone/rclone/backend/imagekit"
_ "github.com/rclone/rclone/backend/internetarchive"
+6 -42
View File
@@ -36,7 +36,6 @@ import (
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fspath"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/sanitize"
)
// Register with Fs
@@ -186,12 +185,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
foundArchive := subArchive(remote)
if foundArchive != nil {
fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
// Archive path, in canonical form so that it compares equal
// to the cleaned entry names inside the archive
foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
if foundArchive.root == "." {
foundArchive.root = ""
}
// Archive path
foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
// Path to the archive
archiveRemote := remote[:len(foundArchive.remote)]
// Remote is archive leaf name
@@ -496,50 +491,24 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, err
}
subEntries, err := subFs.List(ctx, dir)
entries, err = subFs.List(ctx, dir)
if err != nil {
return nil, err
}
entries = subEntries[:0]
skipped := 0
for _, entry := range subEntries {
remote := entry.Remote()
// Only pass on direct children of dir - anything else
// could escape the archive's namespace
if !isDirectChild(dir, remote) {
fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
skipped++
continue
}
for i, entry := range entries {
// Can only unarchive files
if o, ok := entry.(fs.Object); ok {
remote := o.Remote()
archive := f.findArchive(remote)
if archive != nil {
// Overwrite entry with directory
entry = fs.NewDir(remote, o.ModTime(ctx))
entries[i] = fs.NewDir(remote, o.ModTime(ctx))
}
}
entries = append(entries, entry)
}
if skipped > 0 {
fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
}
return entries, nil
}
// isDirectChild reports whether remote names an entry directly in dir
// ("" being the root), with no ".." or other components in between.
func isDirectChild(dir, remote string) bool {
if dir != "" {
var ok bool
remote, ok = strings.CutPrefix(remote, dir+"/")
if !ok {
return false
}
}
return sanitize.Leaf(remote) == nil
}
// NewObject creates a new remote archive file object
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
@@ -557,11 +526,6 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
if err != nil {
return nil, err
}
// Don't trust the archiver to have returned the object asked for
if o.Remote() != remote {
fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
return nil, fs.ErrorObjectNotFound
}
return o, nil
}
-184
View File
@@ -3,11 +3,9 @@
package archive
import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"path"
@@ -16,7 +14,6 @@ import (
"strings"
"testing"
"github.com/rclone/rclone/backend/archive/archiver"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
@@ -222,184 +219,3 @@ func TestArchiveSquashfs(t *testing.T) {
run(t, "mksquashfs", input, output)
})
}
// TestArchiveSquashfsIssue9004 lists and reads squashfs images that exercise
// two layouts go-diskfs used to choke on (fixed in go-diskfs v1.9.4):
//
// - 1.sqfs: a single empty directory, so the image has no fragment table
// (its fragment-table start holds the "not present" sentinel).
// - 2.sqfs: a small tree whose superblock has the NO_XATTRS flag set while
// inodes still carry a (non-sentinel) xattr index - the shape squashfs-
// tools-ng can emit. Built by packing a two-file tree with xattrs via
// `gensquashfs -x`, then setting the NO_XATTRS superblock flag; the tree
// content is trivial placeholder data.
//
// Both images used to fail to list. Regression test for #9004.
func TestArchiveSquashfsIssue9004(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
testdata, err := filepath.Abs(filepath.Join("squashfs", "testdata"))
require.NoError(t, err)
archiveFor := func(t *testing.T, name string) fs.Fs {
f, err := cache.Get(ctx, ":archive:"+filepath.Join(testdata, name))
require.NoError(t, err)
return f
}
t.Run("EmptyDir", func(t *testing.T) {
// 1.sqfs is a single empty directory - it must list without error.
entries, err := archiveFor(t, "1.sqfs").List(ctx, "")
require.NoError(t, err)
assert.Equal(t, 0, len(entries))
})
t.Run("NoXattrTree", func(t *testing.T) {
f := archiveFor(t, "2.sqfs")
entries, err := f.List(ctx, "")
require.NoError(t, err)
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, path.Base(e.Remote()))
}
assert.Contains(t, names, "alpha")
assert.Contains(t, names, "beta")
// A file in the tree must be readable with its real content.
obj, err := f.NewObject(ctx, "beta/sample.xml")
require.NoError(t, err)
assert.Greater(t, obj.Size(), int64(0))
rc, err := obj.Open(ctx)
require.NoError(t, err)
data, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, int(obj.Size()), len(data))
assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
})
}
// TestArchiveUncleanRoot checks that a path into an archive which isn't
// in canonical form (with "./" or doubled slashes) still finds its
// directory.
func TestArchiveUncleanRoot(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
w, err := zw.Create("sub/dir/a.txt")
require.NoError(t, err)
_, err = w.Write([]byte("data"))
require.NoError(t, err)
require.NoError(t, zw.Close())
zipPath := filepath.Join(t.TempDir(), "test.zip")
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
t.Run(root, func(t *testing.T) {
f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, "a.txt", entries[0].Remote())
})
}
}
// escapingObject is an object whose remote is not where it was asked for.
type escapingObject struct {
fs.Object
remote string
}
func (o *escapingObject) Remote() string { return o.remote }
func (o *escapingObject) String() string { return o.remote }
// escapingFs stands in for a badly behaved archiver which exposes entry
// names outside the directory being listed.
type escapingFs struct {
fs.Fs
prefix string
}
func (f *escapingFs) Name() string { return "escaping" }
func (f *escapingFs) Root() string { return "" }
func (f *escapingFs) String() string { return "escaping" }
func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
return fs.DirEntries{
&escapingObject{remote: path.Join(dir, "good.txt")},
fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
&escapingObject{remote: path.Join(dir, "../escape.txt")},
&escapingObject{remote: "../../escape.txt"},
&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
}, nil
}
func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
return &escapingObject{remote: "../escape.txt"}, nil
}
// TestArchiveEscapingArchiver checks that the archive backend does not
// pass on entries from an archiver which escape the directory being
// listed, whatever the archiver does.
func TestArchiveEscapingArchiver(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
archiver.Register(archiver.Archiver{
New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
return &escapingFs{prefix: prefix}, nil
},
Extension: ".escaping",
})
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
f, err := cache.Get(ctx, ":archive:"+dir)
require.NoError(t, err)
// Archives are discovered when their parent directory is listed
_, err = f.List(ctx, "")
require.NoError(t, err)
entries, err := f.List(ctx, "test.escaping")
require.NoError(t, err)
var remotes []string
for _, entry := range entries {
remotes = append(remotes, entry.Remote())
}
assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
_, err = f.NewObject(ctx, "test.escaping/file.txt")
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}
// TestIsDirectChild checks the guard which decides whether an entry
// returned by an archiver belongs directly in the directory listed.
func TestIsDirectChild(t *testing.T) {
for _, test := range []struct {
dir, remote string
want bool
}{
{"", "a.txt", true},
{"", "/a.txt", false},
{"", "a.txt/", false},
{"", "../a.txt", false},
{"", "sub/a.txt", false},
{"d", "d/a.txt", true},
{"d", "d", false},
{"d", "d/", false},
{"d", "d//a.txt", false},
{"d", "d/../a.txt", false},
{"d", "dd/a.txt", false},
{"d", "a.txt", false},
} {
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
}
}
-5
View File
@@ -8,11 +8,6 @@ import (
)
// Archiver describes an archive package
//
// Entry names inside an archive are attacker controlled. An archiver
// must not expose an entry whose remote escapes the archive's own
// namespace (for example one with a ".." component), typically by
// validating names with lib/sanitize.
type Archiver struct {
// New constructs an Fs from the (wrappedFs, remote) with the objects
// prefix with prefix and rooted at root
+1 -1
View File
@@ -36,7 +36,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs
// FIXME vfs cache?
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
VFS := vfs.New(ctx, wrappedFs, nil)
VFS := vfs.New(wrappedFs, nil)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
-5
View File
@@ -161,10 +161,5 @@ func (c *cache) Writable() (backend.WritableFile, error) {
return nil, errCacheNotImplemented
}
// Path returns the path to the underlying storage - not applicable here
func (c *cache) Path() string {
return ""
}
// check interfaces
var _ backend.Storage = (*cache)(nil)
+31 -81
View File
@@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
"os"
"path"
"strings"
"time"
@@ -16,7 +15,6 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/sanitize"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
)
@@ -44,38 +42,15 @@ type Fs struct {
root string // position to read from within the archive
}
// recoverParsePanic converts a panic raised while parsing a squashfs image
// into an error assigned through err.
func recoverParsePanic(err *error) {
if r := recover(); r != nil {
*err = fmt.Errorf("invalid or corrupt squashfs image: %v", r)
}
}
// recoveringReadCloser wraps a reader returned by the go-diskfs parser so
// panics raised while reading become errors.
//
// The parser reads file data lazily, so an image which opens cleanly can
// still panic on the first Read.
type recoveringReadCloser struct {
io.ReadCloser
}
func (r recoveringReadCloser) Read(p []byte) (n int, err error) {
defer recoverParsePanic(&err)
return r.ReadCloser.Read(p)
}
// New constructs an Fs from the (wrappedFs, remote) with the objects
// prefix with prefix and rooted at root
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ fs.Fs, err error) {
defer recoverParsePanic(&err)
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
// FIXME vfs cache?
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
vfsOpt := vfscommon.Opt
vfsOpt.ReadWait = 0
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
VFS := vfs.New(wrappedFs, &vfsOpt)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
@@ -181,17 +156,6 @@ func (f *Fs) toNative(remote string) (string, error) {
return native, nil
}
// This turns a native path with a leading / into a path suitable for
// the squashfs library which requires io/fs.ValidPath paths with no
// leading / and "." for the root directory.
func toIOFS(native string) string {
native = strings.Trim(native, "/")
if native == "" {
return "."
}
return native
}
// Turn a (nativeDir, leaf) into a remote
func (f *Fs) fromNative(nativeDir string, leaf string) string {
// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
@@ -205,12 +169,13 @@ func (f *Fs) fromNative(nativeDir string, leaf string) string {
}
// Convert a FileInfo into an Object from native dir
func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {
return &Object{
fs: f,
remote: f.fromNative(nativeDir, item.Name()),
size: item.Size(),
modTime: item.ModTime(),
item: item,
}
}
@@ -225,69 +190,55 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
defer recoverParsePanic(&err)
nativeDir, err := f.toNative(dir)
if err != nil {
return nil, err
}
items, err := f.sqfs.ReadDir(toIOFS(nativeDir))
items, err := f.sqfs.ReadDir(nativeDir)
if err != nil {
return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
}
entries = make(fs.DirEntries, 0, len(items))
skipped := 0
for _, item := range items {
// fs.Debugf(item.Name(), "entry = %#v", item)
// Skip entry names that aren't safe
if err := sanitize.Leaf(item.Name()); err != nil {
fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
skipped++
continue
for _, fi := range items {
item, ok := fi.(squashfs.FileStat)
if !ok {
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
}
// fs.Debugf(item.Name(), "entry = %#v", item)
var entry fs.DirEntry
if err != nil {
return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)
}
if item.IsDir() {
remote := f.fromNative(nativeDir, item.Name())
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
entry = fs.NewDir(remote, info.ModTime())
} else if item.Type().IsRegular() {
info, err := item.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
}
entry = f.objectFromFileInfo(nativeDir, info)
var remote = f.fromNative(nativeDir, item.Name())
entry = fs.NewDir(remote, item.ModTime())
} else {
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
continue
if item.Mode().IsRegular() {
entry = f.objectFromFileInfo(nativeDir, item)
} else {
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
continue
}
}
entries = append(entries, entry)
}
// Log once per listing rather than per entry so a crafted image
// can't flood the log
if skipped > 0 {
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
}
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
return entries, nil
}
// newObjectNative finds the object at the native path passed in
func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
defer recoverParsePanic(&err)
// get the path and filename
dir, leaf := path.Split(nativePath)
dir = strings.TrimRight(dir, "/")
leaf = strings.Trim(leaf, "/")
// FIXME need to detect directory not found
fis, err := f.sqfs.ReadDir(toIOFS(dir))
fis, err := f.sqfs.ReadDir(dir)
if err != nil {
return nil, fs.ErrorObjectNotFound
@@ -298,11 +249,11 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
if fi.IsDir() {
return nil, fs.ErrorNotAFile
}
info, err := fi.Info()
if err != nil {
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
item, ok := fi.(squashfs.FileStat)
if !ok {
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
}
o = f.objectFromFileInfo(dir, info)
o = f.objectFromFileInfo(dir, item)
break
}
}
@@ -377,6 +328,7 @@ type Object struct {
remote string
size int64
modTime time.Time
item squashfs.FileStat
}
// Fs returns read only access to the Fs that this object is part of
@@ -433,7 +385,6 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
// Open opens the file for read. Call Close() on the returned io.ReadCloser
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
defer recoverParsePanic(&err)
var offset, limit int64 = 0, -1
for _, option := range options {
switch x := option.(type) {
@@ -454,7 +405,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
}
fs.Debugf(o, "Opening %q", remote)
fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)
//fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)
fh, err := o.item.Open()
if err != nil {
return nil, err
}
@@ -466,15 +418,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
return nil, err
}
}
rc = recoveringReadCloser{fh}
// If limited then don't return everything
if limit >= 0 {
fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
return readers.NewLimitedReadCloser(rc, limit), nil
return readers.NewLimitedReadCloser(fh, limit), nil
}
return rc, nil
return fh, nil
}
// Update in to the object with the modTime given of the given size
@@ -1,32 +0,0 @@
package squashfs
import (
"context"
"testing"
_ "github.com/rclone/rclone/backend/local"
fscache "github.com/rclone/rclone/fs/cache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A backslash is an ordinary character in a file name on the systems
// squashfs images are made on, so an entry containing one must be
// listed and openable like any other file.
func TestBackslashInName(t *testing.T) {
ctx := context.Background()
localFs, err := fscache.Get(ctx, "testdata")
require.NoError(t, err)
f, err := New(ctx, localFs, "backslash.sqfs", "", "")
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 1)
assert.Equal(t, `back\slash.txt`, entries[0].Remote())
o, err := f.NewObject(ctx, `back\slash.txt`)
require.NoError(t, err)
assert.Equal(t, int64(6), o.Size())
}
-25
View File
@@ -1,25 +0,0 @@
package squashfs
import (
"io"
"testing"
"github.com/stretchr/testify/assert"
)
// panickingReader panics on Read, standing in for the go-diskfs parser
// hitting an unchecked slice while reading a malformed image.
type panickingReader struct{}
func (panickingReader) Read([]byte) (int, error) { panic("boom") }
func (panickingReader) Close() error { return nil }
// Test that a panic raised while reading file data becomes an error. The
// parser reads lazily, so an image can open cleanly and only panic here.
func TestRecoveringReadCloser(t *testing.T) {
var rc io.ReadCloser = recoveringReadCloser{panickingReader{}}
_, err := rc.Read(make([]byte, 16))
assert.ErrorContains(t, err, "invalid or corrupt squashfs image")
assert.ErrorContains(t, err, "boom")
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,89 +0,0 @@
//go:build !plan9
package archive
import (
"context"
"encoding/binary"
"os"
"path/filepath"
"testing"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// squashfsSuperblock builds a minimal 96-byte squashfs superblock with the
// given attacker-controllable fields. It mirrors the on-disk layout the
// go-diskfs parser reads, and is only used to construct deliberately malformed
// images for the tests below.
func squashfsSuperblock(blockSize uint32, blockLog, compression uint16, rootInode, inodeTableStart uint64) []byte {
const u64max = ^uint64(0)
b := make([]byte, 96)
binary.LittleEndian.PutUint32(b[0:], 0x73717368) // "hsqs" magic
binary.LittleEndian.PutUint32(b[4:], 1) // inode count
binary.LittleEndian.PutUint32(b[8:], 0) // modtime
binary.LittleEndian.PutUint32(b[12:], blockSize)
binary.LittleEndian.PutUint32(b[16:], 0) // fragment count
binary.LittleEndian.PutUint16(b[20:], compression)
binary.LittleEndian.PutUint16(b[22:], blockLog)
binary.LittleEndian.PutUint16(b[24:], 0x0210) // no fragments/xattrs
binary.LittleEndian.PutUint16(b[26:], 0) // id count
binary.LittleEndian.PutUint16(b[28:], 4) // version major
binary.LittleEndian.PutUint16(b[30:], 0) // version minor
binary.LittleEndian.PutUint64(b[32:], rootInode)
binary.LittleEndian.PutUint64(b[40:], 96) // bytes used
binary.LittleEndian.PutUint64(b[48:], u64max)
binary.LittleEndian.PutUint64(b[56:], u64max)
binary.LittleEndian.PutUint64(b[64:], inodeTableStart)
binary.LittleEndian.PutUint64(b[72:], inodeTableStart)
binary.LittleEndian.PutUint64(b[80:], u64max)
binary.LittleEndian.PutUint64(b[88:], u64max)
return b
}
// TestArchiveSquashfsMalformed feeds the archive backend squashfs images whose
// superblock/metadata make the go-diskfs parser panic. See GHSA-6jcg-q3wp-x2f4.
func TestArchiveSquashfsMalformed(t *testing.T) {
fstest.Initialise()
ctx := context.Background()
// Variant 1: zero block size reaches an integer division in the parser.
zeroBlock := squashfsSuperblock(0, 0, 1, 0, 96)
// Variant 2: an out-of-range root inode byte offset reaches an unchecked
// slice against an empty, uncompressed metadata block.
oob := append(squashfsSuperblock(65536, 16, 1, 12336, 96), 0x00, 0x80)
binary.LittleEndian.PutUint64(oob[40:], uint64(len(oob)))
for _, tc := range []struct {
name string
image []byte
}{
{"ZeroBlockSize", zeroBlock},
{"OutOfRangeInodeOffset", oob},
} {
t.Run(tc.name, func(t *testing.T) {
// A directory per image: the fs cache keys on the parent
// directory, so sharing one would hand the second subtest a
// cached listing which does not contain its image.
path := filepath.Join(t.TempDir(), tc.name+".sqfs")
require.NoError(t, os.WriteFile(path, tc.image, 0o644))
// The whole point is that this returns rather than panicking and
// killing the test binary (and, in production, the serve daemon).
f, err := cache.Get(ctx, ":archive:"+path)
if err == nil {
// The image parsed, so the panic must surface from a later
// lazily-parsed operation instead.
_, err = f.List(ctx, "")
}
require.Error(t, err, "a malformed squashfs image must error, not panic")
assert.Contains(t, err.Error(), "invalid or corrupt squashfs image",
"error should come from the recover, not an unrelated failure")
})
}
}
+8 -27
View File
@@ -9,7 +9,6 @@ import (
"io"
"os"
"path"
"slices"
"strings"
"time"
@@ -19,7 +18,6 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/sanitize"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
)
@@ -54,7 +52,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
vfsOpt := vfscommon.Opt
vfsOpt.ReadWait = 0
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
VFS := vfs.New(wrappedFs, &vfsOpt)
node, err := VFS.Stat(remote)
if err != nil {
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
@@ -137,27 +135,15 @@ func (f *Fs) readZip() (singleObject bool, err error) {
return singleObject, fmt.Errorf("failed to read zip file: %w", err)
}
dt := dirtree.New()
skipped := 0
for _, file := range zr.File {
// Skip entries whose name escapes the archive's own namespace
remote, err := sanitize.Path(file.Name)
if err != nil {
skipped++
continue
}
isDir := strings.HasSuffix(file.Name, "/")
// A file entry whose last component is "", "." or ".." names
// a directory rather than a file, so skip
if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
skipped++
continue
remote := strings.Trim(path.Clean(file.Name), "/")
if remote == "." {
remote = ""
}
remote = path.Join(f.prefix, remote)
if f.root != "" {
// Ignore all files outside the root, requiring a path
// boundary so that root "foo" does not also match a
// sibling entry such as "foobar"
if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
// Ignore all files outside the root
if !strings.HasPrefix(remote, f.root) {
continue
}
if remote == f.root {
@@ -166,12 +152,11 @@ func (f *Fs) readZip() (singleObject bool, err error) {
remote = strings.TrimPrefix(remote, f.root+"/")
}
}
if isDir {
if strings.HasSuffix(file.Name, "/") {
dir := fs.NewDir(remote, file.Modified)
dt.AddDir(dir)
} else {
if remote == "" {
// A file at the root is the archive member f.root points at
remote = path.Base(f.root)
singleObject = true
dt = dirtree.New()
@@ -188,9 +173,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
}
}
}
if skipped > 0 {
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
}
dt.CheckParents("")
dt.Sort()
f.dt = dt
@@ -218,8 +200,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return nil, fs.ErrorDirNotFound
}
fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
// Return a copy as callers may filter the listing in place
return slices.Clone(entries), nil
return entries, nil
}
// NewObject finds the Object at remote.
-172
View File
@@ -1,172 +0,0 @@
package zip
import (
"archive/zip"
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"testing"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// writeZip builds a zip file with the given entry names (a name ending
// in "/" is a directory) in dir and returns its leaf name.
func writeZip(t *testing.T, dir, name string, names ...string) string {
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for _, n := range names {
w, err := zw.Create(n)
require.NoError(t, err)
if !strings.HasSuffix(n, "/") {
_, err = w.Write([]byte("data for " + n))
require.NoError(t, err)
}
}
require.NoError(t, zw.Close())
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
return name
}
// allRemotes returns every Object remote in the mounted archive's dirtree.
func allRemotes(t *testing.T, f fs.Fs) []string {
zf, ok := f.(*Fs)
require.True(t, ok)
var remotes []string
for _, entries := range zf.dt {
for _, entry := range entries {
if o, ok := entry.(*Object); ok {
remotes = append(remotes, o.Remote())
}
}
}
// dt iteration order is nondeterministic, so sort for stable comparison.
sort.Strings(remotes)
return remotes
}
// A malicious zip whose entry names escape the archive's own namespace
// must not be exposed by the zip backend (Zip Slip).
func TestReadZipSlip(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "evil.zip",
"good.txt",
"../../etc/cron.d/evil",
"../escape.txt",
"sub/../../above.txt",
`..\..\evil.exe`,
)
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "")
require.NoError(t, err)
remotes := allRemotes(t, f)
// The one benign entry must survive.
assert.Contains(t, remotes, "good.txt")
// No escaping entry may be exposed.
for _, remote := range remotes {
assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
"escaping remote exposed: %q", remote)
}
assert.Equal(t, []string{"good.txt"}, remotes)
}
// Mounting with a non-empty root must only expose entries within that
// root directory, not sibling directories that merely share a name
// prefix (root "foo" must not match "foobar").
func TestReadZipRootBoundary(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "test.zip",
"foo/a.txt",
"foobar/b.txt",
)
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "foo")
require.NoError(t, err)
remotes := allRemotes(t, f)
assert.Equal(t, []string{"a.txt"}, remotes)
}
// A file entry whose name refers to a directory (".", "/", "./", "" or
// "sub/.") must be skipped, not turn the archive or the directory
// mounted as its root into a single file which hides every other
// entry, or become an object named for the archive itself when the
// archive is found by listing its directory.
func TestReadZipDirNamedFileEntry(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "dot.zip",
".",
"/",
"./",
"",
"good.txt",
"sub/",
"sub/.",
"sub/good.txt",
)
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
for _, test := range []struct {
prefix, root string
want []string
}{
{"", "", []string{"good.txt", "sub/good.txt"}},
{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
{"", "sub", []string{"good.txt"}},
} {
t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
f, err := New(ctx, localFs, name, test.prefix, test.root)
require.NoError(t, err)
assert.Equal(t, test.want, allRemotes(t, f))
})
}
}
// Listings are served from a cache which must survive callers
// filtering the returned slice in place, as fs/list does.
func TestListCacheNotAliased(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
localFs, err := cache.Get(ctx, dir)
require.NoError(t, err)
f, err := New(ctx, localFs, name, "", "")
require.NoError(t, err)
entries, err := f.List(ctx, "")
require.NoError(t, err)
require.Len(t, entries, 3)
// Compact in place, dropping the first entry
copy(entries, entries[1:])
entries, err = f.List(ctx, "")
require.NoError(t, err)
var remotes []string
for _, entry := range entries {
remotes = append(remotes, entry.Remote())
}
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
}
-169
View File
@@ -1,169 +0,0 @@
//go:build !plan9 && !js
// Parallel listing support for azureblob, built on the Apache Arrow
// listing format.
//
// The Arrow listing exposes the server-side startFrom (inclusive) and endBefore
// (exclusive) name-range parameters, which let a listing be sharded by
// blob-name range and listed concurrently.
package azureblob
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/fs"
"golang.org/x/sync/errgroup"
)
// listArrowParallel lists (containerName, directory) by splitting the blob name
// keyspace into shards and listing them concurrently. opts must already request
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
// returns the number of raw items seen. directory/prefix are already normalised
// by the caller.
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
// Parallel listing needs the arrowlist client (server-side endBefore is
// only honoured on the Arrow path) - without it fall back to sequential.
if _, err := f.arrowCntSVC(containerName); err != nil {
fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
// Split the keyspace by a single-case character ladder (see
// arrowLadderBoundaries for why single-case matters).
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
if len(points) < 1 {
// Couldn't usefully split - list sequentially.
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
// Build half-open [startFrom, endBefore) shards from the sorted boundary
// points p0..pk: ["", p0), [p0, p1), ..., [pk, "").
type shard struct{ startFrom, endBefore string }
shards := make([]shard, 0, len(points)+1)
prev := ""
for _, p := range points {
shards = append(shards, shard{prev, p})
prev = p
}
shards = append(shards, shard{prev, ""})
// fn (and the list.Helper behind it) is not concurrency-safe, so serialise
// delivery. Only the cheap callback is locked, not the HTTP/decode work.
var fnMu sync.Mutex
safeFn := func(remote string, item *container.BlobItem, isDir bool) error {
fnMu.Lock()
defer fnMu.Unlock()
return fn(remote, item, isDir)
}
// Schedule every shard; SetLimit bounds concurrency, so more shards than
// workers gives free load balancing as workers pick up the next shard.
var total atomic.Int64
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(f.opt.ListParallelism)
for _, sh := range shards {
if gCtx.Err() != nil {
break
}
g.Go(func() error {
// Clone opts per shard so concurrent shards don't share StartFrom/EndBefore.
shardOpts := *opts
if sh.startFrom != "" {
startFrom := sh.startFrom
shardOpts.StartFrom = &startFrom
} else {
shardOpts.StartFrom = nil
}
if sh.endBefore != "" {
endBefore := sh.endBefore
shardOpts.EndBefore = &endBefore
} else {
shardOpts.EndBefore = nil
}
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
total.Add(int64(n))
return err
})
}
err := g.Wait()
found := int(total.Load())
if isEndBeforeUnsupported(err) {
// The account can't honour the shards' endBefore bounds, so parallel
// listing can't be used on it.
f.arrowXMLFallback.Store(true)
if found == 0 {
// No shard delivered anything yet (the normal case - Arrow listing
// enablement is per account so every shard fails on its first
// page) so it is safe to relist sequentially.
fs.Debugf(f, "Parallel Arrow listing not supported by this account (%v) - listing sequentially", err)
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
// Some entries were already delivered so a rerun would duplicate them.
return found, fmt.Errorf("parallel listing aborted after partial results (retry, or unset list_parallelism): %w", err)
}
return found, err
}
// isEndBeforeUnsupported reports whether err means the service can't honour
// the endBefore listing bound, so sharded parallel listing must not be used.
//
// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
// server answered XML with endBefore set), or the service rejecting the
// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
// requested operation is not allowed as EndBefore parameter support is
// missing on this account" - endBefore is only supported on the Arrow
// listing path of accounts with Blob Listing with Apache Arrow enabled.
func isEndBeforeUnsupported(err error) bool {
if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
return true
}
var respErr *azcore.ResponseError
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
}
// arrowShardTarget is the number of ladder boundaries to use for the given
// parallelism (a few per worker for load balancing).
func arrowShardTarget(parallelism int) int {
return parallelism * 4
}
// arrowLadderBoundaries returns up to target sorted ENCODED-space boundaries of
// the form directory+c, used to split the keyspace into shards.
//
// The partition is always correct (no dup/gap) regardless of the alphabet: keys
// whose next byte is outside the alphabet simply fall into an adjacent shard;
// the alphabet only affects balance. A single-case (digit+lowercase) alphabet
// is deliberate: the boundaries are then monotonic under BOTH byte order (the
// listing order) AND the service's case-insensitive startFrom/endBefore
// validation, so server-side endBefore is accepted. (A mixed-case ladder gives
// pairs like [".../Z", ".../a") which are byte-ascending but rejected 400 as
// "endbefore precedes startfrom".) directory is already encoded; the alphabet
// is ASCII (encoder-invariant) and ascending, so the result is sorted.
func arrowLadderBoundaries(directory string, target int) []string {
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
if target < 1 {
return nil
}
if target > len(alphabet) {
target = len(alphabet)
}
points := make([]string, 0, target)
last := -1
for i := 0; i < target; i++ {
idx := i * len(alphabet) / target
if idx == last {
continue
}
last = idx
points = append(points, directory+string(alphabet[idx]))
}
return points
}
-221
View File
@@ -1,221 +0,0 @@
//go:build !plan9 && !js
package azureblob
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/random"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// testArrowList exercises the experimental Apache Arrow listing.
//
// It is skipped automatically if Arrow listing is not enabled on the test
// account (the server returns XML, or 409 on HNS accounts).
func (f *Fs) testArrowList(t *testing.T) {
ctx := context.Background()
containerName, _ := f.split("probe")
if containerName == "" {
t.Skip("Arrow list test needs a container in the remote root")
}
// withArrow runs fn with UseArrowList temporarily set.
withArrow := func(on bool, fn func()) {
old := f.opt.UseArrowList
f.opt.UseArrowList = on
defer func() { f.opt.UseArrowList = old }()
fn()
}
// withParallel runs fn with ListParallelism temporarily set.
withParallel := func(n int, fn func()) {
old := f.opt.ListParallelism
f.opt.ListParallelism = n
defer func() { f.opt.ListParallelism = old }()
fn()
}
// Create a set of known objects under a unique directory: flat files plus
// one in a subdirectory (so we exercise both the flat and delimited paths).
dir := "arrow-list-" + random.String(8)
// Includes upper-case and digit first-characters: these byte-sort between
// the digit and lowercase ladder boundaries, so they verify the server's
// endBefore filtering is byte-ordered (not case-insensitive, which would
// drop them from a shard and make parallel != sequential).
objects := []string{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt", "sub/x.txt", "Foo.txt", "Zoo.txt", "0num.txt", "9end.txt"}
wantSizes := map[string]int64{}
created := make([]string, 0, len(objects))
for i, n := range objects {
contents := random.String(10 + i)
remote := dir + "/" + n
item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
_ = fstests.PutTestContents(ctx, t, f, &item, contents, true)
created = append(created, remote)
wantSizes[remote] = int64(len(contents))
}
defer func() {
for _, remote := range created {
if o, err := f.NewObject(ctx, remote); err == nil {
_ = o.Remove(ctx)
}
}
}()
// listing returns remote->size for dir (dirs reported as size -1).
listing := func(recurse bool) (map[string]int64, error) {
got := map[string]int64{}
add := func(entries fs.DirEntries) error {
for _, e := range entries {
if o, ok := e.(fs.Object); ok {
got[e.Remote()] = o.Size()
} else {
got[e.Remote()] = -1
}
}
return nil
}
if recurse {
return got, f.ListR(ctx, dir, add)
}
entries, err := f.List(ctx, dir)
if err != nil {
return nil, err
}
_ = add(entries)
return got, nil
}
// Probe: is Arrow available? If not, skip cleanly.
var probeErr error
withArrow(true, func() { _, probeErr = listing(true) })
if probeErr != nil {
if strings.Contains(probeErr.Error(), "OperationNotSupportedWithFeatureMissing") || strings.Contains(probeErr.Error(), "409") {
t.Skipf("Arrow listing not available on this account: %v", probeErr)
}
require.NoError(t, probeErr)
}
// Arrow listing must equal XML listing, recursively and per-directory
// (including the "sub" subdirectory).
t.Run("EqualsXML", func(t *testing.T) {
for _, recurse := range []bool{true, false} {
var xml, arrow map[string]int64
var err error
withArrow(false, func() { xml, err = listing(recurse) })
require.NoError(t, err)
withArrow(true, func() { arrow, err = listing(recurse) })
require.NoError(t, err)
assert.Equal(t, xml, arrow, "recurse=%v: Arrow listing differs from XML", recurse)
}
// Recursive Arrow listing contains every object with the right size.
var arrow map[string]int64
withArrow(true, func() { arrow, _ = listing(true) })
for remote, size := range wantSizes {
assert.Equal(t, size, arrow[remote], remote)
}
// Single-directory Arrow listing reports "sub" as a directory.
var entries fs.DirEntries
withArrow(true, func() {
var err error
entries, err = f.List(ctx, dir)
require.NoError(t, err)
})
foundSubDir := false
for _, e := range entries {
if e.Remote() == dir+"/sub" {
_, isObj := e.(fs.Object)
foundSubDir = !isObj
}
}
assert.True(t, foundSubDir, "expected %q to be listed as a directory", dir+"/sub")
})
// Parallel listing must equal sequential listing, recursively and
// per-directory, including the "sub" subdirectory.
t.Run("ParallelEqualsSequential", func(t *testing.T) {
for _, recurse := range []bool{true, false} {
var seq, par map[string]int64
var err error
withArrow(true, func() {
withParallel(0, func() { seq, err = listing(recurse) })
require.NoError(t, err)
withParallel(8, func() { par, err = listing(recurse) })
require.NoError(t, err)
})
assert.Equal(t, seq, par, "recurse=%v: parallel listing differs from sequential", recurse)
}
})
// A tiny list_chunk forces each shard to span multiple Arrow pages, so this
// exercises pagination (startFrom+endBefore+marker resent together) inside a
// shard. The "sub/" boundary puts a.txt..e.txt in one shard => >1 page at
// chunk size 2.
t.Run("ParallelMultiPage", func(t *testing.T) {
oldChunk := f.opt.ListChunkSize
f.opt.ListChunkSize = 2
defer func() { f.opt.ListChunkSize = oldChunk }()
var seq, par map[string]int64
var err error
withArrow(true, func() {
withParallel(0, func() { seq, err = listing(true) })
require.NoError(t, err)
withParallel(8, func() { par, err = listing(true) })
require.NoError(t, err)
})
assert.Equal(t, seq, par)
for remote, size := range wantSizes {
assert.Equal(t, size, par[remote], remote)
}
})
}
func TestArrowLadderBoundaries(t *testing.T) {
for _, tc := range []struct {
directory string
target int
}{
{"", 8},
{"dir/", 16},
{"dir/", 100}, // capped at alphabet size
{"dir/", 1},
} {
points := arrowLadderBoundaries(tc.directory, tc.target)
require.NotEmpty(t, points)
assert.LessOrEqual(t, len(points), 62)
for i, p := range points {
assert.True(t, strings.HasPrefix(p, tc.directory), "boundary %q lacks prefix %q", p, tc.directory)
if i > 0 {
assert.Less(t, points[i-1], p, "boundaries not strictly ascending")
}
}
}
assert.Nil(t, arrowLadderBoundaries("dir/", 0))
// alphabet is single-case (digit+lowercase) so boundaries are monotonic
// under both byte order and case-insensitive order
pts := arrowLadderBoundaries("dir/", 36)
for _, p := range pts {
assert.Equal(t, p, strings.ToLower(p), "boundary must be lowercase: %q", p)
}
}
func TestIsEndBeforeUnsupported(t *testing.T) {
assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
assert.False(t, isEndBeforeUnsupported(nil))
}
-421
View File
@@ -1,421 +0,0 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
// public type aliases so it compiles against the released azblob module.
package arrowlist
import (
"encoding/base64"
"fmt"
"io"
"net/http"
"strconv"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
"github.com/apache/arrow-go/v18/arrow"
"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/ipc"
"github.com/apache/arrow-go/v18/arrow/memory"
)
const (
// ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
ArrowContentType = "application/vnd.apache.arrow.stream"
// ArrowAcceptHeader is the Accept header value to request Arrow format.
ArrowAcceptHeader = ArrowContentType
// resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
// in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
// in Arrow, all rows share the same schema and are distinguished by this field value.
resourceTypeBlobPrefix = "blobprefix"
)
// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
result := container.ListBlobsFlatResponse{}
// Extract response headers
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
// Parse Arrow IPC stream
items, nextMarker, err := parseArrowStream(resp.Body)
if err != nil {
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
}
result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
Segment: &container.BlobFlatListSegment{
BlobItems: items,
},
NextMarker: nextMarker,
}
return result, nil
}
// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
result := container.ListBlobsHierarchyResponse{}
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
items, nextMarker, err := parseArrowStream(resp.Body)
if err != nil {
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
}
// Separate blob items from blob prefixes based on ResourceType field.
var blobItems []*container.BlobItem
var blobPrefixes []*container.BlobPrefix
for _, item := range items {
if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
Name: item.Name,
Properties: item.Properties,
})
} else {
blobItems = append(blobItems, item)
}
}
result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
Segment: &container.BlobHierarchyListSegment{
BlobItems: blobItems,
BlobPrefixes: blobPrefixes,
},
NextMarker: nextMarker,
}
return result, nil
}
func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
*clientRequestID = &val
}
if val := resp.Header.Get("Content-Type"); val != "" {
*contentType = &val
}
if val := resp.Header.Get("Date"); val != "" {
if t, err := time.Parse(time.RFC1123, val); err == nil {
*date = &t
}
}
if val := resp.Header.Get("x-ms-request-id"); val != "" {
*requestID = &val
}
if val := resp.Header.Get("x-ms-version"); val != "" {
*version = &val
}
}
// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
if err != nil {
return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
}
defer reader.Release()
// Extract NextMarker from schema metadata
var nextMarker *string
var numRecords int
if md := reader.Schema().Metadata(); md.Len() > 0 {
if idx := md.FindKey("NextMarker"); idx >= 0 {
val := md.Values()[idx]
if val != "" {
nextMarker = &val
}
}
if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
numRecords = n
}
}
}
// Build column index for the schema
schema := reader.Schema()
colIndex := buildColumnIndex(schema)
// Pre-allocate if we know the count
items := make([]*container.BlobItem, 0, numRecords)
// Iterate over record batches
for reader.Next() {
rec := reader.RecordBatch()
rows := int(rec.NumRows())
for row := range rows {
item := &container.BlobItem{
Properties: &container.BlobProperties{},
}
populateBlobItem(item, rec, row, colIndex)
items = append(items, item)
}
}
if err := reader.Err(); err != nil {
return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
}
return items, nextMarker, nil
}
// columnIndex maps column names to their indices in the record batch for O(1) lookup.
type columnIndex map[string]int
func buildColumnIndex(schema *arrow.Schema) columnIndex {
idx := make(columnIndex, len(schema.Fields()))
for i, f := range schema.Fields() {
idx[f.Name] = i
}
return idx
}
// populateBlobItem fills a BlobItem from a single row across all columns.
func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
// Top-level BlobItem fields
item.Name = getStringField(rec, row, colIdx, "Name")
item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
item.VersionID = getStringField(rec, row, colIdx, "VersionId")
item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
// Metadata maps
item.Metadata = getMapField(rec, row, colIdx, "Metadata")
item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
// Tags
if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
var blobTags []*container.BlobTag
for k, v := range tags {
key := k
blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
}
item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
}
// BlobProperties fields
p := item.Properties
p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
p.ETag = getETagField(rec, row, colIdx, "Etag")
p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
p.CopyID = getStringField(rec, row, colIdx, "CopyId")
p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
p.CopySource = getStringField(rec, row, colIdx, "CopySource")
p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
// Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
// The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
// It is silently ignored for forward compatibility.
}
// Field accessor helpers that safely handle missing columns and null values.
func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.String); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Boolean); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Timestamp); ok {
val := arr.Value(row)
// Arrow timestamps are stored as int64 with a unit; convert to time.Time
dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
return &dt
}
return nil
}
func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Uint64); ok {
val := int64(arr.Value(row))
return &val
}
if arr, ok := col.(*array.Int64); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
if arr, ok := col.(*array.Uint64); ok {
val := int32(arr.Value(row))
return &val
}
if arr, ok := col.(*array.Int32); ok {
val := arr.Value(row)
return &val
}
return nil
}
func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
s := getStringField(rec, row, colIdx, name)
if s == nil {
return nil
}
etag := azcore.ETag(*s)
return &etag
}
func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
s := getStringField(rec, row, colIdx, name)
if s == nil {
return nil
}
decoded, err := base64.StdEncoding.DecodeString(*s)
if err != nil {
return nil
}
return decoded
}
// getEnumField returns a pointer to a string-typed enum value.
type stringEnum interface {
~string
}
func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
s := getStringField(rec, row, colIdx, name)
if s == nil {
return nil
}
val := T(*s)
return &val
}
func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
idx, ok := colIdx[name]
if !ok {
return nil
}
col := rec.Column(idx)
if col.IsNull(row) {
return nil
}
mapArr, ok := col.(*array.Map)
if !ok {
return nil
}
// Get the offsets for this row's map entries
start := int(mapArr.Offsets()[row])
end := int(mapArr.Offsets()[row+1])
if start == end {
return nil
}
keys := mapArr.Keys().(*array.String)
values := mapArr.Items().(*array.String)
result := make(map[string]*string, end-start)
for i := start; i < end; i++ {
k := keys.Value(i)
if values.IsNull(i) {
result[k] = nil
} else {
v := values.Value(i)
result[k] = &v
}
}
return result
}
-374
View File
@@ -1,374 +0,0 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
// c6fa341ca22b (branch feature/storage/bifrost).
package arrowlist
import (
"bytes"
"io"
"net/http"
"testing"
"github.com/apache/arrow-go/v18/arrow"
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/ipc"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/stretchr/testify/require"
)
// coreSchema returns a basic Arrow schema for testing with common blob fields.
func coreSchema(md *arrow.Metadata) *arrow.Schema {
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
{Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
{Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
{Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
{Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
}
return arrow.NewSchema(fields, md)
}
// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
t.Helper()
alloc := memory.DefaultAllocator
var buf bytes.Buffer
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
builder := arrowArray.NewRecordBuilder(alloc, schema)
defer builder.Release()
populate(builder)
rec := builder.NewRecordBatch()
defer rec.Release()
require.NoError(t, w.Write(rec))
require.NoError(t, w.Close())
return buf.Bytes()
}
func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
h := http.Header{}
h.Set("Content-Type", ArrowContentType)
for k, v := range headers {
h.Set(k, v)
}
return &http.Response{
StatusCode: http.StatusOK,
Header: h,
Body: io.NopCloser(bytes.NewReader(data)),
}
}
func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
schema := coreSchema(&md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
b.Field(9).(*arrowArray.StringBuilder).Append("available")
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
b.Field(9).(*arrowArray.StringBuilder).Append("available")
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.NotNil(t, result.NextMarker)
require.Equal(t, "marker123", *result.NextMarker)
require.Len(t, result.Segment.BlobItems, 2)
item := result.Segment.BlobItems[0]
require.Equal(t, "blob1.txt", *item.Name)
require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
require.Equal(t, int64(1024), *item.Properties.ContentLength)
require.Equal(t, "Hot", string(*item.Properties.AccessTier))
}
func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
schema := coreSchema(&md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
// No rows added
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Nil(t, result.NextMarker)
require.Empty(t, result.Segment.BlobItems)
}
func TestHandleFlatListResponse_NullFields(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
schema := coreSchema(&md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
b.Field(3).(*arrowArray.StringBuilder).AppendNull()
b.Field(4).(*arrowArray.StringBuilder).AppendNull()
b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
b.Field(6).(*arrowArray.StringBuilder).AppendNull()
b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
b.Field(8).(*arrowArray.StringBuilder).AppendNull()
b.Field(9).(*arrowArray.StringBuilder).AppendNull()
b.Field(10).(*arrowArray.StringBuilder).AppendNull()
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
item := result.Segment.BlobItems[0]
require.Equal(t, "blob.txt", *item.Name)
require.Nil(t, item.Properties.CreationTime)
require.Nil(t, item.Properties.BlobType)
require.Nil(t, item.Properties.ETag)
require.Nil(t, item.Properties.ContentLength)
require.Nil(t, item.Properties.ContentType)
require.Nil(t, item.Properties.ServerEncrypted)
}
func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
}
func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
alloc := memory.DefaultAllocator
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
schema := coreSchema(&md)
var buf bytes.Buffer
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
// First batch: 2 rows
builder := arrowArray.NewRecordBuilder(alloc, schema)
for range 2 {
builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
for j := 1; j < 11; j++ {
builder.Field(j).AppendNull()
}
}
rec := builder.NewRecordBatch()
require.NoError(t, w.Write(rec))
rec.Release()
builder.Release()
// Second batch: 1 row
builder = arrowArray.NewRecordBuilder(alloc, schema)
builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
for j := 1; j < 11; j++ {
builder.Field(j).AppendNull()
}
rec = builder.NewRecordBatch()
require.NoError(t, w.Write(rec))
rec.Release()
builder.Release()
require.NoError(t, w.Close())
resp := makeHTTPResponse(buf.Bytes(), nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 3)
}
func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
// A regular blob
b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
b.Field(1).(*arrowArray.StringBuilder).AppendNull()
// A blob prefix (virtual directory)
b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleHierarchyListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
require.Len(t, result.Segment.BlobPrefixes, 1)
require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
}
func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
})
resp := makeHTTPResponse(data, map[string]string{
"x-ms-request-id": "req-123",
"x-ms-version": "2026-10-06",
"x-ms-client-request-id": "client-req-456",
"Date": "Mon, 01 Jan 2024 00:00:00 GMT",
})
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Equal(t, "req-123", *result.RequestID)
require.Equal(t, "2026-10-06", *result.Version)
require.Equal(t, "client-req-456", *result.ClientRequestID)
require.NotNil(t, result.Date)
}
func TestHandleFlatListResponse_MapFields(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
{Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
metaBuilder.Append(true)
metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
tagBuilder.Append(true)
tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
require.Len(t, result.Segment.BlobItems, 1)
item := result.Segment.BlobItems[0]
require.NotNil(t, item.Metadata)
require.Equal(t, "value1", *item.Metadata["key1"])
require.NotNil(t, item.BlobTags)
require.Len(t, item.BlobTags.BlobTagSet, 1)
}
func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
item := result.Segment.BlobItems[0]
require.True(t, *item.Deleted)
require.False(t, *item.IsCurrentVersion)
require.True(t, *item.Properties.ServerEncrypted)
}
func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
})
resp := makeHTTPResponse(data, nil)
result, err := HandleFlatListResponse(resp)
require.NoError(t, err)
item := result.Segment.BlobItems[0]
require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
require.Equal(t, "snap-123", *item.Snapshot)
require.True(t, *item.IsCurrentVersion)
}
-259
View File
@@ -1,259 +0,0 @@
//go:build !plan9 && !js
// Package arrowlist implements the experimental "Blob Listing with Apache
// Arrow" feature on top of the released Azure azblob SDK.
//
// The Azure SDK for Go has support for Arrow listing on the unreleased
// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
// response types. Until that ships in a tagged azblob release this package
// fills the gap: it exposes the same options and pager interface as the
// experimental SDK, built only on the public API of the released SDK plus
// copies of two small internal auth policies (see sharedkey.go and
// challenge_policy.go).
//
// This package is temporary. When Arrow listing ships in a released azblob
// the whole package should be deleted and its callers pointed back at
// container.Client.NewListBlobsHierarchyPager - the option and response
// types here are deliberately source compatible with the SDK's to make that
// a mechanical change.
package arrowlist
import (
"context"
"errors"
"net/http"
"strconv"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
)
const (
// moduleName and moduleVersion identify this package in the azcore
// telemetry policy's User-Agent fragment.
moduleName = "github.com/rclone/rclone/backend/azureblob/arrowlist"
moduleVersion = "v0.0.1"
// serviceVersion is the x-ms-version sent with listing requests. Arrow
// responses and the startFrom/endBefore parameters need at least
// 2026-06-06.
serviceVersion = "2026-06-06"
// tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
// own clients.
tokenScope = "https://storage.azure.com/.default"
)
// ErrEndBeforeXMLFallback is returned by the pager when the server answered
// with XML instead of Arrow while EndBefore was set. The service only honours
// endBefore on the Arrow listing path, so an XML page could silently contain
// blobs outside the requested range and cannot be used.
var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
// ListBlobsHierarchyOptions provides the configuration for
// Client.NewListBlobsHierarchyPager.
//
// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
// the released SDK's options are embedded (providing Include, Marker,
// MaxResults, Prefix and StartFrom by promotion) with the two fields the
// released SDK lacks added alongside.
type ListBlobsHierarchyOptions struct {
container.ListBlobsHierarchyOptions
// EndBefore limits listing to blobs whose full container path is
// lexically before it (exclusive). Only honoured on Arrow responses.
EndBefore *string
// UseArrowFormat requests the response as an Apache Arrow IPC stream.
UseArrowFormat *bool
}
// ClientOptions contains the optional parameters for client creation.
type ClientOptions struct {
azcore.ClientOptions
}
// Client issues Arrow listing requests to a single container.
type Client struct {
endpoint string
azClient *azcore.Client
}
// newClient creates a Client for the container at containerURL sending
// requests through a pipeline with the given auth policy (which may be nil
// for anonymous or SAS access).
func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
if options == nil {
options = &ClientOptions{}
}
plOpts := runtime.PipelineOptions{}
if authPolicy != nil {
plOpts.PerRetry = []policy.Policy{authPolicy}
}
azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
if err != nil {
return nil, err
}
return &Client{
endpoint: containerURL,
azClient: azClient,
}, nil
}
// NewClient creates a Client authenticating with a token credential.
func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
}
// NewClientWithNoCredential creates a Client with no authentication, for
// anonymous access or a containerURL carrying a SAS token in its query.
func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
return newClient(containerURL, nil, options)
}
// NewClientWithSharedKeyCredential creates a Client signing requests with the
// account's shared key.
func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
}
// NewListBlobsHierarchyPager returns a pager over the container's blobs,
// requesting Arrow responses when o.UseArrowFormat is set and returning the
// same response type as container.Client.NewListBlobsHierarchyPager.
//
// If the service does not support Arrow listing (the feature not enabled on
// the account, or an HNS account) it answers with XML, which is decoded
// transparently - unless EndBefore is set, in which case the fetcher returns
// ErrEndBeforeXMLFallback (see there).
func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
// Copy the options so advancing the Marker doesn't mutate the caller's struct.
opts := ListBlobsHierarchyOptions{}
if o != nil {
opts = *o
}
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
More: func(page container.ListBlobsHierarchyResponse) bool {
return page.NextMarker != nil && len(*page.NextMarker) > 0
},
Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
if page != nil {
opts.Marker = page.NextMarker
}
req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
if err != nil {
return container.ListBlobsHierarchyResponse{}, err
}
resp, err := c.azClient.Pipeline().Do(req)
if err != nil {
return container.ListBlobsHierarchyResponse{}, err
}
if !runtime.HasStatusCode(resp, http.StatusOK) {
return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
}
if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
return HandleHierarchyListResponse(resp)
}
if opts.EndBefore != nil {
return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
}
return handleXMLResponse(resp)
},
})
}
// listCreateRequest builds the ListBlobs request, mirroring the generated
// request builders in the SDK.
func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
if err != nil {
return nil, err
}
// Reading the existing query first preserves any SAS token in the endpoint.
reqQP := req.Raw().URL.Query()
reqQP.Set("comp", "list")
reqQP.Set("restype", "container")
reqQP.Set("delimiter", delimiter)
if include := formatInclude(opts.Include); include != "" {
reqQP.Set("include", include)
}
if opts.Marker != nil {
reqQP.Set("marker", *opts.Marker)
}
if opts.MaxResults != nil {
reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
}
if opts.Prefix != nil {
reqQP.Set("prefix", *opts.Prefix)
}
if opts.StartFrom != nil {
reqQP.Set("startFrom", *opts.StartFrom)
}
if useArrow && opts.EndBefore != nil {
reqQP.Set("endBefore", *opts.EndBefore)
}
req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
if useArrow {
// Stream the Arrow body rather than buffering it in the pipeline.
runtime.SkipBodyDownload(req)
req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
} else {
req.Raw().Header["Accept"] = []string{"application/xml"}
}
req.Raw().Header["x-ms-version"] = []string{serviceVersion}
return req, nil
}
// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
// generated ListBlobHierarchySegmentHandleResponse.
func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
result := container.ListBlobsHierarchyResponse{}
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
return container.ListBlobsHierarchyResponse{}, err
}
return result, nil
}
// formatInclude renders the include datasets as the comma separated query
// parameter value, in the same order as the SDK's unexported
// ListBlobsInclude.format.
func formatInclude(l container.ListBlobsInclude) string {
var include []string
if l.Copy {
include = append(include, "copy")
}
if l.Deleted {
include = append(include, "deleted")
}
if l.DeletedWithVersions {
include = append(include, "deletedwithversions")
}
if l.ImmutabilityPolicy {
include = append(include, "immutabilitypolicy")
}
if l.LegalHold {
include = append(include, "legalhold")
}
if l.Metadata {
include = append(include, "metadata")
}
if l.Snapshots {
include = append(include, "snapshots")
}
if l.Tags {
include = append(include, "tags")
}
if l.UncommittedBlobs {
include = append(include, "uncommittedblobs")
}
if l.Versions {
include = append(include, "versions")
}
if l.Permissions {
include = append(include, "permissions")
}
return strings.Join(include, ",")
}
@@ -1,198 +0,0 @@
//go:build !plan9 && !js
package arrowlist
import (
"context"
"encoding/base64"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/apache/arrow-go/v18/arrow"
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// arrowPage builds a one row Arrow IPC listing page with the given blob name
// and NextMarker.
func arrowPage(t *testing.T, name, nextMarker string) []byte {
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
fields := []arrow.Field{
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
}
schema := arrow.NewSchema(fields, &md)
return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
b.Field(0).(*arrowArray.StringBuilder).Append(name)
b.Field(1).(*arrowArray.Uint64Builder).Append(42)
})
}
func TestPagerArrow(t *testing.T) {
var requests []*http.Request
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests = append(requests, r.Clone(context.Background()))
w.Header().Set("Content-Type", ArrowContentType)
if r.URL.Query().Get("marker") == "" {
_, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
} else {
_, _ = w.Write(arrowPage(t, "blob2.txt", ""))
}
}))
defer srv.Close()
// SAS style query on the endpoint must be preserved in requests.
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
require.NoError(t, err)
opts := &ListBlobsHierarchyOptions{
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
Include: container.ListBlobsInclude{Metadata: true, Tags: true},
Prefix: new("dir/"),
MaxResults: new(int32(1000)),
StartFrom: new("testcontainer/dir/a"),
},
EndBefore: new("testcontainer/dir/n"),
UseArrowFormat: new(true),
}
pager := client.NewListBlobsHierarchyPager("/", opts)
var names []string
for pager.More() {
page, err := pager.NextPage(context.Background())
require.NoError(t, err)
assert.Equal(t, ArrowContentType, *page.ContentType)
for _, item := range page.Segment.BlobItems {
names = append(names, *item.Name)
assert.Equal(t, int64(42), *item.Properties.ContentLength)
}
}
assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
require.Len(t, requests, 2)
q := requests[0].URL.Query()
assert.Equal(t, "list", q.Get("comp"))
assert.Equal(t, "container", q.Get("restype"))
assert.Equal(t, "/", q.Get("delimiter"))
assert.Equal(t, "metadata,tags", q.Get("include"))
assert.Equal(t, "dir/", q.Get("prefix"))
assert.Equal(t, "1000", q.Get("maxresults"))
assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
assert.Equal(t, "fakesas", q.Get("sv"))
assert.Empty(t, q.Get("marker"))
assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
// The caller's options must not have been mutated by paging.
assert.Nil(t, opts.Marker)
}
const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
<Blobs>
<Blob>
<Name>file.txt</Name>
<Properties>
<Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
<Etag>0x8D1</Etag>
<Content-Length>7</Content-Length>
<BlobType>BlockBlob</BlobType>
</Properties>
</Blob>
<BlobPrefix>
<Name>subdir/</Name>
</BlobPrefix>
</Blobs>
<NextMarker />
</EnumerationResults>`
func TestPagerXMLFallback(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(xmlListResponse))
}))
defer srv.Close()
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
UseArrowFormat: new(true),
})
page, err := pager.NextPage(context.Background())
require.NoError(t, err)
assert.False(t, pager.More())
assert.Equal(t, "application/xml", *page.ContentType)
require.Len(t, page.Segment.BlobItems, 1)
assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
require.Len(t, page.Segment.BlobPrefixes, 1)
assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
}
func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(xmlListResponse))
}))
defer srv.Close()
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
EndBefore: new("testcontainer/n"),
UseArrowFormat: new(true),
})
_, err = pager.NextPage(context.Background())
require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
}
func TestPagerSharedKeyAuth(t *testing.T) {
var gotAuth, gotDate string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
gotDate = r.Header.Get("x-ms-date")
w.Header().Set("Content-Type", ArrowContentType)
_, _ = w.Write(arrowPage(t, "blob.txt", ""))
}))
defer srv.Close()
cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
require.NoError(t, err)
client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
UseArrowFormat: new(true),
})
_, err = pager.NextPage(context.Background())
require.NoError(t, err)
assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
assert.NotEmpty(t, gotDate)
}
func TestPagerResponseError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
w.WriteHeader(http.StatusNotFound)
}))
defer srv.Close()
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
require.NoError(t, err)
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
UseArrowFormat: new(true),
})
_, err = pager.NextPage(context.Background())
require.Error(t, err)
assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
}
@@ -1,121 +0,0 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
// released SDK does not export its storage bearer-challenge policy, which is
// needed for token authentication with the same semantics as the SDK's own
// clients.
package arrowlist
import (
"errors"
"net/http"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
)
type storageAuthorizer struct {
scopes []string
tenantID string
}
// NewStorageChallengePolicy returns a bearer token policy for cred which
// handles the storage service's authentication challenges.
func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
s := storageAuthorizer{scopes: []string{audience}}
return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
AuthorizationHandler: policy.AuthorizationHandler{
OnRequest: s.onRequest,
OnChallenge: s.onChallenge,
},
InsecureAllowCredentialWithHTTP: allowHTTP,
})
}
func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
}
func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
// parse the challenge
err := s.parseChallenge(resp)
if err != nil {
return err
}
// TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
}
type challengePolicyError struct {
err error
}
func (c *challengePolicyError) Error() string {
return c.err.Error()
}
func (*challengePolicyError) NonRetriable() {
// marker method
}
func (c *challengePolicyError) Unwrap() error {
return c.err
}
// parses Tenant ID from auth challenge
// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
func parseTenant(url string) string {
if url == "" {
return ""
}
parts := strings.Split(url, "/")
if len(parts) >= 3 {
tenant := parts[3]
tenant = strings.ReplaceAll(tenant, ",", "")
return tenant
}
return ""
}
func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
authHeader := resp.Header.Get("WWW-Authenticate")
if authHeader == "" {
return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
}
// Strip down to auth and resource
// Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
parts := strings.Split(authHeader, " ")
vals := map[string]string{}
for _, part := range parts {
subParts := strings.Split(part, "=")
if len(subParts) == 2 {
stripped := strings.ReplaceAll(subParts[1], "\"", "")
stripped = strings.TrimSuffix(stripped, ",")
vals[subParts[0]] = stripped
}
}
s.tenantID = parseTenant(vals["authorization_uri"])
scope := vals["resource_id"]
if scope == "" {
return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
}
if !strings.HasSuffix(scope, "/.default") {
scope += "/.default"
}
s.scopes = []string{scope}
return nil
}
-318
View File
@@ -1,318 +0,0 @@
//go:build !plan9 && !js
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.
//
// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
// with the internal header constants inlined. The released SDK does not
// export its shared key signing policy, so a copy is needed to sign the
// Arrow listing requests this package sends through its own pipeline.
package arrowlist
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"fmt"
"net/http"
"net/url"
"sort"
"strings"
"sync/atomic"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/rclone/rclone/fs"
)
// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
// storage account's name and either its primary or secondary key.
func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
c := SharedKeyCredential{accountName: accountName}
if err := c.SetAccountKey(accountKey); err != nil {
return nil, err
}
return &c, nil
}
// SharedKeyCredential contains an account's name and its primary or secondary key.
type SharedKeyCredential struct {
// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
accountName string
accountKey atomic.Value // []byte
}
// AccountName returns the Storage account's name.
func (c *SharedKeyCredential) AccountName() string {
return c.accountName
}
// SetAccountKey replaces the existing account key with the specified account key.
func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
_bytes, err := base64.StdEncoding.DecodeString(accountKey)
if err != nil {
return fmt.Errorf("decode account key: %w", err)
}
c.accountKey.Store(_bytes)
return nil
}
// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
_, err := h.Write([]byte(message))
return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
}
func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
headers := req.Header
contentLength := getHeader("Content-Length", headers)
if contentLength == "0" {
contentLength = ""
}
canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
if err != nil {
return "", err
}
stringToSign := strings.Join([]string{
req.Method,
getHeader("Content-Encoding", headers),
getHeader("Content-Language", headers),
contentLength,
getHeader("Content-MD5", headers),
getHeader("Content-Type", headers),
"", // Empty date because x-ms-date is expected (as per web page above)
getHeader("If-Modified-Since", headers),
getHeader("If-Match", headers),
getHeader("If-None-Match", headers),
getHeader("If-Unmodified-Since", headers),
getHeader("Range", headers),
c.buildCanonicalizedHeader(headers),
canonicalizedResource,
}, "\n")
return stringToSign, nil
}
func getHeader(key string, headers map[string][]string) string {
if headers == nil {
return ""
}
if v, ok := headers[key]; ok {
if len(v) > 0 {
return v[0]
}
}
return ""
}
func getWeightTables() [][]int {
tableLv0 := [...]int{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
}
tableLv2 := [...]int{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
}
tables := [][]int{tableLv0[:], tableLv2[:]}
return tables
}
// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
func compareHeaders(lhs, rhs string, tables [][]int) int {
currLevel, i, j := 0, 0, 0
n := len(tables)
lhsLen := len(lhs)
rhsLen := len(rhs)
for currLevel < n {
if currLevel == (n-1) && i != j {
if i > j {
return -1
}
if i < j {
return 1
}
return 0
}
var w1, w2 int
// Check bounds before accessing lhs[i]
if i < lhsLen {
w1 = tables[currLevel][lhs[i]]
} else {
w1 = 0x1
}
// Check bounds before accessing rhs[j]
if j < rhsLen {
w2 = tables[currLevel][rhs[j]]
} else {
w2 = 0x1
}
if w1 == 0x1 && w2 == 0x1 {
i = 0
j = 0
currLevel++
} else if w1 == w2 {
i++
j++
} else if w1 == 0 {
i++
} else if w2 == 0 {
j++
} else {
if w1 < w2 {
return -1
}
if w1 > w2 {
return 1
}
return 0
}
}
return 0
}
func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
cm := map[string][]string{}
for k, v := range headers {
headerName := strings.TrimSpace(strings.ToLower(k))
if strings.HasPrefix(headerName, "x-ms-") {
cm[headerName] = v // NOTE: the value must not have any whitespace around it.
}
}
if len(cm) == 0 {
return ""
}
keys := make([]string, 0, len(cm))
for key := range cm {
keys = append(keys, key)
}
tables := getWeightTables()
// Sort the keys using the custom comparator
sort.Slice(keys, func(i, j int) bool {
return compareHeaders(keys[i], keys[j], tables) < 0
})
ch := bytes.NewBufferString("")
for i, key := range keys {
if i > 0 {
ch.WriteRune('\n')
}
ch.WriteString(key)
ch.WriteRune(':')
ch.WriteString(strings.Join(cm[key], ","))
}
return ch.String()
}
func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
cr := bytes.NewBufferString("/")
cr.WriteString(c.accountName)
if len(u.Path) > 0 {
// Any portion of the CanonicalizedResource string that is derived from
// the resource's URI should be encoded exactly as it is in the URI.
// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
cr.WriteString(u.EscapedPath())
} else {
// a slash is required to indicate the root path
cr.WriteString("/")
}
// params is a map[string][]string; param name is key; params values is []string
params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
if err != nil {
return "", fmt.Errorf("failed to parse query params: %w", err)
}
if len(params) > 0 { // There is at least 1 query parameter
var paramNames []string // We use this to sort the parameter key names
for paramName := range params {
paramNames = append(paramNames, paramName) // paramNames must be lowercase
}
sort.Strings(paramNames)
for _, paramName := range paramNames {
paramValues := params[paramName]
sort.Strings(paramValues)
// Join the sorted key values separated by ','
// Then prepend "keyName:"; then add this string to the buffer
cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
}
}
return cr.String(), nil
}
// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
return cred.computeHMACSHA256(message)
}
// SharedKeyCredPolicy is a pipeline policy that signs each request with the
// account's shared key.
type SharedKeyCredPolicy struct {
cred *SharedKeyCredential
}
// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
return &SharedKeyCredPolicy{cred: cred}
}
// Do signs the request and sends it via the next policy in the pipeline.
func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
// skip adding the authorization header if no SharedKeyCredential was provided.
// this prevents a panic that might be hard to diagnose and allows testing
// against http endpoints that don't require authentication.
if s.cred == nil {
return req.Next()
}
if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
}
stringToSign, err := s.cred.buildStringToSign(req.Raw())
if err != nil {
return nil, err
}
signature, err := s.cred.computeHMACSHA256(stringToSign)
if err != nil {
return nil, err
}
authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
req.Raw().Header.Set("Authorization", authHeader)
response, err := req.Next()
if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
// Service failed to authenticate request, log it
fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
}
return response, err
}
-11
View File
@@ -9,7 +9,6 @@ import (
"net/http"
"net/url"
"os"
"strings"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
@@ -284,9 +283,6 @@ type transporter struct {
// Make a new transporter
func newTransporter(ctx context.Context) transporter {
// Set the User-Agent to include the Microsoft Partner Network prefix
ctx, ci := fs.AddConfig(ctx)
ci.UserAgent = "APN/1.0 rclone/1.0 rclone/" + strings.TrimPrefix(fs.Version, "v")
return transporter{
RoundTripper: fshttp.NewTransport(ctx),
}
@@ -297,13 +293,6 @@ func (tr transporter) Do(req *http.Request) (*http.Response, error) {
return tr.RoundTripper.RoundTrip(req)
}
// Transporter returns the policy.Transporter rclone uses for Azure SDK
// clients (an fshttp based transport with the APN user agent), for callers
// which build their own azcore pipelines.
func Transporter(ctx context.Context) policy.Transporter {
return newTransporter(ctx)
}
// NewClientOpts should be passed to configure NewClient
type NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct {
DefaultBaseURL string // Base URL, eg blob.core.windows.net
+85 -314
View File
@@ -1,4 +1,4 @@
//go:build !plan9 && !js
//go:build !plan9 && !solaris && !js
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
package azureblob
@@ -19,21 +19,19 @@ import (
"path"
"slices"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/backend/azureblob/auth"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
@@ -49,8 +47,6 @@ import (
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/rclone/rclone/lib/transferaccounter"
"golang.org/x/sync/errgroup"
)
@@ -69,19 +65,8 @@ const (
defaultChunkSize = 4 * fs.Mebi
defaultAccessTier = blob.AccessTier("") // FIXME AccessTierNone
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
sasCopyStartSkew = 15 * time.Minute // how far back SAS should start to allow for clock skew
)
// setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
// on every request. This prevents the Go HTTP transport from automatically
// decompressing gzip-encoded blobs on download.
type setAcceptEncodingGzip struct{}
func (p setAcceptEncodingGzip) Do(req *policy.Request) (*http.Response, error) {
req.Raw().Header.Set("Accept-Encoding", "gzip")
return req.Next()
}
var (
errCantUpdateArchiveTierBlobs = fserrors.NoRetryError(errors.New("can't update archive tier blob without --azureblob-archive-tier-delete"))
@@ -239,42 +224,6 @@ minutes per megabyte on average, it will time out (
avoid the time out.`,
Default: maxListChunkSize,
Advanced: true,
}, {
Name: "use_arrow_list",
Help: `Use the experimental Apache Arrow listing format.
If set, directory listings are fetched using the ListBlobs Apache
Arrow response format instead of XML. This can be faster for very
large containers.
This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
preview feature to be enabled on the storage account. It is NOT
supported on accounts with a hierarchical namespace (ADLS Gen2) -
those return a 409 error. If the feature is not enabled the server
returns XML and the listing transparently falls back to the normal XML
path (logged at debug level).
Not supported with connection_string auth - falls back to normal
listing.`,
Default: false,
Advanced: true,
Hide: fs.OptionHideBoth,
}, {
Name: "list_parallelism",
Help: `Number of parallel shards to list a directory with.
EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
directory is split into this many ranges which are listed concurrently
using the Arrow startFrom/endBefore range parameters. This can
dramatically speed up listing containers with millions of objects, for
both recursive (ListR) and single directory listings.
This has no effect unless "use_arrow_list" is also set, as Arrow is the
only listing path that supports server-side name ranges. The default of
0 (or 1) lists sequentially.`,
Default: 0,
Advanced: true,
Hide: fs.OptionHideBoth,
}, {
Name: "access_tier",
Help: `Access tier of blob: hot, cool, cold or archive.
@@ -401,19 +350,6 @@ rclone does if you know the container exists already.
Default: "",
Exclusive: true,
Advanced: true,
}, {
Name: "decompress",
Help: `If set this will decompress gzip encoded objects.
It is possible to upload objects to Azure Blob Storage with "Content-Encoding: gzip"
set. Normally rclone will download these files as compressed objects.
If this flag is set then rclone will decompress these files with
"Content-Encoding: gzip" as they are received. This means that rclone
can't check the size and hash but the file contents will be decompressed.
`,
Advanced: true,
Default: false,
}}),
})
}
@@ -428,8 +364,6 @@ type Options struct {
UseCopyBlob bool `config:"use_copy_blob"`
UploadConcurrency int `config:"upload_concurrency"`
ListChunkSize uint `config:"list_chunk"`
UseArrowList bool `config:"use_arrow_list"`
ListParallelism int `config:"list_parallelism"`
AccessTier string `config:"access_tier"`
ArchiveTierDelete bool `config:"archive_tier_delete"`
DisableCheckSum bool `config:"disable_checksum"`
@@ -439,7 +373,6 @@ type Options struct {
NoCheckContainer bool `config:"no_check_container"`
NoHeadObject bool `config:"no_head_object"`
DeleteSnapshots string `config:"delete_snapshots"`
Decompress bool `config:"decompress"`
}
// Fs represents a remote azure server
@@ -449,11 +382,8 @@ type Fs struct {
opt Options // parsed config options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache and arrowCntSVCcache
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache
cntSVCcache map[string]*container.Client // reference to containerClient per container
arrowCntSVCcache map[string]*arrowlist.Client // reference to arrowlist client per container
arrowClientOpts *arrowlist.ClientOptions // client options for the arrowlist clients
arrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retried
svc *service.Client // client to access azblob
cred azcore.TokenCredential // how to generate tokens (may be nil)
usingSharedKeyCred bool // set if using shared key credentials
@@ -467,8 +397,6 @@ type Fs struct {
copyToken *pacer.TokenDispenser // global multipart copy concurrency limiter
publicAccess container.PublicAccessType // Container Public Access Level
warnCompressed sync.Once // warn once about compressed files
// user delegation cache
userDelegationMu sync.Mutex
userDelegation *service.UserDelegationCredential
@@ -477,16 +405,15 @@ type Fs struct {
// Object describes an azure object
type Object struct {
fs *Fs // what this object is part of
remote string // The remote path
modTime time.Time // The modified time of the object if known
md5 string // MD5 hash if known
size int64 // Size of the object
mimeType string // Content-Type of the object
accessTier blob.AccessTier // Blob Access Tier
meta map[string]string // blob metadata - take metadataMu when accessing
tags map[string]string // blob tags
contentEncoding *string // Content-Encoding of the object
fs *Fs // what this object is part of
remote string // The remote path
modTime time.Time // The modified time of the object if known
md5 string // MD5 hash if known
size int64 // Size of the object
mimeType string // Content-Type of the object
accessTier blob.AccessTier // Blob Access Tier
meta map[string]string // blob metadata - take metadataMu when accessing
tags map[string]string // blob tags
}
// ------------------------------------------------------------
@@ -575,7 +502,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
var storageErr *azcore.ResponseError
if errors.As(err, &storageErr) {
// General errors from:
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
// Blob specific errors from:
@@ -652,9 +580,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if opt.ListChunkSize > maxListChunkSize {
return nil, fmt.Errorf("blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
}
if opt.ListParallelism > 1 && !opt.UseArrowList {
fs.Logf(nil, "azureblob: list_parallelism has no effect without use_arrow_list - listing sequentially")
}
if opt.AccessTier == "" {
opt.AccessTier = string(defaultAccessTier)
@@ -670,15 +595,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
opt: *opt,
ci: ci,
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
cache: bucket.NewCache(),
cntSVCcache: make(map[string]*container.Client, 1),
arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
name: name,
opt: *opt,
ci: ci,
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
cache: bucket.NewCache(),
cntSVCcache: make(map[string]*container.Client, 1),
}
f.publicAccess = container.PublicAccessType(opt.PublicAccess)
f.setRoot(root)
@@ -710,9 +634,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential,
NewSharedKeyCredential: service.NewSharedKeyCredential,
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
// Override the automatic decompression in the transport
// to download compressed files as-is
policyClientOptions.PerCallPolicies = append(policyClientOptions.PerCallPolicies, setAcceptEncodingGzip{})
options.ClientOptions = policyClientOptions
},
}
@@ -725,15 +646,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f.usingSharedKeyCred = res.UsingSharedKeyCred
f.anonymous = res.Anonymous
// Client options for the arrowlist clients, using the same transport and
// gzip policy as the SDK clients built above.
f.arrowClientOpts = &arrowlist.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: auth.Transporter(ctx),
PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
},
}
// if using Container level SAS put the container client into the cache
if opt.SASURL != "" && res.Container != "" {
_ = f.cntSVC(res.Container)
@@ -772,44 +684,6 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
return containerClient
}
// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
// credentials can't be used for Arrow listing.
var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
// return the arrowlist client for the container passed in
//
// Returns errArrowAuthUnsupported if the configured credentials can't be
// reused for the arrowlist client's pipeline.
func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
// The container URL includes any SAS token in its query
url := f.cntSVC(containerName).URL()
f.cntSVCcacheMu.Lock()
defer f.cntSVCcacheMu.Unlock()
if client, ok := f.arrowCntSVCcache[containerName]; ok {
return client, nil
}
switch {
case f.usingSharedKeyCred:
// Covers account+key and the emulator (auth fills in Account/Key)
var cred *arrowlist.SharedKeyCredential
cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
if err == nil {
client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
}
case f.cred != nil:
client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
case f.anonymous || f.opt.SASURL != "":
client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
default:
return nil, errArrowAuthUnsupported
}
if err != nil {
return nil, err
}
f.arrowCntSVCcache[containerName] = client
return client, nil
}
// Return an Object from a path
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -904,15 +778,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
lowerKey := strings.ToLower(k)
switch lowerKey {
case "cache-control":
headers.BlobCacheControl = new(v)
headers.BlobCacheControl = pString(v)
case "content-disposition":
headers.BlobContentDisposition = new(v)
headers.BlobContentDisposition = pString(v)
case "content-encoding":
headers.BlobContentEncoding = new(v)
headers.BlobContentEncoding = pString(v)
case "content-language":
headers.BlobContentLanguage = new(v)
headers.BlobContentLanguage = pString(v)
case "content-type":
headers.BlobContentType = new(v)
headers.BlobContentType = pString(v)
case "x-ms-tags":
parsed, perr := parseXMsTags(v)
if perr != nil {
@@ -1186,77 +1060,19 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
delimiter = "/"
}
opts := &arrowlist.ListBlobsHierarchyOptions{
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
Include: container.ListBlobsInclude{
Copy: false,
Metadata: true,
Snapshots: false,
UncommittedBlobs: false,
Deleted: false,
},
Prefix: &directory,
MaxResults: &maxResults,
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &container.ListBlobsHierarchyOptions{
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
Include: container.ListBlobsInclude{
Copy: false,
Metadata: true,
Snapshots: false,
UncommittedBlobs: false,
Deleted: false,
},
}
// Experimental: request the Apache Arrow listing format. The arrowlist
// pager requests an Arrow IPC stream and decodes it, falling back to XML
// if the account doesn't have Arrow listing enabled. Skip the
// maxResults==1 probe (isEmpty) which doesn't benefit.
useArrow := f.opt.UseArrowList && maxResults != 1
if useArrow {
opts.UseArrowFormat = new(true)
}
var foundItems int
var err error
if useArrow && f.opt.ListParallelism > 1 && !f.arrowXMLFallback.Load() {
// Arrow exposes server-side startFrom/endBefore name ranges, so the
// keyspace can be sharded and listed concurrently.
foundItems, err = f.listArrowParallel(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
} else {
foundItems, err = f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
}
if err != nil {
return err
}
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
// Determine whether the directory exists or not by whether it has a marker
_, err := f.readMetaData(ctx, containerName, directory)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
}
return nil
}
// listBlobsPager runs the hierarchy listing described by opts, calling fn for
// each blob and subdirectory, and returns the number of raw items seen.
// delimiter selects flat (recurse) vs hierarchical listing. If opts requests
// the Apache Arrow format the listing goes through the arrowlist pager
// (falling back to the SDK pager if the credentials don't support it); if the
// service then answers with XML (Arrow listing not enabled) a debug message
// is logged.
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
if useArrow {
arrowSVC, err := f.arrowCntSVC(containerName)
if err != nil {
fs.Debugf(f, "Not using Arrow listing: %v", err)
useArrow = false
} else {
pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
}
}
if pager == nil {
pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
}
checkedArrow := false
Prefix: &directory,
MaxResults: &maxResults,
})
foundItems := 0
for pager.More() {
var response container.ListBlobsHierarchyResponse
err := f.pacer.Call(func() (bool, error) {
@@ -1266,22 +1082,12 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
return f.shouldRetry(ctx, err)
})
// If Arrow was requested but the service answered with XML, Arrow
// listing is not enabled on this account; the listing is still correct
// but not accelerated.
if useArrow && !checkedArrow && err == nil {
checkedArrow = true
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
}
}
if err != nil {
// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
if storageErr, ok := err.(*azcore.ResponseError); ok && (storageErr.ErrorCode == string(bloberror.ContainerNotFound) || storageErr.StatusCode == http.StatusNotFound) {
return foundItems, fs.ErrorDirNotFound
return fs.ErrorDirNotFound
}
return foundItems, err
return err
}
// Advance marker to next
// marker = response.NextMarker
@@ -1301,12 +1107,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
fs.Debugf(f, "Odd name received %q", remote)
continue
}
// ContentLength is documented as nullable in the Arrow listing schema
size := int64(-1)
if file.Properties.ContentLength != nil {
size = *file.Properties.ContentLength
}
isDirectory := isDirectoryMarker(size, file.Metadata, remote)
isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
if isDirectory {
// Don't insert the root directory
if remote == f.opt.Enc.ToStandardPath(directory) {
@@ -1322,7 +1123,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
// Send object
err = fn(remote, file, isDirectory)
if err != nil {
return foundItems, err
return err
}
}
// Send the subdirectories
@@ -1337,13 +1138,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
fs.Debugf(f, "Odd directory name received %q", remote)
continue
}
// Don't insert the root directory. The Arrow listing returns a
// directory marker blob with the same name as the listed directory
// as a BlobPrefix, whereas XML returns it as a blob (handled
// above).
if remote == f.opt.Enc.ToStandardPath(directory) {
continue
}
remote = remote[len(prefix):]
// Trim one slash off the remote name
remote, _ = strings.CutSuffix(remote, "/")
@@ -1356,11 +1150,21 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
// Send object
err = fn(remote, nil, true)
if err != nil {
return foundItems, err
return err
}
}
}
return foundItems, nil
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
// Determine whether the directory exists or not by whether it has a marker
_, err := f.readMetaData(ctx, containerName, directory)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
}
return nil
}
// Convert a list item into a DirEntry
@@ -1826,7 +1630,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
}
// Validity window
start := time.Now().UTC().Add(-sasCopyStartSkew)
start := time.Now().UTC()
expiry := start.Add(2 * sasCopyValidity)
startStr := start.Format(time.RFC3339)
expiryStr := expiry.Format(time.RFC3339)
@@ -1875,7 +1679,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
// Build the SAS values
perms := sas.BlobPermissions{Read: true}
container, containerPath := o.split()
start := time.Now().UTC().Add(-sasCopyStartSkew)
start := time.Now().UTC()
expiry := start.Add(sasCopyValidity)
vals := sas.BlobSignatureValues{
StartTime: start,
@@ -2197,10 +2001,6 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
if t != hash.MD5 {
return "", hash.ErrUnsupported
}
// If decompressing, erase the hash
if o.size < 0 {
return "", nil
}
// Convert base64 encoded md5 into lower case hex
if o.md5 == "" {
return "", nil
@@ -2326,13 +2126,6 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *blob.GetPropertiesRe
o.accessTier = blob.AccessTier(*info.AccessTier)
}
o.setMetadata(metadata)
o.contentEncoding = info.ContentEncoding
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
o.size = -1
o.md5 = ""
}
return nil
}
@@ -2345,31 +2138,12 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
} else {
size = *info.ContentLength
}
// On a range request Content-Length is the length of the range, not the object, so take the
// object's size from the total in Content-Range instead.
if info.ContentRange != nil {
contentRange, err := rest.ParseContentRange(*info.ContentRange)
if err != nil {
fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
} else if contentRange.Size >= 0 {
size = contentRange.Size
}
}
if isDirectoryMarker(size, metadata, o.remote) {
return fs.ErrorNotAFile
}
// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
// this as base64 encoded string.
//
// On a range request the service returns the whole-blob MD5 in the
// x-ms-blob-content-md5 header (BlobContentMD5) and leaves Content-MD5
// (ContentMD5) empty, so prefer BlobContentMD5. Never overwrite a known
// hash with an empty one, or a ranged read would drop the object's MD5.
if md5 := info.BlobContentMD5; len(md5) > 0 {
o.md5 = base64.StdEncoding.EncodeToString(md5)
} else if len(info.ContentMD5) > 0 {
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
}
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
if info.ContentType == nil {
o.mimeType = ""
} else {
@@ -2388,12 +2162,21 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
// o.accessTier = blob.AccessTier(*info.AccessTier)
// }
o.setMetadata(metadata)
o.contentEncoding = info.ContentEncoding
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
o.size = -1
o.md5 = ""
// If it was a Range request, the size is wrong, so correct it
if info.ContentRange != nil {
contentRange := *info.ContentRange
slash := strings.IndexRune(contentRange, '/')
if slash >= 0 {
i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
if err == nil {
o.size = i
} else {
fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
}
} else {
fs.Debugf(o, "Failed to find length in %q", contentRange)
}
}
return nil
@@ -2433,13 +2216,6 @@ func (o *Object) decodeMetaDataFromBlob(info *container.BlobItem) (err error) {
o.accessTier = *info.Properties.AccessTier
}
o.setMetadata(metadata)
o.contentEncoding = info.Properties.ContentEncoding
// If decompressing then size and md5sum are unknown
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
o.size = -1
o.md5 = ""
}
return nil
}
@@ -2615,20 +2391,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
if err != nil {
return nil, fmt.Errorf("failed to decode metadata for download: %w", err)
}
// Decompress body if necessary
if downloadResponse.ContentEncoding != nil && *downloadResponse.ContentEncoding == "gzip" {
if o.fs.opt.Decompress {
return readers.NewGzipReader(downloadResponse.Body)
}
o.fs.warnCompressed.Do(func() {
fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --azureblob-decompress to override")
})
}
return downloadResponse.Body, nil
}
// Converts a string into a pointer to a string
func pString(s string) *string {
return &s
}
// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
type readSeekCloser struct {
io.Reader
@@ -2783,7 +2553,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
// isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
// returning true if it is found
func isInvalidBlockOrBlob(err error) bool {
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
var storageErr *azcore.ResponseError
if errors.As(err, &storageErr) {
return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
}
return false
@@ -3155,7 +2926,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
// Start with default content-type based on source
ui.httpHeaders = blob.HTTPHeaders{
BlobContentType: new(fs.MimeType(ctx, src)),
BlobContentType: pString(fs.MimeType(ctx, src)),
}
// Apply mapped metadata/headers/tags if requested
@@ -3201,15 +2972,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
o.tags[parts[0]] = parts[1]
}
case "cache-control":
ui.httpHeaders.BlobCacheControl = new(value)
ui.httpHeaders.BlobCacheControl = pString(value)
case "content-disposition":
ui.httpHeaders.BlobContentDisposition = new(value)
ui.httpHeaders.BlobContentDisposition = pString(value)
case "content-encoding":
ui.httpHeaders.BlobContentEncoding = new(value)
ui.httpHeaders.BlobContentEncoding = pString(value)
case "content-language":
ui.httpHeaders.BlobContentLanguage = new(value)
ui.httpHeaders.BlobContentLanguage = pString(value)
case "content-type":
ui.httpHeaders.BlobContentType = new(value)
ui.httpHeaders.BlobContentType = pString(value)
}
}
+1 -113
View File
@@ -1,12 +1,9 @@
//go:build !plan9 && !js
//go:build !plan9 && !solaris && !js
package azureblob
import (
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"encoding/base64"
"fmt"
"net/http"
@@ -17,7 +14,6 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
@@ -54,58 +50,6 @@ func TestBlockIDCreator(t *testing.T) {
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
}
func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
pInt64 := func(i int64) *int64 { return &i }
pString := func(s string) *string { return &s }
newTestObject := func() *Object {
return &Object{
fs: &Fs{},
remote: "test.bin",
size: -2, // sentinel to check it gets set
}
}
t.Run("WholeBlob", func(t *testing.T) {
o := newTestObject()
info := blob.DownloadStreamResponse{}
info.ContentLength = pInt64(12345)
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(12345), o.size)
})
t.Run("RangeRequest", func(t *testing.T) {
// On a ranged download Content-Length is the chunk length so the
// size must come from the total in Content-Range
o := newTestObject()
info := blob.DownloadStreamResponse{}
info.ContentLength = pInt64(67108864)
info.ContentRange = pString("bytes 67108864-134217727/169721004032")
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(169721004032), o.size)
})
t.Run("BadContentRange", func(t *testing.T) {
o := newTestObject()
info := blob.DownloadStreamResponse{}
info.ContentLength = pInt64(67108864)
info.ContentRange = pString("potato")
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(67108864), o.size)
})
t.Run("UnknownLength", func(t *testing.T) {
o := newTestObject()
info := blob.DownloadStreamResponse{}
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
assert.Equal(t, int64(-1), o.size)
})
}
func TestCopySASTimingConstants(t *testing.T) {
require.Greater(t, sasCopyStartSkew, time.Duration(0))
require.Greater(t, sasCopyValidity, sasCopyStartSkew)
}
func (f *Fs) testFeatures(t *testing.T) {
// Check first feature flags are set on this remote
enabled := f.Features().SetTier
@@ -206,66 +150,10 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
require.NoError(t, dst.Remove(ctx))
}
func gz(t *testing.T, s string) string {
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
_, err := zw.Write([]byte(s))
require.NoError(t, err)
err = zw.Close()
require.NoError(t, err)
return buf.String()
}
func md5sum(t *testing.T, s string) string {
hash := md5.Sum([]byte(s))
return fmt.Sprintf("%x", hash)
}
func (f *Fs) testGzipEncoding(t *testing.T) {
ctx := context.Background()
original := random.String(1000)
contents := gz(t, original)
item := fstest.NewItem("test-gzip", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
metadata := fs.Metadata{
"content-encoding": "gzip",
"content-type": "text/plain",
}
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", metadata)
defer func() {
assert.NoError(t, obj.Remove(ctx))
}()
o := obj.(*Object)
// Test that the gzipped file we uploaded can be
// downloaded with and without decompression
checkDownload := func(wantContents string, wantSize int64, wantHash string) {
gotContents := fstests.ReadObject(ctx, t, o, -1)
assert.Equal(t, wantContents, gotContents)
assert.Equal(t, wantSize, o.Size())
gotHash, err := o.Hash(ctx, hash.MD5)
require.NoError(t, err)
assert.Equal(t, wantHash, gotHash)
}
t.Run("NoDecompress", func(t *testing.T) {
checkDownload(contents, int64(len(contents)), md5sum(t, contents))
})
t.Run("Decompress", func(t *testing.T) {
f.opt.Decompress = true
defer func() {
f.opt.Decompress = false
}()
checkDownload(original, -1, "")
})
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("Features", f.testFeatures)
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
t.Run("Metadata", f.testMetadataPaths)
t.Run("GzipEncoding", f.testGzipEncoding)
t.Run("ArrowList", f.testArrowList)
}
// helper to read blob properties for an object
+1 -1
View File
@@ -1,6 +1,6 @@
// Test AzureBlob filesystem interface
//go:build !plan9 && !js
//go:build !plan9 && !solaris && !js
package azureblob
+1 -1
View File
@@ -1,7 +1,7 @@
// Build for azureblob for unsupported platforms to stop go complaining
// about "no buildable Go source files "
//go:build plan9 || js
//go:build plan9 || solaris || js
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
package azureblob
+21 -41
View File
@@ -58,9 +58,6 @@ const (
maxFileSize = 4 * fs.Tebi
defaultChunkSize = 4 * fs.Mebi
storageDefaultBaseURL = "file.core.windows.net"
// smbTimePrecision is the precision the server stores
// modtimes with (100 ns FILETIME ticks)
smbTimePrecision = 100 * time.Nanosecond
)
func init() {
@@ -176,10 +173,6 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
NewSharedKeyCredential: service.NewSharedKeyCredential,
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
options.ClientOptions = policyClientOptions
// FileRequestIntent is required for TokenCredential (OAuth)
// auth; the SDK omits it for shared-key / SAS.
backup := service.ShareTokenIntentBackup
options.FileRequestIntent = &backup
},
}
res, err := auth.NewClient(ctx, conf, &opt.Options)
@@ -253,10 +246,10 @@ func (f *Fs) Features() *fs.Features {
// Precision return the precision of this Fs
//
// The SMB last write time properties are ISO 8601 with 100 ns (FILETIME) precision
// One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers
func (f *Fs) Precision() time.Duration {
return smbTimePrecision
return time.Second
}
// Hashes returns the supported hash sets.
@@ -601,10 +594,7 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
if err != nil {
return fmt.Errorf("unable to set modTime: %w", err)
}
// Truncate to the precision the server stores the modtime with
// to keep the in-memory modtime identical to the one a fresh
// listing returns.
o.modTime = t.Truncate(smbTimePrecision)
o.modTime = t
return nil
}
@@ -650,6 +640,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
return resp.Body, nil
}
// Returns a pointer to t - useful for returning pointers to constants
func ptr[T any](t T) *T {
return &t
}
var warnStreamUpload sync.Once
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
@@ -691,9 +686,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
}
// Count the bytes read so short reads from the source can be detected
counter = readers.NewCountingReader(in)
in = counter
// Measure the size if it is unknown
if sizeUnknown {
counter = readers.NewCountingReader(in)
in = counter
}
// Check we have a source MD5 hash...
if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
@@ -728,21 +725,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if sizeUnknown {
// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
size = int64(counter.BytesRead())
} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
// The file was created at the declared size so a short source
// leaves the remainder padded with zeroes
if isNewlyCreated {
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
}
} else if bytesRead < size {
// Truncate the file to the bytes actually received so it
// is not left padded to the declared size with zeroes
if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
}
}
return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
}
if hashUnknown {
md5Hash = hasher.Sum(nil)
@@ -783,12 +765,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return fmt.Errorf("update: failed to set properties: %w", err)
}
// Make sure Object is in sync, truncating the modtime to the
// precision the server stores it with so it is identical to the
// one a fresh listing returns
// Make sure Object is in sync
o.size = size
o.md5 = md5Hash
o.modTime = modTime.Truncate(smbTimePrecision)
o.modTime = modTime
o.contentType = contentType
return nil
}
@@ -813,8 +793,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
}
opt := file.RenameOptions{
IgnoreReadOnly: new(true),
ReplaceIfExists: new(true),
IgnoreReadOnly: ptr(true),
ReplaceIfExists: ptr(true),
}
dstAbsPath := f.absPath(remote)
fc := srcObj.fileClient()
@@ -859,8 +839,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
opt := directory.RenameOptions{
IgnoreReadOnly: new(false),
ReplaceIfExists: new(false),
IgnoreReadOnly: ptr(false),
ReplaceIfExists: ptr(false),
}
dstAbsPath := dstFs.absPath(dstRemote)
dirClient := srcFs.dirClient(srcRemote)
@@ -899,8 +879,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
ChangeTime: file.SourceCopyFileChangeTime{},
CreationTime: file.SourceCopyFileCreationTime{},
LastWriteTime: file.SourceCopyFileLastWriteTime{},
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
IgnoreReadOnly: new(true),
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
IgnoreReadOnly: ptr(true),
},
}
srcURL := srcObj.fileClient().URL()
+4 -11
View File
@@ -45,7 +45,6 @@ import (
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/youmark/pkcs8"
)
@@ -1461,7 +1460,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
// Item in the cache has the same or newer SequenceID than
// this event. Ignore this event, it must be old.
f.itemMetaCacheMu.Unlock()
fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
continue
}
@@ -1692,7 +1691,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
// upload does a single non-multipart upload
//
// This is recommended for less than 50 MiB of content
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
upload := api.UploadFile{
Name: o.fs.opt.Enc.FromStandardName(leaf),
ContentModifiedAt: api.Time(modTime),
@@ -1704,10 +1703,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
var resp *http.Response
var result api.FolderItems
counter := readers.NewCountingReader(in)
opts := rest.Opts{
Method: "POST",
Body: counter,
Body: in,
MultipartMetadataName: "attributes",
MultipartContentName: "contents",
MultipartFileName: upload.Name,
@@ -1727,11 +1725,6 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
if err != nil {
return err
}
// Check the source supplied the number of bytes it declared
// otherwise a truncated file would be stored as a good upload.
if size >= 0 && int64(counter.BytesRead()) != size {
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
}
if result.TotalCount != 1 || len(result.Entries) != 1 {
return fmt.Errorf("failed to upload %v - not sure why", o)
}
@@ -1761,7 +1754,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Upload with simple or multipart
if size <= int64(o.fs.opt.UploadCutoff) {
err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
} else {
err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
}
+15 -23
View File
@@ -3,6 +3,7 @@
package box
import (
"bytes"
"context"
"crypto/sha1"
"encoding/base64"
@@ -19,7 +20,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/rest"
)
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
}
// uploadPart uploads a part in an upload session
//
// chunk must contain exactly the part's data and is read from the start
// on each attempt.
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
hasher := sha1.New()
chunkSize, err := io.Copy(hasher, chunk)
if err != nil {
return nil, err
}
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
chunkSize := int64(len(chunk))
sha1sum := sha1.Sum(chunk)
opts := rest.Opts{
Method: "PUT",
Path: "/files/upload_sessions/" + SessionID,
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
Options: options,
ExtraHeaders: map[string]string{
"Digest": sha1Digest(hasher.Sum(nil)),
"Digest": sha1Digest(sha1sum[:]),
},
}
var resp *http.Response
err = o.fs.pacer.Call(func() (bool, error) {
_, err = chunk.Seek(0, io.SeekStart)
if err != nil {
return false, err
}
opts.Body = wrap(chunk)
opts.Body = wrap(bytes.NewReader(chunk))
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
return shouldRetry(ctx, resp, err)
})
@@ -223,25 +213,27 @@ outer:
reqSize := min(remaining, chunkSize)
// Read the chunk into memory from the global pool, hashing
// it for the whole file hash (must be done sequentially)
rw := multipart.NewRW().Reserve(reqSize)
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
// Make a block of memory
buf := make([]byte, reqSize)
// Read the chunk
_, err = io.ReadFull(in, buf)
if err != nil {
_ = rw.Close()
err = fmt.Errorf("multipart upload failed to read source: %w", err)
break outer
}
// Make the global hash (must be done sequentially)
_, _ = hash.Write(buf)
// Transfer the chunk
wg.Add(1)
o.fs.uploadToken.Get()
go func(part int, position int64) {
defer wg.Done()
defer o.fs.uploadToken.Put()
defer func() { _ = rw.Close() }()
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
if err != nil {
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
select {
+5 -10
View File
@@ -503,19 +503,15 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
}
go func() {
// Wait on the timer and the stop channel together so that a stop
// signalled by StopBackgroundRunners is honoured immediately.
timer := time.NewTimer(time.Duration(f.opt.ChunkCleanInterval))
defer timer.Stop()
for {
time.Sleep(time.Duration(f.opt.ChunkCleanInterval))
select {
case <-f.cleanupChan:
fs.Infof(f, "stopping cleanup")
return
case <-timer.C:
default:
fs.Debugf(f, "starting cleanup")
f.CleanUpCache(false)
timer.Reset(time.Duration(f.opt.ChunkCleanInterval))
}
}
}()
@@ -567,10 +563,9 @@ Eg
})
rc.Add(rc.Call{
Path: "cache/stats",
NoAuth: true,
Fn: f.httpStats,
Title: "Get cache stats",
Path: "cache/stats",
Fn: f.httpStats,
Title: "Get cache stats",
Help: `
Show statistics for the cache remote.
`,
+1 -1
View File
@@ -43,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
type settings map[string]any
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
configMap := configmap.Simple{}
for key, val := range opts {
configMap[key] = fmt.Sprintf("%v", val)
+9 -13
View File
@@ -34,13 +34,6 @@ import (
"github.com/zeebo/blake3"
)
// Pacer settings - tuned to match other rclone backends
const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
)
// Cloudinary shouldn't have a trailing dot if there is no path
func cldPathDir(somePath string) string {
if somePath == "" || somePath == "." {
@@ -196,7 +189,7 @@ func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (f
root: root,
opt: *opt,
cld: cld,
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
srv: rest.NewClient(client),
}
@@ -472,8 +465,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
if updateOptions, ok := option.(*api.UpdateOptions); ok {
if updateOptions.PublicID != "" {
updateObject = true
params.Overwrite = new(true)
params.Invalidate = new(true)
params.Overwrite = SDKApi.Bool(true)
params.Invalidate = SDKApi.Bool(true)
params.PublicID = updateOptions.PublicID
params.ResourceType = updateOptions.ResourceType
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
@@ -593,18 +586,21 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
if fserrors.ContextError(ctx, &err) {
return false, err
}
// Cloudinary returns rate-limit windows as "Try again on YYYY-MM-DD HH:MM:SS UTC"
// embedded in the error message - honour it as a retry-after.
if err != nil {
tryAgain := "Try again on "
if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
layout := "2006-01-02 15:04:05 UTC"
dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
if timestamp, err2 := time.Parse(layout, dateStr); err2 == nil {
timestamp, err2 := time.Parse(layout, dateStr)
if err2 == nil {
return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
}
}
fs.Debugf(nil, "Retrying API error %v", err)
return true, err
}
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
}
+1 -1
View File
@@ -1104,7 +1104,7 @@ func (o *Object) Remote() string {
newPath, err := o.u.pathAdjustment.do(o.Object.String())
if err != nil {
fs.Errorf(o.Object, "Bad object: %v", err)
return ""
return err.Error()
}
return newPath
}
+5 -61
View File
@@ -31,7 +31,6 @@ import (
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/multipart"
)
// Globals
@@ -564,19 +563,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
// cache small files in memory and do normal upload
rw := multipart.NewRW()
defer fs.CheckClose(rw, &err)
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
if err == io.EOF {
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
return f.Fs.Put(ctx, rw, src, options...)
} else if err != nil {
return nil, err
buf := make([]byte, f.opt.RAMCacheLimit)
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
}
// Need to include what we already read
in = &ReadCloserWrapper{
Reader: io.MultiReader(rw, in),
Reader: io.MultiReader(bytes.NewReader(buf), in),
Closer: in,
}
@@ -703,16 +698,6 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
return nil, err
}
// Check the source supplied the number of bytes it declared -
// compressed data is stored under a name containing the
// original size so a mismatch makes the object unreadable.
if size := src.Size(); size >= 0 && meta.Size != size {
if removeErr := dataObject.Remove(ctx); removeErr != nil {
fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
}
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
}
mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
// meta data upload may fail. in this case we try to remove the original object
@@ -1085,35 +1070,6 @@ type ObjectMetadata struct {
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
}
// validate checks metadata read from the wrapped remote before it is used to
// construct a compressed reader.
func (meta *ObjectMetadata) validate() error {
if meta.Mode != Gzip {
return nil
}
if meta.CompressionMetadataGzip == nil {
return errors.New("missing gzip metadata")
}
if meta.CompressionMetadataGzip.BlockSize <= 0 {
return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
}
if meta.Size != meta.CompressionMetadataGzip.Size {
return errors.New("gzip metadata size does not match object size")
}
if meta.Size < 0 {
return errors.New("invalid gzip object size")
}
blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
blocks := meta.Size / blockSize
if meta.Size%blockSize != 0 {
blocks++
}
if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
return errors.New("gzip block data is incomplete")
}
return nil
}
// Object with external metadata
type Object struct {
fs.Object // Wraps around data object for this object
@@ -1137,9 +1093,6 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
if err = jr.Decode(meta); err != nil {
return nil, err
}
if err = meta.validate(); err != nil {
return nil, err
}
return meta, nil
}
@@ -1157,15 +1110,6 @@ func (o *Object) Remove(ctx context.Context) error {
return objErr
}
// countingDiscard is an io.Writer which discards its input and counts
// the bytes written
type countingDiscard int64
func (c *countingDiscard) Write(p []byte) (int, error) {
*c += countingDiscard(len(p))
return len(p), nil
}
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
type ReadCloserWrapper struct {
io.Reader
-89
View File
@@ -6,8 +6,6 @@ import (
"path/filepath"
"testing"
"github.com/buengese/sgzip"
_ "github.com/rclone/rclone/backend/drive"
_ "github.com/rclone/rclone/backend/local"
_ "github.com/rclone/rclone/backend/s3"
@@ -16,93 +14,6 @@ import (
"github.com/rclone/rclone/fstest/fstests"
)
func TestObjectMetadataValidate(t *testing.T) {
tests := []struct {
name string
meta ObjectMetadata
want bool
}{
{
name: "nil gzip metadata",
meta: ObjectMetadata{Mode: Gzip},
want: true,
},
{
name: "zero block size",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 0,
Size: 1024,
BlockData: []uint32{0},
},
},
want: true,
},
{
name: "negative block size",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: -1,
Size: 1024,
BlockData: []uint32{0},
},
},
want: true,
},
{
name: "mismatched sizes",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1024,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 2048,
BlockData: []uint32{0, 1},
},
},
want: true,
},
{
name: "short block data",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1025,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 1025,
BlockData: []uint32{0, 1},
},
},
want: true,
},
{
name: "valid",
meta: ObjectMetadata{
Mode: Gzip,
Size: 1025,
CompressionMetadataGzip: &sgzip.GzipMetadata{
BlockSize: 512,
Size: 1025,
BlockData: []uint32{0, 1, 2},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := test.meta.validate()
if (err != nil) != test.want {
t.Fatalf("validate() error = %v, want error: %v", err, test.want)
}
})
}
}
var defaultOpt = fstests.Opt{
RemoteName: "TestCompress:",
NilObject: (*Object)(nil),
+3 -2
View File
@@ -2,6 +2,7 @@ package compress
import (
"bufio"
"bytes"
"context"
"crypto/md5"
"encoding/hex"
@@ -22,7 +23,7 @@ type gzipModeHandler struct{}
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
// the configured threshold
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
var b countingDiscard
var b bytes.Buffer
var n int64
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
if err != nil {
@@ -36,7 +37,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
if err != nil {
return false, err
}
ratio := float64(n) / float64(b)
ratio := float64(n) / float64(b.Len())
return ratio > minCompressionRatio, nil
}
-137
View File
@@ -1,137 +0,0 @@
package compress
import (
"bytes"
"context"
"crypto/rand"
"io"
"testing"
"time"
_ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/random"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
// PutStream. It reads each body twice when it is seekable to model a
// backend retrying an upload.
type putRecorder struct {
fs.Fs
t *testing.T
stream bool // expose PutStream from the wrapped Fs
size int64
seekable bool
body []byte
streamed bool
}
func (r *putRecorder) Features() *fs.Features {
features := *r.Fs.Features()
features.PutStream = nil
if r.stream {
features.PutStream = r.PutStream
}
return &features
}
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
r.size = src.Size()
body, err := io.ReadAll(in)
require.NoError(r.t, err)
if seeker, ok := in.(io.Seeker); ok {
r.seekable = true
_, err = seeker.Seek(0, io.SeekStart)
require.NoError(r.t, err)
again, err := io.ReadAll(in)
require.NoError(r.t, err)
assert.Equal(r.t, body, again, "retried body should be identical")
}
r.body = body
return body
}
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
body := r.record(in, src)
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
}
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
r.streamed = true
return r.Put(ctx, in, src, options...)
}
// TestRcat checks the three upload paths rcat takes: small files buffered
// in memory and uploaded with a known size and a seekable body, large
// files streamed when the wrapped remote supports it, and large files
// spooled to a temporary file when it doesn't. Pool pages taken for the
// buffer must be returned on every path.
func TestRcat(t *testing.T) {
ctx := context.Background()
const limit = 1024
for _, test := range []struct {
name string
size int
stream bool
wantSize int64
}{
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
{name: "Empty", size: 0, stream: true, wantSize: 0},
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
} {
t.Run(test.name, func(t *testing.T) {
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
require.NoError(t, err)
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
f := &Fs{Fs: rec}
f.opt.RAMCacheLimit = limit
content := []byte(random.String(test.size))
inUse := pool.Global().InUse()
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
require.NoError(t, err)
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
assert.Equal(t, test.wantSize, rec.size)
assert.Equal(t, content, rec.body)
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
if test.size < limit {
assert.True(t, rec.seekable, "small file body should be seekable")
}
rc, err := o.Open(ctx)
require.NoError(t, err)
got, err := io.ReadAll(rc)
require.NoError(t, err)
require.NoError(t, rc.Close())
assert.Equal(t, content, got)
})
}
}
// TestIsCompressible checks the compressibility heuristic on data at
// both ends of the scale for every compression mode which has one.
func TestIsCompressible(t *testing.T) {
compressible := bytes.Repeat([]byte("compress me "), 4096)
incompressible := make([]byte, len(compressible))
_, err := io.ReadFull(rand.Reader, incompressible)
require.NoError(t, err)
for name, handler := range map[string]compressionModeHandler{
"gzip": &gzipModeHandler{},
"zstd": &zstdModeHandler{},
} {
t.Run(name, func(t *testing.T) {
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
require.NoError(t, err)
assert.True(t, ok)
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
require.NoError(t, err)
assert.False(t, ok)
})
}
}
+3 -2
View File
@@ -2,6 +2,7 @@ package compress
import (
"bufio"
"bytes"
"context"
"crypto/md5"
"encoding/hex"
@@ -22,7 +23,7 @@ type zstdModeHandler struct{}
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
// the configured threshold
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
var b countingDiscard
var b bytes.Buffer
var n int64
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
if err != nil {
@@ -36,7 +37,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
if err != nil {
return false, err
}
ratio := float64(n) / float64(b)
ratio := float64(n) / float64(b.Len())
return ratio > minCompressionRatio, nil
}
+8 -35
View File
@@ -11,7 +11,6 @@ import (
"errors"
"fmt"
"io"
"path"
"strconv"
"strings"
"sync"
@@ -487,12 +486,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
}
// encryptFileName encrypts a file path
//
// If stripVersion is set, a version string (as used by --b2-versions)
// on the last segment is kept in plain text. Set it for file names
// only: directory names must encrypt identically whether passed alone
// or as the parent of a file name.
func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
func (c *Cipher) encryptFileName(in string) string {
segments := strings.Split(in, "/")
for i := range segments {
// Skip directory name encryption if the user chose to
@@ -505,7 +499,7 @@ func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
// of the file name gets encrypted/obfuscated
hasVersion := false
var t time.Time
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
if i == (len(segments)-1) && version.Match(segments[i]) {
var s string
t, s = version.Remove(segments[i])
// version.Remove can fail, in which case it returns segments[i]
@@ -535,24 +529,19 @@ func (c *Cipher) EncryptFileName(in string) string {
if c.mode == NameEncryptionOff {
return in + c.encryptedSuffix
}
return c.encryptFileName(in, true)
return c.encryptFileName(in)
}
// EncryptDirName encrypts a directory path
//
// Unlike EncryptFileName, a version string on the last segment is
// encrypted along with the rest of the name.
func (c *Cipher) EncryptDirName(in string) string {
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
return in
}
return c.encryptFileName(in, false)
return c.encryptFileName(in)
}
// decryptFileName decrypts a file path
//
// stripVersion is as for encryptFileName.
func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
func (c *Cipher) decryptFileName(in string) (string, error) {
segments := strings.Split(in, "/")
for i := range segments {
var err error
@@ -566,7 +555,7 @@ func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
// of the file name gets decrypted/deobfuscated
hasVersion := false
var t time.Time
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
if i == (len(segments)-1) && version.Match(segments[i]) {
var s string
t, s = version.Remove(segments[i])
// version.Remove can fail, in which case it returns segments[i]
@@ -612,31 +601,15 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
// Leave the version string on, if it was there
return decrypted, nil
}
return c.decryptFileName(in, true)
return c.decryptFileName(in)
}
// DecryptDirName decrypts a directory path
//
// Unlike DecryptFileName, a version string on the last segment is
// expected to be part of the encrypted name. Directory names created
// by rclone before v1.76 had a version-like suffix left in plain
// text; these are still decrypted so that they appear in listings,
// but they can't be opened or removed until renamed on the
// underlying remote.
func (c *Cipher) DecryptDirName(in string) (string, error) {
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
return in, nil
}
out, err := c.decryptFileName(in, false)
if err != nil && version.Match(path.Base(in)) {
var legacyErr error
out, legacyErr = c.decryptFileName(in, true)
if legacyErr == nil {
fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
return out, nil
}
}
return out, err
return c.decryptFileName(in)
}
// NameEncryptionMode returns the encryption mode in use for names
-38
View File
@@ -8,7 +8,6 @@ import (
"errors"
"fmt"
"io"
"path"
"strings"
"testing"
@@ -683,43 +682,6 @@ func TestNonStandardDecryptDirName(t *testing.T) {
}
}
// Test directories whose name looks like it has a version string -
// these encrypt verbatim, so that EncryptDirName agrees with the same
// directory encrypted as the parent of a file name, whereas file names
// keep the version string in plain text
func TestVersionedDirName(t *testing.T) {
const dir = "dir-v2001-02-03-040506-123"
enc, err := NewNameEncoding("base32")
require.NoError(t, err)
for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
c, err := newCipher(mode, "", "", true, enc)
require.NoError(t, err)
what := fmt.Sprintf("Testing mode=%v", mode)
// Check EncryptDirName matches the parent of an encrypted file name
encryptedDir := c.EncryptDirName(dir)
encryptedFile := c.EncryptFileName(dir + "/file.txt")
assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
// Check the encrypted directory name round trips OK
decryptedDir, err := c.DecryptDirName(encryptedDir)
assert.NoError(t, err, what)
assert.Equal(t, dir, decryptedDir, what)
// Check a file with the same name keeps its version string in plain text
encryptedFile = c.EncryptFileName(dir)
assert.NotEqual(t, encryptedDir, encryptedFile, what)
assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
decryptedFile, err := c.DecryptFileName(encryptedFile)
assert.NoError(t, err, what)
assert.Equal(t, dir, decryptedFile, what)
// Check a directory with a legacy encrypted name (the file form) still decrypts
decryptedDir, err = c.DecryptDirName(encryptedFile)
assert.NoError(t, err, what)
assert.Equal(t, dir, decryptedDir, what)
// but a name which is bad in both forms is still an error
_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
assert.Error(t, err, what)
}
}
func TestEncryptedSize(t *testing.T) {
c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
for _, test := range []struct {
-5
View File
@@ -1179,11 +1179,6 @@ func (o *ObjectInfo) Size() int64 {
// Hash returns the selected checksum of the file
// If no checksum is available it returns ""
func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
// If the data is unchanged then the hash of the source is the
// hash of the object which will be uploaded
if o.f.opt.NoDataEncryption {
return o.ObjectInfo.Hash(ctx, hash)
}
var srcObj fs.Object
var ok bool
// Get the underlying object if there is one
-4
View File
@@ -84,10 +84,6 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
// Test ObjectInfo.Hash
wantHash := md5.Sum(outBuf.Bytes())
if f.opt.NoDataEncryption {
// If the data isn't encrypted, the hash should be that of the plaintext
wantHash = md5.Sum([]byte(contents))
}
gotHash, err := src.Hash(ctx, hash.MD5)
require.NoError(t, err)
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
+6 -5
View File
@@ -21,7 +21,7 @@ type User struct {
Avatar string `json:"avatar"`
ModelType string `json:"model_type"`
OwnsEntry bool `json:"owns_entry"`
EntryPermissions any `json:"entry_permissions"`
EntryPermissions []any `json:"entry_permissions"`
DisplayName string `json:"display_name"`
}
@@ -57,6 +57,7 @@ type Item struct {
FileHash any `json:"file_hash"`
Public bool `json:"public"`
Thumbnail bool `json:"thumbnail"`
MuxStatus any `json:"mux_status"`
ThumbnailURL any `json:"thumbnail_url"`
WorkspaceID int `json:"workspace_id"`
IsEncrypted int `json:"is_encrypted"`
@@ -76,9 +77,9 @@ type Listing struct {
Data []Item `json:"data"`
From int `json:"from"`
LastPage int `json:"last_page"`
NextPageURL string `json:"next_page_url"`
NextPage int `json:"next_page"`
PerPage int `json:"per_page"`
PrevPageURL string `json:"prev_page_url"`
PrevPage int `json:"prev_page"`
To int `json:"to"`
Total int `json:"total"`
}
@@ -170,7 +171,7 @@ type MultiPartCreateRequest struct {
Mime string `json:"mime"`
Size int64 `json:"size"`
Extension string `json:"extension"`
ParentID json.Number `json:"parentId"`
ParentID json.Number `json:"parent_id"`
RelativePath string `json:"relativePath"`
WorkspaceID string `json:"workspaceId,omitempty"`
}
@@ -221,7 +222,7 @@ type MultiPartEntriesRequest struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
ClientExtension string `json:"clientExtension"`
ParentID json.Number `json:"parentId"`
ParentID json.Number `json:"parent_id"`
RelativePath string `json:"relativePath"`
WorkspaceID string `json:"workspaceId,omitempty"`
}
+19 -90
View File
@@ -54,7 +54,7 @@ const (
rootURL = baseURL + "api/v1"
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
minChunkSize = fs.SizeSuffix(1024 * 1024 * 5)
defaultUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024) // as per https://docs.drime.cloud/uploads-guide
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
)
// Register with Fs
@@ -146,6 +146,14 @@ and these uploads do not fully utilize your bandwidth, then increasing
this may help to speed up the transfers.`,
Default: 4,
Advanced: true,
}, {
Name: "upload_cutoff",
Help: `Cutoff for switching to chunked upload.
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5 GiB.`,
Default: defaultUploadCutoff,
Advanced: true,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
@@ -270,11 +278,6 @@ var retryErrorCodes = []int{
503, // Service Unavailable
504, // Gateway Timeout
509, // Bandwidth Limit Exceeded
520, // Cloudflare: Web server returns an unknown error
521, // Cloudflare: Web server is down
522, // Cloudflare: Connection timed out
523, // Cloudflare: Origin is unreachable
524, // Cloudflare: A timeout occurred
}
// shouldRetry returns a boolean as to whether this resp and err
@@ -524,7 +527,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
Parameters: url.Values{},
}
if dirID != "" {
opts.Parameters.Add("folderId", dirID)
opts.Parameters.Add("parentIds", dirID)
}
if directoriesOnly {
opts.Parameters.Add("type", api.ItemTypeFolder)
@@ -562,10 +565,10 @@ OUTER:
break OUTER
}
}
if result.CurrentPage == result.LastPage {
if result.NextPage == 0 {
break
}
page = result.CurrentPage + 1
page = result.NextPage
}
return found, err
}
@@ -706,11 +709,6 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
return shouldRetry(ctx, resp, err)
})
if err != nil {
// If Drime returns 422 with invalid entry ID, the object is already gone.
// Map this to a standard not-found error
if strings.Contains(err.Error(), "entry ids is invalid") {
return fs.ErrorObjectNotFound
}
return fmt.Errorf("failed to delete item: %w", err)
}
// Check the individual result codes also
@@ -784,16 +782,10 @@ func (f *Fs) patch(ctx context.Context, id, attribute string, value string) (ite
Name: value,
}
var result api.UpdateItemResponse
// The drime origin returns a malformed response (seen by Cloudflare as
// a 520) for a literal PUT to this endpoint, so use a POST with Laravel
// method spoofing instead - the API routes it to the same handler.
opts := rest.Opts{
Method: "POST",
Method: "PUT",
Path: "/file-entries/" + id,
Parameters: url.Values{},
ExtraHeaders: map[string]string{
"X-HTTP-Method-Override": "PUT",
},
}
if f.opt.WorkspaceID != "" {
opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
@@ -927,48 +919,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return dstObj, nil
}
// countDirChildren returns the number of direct children of dirID.
func (f *Fs) countDirChildren(ctx context.Context, dirID string) (int, error) {
n := 0
_, err := f.listAll(ctx, dirID, false, false, "", func(*api.Item) bool {
n++
return false
})
return n, err
}
// waitForDirChildren polls dirID until it lists at least want direct
// children, or the timeout elapses. Drime's folder rename has visible
// eventual-consistency on the children listing - the rename returns
// success immediately but the renamed folder lists as empty for a
// short window - so callers that rename a folder use this to wait for
// the listing to settle before returning (see issue #9450).
func (f *Fs) waitForDirChildren(ctx context.Context, dirID string, want int) error {
const timeout = 30 * time.Second
deadline := time.Now().Add(timeout)
backoff := 100 * time.Millisecond
for {
got, err := f.countDirChildren(ctx, dirID)
if err != nil {
return err
}
if got >= want {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("timed out waiting for directory listing to settle: got %d, want %d", got, want)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(backoff):
}
if backoff < 2*time.Second {
backoff *= 2
}
}
}
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
@@ -989,25 +939,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
return err
}
// Count children before the move so we can wait for the server's
// children listing to reflect the rename - see waitForDirChildren.
preCount, err := f.countDirChildren(ctx, srcID)
if err != nil {
return fmt.Errorf("DirMove: count source children: %w", err)
}
// Do the move
info, err := f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
_, err = f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
if err != nil {
return err
}
if preCount > 0 {
if err := f.waitForDirChildren(ctx, info.ID.String(), preCount); err != nil {
fs.Logf(f, "DirMove: %v (continuing)", err)
}
}
srcFs.dirCache.FlushDir(srcRemote)
return nil
}
@@ -1160,11 +1096,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
return info, nil, err
}
// Send just the leaf as relativePath, matching the single-part /uploads
// convention. The file is placed by parentId; sending an absolute path
// here makes the server build folders from it and drop "0" path segments.
relPath := f.opt.Enc.FromStandardName(leaf)
// Temporary Object under construction
o := &Object{
fs: f,
@@ -1198,7 +1129,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
Size: createSize,
Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
ParentID: json.Number(directoryID),
RelativePath: relPath,
RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
WorkspaceID: f.opt.WorkspaceID,
}
@@ -1225,6 +1156,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
if ext == "" {
ext = "bin"
}
rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
chunkWriter := &drimeChunkWriter{
uploadID: resp.UploadID,
key: resp.Key,
@@ -1237,7 +1170,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
mime: mime,
extension: ext,
parentID: json.Number(directoryID),
relativePath: relPath,
relativePath: rel,
}
info = fs.ChunkWriterInfo{
ChunkSize: int64(chunkSize),
@@ -1597,11 +1530,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
fs.Debugf(o, "Removing old object on successful upload")
deleteErr := o.fs.deleteObject(ctx, id)
if deleteErr != nil {
if errors.Is(deleteErr, fs.ErrorObjectNotFound) {
fs.Debugf(o, "Old object already deleted, safely ignoring")
} else {
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
}
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
}
}()
}
+15 -82
View File
@@ -144,7 +144,6 @@ var (
_importFormats map[string][]string // allowed import MIME type conversions
templatesOnce sync.Once // parse link templates only once
_linkTemplates map[string]*template.Template // available link types
gdocsWarnOnce sync.Once // warn once about skipped non-exportable Google documents
)
// rwChoices type for fs.Bits
@@ -208,10 +207,7 @@ func driveOAuthOptions() []fs.Option {
opts := []fs.Option{}
for _, opt := range oauthutil.SharedOptions {
if opt.Name == config.ConfigClientID {
opt.Help = "Google Application Client Id\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nThe shared client_id is being retired and will stop working during 2026, so creating your own is now strongly recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own."
}
if opt.Name == config.ConfigClientSecret {
opt.Help = "Google Application Client Secret\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nIf you created your own client_id then enter its client secret here."
opt.Help = "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance."
}
opts = append(opts, opt)
}
@@ -225,7 +221,7 @@ func init() {
Description: "Google Drive",
NewFs: NewFs,
CommandHelp: commandHelp,
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
@@ -233,7 +229,7 @@ func init() {
return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
}
switch configIn.State {
switch config.State {
case "":
// Fill in the scopes
driveConfig.Scopes = driveScopes(opt.Scope)
@@ -244,43 +240,24 @@ func init() {
}
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
return fs.ConfigGoto("client_id")
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
OAuth2Config: driveConfig,
})
}
return fs.ConfigGoto("teamdrive")
case "client_id":
if clientID, _ := m.Get(config.ConfigClientID); clientID != "" {
return fs.ConfigGoto("oauth")
}
return oauthutil.SharedClientIDConfigConfirm("client_id_warning", "Google Drive", "https://rclone.org/drive/#making-your-own-client-id")
case "client_id_warning":
if configIn.Result == "true" {
// Continue using the shared client_id
return fs.ConfigGoto("oauth")
}
return fs.ConfigInput("client_id_set", config.ConfigClientID, "Google Application Client Id")
case "client_id_set":
m.Set(config.ConfigClientID, configIn.Result)
return fs.ConfigInput("client_secret_set", config.ConfigClientSecret, "Google Application Client Secret")
case "client_secret_set":
m.Set(config.ConfigClientSecret, configIn.Result)
return fs.ConfigGoto("oauth")
case "oauth":
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
OAuth2Config: driveConfig,
})
case "teamdrive":
if opt.TeamDriveID == "" {
return fs.ConfigConfirm("teamdrive_ok", false, "config_change_team_drive", "Configure this as a Shared Drive (Team Drive)?\n")
}
return fs.ConfigConfirm("teamdrive_change", false, "config_change_team_drive", fmt.Sprintf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID))
case "teamdrive_ok":
if configIn.Result == "false" {
if config.Result == "false" {
m.Set("team_drive", "")
return nil, nil
}
return fs.ConfigGoto("teamdrive_config")
case "teamdrive_change":
if configIn.Result == "false" {
if config.Result == "false" {
return nil, nil
}
return fs.ConfigGoto("teamdrive_config")
@@ -301,14 +278,14 @@ func init() {
return teamDrive.Id, teamDrive.Name
})
case "teamdrive_final":
driveID := configIn.Result
driveID := config.Result
m.Set("team_drive", driveID)
m.Set("root_folder_id", "")
opt.TeamDriveID = driveID
opt.RootFolderID = ""
return nil, nil
}
return nil, fmt.Errorf("unknown state %q", configIn.State)
return nil, fmt.Errorf("unknown state %q", config.State)
},
MetadataInfo: &fs.MetadataInfo{
System: systemMetadataInfo,
@@ -1325,7 +1302,6 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
return nil, fmt.Errorf("failed to create client from environment: %w", err)
}
} else {
oauthutil.SharedClientIDWarning(name, "Google Drive", "https://rclone.org/drive/#making-your-own-client-id", m)
oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
if err != nil {
return nil, fmt.Errorf("failed to create oauth client: %w", err)
@@ -1710,9 +1686,6 @@ func (f *Fs) newObjectWithExportInfo(
// If item MimeType is in the ExportFormats then it is a google doc
if !isDocument {
fs.Debugf(remote, "Ignoring unknown document type %q", info.MimeType)
gdocsWarnOnce.Do(func() {
fs.Logf(remote, "Skipping unexportable google document %q. Use --drive-show-all-gdocs to include them in server side copy and move", info.MimeType)
})
return nil, fs.ErrorObjectNotFound
}
if extension == "" {
@@ -2439,40 +2412,12 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
return newItem, nil
}
// shortcutLoop reports whether a folder shortcut at remote pointing to
// the directory targetID would create a recursive loop.
//
// This is the case when targetID is the ID of remote's parent or any
// other ancestor directory, as descending into it would list an
// ancestor again forever. The ancestor IDs are looked up in the
// directory cache which is always populated for the parents of remote
// as they are listed before their children.
func (f *Fs) shortcutLoop(remote, targetID string) bool {
for dir := path.Dir(remote); ; dir = path.Dir(dir) {
if dir == "." {
dir = ""
}
if id, ok := f.dirCache.Get(dir); ok && actualID(id) == targetID {
return true
}
if dir == "" {
return false
}
}
}
// itemToDirEntry converts a drive.File to an fs.DirEntry.
// When the drive.File cannot be represented as an fs.DirEntry
// (nil, nil) is returned.
func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
switch {
case item.MimeType == driveFolderType:
// A folder shortcut pointing at one of its own ancestors
// would make rclone recurse forever, so drop it.
if isShortcutID(item.Id) && f.shortcutLoop(remote, actualID(item.Id)) {
fs.Errorf(remote, "Ignoring folder shortcut as it points to an ancestor directory creating a loop")
return nil, nil
}
// cache the directory ID for later lookups
f.dirCache.Put(remote, item.Id)
// cache the resource key for later lookups
@@ -2766,12 +2711,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
}
trashedFiles := false
if check {
// trashedFiles is only consulted to decide between trashing and hard deleting
// below, so when the directory is going to be trashed anyway there is no reason
// to enumerate trashed children: let the server filter them out instead of
// paging through them. Hard deletes still need to see them (#1040).
includeAll := !f.opt.UseTrash || f.opt.TrashedOnly
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, includeAll, func(item *drive.File) bool {
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
if !item.Trashed {
fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
return true
@@ -3094,18 +3034,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
remote = remote[:len(remote)-len(ext)]
}
// Find the ID of the parent to remove the file from.
var srcParentID string
if len(srcObj.parents) == 1 {
srcParentID = srcObj.parents[0]
} else {
var err error
_, srcParentID, err = srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
if err != nil {
return nil, err
}
srcParentID = actualID(srcParentID)
_, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
if err != nil {
return nil, err
}
srcParentID = actualID(srcParentID)
// Temporary Object under construction
dstInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
-211
View File
@@ -8,14 +8,10 @@ import (
"fmt"
"io"
"mime"
"net/http"
"net/http/httptest"
"os"
"path"
"path/filepath"
"strings"
stdsync "sync"
"sync/atomic"
"testing"
"time"
@@ -28,114 +24,13 @@ import (
"github.com/rclone/rclone/fs/sync"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"google.golang.org/api/option"
)
func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
t.Helper()
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
service, err := drive.NewService(
context.Background(),
option.WithHTTPClient(server.Client()),
option.WithEndpoint(server.URL+"/"),
)
require.NoError(t, err)
return &Fs{
svc: service,
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
permissionsMu: new(stdsync.Mutex),
permissions: make(map[string]*drive.Permission),
}
}
func TestInternalGetPermissionConcurrent(t *testing.T) {
const requestCount = 2
arrived := make(chan string, requestCount)
release := make(chan struct{})
var releaseOnce stdsync.Once
releaseRequests := func() {
releaseOnce.Do(func() { close(release) })
}
defer releaseRequests()
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
permissionID := path.Base(r.URL.Path)
arrived <- permissionID
<-release
w.Header().Set("Content-Type", "application/json")
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
assert.NoError(t, err)
}))
type permissionResult struct {
permission *drive.Permission
inherited bool
err error
}
results := make(chan permissionResult, requestCount)
permissionIDs := []string{"permission-1", "permission-2"}
for _, permissionID := range permissionIDs {
go func() {
permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
results <- permissionResult{permission: permission, inherited: inherited, err: err}
}()
}
arrivedIDs := make(map[string]bool, requestCount)
timer := time.NewTimer(time.Second)
defer timer.Stop()
for len(arrivedIDs) < requestCount {
select {
case permissionID := <-arrived:
arrivedIDs[permissionID] = true
case <-timer.C:
releaseRequests()
t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
}
}
releaseRequests()
assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
returnedIDs := make(map[string]bool, requestCount)
for range requestCount {
result := <-results
require.NoError(t, result.err)
require.NotNil(t, result.permission)
assert.False(t, result.inherited)
returnedIDs[result.permission.Id] = true
}
assert.Equal(t, arrivedIDs, returnedIDs)
}
func TestInternalGetPermissionCacheHit(t *testing.T) {
var requests atomic.Int32
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
permissionID := path.Base(r.URL.Path)
w.Header().Set("Content-Type", "application/json")
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
assert.NoError(t, err)
}))
first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
require.NoError(t, err)
assert.False(t, inherited)
second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
require.NoError(t, err)
assert.False(t, inherited)
assert.Same(t, first, second)
assert.Equal(t, int32(1), requests.Load())
}
func TestDriveScopes(t *testing.T) {
for _, test := range []struct {
in string
@@ -250,34 +145,6 @@ func TestInternalFindExportFormat(t *testing.T) {
}
}
func TestShortcutLoop(t *testing.T) {
f := &Fs{}
f.dirCache = dircache.New("", "root", f)
// root is FolderX with ID "X"
f.dirCache.Put("", "X")
// a real subfolder
f.dirCache.Put("sub", "sub-id")
// a nested subfolder reached via a shortcut, composite ID
f.dirCache.Put("sub/nested", joinID("nested-id", "shortcut-id"))
for _, test := range []struct {
name string
remote string
targetID string
want bool
}{
{"shortcut to the root itself", "loop", "X", true},
{"shortcut to the parent", "sub/loop", "sub-id", true},
{"shortcut to a grandparent", "sub/nested/loop", "X", true},
{"shortcut to a composite ancestor", "sub/nested/loop", "nested-id", true},
{"shortcut to an unrelated folder", "sub/loop", "other-id", false},
{"shortcut to a sibling", "sub/loop", "sub-id2", false},
} {
got := f.shortcutLoop(test.remote, test.targetID)
assert.Equal(t, test.want, got, test.name)
}
}
func TestMimeTypesToExtension(t *testing.T) {
for mimeType, extension := range _mimeTypeToExtension {
extensions, err := mime.ExtensionsByType(mimeType)
@@ -814,83 +681,6 @@ func (f *Fs) InternalTestSingleQuoteFolder(t *testing.T) {
assert.NoError(t, err)
}
// TestIntegration/FsMkdir/FsPutFiles/Internal/MoveDuplicateParent
//
// Check that moving a file removes it from its real parent even when there
// are two directories with the same name and the directory cache resolves the
// path to the wrong duplicate. Otherwise the file ends up with two parents
// which fails on shared drives with a teamDrivesParentLimit error (#9472).
func (f *Fs) InternalTestMoveDuplicateParent(t *testing.T) {
ctx := context.Background()
if f.Features().Move == nil {
t.Skip("Move not supported")
}
rootID, err := f.dirCache.RootID(ctx, false)
require.NoError(t, err)
const dupName = "dup9472"
const dstName = "dst9472"
const fileName = "michael.txt"
// Create the first "dup9472" directory - this is the one the directory
// cache will resolve the path to.
dirA, err := f.createDir(ctx, rootID, dupName, nil)
require.NoError(t, err)
f.dirCache.Put(dupName, dirA.Id)
// Create a second directory with the SAME name and parent: a duplicate
// the directory cache does not know about.
dirB, err := f.createDir(ctx, rootID, dupName, nil)
require.NoError(t, err)
// Destination directory
dirDst, err := f.createDir(ctx, rootID, dstName, nil)
require.NoError(t, err)
defer func() {
_ = f.delete(ctx, dirA.Id, false)
_ = f.delete(ctx, dirB.Id, false)
_ = f.delete(ctx, dirDst.Id, false)
f.dirCache.Flush()
}()
// Upload a file directly into dirB (the duplicate the cache doesn't know).
createInfo := &drive.File{
Name: fileName,
Parents: []string{dirB.Id},
}
var fileInfo *drive.File
err = f.pacer.Call(func() (bool, error) {
fileInfo, err = f.svc.Files.Create(createInfo).
Media(strings.NewReader("9472 duplicate parent test")).
Fields(f.getFileFields(ctx)).
SupportsAllDrives(true).
Context(ctx).Do()
return f.shouldRetry(ctx, err)
})
require.NoError(t, err)
require.Equal(t, []string{dirB.Id}, fileInfo.Parents)
// Sanity check: the directory cache resolves the path to dirA, not the
// dirB the file actually lives in.
gotDirID, err := f.dirCache.FindDir(ctx, dupName, false)
require.NoError(t, err)
require.Equal(t, dirA.Id, gotDirID, "directory cache should resolve to the wrong duplicate")
// Build the source object pointing at the file in dirB and move it.
srcObj, err := f.newObjectWithInfo(ctx, dupName+"/"+fileName, fileInfo)
require.NoError(t, err)
dstObj, err := f.Move(ctx, srcObj, dstName+"/"+fileName)
require.NoError(t, err)
// The moved file must have exactly one parent: the destination.
movedInfo, err := f.getFile(ctx, dstObj.(fs.IDer).ID(), "id,parents")
require.NoError(t, err)
assert.Equal(t, []string{dirDst.Id}, movedInfo.Parents)
}
func (f *Fs) InternalTest(t *testing.T) {
// These tests all depend on each other so run them as nested tests
t.Run("DocumentImport", func(t *testing.T) {
@@ -911,7 +701,6 @@ func (f *Fs) InternalTest(t *testing.T) {
t.Run("Query", f.InternalTestQuery)
t.Run("AgeQuery", f.InternalTestAgeQuery)
t.Run("SingleQuoteFolder", f.InternalTestSingleQuoteFolder)
t.Run("MoveDuplicateParent", f.InternalTestMoveDuplicateParent)
t.Run("ShouldRetry", f.InternalTestShouldRetry)
}
+2 -4
View File
@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
// getPermission returns permissions for the fileID and permissionID passed in
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
f.permissionsMu.Lock()
defer f.permissionsMu.Unlock()
if useCache {
f.permissionsMu.Lock()
perm = f.permissions[permissionID]
f.permissionsMu.Unlock()
if perm != nil {
return perm, false, nil
}
@@ -131,9 +131,7 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
cleanPermission(perm)
// cache the permission
f.permissionsMu.Lock()
f.permissions[permissionID] = perm
f.permissionsMu.Unlock()
return perm, inherited, err
}
+18 -30
View File
@@ -11,6 +11,7 @@
package drive
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -21,7 +22,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/readers"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
@@ -112,13 +112,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
}
// Make an http.Request for the range passed in
//
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
// to an io.Closer and close it after each attempt — the chunk buffer is
// pool-backed, so an early close would free its pages and a retry would then
// read a dead buffer. The upload loop owns the buffer's lifetime.
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
req.ContentLength = reqSize
totalSize := "*"
if rx.ContentLength >= 0 {
@@ -173,40 +168,37 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
start := int64(0)
var StatusCode int
var err error
chunkSize := int64(rx.f.opt.ChunkSize)
buf := make([]byte, int(rx.f.opt.ChunkSize))
for finished := false; !finished; {
reqSize := chunkSize
var reqSize int64
var chunk io.ReadSeeker
if rx.ContentLength >= 0 {
// If size known use repeatable reader for smoother bwlimit
if start >= rx.ContentLength {
break
}
reqSize = min(rx.ContentLength-start, chunkSize)
}
// Buffer the chunk in memory from the global pool so reads are
// repeatable for retries
rw := multipart.NewRW()
var n int64
n, err = io.CopyN(rw, rx.Media, reqSize)
if rx.ContentLength < 0 {
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
} else {
// If size unknown read into buffer
var n int
n, err = readers.ReadFill(rx.Media, buf)
if err == io.EOF {
// Send the last chunk with the correct ContentLength
// otherwise Google doesn't know we've finished
rx.ContentLength = start + n
rx.ContentLength = start + int64(n)
finished = true
err = nil
} else if err != nil {
return nil, err
}
reqSize = n
}
if err != nil {
_ = rw.Close()
return nil, err
reqSize = int64(n)
chunk = bytes.NewReader(buf[:reqSize])
}
// Transfer the chunk
err = rx.f.pacer.Call(func() (bool, error) {
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
again, err := rx.f.shouldRetry(ctx, err)
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
again = false
@@ -214,13 +206,9 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
}
return again, err
})
closeErr := rw.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
start += reqSize
}
-57
View File
@@ -1,57 +0,0 @@
package drive
import (
"bytes"
"context"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
// retried successfully. The chunk buffer is pool-backed and implements
// io.Closer, so if it reaches the http transport unwrapped the transport
// closes it after the failed attempt, returning its pages to the pool, and
// the retry then reads a freed buffer.
func TestResumableUploadRetry(t *testing.T) {
content := bytes.Repeat([]byte("resumable"), 512)
var attempts atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
require.NoError(t, err)
if attempts.Add(1) == 1 {
w.WriteHeader(http.StatusInternalServerError)
return
}
assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
}))
defer server.Close()
f := &Fs{
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
client: server.Client(),
}
f.opt.ChunkSize = fs.SizeSuffix(len(content))
rx := &resumableUpload{
f: f,
remote: "remote",
URI: server.URL,
Media: bytes.NewReader(content),
MediaType: "application/octet-stream",
ContentLength: int64(len(content)),
}
info, err := rx.Upload(context.Background())
require.NoError(t, err)
require.NotNil(t, info)
assert.Equal(t, "fake-id", info.Id)
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
}
+1 -1
View File
@@ -19,7 +19,7 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
Entries: items,
}
err = f.pacer.Call(func() (bool, error) {
complete, err = f.srv.UploadSessionFinishBatchV2Context(ctx, arg)
complete, err = f.srv.UploadSessionFinishBatchV2(arg)
if retry, err := shouldRetryExclude(ctx, err); !retry {
return retry, err
}
+115 -275
View File
@@ -51,7 +51,6 @@ import (
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/batcher"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/readers"
@@ -151,20 +150,14 @@ var (
// Gets an oauth config with the right scopes
func getOauthConfig(m configmap.Mapper) *oauthutil.Config {
impersonate, _ := m.Get("impersonate")
impersonateAdmin, _ := m.Get("impersonate_admin")
// If not impersonating, use standard scopes
if impersonate == "" && impersonateAdmin == "" {
if impersonate, _ := m.Get("impersonate"); impersonate == "" {
return dropboxConfig
}
// Make a copy of the config
config := *dropboxConfig
// Make a copy of the scopes with extra scopes requires appended
config.Scopes = append(config.Scopes, "team_data.member")
if impersonate != "" {
config.Scopes = append(config.Scopes, "members.read")
}
config.Scopes = append(config.Scopes, "members.read", "team_data.member")
return &config
}
@@ -213,29 +206,6 @@ You will have to use your own App (setting your own client_id and
client_secret) to use this option as currently rclone's default set of
permissions doesn't include "members.read". This can be added once
v1.55 or later is in use everywhere.
`,
Default: "",
Advanced: true,
Sensitive: true,
}, {
Name: "impersonate_admin",
Help: `Team admin ID to use when performing actions as a team administrator.
This sets the Dropbox-API-Select-Admin header with the given team
member ID (for example "dbmid:...").
This takes a team member ID directly rather than an email address.
Note that if you want to use impersonate_admin, you should make sure this
flag is set when running "rclone config" as this will cause rclone to
request the "team_data.member" scope which it won't normally.
Using the "team_data.member" scope will require a Dropbox Team Admin
to approve during the OAuth flow.
You will have to use your own App (setting your own client_id and
client_secret) to use this option as currently rclone's default set of
permissions doesn't include "team_data.member".
`,
Default: "",
Advanced: true,
@@ -268,36 +238,6 @@ See also --dropbox-root-namespace for an alternative way to work with shared
folders.`,
Default: false,
Advanced: true,
}, {
Name: "skip_shared_folders",
Help: `Instructs rclone to skip all shared folders.
When set, any folder that is a shared folder mount point will be
excluded from directory listings, regardless of ownership.
This is useful if you prefer to back up shared folders separately
using a separate remote configured with the shared folder namespace.`,
Default: false,
Advanced: true,
}, {
Name: "skip_unowned_folders",
Help: `Instructs rclone to skip shared folders not owned by the current user.
When set, any folder that is a shared folder mount point and not
owned by the current user will be excluded from directory listings.
This is useful when backing up multiple Dropbox accounts that share
common folders, to avoid duplicating the shared data across accounts.
Note: In Dropbox Business, 'Team Folders' are owned by the Team.
For standard team members, these folders evaluate as 'unowned'
(editor/viewer access) and will be excluded by this flag. To back up
Team Folders, do not use this flag or run the backup using a Team Admin
account.
If --dropbox-skip-shared-folders is also enabled, this flag has no effect.
This makes an extra API call per shared folder mount point.`,
Default: false,
Advanced: true,
}, {
Name: "pacer_min_sleep",
Default: defaultMinSleep,
@@ -362,41 +302,38 @@ will fail to download them.
// Options defines the configuration for this backend
type Options struct {
ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"`
ImpersonateAdmin string `config:"impersonate_admin"`
SharedFiles bool `config:"shared_files"`
SharedFolders bool `config:"shared_folders"`
SkipSharedFolders bool `config:"skip_shared_folders"`
SkipUnownedFolders bool `config:"skip_unowned_folders"`
BatchMode string `config:"batch_mode"`
BatchSize int `config:"batch_size"`
BatchTimeout fs.Duration `config:"batch_timeout"`
AsyncBatch bool `config:"async_batch"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
Enc encoder.MultiEncoder `config:"encoding"`
RootNsid string `config:"root_namespace"`
ExportFormats fs.CommaSepList `config:"export_formats"`
SkipExports bool `config:"skip_exports"`
ShowAllExports bool `config:"show_all_exports"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
Impersonate string `config:"impersonate"`
SharedFiles bool `config:"shared_files"`
SharedFolders bool `config:"shared_folders"`
BatchMode string `config:"batch_mode"`
BatchSize int `config:"batch_size"`
BatchTimeout fs.Duration `config:"batch_timeout"`
AsyncBatch bool `config:"async_batch"`
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
Enc encoder.MultiEncoder `config:"encoding"`
RootNsid string `config:"root_namespace"`
ExportFormats fs.CommaSepList `config:"export_formats"`
SkipExports bool `config:"skip_exports"`
ShowAllExports bool `config:"show_all_exports"`
}
// Fs represents a remote dropbox server
type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv files.ContextClient // the connection to the dropbox server
svc files.ContextClient // the connection to the dropbox server (unauthorized)
sharing sharing.ContextClient // as above, but for generating sharing links
users users.ContextClient // as above, but for accessing user information
team team.ContextClient // for the Teams API
slashRoot string // root with "/" prefix
slashRootSlash string // root with "/" prefix and postfix
pacer *fs.Pacer // To pace the API calls
ns string // The namespace we are using or "" for none
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv files.Client // the connection to the dropbox server
svc files.Client // the connection to the dropbox server (unauthorized)
sharing sharing.Client // as above, but for generating sharing links
users users.Client // as above, but for accessing user information
team team.Client // for the Teams API
slashRoot string // root with "/" prefix, lowercase
slashRootSlash string // root with "/" prefix and postfix, lowercase
pacer *fs.Pacer // To pace the API calls
ns string // The namespace we are using or "" for none
batcher *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
exportExts []exportExtension
}
@@ -578,7 +515,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
f.team = team.NewContext(cfg)
f.team = team.New(cfg)
if opt.Impersonate != "" {
user := team.UserSelectorArg{
@@ -589,7 +526,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
members := []*team.UserSelectorArg{&user}
args := team.NewMembersGetInfoArgs(members)
memberIDs, err := f.team.MembersGetInfoContext(ctx, args)
memberIDs, err := f.team.MembersGetInfo(args)
if err != nil {
return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
}
@@ -600,14 +537,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId
}
if opt.ImpersonateAdmin != "" {
cfg.AsAdminID = opt.ImpersonateAdmin
}
f.srv = files.NewContext(cfg)
f.svc = files.NewContext(ucfg)
f.sharing = sharing.NewContext(cfg)
f.users = users.NewContext(cfg)
f.srv = files.New(cfg)
f.svc = files.New(ucfg)
f.sharing = sharing.New(cfg)
f.users = users.New(cfg)
f.features = (&fs.Features{
CaseInsensitive: true,
ReadMimeType: false,
@@ -634,8 +567,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return f, nil // our root it empty so we probably want to list shared folders
}
dir := path.Dir(f.root)
if dir == "." {
dir = f.root
}
// root is not empty so we have find the right shared folder if it exists
id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
id, err := f.findSharedFolder(ctx, dir)
if err != nil {
// if we didn't find the specified shared folder we have to bail out here
return nil, err
@@ -669,7 +607,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// If root starts with / then use the actual root
var acc *users.FullAccount
err = f.pacer.Call(func() (bool, error) {
acc, err = f.users.GetCurrentAccountContext(ctx)
acc, err = f.users.GetCurrentAccount()
return shouldRetry(ctx, err)
})
if err != nil {
@@ -689,7 +627,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// See if the root is actually an object
if f.root != "" {
_, _, err = f.getFileMetadata(ctx, f.slashRoot)
_, err = f.getFileMetadata(ctx, f.slashRoot)
if err == nil {
newRoot := path.Dir(f.root)
if newRoot == "." {
@@ -724,16 +662,15 @@ func (f *Fs) setRoot(root string) {
}
type getMetadataResult struct {
entry files.IsMetadata
remoteIsExportPath bool
notFound bool
err error
entry files.IsMetadata
notFound bool
err error
}
// getMetadata gets the metadata for a file or directory
func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
res.err = f.pacer.Call(func() (bool, error) {
res.entry, res.err = f.srv.GetMetadataContext(ctx, &files.GetMetadataArg{
res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
Path: f.opt.Enc.FromStandardPath(objPath),
})
return shouldRetry(ctx, res.err)
@@ -776,7 +713,6 @@ func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportE
ch <- getMetadataResult{notFound: true}
return
}
res.remoteIsExportPath = true
}
// Return our real result or error
@@ -822,7 +758,7 @@ func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-ch
}
// getFileMetadata gets the metadata for a file
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, bool, error) {
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
var res getMetadataResult
// Try all possible metadatas
@@ -831,7 +767,7 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
res = <-ch
if res.err != nil {
return nil, false, res.err
return nil, res.err
}
if !res.notFound {
break
@@ -839,17 +775,17 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
}
if res.notFound {
return nil, false, fs.ErrorObjectNotFound
return nil, fs.ErrorObjectNotFound
}
fileInfo, ok := res.entry.(*files.FileMetadata)
if !ok {
if _, ok = res.entry.(*files.FolderMetadata); ok {
return nil, false, fs.ErrorIsDir
return nil, fs.ErrorIsDir
}
return nil, false, fs.ErrorNotAFile
return nil, fs.ErrorNotAFile
}
return fileInfo, res.remoteIsExportPath, nil
return fileInfo, nil
}
// getDirMetadata gets the metadata for a directory
@@ -908,7 +844,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
Limit: 100,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFoldersContext(ctx, &arg)
res, err = f.sharing.ListFolders(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -920,7 +856,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
Cursor: res.Cursor,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListFoldersContinueContext(ctx, &arg)
res, err = f.sharing.ListFoldersContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -943,21 +879,13 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
return nil
}
// sharedFolderName returns the shared folder name in root, which is its first
// path component. root must be the trimmed root as produced by setRoot (no
// leading slash).
func sharedFolderName(root string) string {
name, _, _ := strings.Cut(root, "/")
return name
}
// findSharedFolder find the id for a given shared folder name
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
// so our only option is to iterate over all shared folders
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
errFoundFile := errors.New("found file")
err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
if entry.(*fs.Dir).Remote() == name {
id = entry.(*fs.Dir).ID()
return errFoundFile
}
@@ -977,7 +905,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
SharedFolderId: id,
}
err := f.pacer.Call(func() (bool, error) {
_, err := f.sharing.MountFolderContext(ctx, &arg)
_, err := f.sharing.MountFolder(&arg)
return shouldRetry(ctx, err)
})
return err
@@ -994,7 +922,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
Limit: 100,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFilesContext(ctx, &arg)
res, err = f.sharing.ListReceivedFiles(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1006,7 +934,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
Cursor: res.Cursor,
}
err := f.pacer.Call(func() (bool, error) {
res, err = f.sharing.ListReceivedFilesContinueContext(ctx, &arg)
res, err = f.sharing.ListReceivedFilesContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1014,12 +942,13 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
}
}
for _, entry := range res.Entries {
entryPath := f.opt.Enc.ToStandardName(entry.Name)
fmt.Printf("%+v\n", entry)
entryPath := entry.Name
o := &Object{
fs: f,
url: entry.PreviewUrl,
remote: entryPath,
modTime: time.Time(*entry.TimeInvited),
modTime: *entry.TimeInvited,
}
if err != nil {
return err
@@ -1039,7 +968,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
errFoundFile := errors.New("found file")
err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
if strings.EqualFold(entry.(*Object).remote, name) {
if entry.(*Object).remote == name {
o = entry.(*Object)
return errFoundFile
}
@@ -1113,7 +1042,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
arg.Path = "" // Specify root folder as empty string
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContext(ctx, arg)
res, err = f.srv.ListFolder(arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1131,7 +1060,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
Cursor: res.Cursor,
}
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
res, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1159,26 +1088,6 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
remote := path.Join(dir, leaf)
if folderInfo != nil {
if folderInfo.SharingInfo != nil && folderInfo.SharingInfo.SharedFolderId != "" {
if f.opt.SkipSharedFolders {
fs.Debugf(remote, "Skipping shared folder")
continue
}
if f.opt.SkipUnownedFolders {
var sfMeta *sharing.SharedFolderMetadata
err = f.pacer.Call(func() (bool, error) {
var apiErr error
sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
return shouldRetry(ctx, apiErr)
})
if err != nil {
fs.Errorf(remote, "Failed to get shared folder metadata (defaulting to include): %v", err)
} else if sfMeta != nil && sfMeta.AccessType != nil && sfMeta.AccessType.Tag != sharing.AccessLevelOwner {
fs.Debugf(remote, "Skipping unowned shared folder")
continue
}
}
}
d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)
err = list.Add(d)
if err != nil {
@@ -1255,7 +1164,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
return cErr
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.CreateFolderV2Context(ctx, &arg2)
_, err = f.srv.CreateFolderV2(&arg2)
return shouldRetry(ctx, err)
})
return err
@@ -1273,41 +1182,34 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
encRoot := f.opt.Enc.FromStandardPath(root)
if check {
// ListFolder reports a missing path and a path that is a file, so it
// checks that the directory exists and is empty in a single request.
// check directory exists
_, err = f.getDirMetadata(ctx, root)
if err != nil {
return fmt.Errorf("Rmdir: %w", err)
}
// check directory empty
arg := files.NewListFolderArg(encRoot)
arg.Recursive = false
arg.Limit = 1
if root == "/" {
arg.Path = "" // Specify root folder as empty string
}
var res *files.ListFolderResult
err = f.pacer.Call(func() (bool, error) {
res, err = f.srv.ListFolderContext(ctx, arg)
res, err = f.srv.ListFolder(arg)
return shouldRetry(ctx, err)
})
if err != nil {
switch e := err.(type) {
case files.ListFolderAPIError:
if e.EndpointError != nil && e.EndpointError.Path != nil {
switch e.EndpointError.Path.Tag {
case files.LookupErrorNotFound:
err = fs.ErrorDirNotFound
case files.LookupErrorNotFolder:
err = fs.ErrorIsFile
}
}
}
return fmt.Errorf("Rmdir: %w", err)
}
if len(res.Entries) != 0 || res.HasMore {
if len(res.Entries) != 0 {
return errors.New("directory not empty")
}
}
// remove it
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.DeleteV2Context(ctx, &files.DeleteArg{Path: encRoot})
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
return shouldRetry(ctx, err)
})
return err
@@ -1366,7 +1268,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
}
var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.CopyV2Context(ctx, &arg)
result, err = f.srv.CopyV2(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1427,7 +1329,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
var err error
var result *files.RelocationResult
err = f.pacer.Call(func() (bool, error) {
result, err = f.srv.MoveV2Context(ctx, &arg)
result, err = f.srv.MoveV2(&arg)
switch e := err.(type) {
case files.MoveV2APIError:
// There seems to be a bit of eventual consistency here which causes this to
@@ -1475,13 +1377,13 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
},
}
if expire < fs.DurationOff {
expiryTime := dropbox.DBXTime(time.Now().Add(time.Duration(expire)).UTC().Round(time.Second))
expiryTime := time.Now().Add(time.Duration(expire)).UTC().Round(time.Second)
createArg.Settings.Expires = &expiryTime
}
var linkRes sharing.IsSharedLinkMetadata
err = f.pacer.Call(func() (bool, error) {
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
return shouldRetry(ctx, err)
})
@@ -1490,7 +1392,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
fs.Debugf(absPath, "can't create link with expiry, trying without")
createArg.Settings.Expires = nil
err = f.pacer.Call(func() (bool, error) {
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
return shouldRetry(ctx, err)
})
}
@@ -1504,7 +1406,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
}
var listRes *sharing.ListSharedLinksResult
err = f.pacer.Call(func() (bool, error) {
listRes, err = f.sharing.ListSharedLinksContext(ctx, &listArg)
listRes, err = f.sharing.ListSharedLinks(&listArg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1565,7 +1467,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
},
}
err = f.pacer.Call(func() (bool, error) {
_, err = f.srv.MoveV2Context(ctx, &arg)
_, err = f.srv.MoveV2(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1579,7 +1481,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
var q *users.SpaceUsage
err = f.pacer.Call(func() (bool, error) {
q, err = f.users.GetSpaceUsageContext(ctx)
q, err = f.users.GetSpaceUsage()
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1666,7 +1568,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
arg.Path = ""
}
startCursor, err = f.srv.ListFolderGetLatestCursorContext(ctx, arg)
startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
return shouldRetry(ctx, err)
})
@@ -1676,34 +1578,6 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
return startCursor.Cursor, nil
}
// trimPrefixFold returns s with the leading prefix removed, matching
// case insensitively rune by rune so folded runes of differing UTF-8
// length still match.
//
// If s is exactly the root that prefix names (prefix without its
// trailing "/") it returns "". If prefix doesn't match it returns s
// unchanged.
func trimPrefixFold(s, prefix string) string {
if strings.HasSuffix(prefix, "/") && strings.EqualFold(s, strings.TrimSuffix(prefix, "/")) {
return ""
}
offset := 0
for prefix != "" {
if offset >= len(s) {
return s
}
sRune, sSize := utf8.DecodeRuneInString(s[offset:])
prefixRune, prefixSize := utf8.DecodeRuneInString(prefix)
if !strings.EqualFold(string(sRune), string(prefixRune)) {
return s
}
offset += sSize
prefix = prefix[prefixSize:]
}
return s[offset:]
}
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
cursor := startCursor
var res *files.ListFolderLongpollResult
@@ -1727,7 +1601,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
Timeout: timeout,
}
res, err = f.svc.ListFolderLongpollContext(ctx, &args)
res, err = f.svc.ListFolderLongpoll(&args)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1750,7 +1624,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
Cursor: cursor,
}
err = f.pacer.Call(func() (bool, error) {
changeList, err = f.srv.ListFolderContinueContext(ctx, &arg)
changeList, err = f.srv.ListFolderContinue(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -1763,18 +1637,17 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
switch info := entry.(type) {
case *files.FolderMetadata:
entryType = fs.EntryDirectory
entryPath = info.PathDisplay
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
case *files.FileMetadata:
entryType = fs.EntryObject
entryPath = info.PathDisplay
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
case *files.DeletedMetadata:
entryType = fs.EntryObject
entryPath = info.PathDisplay
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
default:
fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
continue
}
entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
if entryPath != "" {
notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
@@ -1886,7 +1759,7 @@ func (o *Object) Size() int64 {
return o.bytes
}
func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPath bool) {
func (o *Object) setMetadataForExport(info *files.FileMetadata) {
o.bytes = -1
o.hash = ""
@@ -1907,10 +1780,8 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
o.exportType = exportExportable
// get rid of any paper extension, if present
o.remote = strings.TrimSuffix(o.remote, paperExtension)
if !remoteIsExportPath {
// add the export extension
o.remote += "." + string(exportExt)
}
// add the export extension
o.remote += "." + string(exportExt)
}
}
@@ -1918,23 +1789,19 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
//
// This isn't a complete set of metadata and has an inaccurate date
func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
return o.setMetadataFromEntryWithExportPath(info, false)
}
func (o *Object) setMetadataFromEntryWithExportPath(info *files.FileMetadata, remoteIsExportPath bool) error {
o.id = info.Id
o.bytes = int64(info.Size)
o.modTime = time.Time(info.ClientModified)
o.modTime = info.ClientModified
o.hash = info.ContentHash
if !info.IsDownloadable {
o.setMetadataForExport(info, remoteIsExportPath)
o.setMetadataForExport(info)
}
return nil
}
// Reads the entry for a file from dropbox
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, bool, error) {
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
return o.fs.getFileMetadata(ctx, o.remotePath())
}
@@ -1944,11 +1811,11 @@ func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
if !o.modTime.IsZero() {
return nil
}
entry, remoteIsExportPath, err := o.readEntry(ctx)
entry, err := o.readEntry(ctx)
if err != nil {
return err
}
return o.setMetadataFromEntryWithExportPath(entry, remoteIsExportPath)
return o.setMetadataFromEntry(entry)
}
// Returns the remote path for the object
@@ -2002,7 +1869,7 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
var exportResult *files.ExportResult
err = o.fs.pacer.Call(func() (bool, error) {
exportResult, in, err = o.fs.srv.ExportContext(ctx, &arg)
exportResult, in, err = o.fs.srv.Export(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -2024,7 +1891,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
Url: o.url,
}
err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.sharing.GetSharedLinkFileContext(ctx, &arg)
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -2044,7 +1911,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
ExtraHeaders: headers,
}
err = o.fs.pacer.Call(func() (bool, error) {
_, in, err = o.fs.srv.DownloadContext(ctx, &arg)
_, in, err = o.fs.srv.Download(&arg)
return shouldRetry(ctx, err)
})
@@ -2068,7 +1935,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
// start upload
var res *files.UploadSessionStartResult
err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.srv.UploadSessionStartContext(ctx, &files.UploadSessionStartArg{}, nil)
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, nil)
return shouldRetry(ctx, err)
})
if err != nil {
@@ -2083,16 +1950,12 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
// write chunks
in := readers.NewCountingReader(in0)
buf := make([]byte, int(chunkSize))
cursor := files.UploadSessionCursor{
SessionId: res.SessionId,
Offset: 0,
}
appendArg := files.UploadSessionAppendArg{
Cursor: &cursor,
// A known-size upload which fits in a single chunk can close the
// session with its only append, saving an empty append request
Close: size >= 0 && size <= chunkSize,
}
appendArg := files.UploadSessionAppendArg{Cursor: &cursor}
for currentChunk := 1; ; currentChunk++ {
cursor.Offset = in.BytesRead()
@@ -2102,21 +1965,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
}
// Buffer the chunk in memory from the global pool for retries
rw := multipart.NewRW()
var n int64
n, err = io.CopyN(rw, in, chunkSize)
if err != nil && err != io.EOF {
_ = rw.Close()
return nil, err
}
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
skip := int64(0)
err = o.fs.pacer.Call(func() (bool, error) {
// seek to the start in case this is a retry
if _, err = rw.Seek(skip, io.SeekStart); err != nil {
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
return false, err
}
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
// after session is started, we retry everything
if err != nil {
// Check for incorrect offset error and retry with new offset
@@ -2128,10 +1984,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
if skip < 0 {
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
} else if skip == n {
} else if skip == chunkSize {
fs.Debugf(o, "%s: chunk received OK - continuing", what)
return false, nil
} else if skip > n {
} else if skip > chunkSize {
// This error should never happen
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
}
@@ -2141,27 +1997,11 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
}
}
}
// Don't waste the low level retries if the context has
// been cancelled
if fserrors.ContextError(ctx, &err) {
return false, err
}
return err != nil, err
})
closeErr := rw.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
if size >= 0 {
// Check for sources which truncate early
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
if in.BytesRead() < expected {
return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
}
}
if appendArg.Close {
break
}
@@ -2191,7 +2031,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
}
err = o.fs.pacer.Call(func() (bool, error) {
entry, err = o.fs.srv.UploadSessionFinishContext(ctx, args, nil)
entry, err = o.fs.srv.UploadSessionFinish(args, nil)
if retry, err := shouldRetryExclude(ctx, err); !retry {
return retry, err
}
@@ -2245,7 +2085,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
commitInfo := files.NewCommitInfo(o.fs.opt.Enc.FromStandardPath(o.remotePath()))
commitInfo.Mode.Tag = "overwrite"
// The Dropbox API only accepts timestamps in UTC with second precision.
clientModified := dropbox.DBXTime(src.ModTime(ctx).UTC().Round(time.Second))
clientModified := src.ModTime(ctx).UTC().Round(time.Second)
commitInfo.ClientModified = &clientModified
// Don't attempt to create filenames that are too long
if cErr := checkPathLength(commitInfo.Path); cErr != nil {
@@ -2259,7 +2099,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
} else {
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
entry, err = o.fs.srv.UploadContext(ctx, &files.UploadArg{CommitInfo: *commitInfo}, in)
entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
return shouldRetry(ctx, err)
})
}
@@ -2271,7 +2111,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// This will only happen if we are uploading async batches
if entry == nil {
o.bytes = size
o.modTime = time.Time(*commitInfo.ClientModified)
o.modTime = *commitInfo.ClientModified
o.hash = "" // we don't have this
return nil
}
@@ -2284,7 +2124,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
return errNotSupportedInSharedMode
}
err = o.fs.pacer.Call(func() (bool, error) {
_, err = o.fs.srv.DeleteV2Context(ctx, &files.DeleteArg{
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
})
return shouldRetry(ctx, err)
+1 -436
View File
@@ -2,155 +2,17 @@ package dropbox
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/batcher"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type paperMetadataClient struct {
files.ContextClient
info *files.FileMetadata
}
func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
if arg.Path == "document" {
return c.info, nil
}
return nil, files.GetMetadataAPIError{
APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
EndpointError: &files.GetMetadataError{
Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
Path: &files.LookupError{
Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
},
},
}
}
func TestInternalGetMetadataCancellation(t *testing.T) {
requestStarted := make(chan struct{})
releaseRequest := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(requestStarted)
<-releaseRequest
}))
defer server.Close()
defer close(releaseRequest)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
f := &Fs{
srv: files.NewContext(dropbox.Config{
Client: server.Client(),
URLGenerator: func(hostType string, namespace string, route string) string {
return server.URL
},
}),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
}
result := make(chan getMetadataResult, 1)
go func() {
result <- f.getMetadata(ctx, "/file")
}()
select {
case <-requestStarted:
case <-time.After(time.Second):
t.Fatal("Dropbox request did not start")
}
cancel()
select {
case res := <-result:
require.ErrorIs(t, res.err, context.Canceled)
case <-time.After(time.Second):
t.Fatal("Dropbox request did not observe cancellation")
}
}
type changeNotifyClient struct {
files.ContextClient
entries []files.IsMetadata
}
func (c changeNotifyClient) ListFolderLongpollContext(context.Context, *files.ListFolderLongpollArg) (*files.ListFolderLongpollResult, error) {
return files.NewListFolderLongpollResult(true), nil
}
func (c changeNotifyClient) ListFolderContinueContext(context.Context, *files.ListFolderContinueArg) (*files.ListFolderResult, error) {
return files.NewListFolderResult(c.entries, "next", false), nil
}
func TestTrimPrefixFold(t *testing.T) {
for _, test := range []struct {
name string
path string
prefix string
want string
}{
{name: "exact casing", path: "/Docs/Sub", prefix: "/Docs/", want: "Sub"},
{name: "different casing", path: "/docs/Sub", prefix: "/Docs/", want: "Sub"},
{name: "different UTF-8 lengths", path: "/K/Sub", prefix: "/K/", want: "Sub"},
{name: "exact root", path: "/docs", prefix: "/Docs/", want: ""},
{name: "sibling boundary", path: "/Docs2/File", prefix: "/Docs/", want: "/Docs2/File"},
{name: "root remote", path: "/File", prefix: "/", want: "File"},
} {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.want, trimPrefixFold(test.path, test.prefix))
})
}
}
func TestChangeNotifyTrimsRootCaseInsensitively(t *testing.T) {
ctx := context.Background()
client := changeNotifyClient{entries: []files.IsMetadata{
&files.FolderMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub"}},
&files.FileMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub/File.TXT"}},
&files.DeletedMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Gone.md"}},
}}
f := &Fs{
ci: fs.GetConfig(ctx),
srv: client,
svc: client,
slashRootSlash: "/Docs/",
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
}
type notification struct {
path string
entryType fs.EntryType
}
var notifications []notification
cursor, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
notifications = append(notifications, notification{path: path, entryType: entryType})
}, "start")
require.NoError(t, err)
assert.Equal(t, "next", cursor)
assert.Equal(t, []notification{
{path: "Sub", entryType: fs.EntryDirectory},
{path: "Sub/File.TXT", entryType: fs.EntryObject},
{path: "Gone.md", entryType: fs.EntryObject},
}, notifications)
}
func TestInternalCheckPathLength(t *testing.T) {
rep := func(n int, r rune) (out string) {
rs := make([]rune, n)
@@ -188,170 +50,6 @@ func TestInternalCheckPathLength(t *testing.T) {
}
}
func TestInternalSharedFolderName(t *testing.T) {
for _, test := range []struct {
root string
want string
}{
{root: "", want: ""},
{root: "SharedFolder", want: "SharedFolder"},
{root: "SharedFolder/subdir", want: "SharedFolder"},
{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
} {
assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
}
}
func TestPaperExportRemote(t *testing.T) {
ctx := context.Background()
info := &files.FileMetadata{
ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
}
f := &Fs{
exportExts: []exportExtension{"md"},
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
srv: paperMetadataClient{info: info},
}
direct, err := f.NewObject(ctx, "document.md")
require.NoError(t, err)
assert.Equal(t, "document.md", direct.Remote())
listed, err := f.newObjectWithInfo(ctx, "document.md", info)
require.NoError(t, err)
assert.Equal(t, "document.md.md", listed.Remote())
legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
require.NoError(t, err)
assert.Equal(t, "document.md", legacy.Remote())
}
// uploadSessionClient is a mock files.ContextClient which records the
// chunked upload calls made to it
type uploadSessionClient struct {
files.ContextClient
appends int // number of UploadSessionAppendV2Context calls
maxAppends int // fail the append after this many calls to stop runaway loops
bytesWritten int64 // bytes received by UploadSessionAppendV2Context
finishCalled bool // set if UploadSessionFinishContext was called
appended func() // if set, called after each successful append
}
var errTooManyAppends = errors.New("too many appends - upload looping?")
func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
return &files.UploadSessionStartResult{SessionId: "session"}, nil
}
func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
// the real client fails the request if the context is cancelled
if err := ctx.Err(); err != nil {
return err
}
c.appends++
if c.appends > c.maxAppends {
return errTooManyAppends
}
n, err := io.Copy(io.Discard, content)
if err != nil {
return err
}
c.bytesWritten += n
if c.appended != nil {
c.appended()
}
return nil
}
func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
c.finishCalled = true
return &files.FileMetadata{}, nil
}
// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
ctx := context.Background()
f := &Fs{
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
srv: srv,
}
f.opt.ChunkSize = chunkSize
batcherOptions := defaultBatcherOptions
batcherOptions.Mode = "off"
var err error
f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
require.NoError(t, err)
return f
}
// endlessReader supplies bytes forever
type endlessReader struct{}
func (endlessReader) Read(p []byte) (int, error) {
for i := range p {
p[i] = 'x'
}
return len(p), nil
}
func TestUploadChunkedEarlyEOF(t *testing.T) {
ctx := context.Background()
t.Run("MultiChunk", func(t *testing.T) {
// The declared size spans 4 chunks but the source ends after 1.5
client := &uploadSessionClient{maxAppends: 8}
f := newUploadTestFs(t, client, 100)
o := &Object{fs: f, remote: "test.bin"}
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.False(t, client.finishCalled, "must not commit a truncated upload")
})
t.Run("SingleChunk", func(t *testing.T) {
// The declared size fits in one chunk but the source ends early
client := &uploadSessionClient{maxAppends: 8}
f := newUploadTestFs(t, client, 500)
o := &Object{fs: f, remote: "test.bin"}
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
require.Error(t, err)
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.False(t, client.finishCalled, "must not commit a truncated upload")
})
t.Run("Complete", func(t *testing.T) {
// A source which supplies exactly the declared size uploads OK
client := &uploadSessionClient{maxAppends: 8}
f := newUploadTestFs(t, client, 100)
o := &Object{fs: f, remote: "test.bin"}
entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
require.NoError(t, err)
require.NotNil(t, entry)
assert.True(t, client.finishCalled)
assert.Equal(t, int64(250), client.bytesWritten)
})
}
func TestUploadChunkedCancel(t *testing.T) {
// Cancelling the context must stop the upload even though every
// append is succeeding
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := &uploadSessionClient{maxAppends: 8}
client.appended = func() {
if client.appends == 2 {
cancel()
}
}
f := newUploadTestFs(t, client, 100)
o := &Object{fs: f, remote: "test.bin"}
_, err := o.uploadChunked(ctx, endlessReader{}, files.NewCommitInfo("/test.bin"), -1)
require.Error(t, err)
assert.ErrorIs(t, err, context.Canceled)
assert.False(t, client.finishCalled)
}
func (f *Fs) importPaperForTest(t *testing.T) {
content := `# test doc
@@ -366,7 +64,7 @@ Lorem ipsum __dolor__ sit amet
var err error
err = f.pacer.Call(func() (bool, error) {
reader := strings.NewReader(content)
_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
_, err = f.srv.PaperCreate(&arg, reader)
return shouldRetry(context.Background(), err)
})
require.NoError(t, err)
@@ -402,136 +100,3 @@ func (f *Fs) InternalTest(t *testing.T) {
}
var _ fstests.InternalTester = (*Fs)(nil)
// newSharingTestFs builds a minimal *Fs whose sharing client talks to a
// local mock server instead of the real Dropbox API.
func newSharingTestFs(t *testing.T, handler http.HandlerFunc) *Fs {
t.Helper()
ts := httptest.NewServer(handler)
t.Cleanup(ts.Close)
cfg := dropbox.Config{
Client: &http.Client{Transport: http.DefaultTransport},
URLGenerator: func(hostType, namespace, route string) string {
return fmt.Sprintf("%s/2/%s/%s", ts.URL, namespace, route)
},
}
return &Fs{
opt: Options{
Enc: encoder.Base |
encoder.EncodeBackSlash |
encoder.EncodeDel |
encoder.EncodeRightSpace |
encoder.EncodeInvalidUtf8,
},
sharing: sharing.NewContext(cfg),
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond))),
}
}
func mustParseDBXTime(t *testing.T, s string) *dropbox.DBXTime {
t.Helper()
tm, err := time.Parse(time.RFC3339, s)
require.NoError(t, err)
return (*dropbox.DBXTime)(&tm)
}
func receivedFilesHandler(t *testing.T, rawName string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.URL.Path, "list_received_files") {
http.NotFound(w, r)
return
}
resp := sharing.ListFilesResult{
Entries: []*sharing.SharedFileMetadata{{
Id: "id:123",
Name: rawName,
PreviewUrl: "https://dropbox.com/preview/123",
Policy: &sharing.FolderPolicy{},
TimeInvited: mustParseDBXTime(t, "2024-01-01T00:00:00Z"),
}},
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(resp)
}
}
// TestListReceivedFilesDecodesName confirms received-file names are decoded
// via Enc.ToStandardName before being returned, matching listSharedFolders'
// handling of shared-folder names.
func TestListReceivedFilesDecodesName(t *testing.T) {
const rawName = "report.txt␠"
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
wantName := f.opt.Enc.ToStandardName(rawName)
require.NotEqual(t, rawName, wantName)
var gotNames []string
err := f.listReceivedFiles(context.Background(), func(entry fs.DirEntry) error {
gotNames = append(gotNames, entry.Remote())
return nil
})
require.NoError(t, err)
require.Len(t, gotNames, 1)
assert.Equal(t, wantName, gotNames[0])
}
// TestFindSharedFileResolvesDecodedName confirms findSharedFile can look up
// a received file by its standard, decoded rclone-visible name.
func TestFindSharedFileResolvesDecodedName(t *testing.T) {
const rawName = "report.txt␠"
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
encodedName := f.opt.Enc.ToStandardName(rawName)
require.NotEqual(t, rawName, encodedName)
o, err := f.findSharedFile(context.Background(), encodedName)
require.NoError(t, err)
assert.Equal(t, encodedName, o.remote)
}
// sharedFoldersHandler serves a single shared folder from list_folders.
func sharedFoldersHandler(name, id string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(r.URL.Path, "list_folders") {
http.NotFound(w, r)
return
}
resp := sharing.ListFoldersResult{
Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(resp)
}
}
// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
// lookup must match regardless of the case of the requested name.
func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))
for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
id, err := f.findSharedFolder(context.Background(), name)
require.NoError(t, err, name)
assert.Equal(t, "folder-id", id, name)
}
_, err := f.findSharedFolder(context.Background(), "no-such-folder")
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
}
// The Dropbox backend advertises CaseInsensitive: true, so a received file
// lookup must match regardless of the case of the requested name.
func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))
for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
o, err := f.findSharedFile(context.Background(), name)
require.NoError(t, err, name)
assert.Equal(t, "TestFile.txt", o.remote, name)
}
_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
}
+17 -15
View File
@@ -222,36 +222,38 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
return nil, err
}
fldMap := map[string]bool{}
for _, folder := range result.Result.Folders {
fldMap[folder.FldID.String()] = true
if f.root == "" && dir == "" && strings.Contains(folder.Path, "/") {
continue
}
folderPath := strings.Trim(folder.Path, "/")
root := strings.Trim(f.root, "/")
remotePathWithoutRoot := folderPath
if root != "" {
if remotePathWithoutRoot == root {
continue
}
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, root+"/")
paths := strings.Split(folder.Path, fullPath+"/")
remote := paths[0]
if len(paths) > 1 {
remote = paths[1]
}
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
if remotePathWithoutRoot == "" {
if strings.Contains(remote, "/") {
continue
}
pathsWithoutRoot := strings.Split(folder.Path, "/"+f.root+"/")
remotePathWithoutRoot := pathsWithoutRoot[0]
if len(pathsWithoutRoot) > 1 {
remotePathWithoutRoot = pathsWithoutRoot[1]
}
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
entries = append(entries, fs.NewDir(remotePathWithoutRoot, time.Now()))
}
for _, file := range result.Result.Files {
if _, ok := fldMap[file.FldID.String()]; ok {
continue
}
remote := path.Join(dir, file.Name)
// trim leading slashes
remote = strings.TrimPrefix(remote, "/")
obj := &Object{
fs: f,
remote: remote,
+55 -59
View File
@@ -1,6 +1,7 @@
package filelu
import (
"bytes"
"context"
"encoding/json"
"errors"
@@ -10,11 +11,9 @@ import (
"net/http"
"net/url"
"path"
"strconv"
"strings"
"github.com/rclone/rclone/fs"
rclonemultipart "github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/rest"
)
@@ -26,9 +25,7 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
}
if dir != "" {
if _, err := f.createFolder(ctx, dir); err != nil {
return fmt.Errorf("failed to create multipart folder: %w", err)
}
_ = f.Mkdir(ctx, dir)
}
folder := strings.Trim(dir, "/")
@@ -48,32 +45,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
server := initResp.Result.Server
objectPath := initResp.Result.ObjectPath
chunkSize := int64(f.opt.ChunkSize)
if chunkSize <= 0 {
return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)
}
for partNo := 1; ; partNo++ {
// Buffer the part in memory from the global pool so it can be
// re-sent on retry
rw := rclonemultipart.NewRW()
n, err := io.CopyN(rw, in, chunkSize)
if err != nil && err != io.EOF {
_ = rw.Close()
return fmt.Errorf("read failed: %w", err)
}
chunkSize := int(f.opt.ChunkSize)
buf := make([]byte, 0, chunkSize)
tmp := make([]byte, 1024*1024)
partNo := 1
for {
n, errRead := in.Read(tmp)
if n > 0 {
uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)
if uploadErr != nil {
_ = rw.Close()
return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)
buf = append(buf, tmp[:n]...)
// If buffer reached chunkSize, upload a full part
if len(buf) >= chunkSize {
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
if err != nil {
return fmt.Errorf("upload part %d failed: %w", partNo, err)
}
partNo++
buf = buf[:0]
}
}
if closeErr := rw.Close(); closeErr != nil {
return closeErr
}
if err == io.EOF {
if errRead == io.EOF {
break
}
if errRead != nil {
return fmt.Errorf("read failed: %w", errRead)
}
}
if len(buf) > 0 {
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
if err != nil {
return fmt.Errorf("upload part %d failed: %w", partNo, err)
}
}
err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
@@ -84,40 +89,31 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
return nil
}
// uploadPart sends a single multipart chunk of size bytes to the upload server.
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {
opts := rest.Opts{
Method: "PUT",
RootURL: server,
Parameters: url.Values{
"partNumber": {strconv.Itoa(partNo)},
"uploadId": {uploadID},
},
Body: r,
ContentLength: &size,
ExtraHeaders: map[string]string{
"X-RC-Upload-Id": uploadID,
"X-RC-Part-No": strconv.Itoa(partNo),
"X-Sess-ID": sessID,
"X-Object-Path": objectPath,
},
NoResponse: true,
// uploadPart sends a single multipart chunk to the upload server.
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
if err != nil {
return err
}
return f.pacer.Call(func() (bool, error) {
// rewind the pooled buffer so each attempt sends the whole chunk
_, err := r.Seek(0, io.SeekStart)
if err != nil {
return false, err
}
resp, err := f.srv.Call(ctx, &opts)
if err != nil {
if resp != nil {
return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)
}
return shouldRetry(err), err
}
return false, nil
})
req.Header.Set("X-RC-Upload-Id", uploadID)
req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
req.Header.Set("X-Sess-ID", sessID)
req.Header.Set("X-Object-Path", objectPath)
resp, err := f.client.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 {
return fmt.Errorf("uploadPart failed: %s", resp.Status)
}
return nil
}
// uploadFile uploads a file to FileLu
-29
View File
@@ -1,29 +0,0 @@
package filelu
import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest/fstests"
)
// SetUploadChunkSize changes the configured chunk size, returning the old value.
//
// It is only called by the integration tests while no transfer is in progress.
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
var old fs.SizeSuffix
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
return old, nil
}
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
//
// It is only called by the integration tests while no transfer is in progress.
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
var old fs.SizeSuffix
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
return old, nil
}
var (
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
_ fstests.SetUploadCutoffer = (*Fs)(nil)
)
-12
View File
@@ -14,7 +14,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
)
@@ -142,8 +141,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
// Update updates the object with new data
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
size := src.Size()
counter := readers.NewCountingReader(in)
in = counter
if size <= int64(o.fs.opt.UploadCutoff) {
err := o.fs.uploadFile(ctx, in, o.remote)
@@ -158,15 +155,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
}
// Check the source supplied the number of bytes it declared
// otherwise a truncated file would be stored as a good upload
if size >= 0 && int64(counter.BytesRead()) != size {
if removeErr := o.Remove(ctx); removeErr != nil {
fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
}
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
}
o.size = size
o.modTime = src.ModTime(ctx)
return nil
-3
View File
@@ -12,8 +12,5 @@ func TestIntegration(t *testing.T) {
RemoteName: "TestFileLu:",
NilObject: nil,
SkipInvalidUTF8: true,
ChunkedUpload: fstests.ChunkedUploadConfig{
MinChunkSize: 1,
},
})
}
+4 -38
View File
@@ -21,11 +21,9 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
"golang.org/x/sync/errgroup"
)
@@ -49,7 +47,7 @@ func init() {
},
{
Name: "api_key",
Help: `API Key for your Filen account
Help: `API Key for your Filen account
Get this using the Filen CLI export-api-key command
You can download the Filen CLI from https://github.com/FilenCloudDienste/filen-cli`,
@@ -163,7 +161,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
filen: filen,
Enc: opt.Encoder,
concurrency: opt.UploadConcurrency,
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
fileSystem.features = (&fs.Features{
@@ -197,13 +194,6 @@ type Options struct {
UploadConcurrency int `config:"upload_concurrency"`
}
// defaults copied from box
const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2 // bigger for slower decay, exponential
)
// Fs represents a virtual filesystem mounted on a specific root folder
type Fs struct {
name string
@@ -212,7 +202,6 @@ type Fs struct {
Enc encoder.MultiEncoder
features *fs.Features
concurrency int
pacer *fs.Pacer
}
// Name of the remote (as passed into NewFs)
@@ -362,7 +351,6 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
type chunkWriter struct {
sdk.FileUpload
filen *sdk.Filen
pacer *fs.Pacer
bucketAndRegion chan client.V3UploadResponse
chunkSize int64
@@ -374,13 +362,6 @@ type chunkWriter struct {
size int64
}
func shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
return fserrors.ShouldRetry(err), err
}
func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
realChunkNumber := int64(chunkNumber) * (cw.chunkSize) / sdk.ChunkSize
chunk := make([]byte, sdk.ChunkSize, sdk.ChunkSize+cw.EncryptionKey.Cipher.Overhead())
@@ -433,15 +414,7 @@ func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader i
if err != nil {
return totalWritten, err
}
var resp *client.V3UploadResponse
err = cw.pacer.Call(func() (bool, error) {
var err error
resp, err = cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
if err != nil {
return shouldRetry(ctx, err)
}
return false, nil
})
resp, err := cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
if err != nil {
return totalWritten, err
}
@@ -521,7 +494,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
return info, &chunkWriter{
FileUpload: *fu,
filen: f.filen,
pacer: f.pacer,
chunkSize: chunkSize,
bucketAndRegion: make(chan client.V3UploadResponse, 1),
knownChunks: make(map[int64][]byte),
@@ -710,10 +682,7 @@ func (o *Object) Storable() bool {
// SetModTime sets the metadata on the object to set the modification date
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
// The server stores modtimes with millisecond precision so round
// here too to keep the in-memory modtime identical to the one a
// fresh listing returns.
o.file.LastModified = t.Round(time.Millisecond)
o.file.LastModified = t
return o.fs.filen.UpdateMeta(ctx, o.file)
}
@@ -755,10 +724,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
fs.Logf(option, "Unsupported mandatory option: %v", option)
}
}
// The server stores modtimes with millisecond precision so round
// here too to keep the in-memory modtime identical to the one a
// fresh listing returns.
newModTime := src.ModTime(ctx).Round(time.Millisecond)
newModTime := src.ModTime(ctx)
newIncomplete, err := o.file.NewFromBase(o.fs.filen.FileEncryptionVersion)
if err != nil {
return err
+12 -32
View File
@@ -19,7 +19,6 @@ import (
"github.com/Files-com/files-sdk-go/v3/file"
file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
"github.com/Files-com/files-sdk-go/v3/folder"
files_sdk_lib "github.com/Files-com/files-sdk-go/v3/lib"
"github.com/Files-com/files-sdk-go/v3/session"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
@@ -177,13 +176,6 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
}
}
// Errors from the upload storage servers are of this type
var httpErr files_sdk_lib.ResponseError
if errors.As(err, &httpErr) && slices.Contains(retryErrorCodes, httpErr.StatusCode) {
fs.Debugf(nil, "Retrying HTTP error %v", err)
return true, err
}
return fserrors.ShouldRetry(err), err
}
@@ -353,7 +345,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
}
for it.Next() {
item := new(it.File())
item := ptr(it.File())
remote := f.opt.Enc.ToStandardPath(item.DisplayName)
remote = path.Join(dir, remote)
if remote == dir {
@@ -424,7 +416,7 @@ func (f *Fs) mkdir(ctx context.Context, path string) error {
params := files_sdk.FolderCreateParams{
Path: path,
MkdirParents: new(true),
MkdirParents: ptr(true),
}
err := f.pacer.Call(func() (bool, error) {
@@ -466,7 +458,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
params := files_sdk.FileDeleteParams{
Path: path,
Recursive: new(!check),
Recursive: ptr(!check),
}
err := f.pacer.Call(func() (bool, error) {
@@ -537,7 +529,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.
params := files_sdk.FileCopyParams{
Path: srcPath,
Destination: dstPath,
Overwrite: new(true),
Overwrite: ptr(true),
}
var action files_sdk.FileAction
@@ -680,7 +672,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
Paths: []string{f.absPath(remote)},
}
if expire < fs.DurationOff {
params.ExpiresAt = new(time.Now().Add(time.Duration(expire)))
params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
}
var bundle files_sdk.Bundle
@@ -842,6 +834,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return
}
// Returns a pointer to t - useful for returning pointers to constants
func ptr[T any](t T) *T {
return &t
}
func isFolderNotEmpty(err error) bool {
var re files_sdk.ResponseError
ok := errors.As(err, &re)
@@ -861,7 +858,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
file.UploadWithProvidedMtime(src.ModTime(ctx)),
}
err := o.fs.pacer.CallNoRetry(func() (bool, error) {
err := o.fs.pacer.Call(func() (bool, error) {
err := o.fs.fileClient.Upload(uploadOpts...)
return shouldRetry(ctx, err)
})
@@ -869,24 +866,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return err
}
// The server computes the MD5 asynchronously after upload so
// retry reading the metadata for a short time until it appears.
const maxTries = 10
for tries := 1; ; tries++ {
err = o.readMetaData(ctx)
if err != nil {
return err
}
if o.md5 != "" || tries >= maxTries {
break
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(tries) * 100 * time.Millisecond):
}
}
return nil
return o.readMetaData(ctx)
}
// Remove an object
+4 -23
View File
@@ -637,18 +637,6 @@ func (f *Fs) drainPool(ctx context.Context) (err error) {
return err
}
// commandEncoding hardens the user-configured filename encoding so that it
// can never restore a raw CR or LF.
//
// The FTP control channel is line oriented and the ftp library writes command
// arguments (paths) straight onto it without escaping, so a filename
// containing CR/LF would otherwise be able to inject an independent FTP
// command. CR/LF are therefore always encoded to safe symbols regardless of
// the configured encoding, which is what the default encoding already does.
func commandEncoding(enc encoder.MultiEncoder) encoder.MultiEncoder {
return enc | encoder.EncodeCrLf
}
// NewFs constructs an Fs from the path, container:path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
@@ -658,7 +646,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
if err != nil {
return nil, err
}
opt.Enc = commandEncoding(opt.Enc)
pass := ""
if opt.AskPassword && opt.Pass == "" {
pass = config.GetPassword("FTP server password")
@@ -1248,10 +1235,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
path = o.fs.opt.Enc.FromStandardPath(path)
err = c.SetTime(path, modTime.In(time.UTC))
if err == nil && o.info != nil {
// The server stores modtimes with second precision so
// truncate here too to keep the in-memory modtime identical
// to the one a fresh listing returns.
o.info.ModTime = modTime.Truncate(time.Second)
o.info.ModTime = modTime
o.info.precise = true
}
o.fs.putFtpConnection(&c, err)
@@ -1406,12 +1390,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
o.fs.putFtpConnection(&c, nil)
if o.fs.opt.NoCheckUpload {
o.info = &FileInfo{
Name: o.remote,
Size: uint64(src.Size()),
// The server stores modtimes with second precision so
// truncate here too to keep the in-memory modtime
// identical to the one a fresh listing returns.
ModTime: src.ModTime(ctx).Truncate(time.Second),
Name: o.remote,
Size: uint64(src.Size()),
ModTime: src.ModTime(ctx),
precise: true,
IsDir: false,
}
+2 -33
View File
@@ -12,38 +12,15 @@ import (
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/readers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// A filename containing raw CR/LF must never survive encoding into an FTP
// command argument, whatever encoding the user configured, otherwise it could
// inject an independent FTP command onto the line-oriented control channel.
func TestCommandEncodingNeutralisesCRLF(t *testing.T) {
// Internal Standard-encoded name as produced by e.g. a Unix local
// source, carrying raw CR/LF.
for _, std := range []string{
"victim\r\nDELE other-secret\r\nNOOP",
"victim␍␊DELE other-secret␍␊NOOP",
} {
for _, configured := range []encoder.MultiEncoder{
encoder.EncodeSlash, // "Slash" - omits Ctl and CrLf
encoder.EncodeZero, // "None"
encoder.Display | encoder.EncodeRightSpace, // the FTP default
} {
got := commandEncoding(configured).FromStandardPath(std)
assert.NotContains(t, got, "\r", "encoding %v leaked raw CR: %q", configured, got)
assert.NotContains(t, got, "\n", "encoding %v leaked raw LF: %q", configured, got)
}
}
}
type settings map[string]any
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
configMap := configmap.Simple{}
for key, val := range opts {
configMap[key] = fmt.Sprintf("%v", val)
@@ -75,21 +52,13 @@ func (f *Fs) testUploadTimeout(t *testing.T) {
ci.Timeout = saveTimeout
}()
ci.LowLevelRetries = 1
ci.Timeout = fs.Duration(idleTimeout)
upload := func(concurrency int, shutTimeout time.Duration) (obj fs.Object, err error) {
// Use the saved (long) timeout while NewFs dials the FTP control
// connection and reads the welcome banner. On slow CI servers this
// can take longer than idleTimeout, which would cause a spurious
// i/o timeout before the test can run. The connection's idle
// deadline is captured at dial time, so the data connection
// dialled later inside Put will still get idleTimeout - which is
// what the test is actually trying to exercise.
ci.Timeout = saveTimeout
fixFs := deriveFs(ctx, t, f, settings{
"concurrency": concurrency,
"shut_timeout": shutTimeout,
})
ci.Timeout = fs.Duration(idleTimeout)
// Make test object
fileTime := fstest.Time("2020-03-08T09:30:00.000000000Z")
+2 -1
View File
@@ -194,7 +194,8 @@ var retryErrorCodes = []int{
// Return true if the api error has the status given
func isAPIErr(err error, status string) bool {
if apiErr, ok := errors.AsType[api.Error](err); ok {
var apiErr api.Error
if errors.As(err, &apiErr) {
return apiErr.Status == status
}
return false
+4 -42
View File
@@ -94,23 +94,6 @@ var (
}
)
// gphotosOAuthOptions returns a copy of oauthutil.SharedOptions with
// the client_id help tailored for Google Photos. It copies the shared
// options rather than mutating the global slice.
func gphotosOAuthOptions() []fs.Option {
opts := []fs.Option{}
for _, opt := range oauthutil.SharedOptions {
if opt.Name == config.ConfigClientID {
opt.Help = "OAuth Client Id.\n\nCreating your own is now strongly recommended.\nIf you leave this blank rclone uses a shared client_id which is being retired and will stop working during 2026.\nSee https://rclone.org/googlephotos/#making-your-own-client-id for how to create your own."
}
if opt.Name == config.ConfigClientSecret {
opt.Help = "OAuth Client Secret.\n\nLeave blank to use rclone's shared client_id.\nIf you created your own client_id then enter its client secret here."
}
opts = append(opts, opt)
}
return opts
}
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
@@ -118,7 +101,7 @@ func init() {
Prefix: "gphotos",
Description: "Google Photos",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
@@ -126,7 +109,7 @@ func init() {
return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
}
switch configIn.State {
switch config.State {
case "":
// Fill in the scopes
if opt.ReadOnly {
@@ -134,25 +117,6 @@ func init() {
} else {
oauthConfig.Scopes = scopesReadWrite
}
return fs.ConfigGoto("client_id")
case "client_id":
if clientID, _ := m.Get(config.ConfigClientID); clientID != "" {
return fs.ConfigGoto("oauth")
}
return oauthutil.SharedClientIDConfigConfirm("client_id_warning", "Google Photos", "https://rclone.org/googlephotos/#making-your-own-client-id")
case "client_id_warning":
if configIn.Result == "true" {
// Continue using the shared client_id
return fs.ConfigGoto("oauth")
}
return fs.ConfigInput("client_id_set", config.ConfigClientID, "OAuth Client Id")
case "client_id_set":
m.Set(config.ConfigClientID, configIn.Result)
return fs.ConfigInput("client_secret_set", config.ConfigClientSecret, "OAuth Client Secret")
case "client_secret_set":
m.Set(config.ConfigClientSecret, configIn.Result)
return fs.ConfigGoto("oauth")
case "oauth":
return oauthutil.ConfigOut("warning1", &oauthutil.Options{
OAuth2Config: oauthConfig,
})
@@ -172,9 +136,9 @@ IMPORTANT: Due to Google policy changes rclone can now only download photos it u
case "warning_done":
return nil, nil
}
return nil, fmt.Errorf("unknown state %q", configIn.State)
return nil, fmt.Errorf("unknown state %q", config.State)
},
Options: append(append(gphotosOAuthOptions(), []fs.Option{{
Options: append(append(oauthutil.SharedOptions, []fs.Option{{
Name: "read_only",
Default: false,
Help: `Set to make the Google Photos backend read only.
@@ -385,8 +349,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
oauthutil.SharedClientIDWarning(name, "Google Photos", "https://rclone.org/googlephotos/#making-your-own-client-id", m)
baseClient := fshttp.NewClient(ctx)
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
if err != nil {
-54
View File
@@ -69,66 +69,12 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {
_ = operations.Purge(ctx, f, dirName)
}
func (f *Fs) testUpdateStoresHash(t *testing.T) {
// make a temporary local remote
tempRoot, err := fstest.LocalRemote()
require.NoError(t, err)
defer func() {
_ = os.RemoveAll(tempRoot)
}()
// make a temporary crypt remote as source
ctx := context.Background()
pass := obscure.MustObscure("crypt")
remote := fmt.Sprintf(`:crypt,remote="%s",password="%s":`, tempRoot, pass)
cryptFs, err := fs.NewFs(ctx, remote)
require.NoError(t, err)
const dirName = "update_hash_1"
const fileName = dirName + "/file_update_1"
const longTime = fs.ModTimeNotSupported
hashType := f.keepHashes.GetOne()
// upload initial file to hasher via Put
src1 := putFile(ctx, t, cryptFs, fileName, "initial content")
in1, err := src1.Open(ctx)
require.NoError(t, err)
dst, err := f.Put(ctx, in1, src1)
require.NoError(t, err)
require.NotNil(t, dst)
// verify hash was stored after Put
var hash1 string
if f.opt.MaxAge > 0 {
hash1, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
assert.NoError(t, err)
assert.NotEmpty(t, hash1)
}
// update the file with new content via Update (this is what sync does for replacements)
src2 := putFile(ctx, t, cryptFs, fileName, "updated content")
in2, err := src2.Open(ctx)
require.NoError(t, err)
err = dst.Update(ctx, in2, src2)
require.NoError(t, err)
// verify hash was stored after Update and is different from the original
if f.opt.MaxAge > 0 {
hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
assert.NoError(t, err)
assert.NotEmpty(t, hash2, "hash should be stored after Update")
assert.NotEqual(t, hash1, hash2, "hash should change after Update with different content")
}
_ = operations.Purge(ctx, f, dirName)
}
// InternalTest dispatches all internal tests
func (f *Fs) InternalTest(t *testing.T) {
if !kv.Supported() {
t.Skip("hasher is not supported on this OS")
}
t.Run("UploadFromCrypt", f.testUploadFromCrypt)
t.Run("UpdateStoresHash", f.testUpdateStoresHash)
}
var _ fstests.InternalTester = (*Fs)(nil)
+41 -57
View File
@@ -131,62 +131,10 @@ func (o *Object) updateHashes(ctx context.Context) error {
return nil
}
// wrapInHashingReader wraps in with a hashing reader if needed and
// returns the (possibly wrapped) reader. After the transfer completes
// successfully, call the returned storeHashes function to store the
// computed hashes.
func (f *Fs) wrapInHashingReader(ctx context.Context, in io.Reader, src fs.ObjectInfo) (wrapIn io.Reader, storeHashes func(o *Object)) {
var (
common hash.Set
rehash bool
hashes hashMap
)
if fsrc := src.Fs(); fsrc != nil {
common = fsrc.Hashes().Overlap(f.keepHashes)
// Rehash if source does not have all required hashes or hashing is slow
rehash = fsrc.Features().SlowHash || common != f.keepHashes
}
wrapIn = in
if rehash {
r, err := f.newHashingReader(ctx, in, func(sums hashMap) {
hashes = sums
})
fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err)
if err == nil {
wrapIn = r
} else {
rehash = false
}
}
storeHashes = func(o *Object) {
if !rehash {
hashes = hashMap{}
for _, ht := range common.Array() {
if h, e := src.Hash(ctx, ht); e == nil && h != "" {
hashes[ht] = h
}
}
}
if len(hashes) > 0 {
err := o.putHashes(ctx, hashes)
fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err)
}
}
return wrapIn, storeHashes
}
// Update the object with the given data, time and size.
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
wrapIn, storeHashes := o.f.wrapInHashingReader(ctx, in, src)
_ = o.f.pruneHash(src.Remote())
err := o.Object.Update(ctx, wrapIn, src, options...)
if err != nil {
return err
}
storeHashes(o)
return nil
return o.Object.Update(ctx, in, src, options...)
}
// Remove an object.
@@ -241,15 +189,51 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (r io.ReadC
// Put data into the remote path with given modTime and size
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
wrapIn, storeHashes := f.wrapInHashingReader(ctx, in, src)
var (
o fs.Object
common hash.Set
rehash bool
hashes hashMap
)
if fsrc := src.Fs(); fsrc != nil {
common = fsrc.Hashes().Overlap(f.keepHashes)
// Rehash if source does not have all required hashes or hashing is slow
rehash = fsrc.Features().SlowHash || common != f.keepHashes
}
wrapIn := in
if rehash {
r, err := f.newHashingReader(ctx, in, func(sums hashMap) {
hashes = sums
})
fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err)
if err == nil {
wrapIn = r
} else {
rehash = false
}
}
_ = f.pruneHash(src.Remote())
oResult, err := f.Fs.Put(ctx, wrapIn, src, options...)
o, err := f.wrapObject(oResult, err)
o, err = f.wrapObject(oResult, err)
if err != nil {
return nil, err
}
storeHashes(o.(*Object))
return o, nil
if !rehash {
hashes = hashMap{}
for _, ht := range common.Array() {
if h, e := src.Hash(ctx, ht); e == nil && h != "" {
hashes[ht] = h
}
}
}
if len(hashes) > 0 {
err := o.(*Object).putHashes(ctx, hashes)
fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err)
}
return o, err
}
type hashingReader struct {
+1 -4
View File
@@ -50,10 +50,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
if err != nil {
return err
}
// Chtimes sets the modtime with second precision so truncate
// here too to keep the in-memory modtime identical to the one a
// fresh listing returns.
o.modTime = modTime.Truncate(time.Second)
o.modTime = modTime
return nil
}
+61 -73
View File
@@ -10,6 +10,7 @@ package hidrive
// to be resolved prior to execution with resolvePath(...).
import (
"bytes"
"context"
"errors"
"io"
@@ -23,9 +24,8 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/ranges"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
@@ -454,17 +454,17 @@ func (f *Fs) deleteObject(ctx context.Context, path string) error {
}
// createFile creates a file at the given path
// with the content of the buffer.
// with the content of the io.ReadSeeker.
// This guarantees that existing files will not be overwritten.
// The maximum size of the content is limited by MaximumUploadBytes.
// The caller remains responsible for closing the buffer.
// The io.ReadSeeker should be resettable by seeking to its start.
// If modTime is not the zero time instant,
// it will be set as the file's modification time after the operation.
//
// This returns fs.ErrorDirNotFound
// if the parent directory of the file is not found.
// This returns ErrorFileExists if a file already exists at the specified path.
func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
parameters := api.NewQueryParameters()
parameters.SetFileInDirectory(path)
if onExist == AutoNameOnExist {
@@ -479,14 +479,12 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
}
}
contentLength := content.Size()
opts := rest.Opts{
Method: "POST",
Path: "/file",
Body: content,
ContentType: "application/octet-stream",
ContentLength: &contentLength,
Parameters: parameters.Values,
Method: "POST",
Path: "/file",
Body: content,
ContentType: "application/octet-stream",
Parameters: parameters.Values,
}
var result api.HiDriveObject
@@ -512,16 +510,16 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
}
// overwriteFile updates the content of the file at the given path
// with the content of the buffer.
// with the content of the io.ReadSeeker.
// If the file does not exist it will be created.
// The maximum size of the content is limited by MaximumUploadBytes.
// The caller remains responsible for closing the buffer.
// The io.ReadSeeker should be resettable by seeking to its start.
// If modTime is not the zero time instant,
// it will be set as the file's modification time after the operation.
//
// This returns fs.ErrorDirNotFound
// if the parent directory of the file is not found.
func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, modTime time.Time) (*api.HiDriveObject, error) {
func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
parameters := api.NewQueryParameters()
parameters.SetFileInDirectory(path)
@@ -533,14 +531,12 @@ func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, m
}
}
contentLength := content.Size()
opts := rest.Opts{
Method: "PUT",
Path: "/file",
Body: content,
ContentType: "application/octet-stream",
ContentLength: &contentLength,
Parameters: parameters.Values,
Method: "PUT",
Path: "/file",
Body: content,
ContentType: "application/octet-stream",
Parameters: parameters.Values,
}
var result api.HiDriveObject
@@ -638,18 +634,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
}
// Read a chunk of data.
var (
chunkReader *pool.RW
bytesRead int64
)
chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize))
if bytesRead < int64(chunkSize) {
chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
if bytesRead < chunkSize {
startMoreTransfers = false
}
if readErr != nil || bytesRead <= 0 {
if chunkReader != nil {
_ = chunkReader.Close()
}
break
}
@@ -659,12 +648,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
// After this upload is done,
// signal that another transfer can be started.
defer transferSemaphore.Release(1)
defer func() { _ = chunkReader.Close() }()
uploadErr := f.patchFile(gCtx, path, chunkReader, chunkOffset, zeroTime)
uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
if uploadErr == nil {
// Remember successfully written chunks.
okChunksMu.Lock()
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: bytesRead})
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
okChunksMu.Unlock()
fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
} else {
@@ -708,10 +696,10 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
// starting at the given offset.
//
// Replaces the file contents starting from the given byte offset
// with the content of the buffer.
// with the content of the io.ReadSeeker.
// If the offset is beyond the file end, the file is extended up to the offset.
// The maximum size of the update is limited by MaximumUploadBytes.
// The caller remains responsible for closing the buffer.
// The io.ReadSeeker should be resettable by seeking to its start.
// If modTime is not the zero time instant,
// it will be set as the file's modification time after the operation.
//
@@ -723,7 +711,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
// usually expand the holes into regions of 0-bytes.
//
// This returns fs.ErrorObjectNotFound if the object is not found.
func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offset uint64, modTime time.Time) error {
func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
parameters := api.NewQueryParameters()
parameters.SetPath(path)
parameters.Set("offset", strconv.FormatUint(offset, 10))
@@ -735,15 +723,13 @@ func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offse
}
}
contentLength := content.Size()
opts := rest.Opts{
Method: "PATCH",
Path: "/file",
Body: content,
ContentType: "application/octet-stream",
ContentLength: &contentLength,
Parameters: parameters.Values,
NoResponse: true,
Method: "PATCH",
Path: "/file",
Body: content,
ContentType: "application/octet-stream",
Parameters: parameters.Values,
NoResponse: true,
}
var resp *http.Response
@@ -853,39 +839,41 @@ func createHiDriveScopes(role string, access string) []string {
return []string{}
}
// unwrapAccounting splits any transfer accounting off reader,
// returning the raw stream and the accounting (nil if there is none),
// so the stream can be buffered and accounted when the buffer is uploaded.
func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Account) {
// Any kind of accounter re-wraps into the one type UnWrapAccounting
// knows how to take the *Account out of.
unwrapped, wrap := accounting.UnWrap(reader)
_, acc = accounting.UnWrapAccounting(wrap(unwrapped))
return unwrapped, acc
}
// readerForChunk reads up to length bytes from reader into a buffer
// from the global memory pool and returns it
// along with the number of bytes that have been read from reader.
// Reaching the end of reader early is not an error.
//
// Any accounting on reader is applied when the buffer is read instead,
// so the upload is what gets accounted.
// The caller must Close the buffer when done with it.
func readerForChunk(reader io.Reader, length int64) (chunkReader *pool.RW, bytesRead int64, err error) {
reader, acc := unwrapAccounting(reader)
chunkReader = multipart.NewRW()
if acc != nil {
chunkReader.SetAccounting(acc.AccountRead)
// cachedReader returns a version of the reader that caches its contents and
// can therefore be reset using Seek.
func cachedReader(reader io.Reader) io.ReadSeeker {
bytesReader, ok := reader.(*bytes.Reader)
if ok {
return bytesReader
}
bytesRead, err = io.CopyN(chunkReader, reader, length)
if err == io.EOF {
repeatableReader, ok := reader.(*readers.RepeatableReader)
if ok {
return repeatableReader
}
return readers.NewRepeatableReader(reader)
}
// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
// Returns a new io.Reader (chunkReader) for that chunk
// and the number of bytes that have been read from reader.
func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
// Unwrap any accounting from the input if present.
reader, wrap := accounting.UnWrap(reader)
// Read a chunk of data.
buffer := make([]byte, length)
bytesRead, err = io.ReadFull(reader, buffer)
if err == io.EOF || err == io.ErrUnexpectedEOF {
err = nil
}
if err != nil {
_ = chunkReader.Close()
return nil, bytesRead, err
}
return chunkReader, bytesRead, nil
// Truncate unused capacity.
buffer = buffer[:bytesRead]
// Use wrap to put any accounting back for chunkReader.
return wrap(bytes.NewReader(buffer)), bytesRead, nil
}
+5 -21
View File
@@ -30,7 +30,6 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/rest"
)
@@ -515,14 +514,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
// (i.e. everything up to the cutoff) in the first request,
// avoids files being created on upload failure for small files.
// (As opposed to creating an empty file and then uploading the content.)
//
// Only buffer as much as the source declares it has,
// so small files do not cost the whole cutoff in memory.
prefixSize := int64(f.opt.UploadCutoff)
if size := src.Size(); size >= 0 && size < prefixSize {
prefixSize = size
}
cutoffReader, bytesRead, err := readerForChunk(in, prefixSize)
tmpReader, bytesRead, err := readerForChunk(in, int(f.opt.UploadCutoff))
cutoffReader := cachedReader(tmpReader)
if err != nil {
return nil, err
}
@@ -547,7 +540,6 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
}
return false, createErr
})
_ = cutoffReader.Close()
if err != nil {
return nil, err
@@ -564,7 +556,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
}
// If there is more left to write, o.Update needs to skip ahead.
// Use a fs.SeekOption with the current offset to do this.
options = append(options, &fs.SeekOption{Offset: bytesRead})
options = append(options, &fs.SeekOption{Offset: int64(bytesRead)})
err = o.Update(ctx, in, src, options...)
if err == nil {
@@ -896,10 +888,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
if err != nil {
return err
}
// The server stores modtimes with second precision so truncate
// here too to keep the in-memory modtime identical to the one a
// fresh listing returns.
o.modTime = modTime.Truncate(time.Second)
o.modTime = modTime
return nil
}
@@ -979,12 +968,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Metadata should be updated even if the upload fails.
info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
} else {
var content *pool.RW
content, _, err = readerForChunk(in, src.Size())
if err == nil {
info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
_ = content.Close()
}
info, err = o.fs.overwriteFile(ctx, resolvedPath, cachedReader(in), modTime)
metaErr = err
}
+2 -65
View File
@@ -57,13 +57,7 @@ The input format is comma separated list of key,value pairs. Standard
For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
The headers are only sent to the host in the configured URL. If the
server redirects to another host (including a subdomain or a different
port) the headers are not sent to it, or to any further hop in that
redirect chain. When headers are set, a redirect from https to http is
refused as it would send them in cleartext.`,
You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.`,
Default: fs.CommaSepList{},
Advanced: true,
}, {
@@ -162,7 +156,6 @@ type Fs struct {
endpoint *url.URL
endpointURL string // endpoint as a string
httpClient *http.Client
fileName string // set if we are pointing to a file
}
// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
@@ -289,11 +282,6 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
}
client := fshttp.NewClient(ctx)
// Without configured headers keep the default policy which
// public mirrors that redirect from https to http rely on.
if len(opt.Headers) > 0 {
client.CheckRedirect = checkRedirect(opt)
}
endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
fs.Debugf(nil, "Root: %s", endpoint)
@@ -309,7 +297,6 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
if isFile {
// Correct root if definitely pointing to a file
f.fileName = path.Base(f.root)
f.root = path.Dir(f.root)
if f.root == "." || f.root == "/" {
f.root = ""
@@ -521,45 +508,6 @@ func addHeaders(req *http.Request, opt *Options) {
}
}
// checkRedirect returns an http.Client.CheckRedirect function which
// follows redirects but refuses an HTTPS to HTTP downgrade with
// rest.ErrHTTPSDowngrade and strips the configured headers when the
// redirect chain has left the originally requested host at any point.
func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
return func(req *http.Request, via []*http.Request) error {
if err := rest.RefuseHTTPSDowngradeRedirectFn(req, via); err != nil {
if errors.Is(err, rest.ErrHTTPSDowngrade) {
err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
}
return err
}
if redirectLeavesHost(req, via) {
for i := 0; i < len(opt.Headers); i += 2 {
req.Header.Del(opt.Headers[i])
}
}
return nil
}
}
// redirectLeavesHost reports whether any hop in the redirect chain
// via plus the pending request req is to a different host from the
// original request via[0].
//
// net/http copies the headers afresh from the original request for
// every hop, so once the chain has visited another host the headers
// must be stripped from every subsequent hop, even one back to the
// original host, as the other host chose the URL.
func redirectLeavesHost(req *http.Request, via []*http.Request) bool {
origin := via[0].URL
for _, hop := range via {
if !rest.SameHost(hop.URL, origin) {
return true
}
}
return !rest.SameHost(req.URL, origin)
}
// Adds the configured headers to the request if any
func (f *Fs) addHeaders(req *http.Request) {
addHeaders(req, &f.opt)
@@ -593,7 +541,7 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
return nil, fmt.Errorf("failed to readDir: %w", err)
}
contentType, _, _ := strings.Cut(res.Header.Get("Content-Type"), ";")
contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
switch contentType {
case "text/html":
names, err = parse(u, res.Body)
@@ -616,17 +564,6 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
// pointed at a single file: only that file is visible
if f.fileName != "" {
if dir != "" {
return nil, fs.ErrorDirNotFound
}
obj, err := f.NewObject(ctx, f.fileName)
if err != nil {
return nil, err
}
return fs.DirEntries{obj}, nil
}
if !strings.HasSuffix(dir, "/") && dir != "" {
dir += "/"
}
+1 -192
View File
@@ -331,19 +331,7 @@ func TestIsAFileRoot(t *testing.T) {
f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
assert.Equal(t, err, fs.ErrorIsFile)
entries, err := f.List(context.Background(), "")
require.NoError(t, err)
require.Equal(t, 1, len(entries))
e := entries[0]
assert.Equal(t, "one%.txt", e.Remote())
assert.Equal(t, int64(5+lineEndSize), e.Size())
_, ok := e.(*Object)
assert.True(t, ok)
_, err = f.List(context.Background(), "anysub")
assert.Equal(t, fs.ErrorDirNotFound, err)
testListRoot(t, f, false)
}
func TestIsAFileSubDir(t *testing.T) {
@@ -581,182 +569,3 @@ func TestFsNoSlashRoots(t *testing.T) {
}
}
}
// readRedirected makes an http remote with the configured headers
// pointing at url and reads file.txt from it, checking the content
func readRedirected(t *testing.T, url string) {
configfile.Install()
m := configmap.Simple{
"type": "http",
"url": url + "/",
"headers": strings.Join(headers, ","),
}
f, err := NewFs(context.Background(), remoteName, "", m)
require.NoError(t, err)
o, err := f.NewObject(context.Background(), "file.txt")
require.NoError(t, err)
fd, err := o.Open(context.Background())
require.NoError(t, err)
data, err := io.ReadAll(fd)
require.NoError(t, err)
require.NoError(t, fd.Close())
assert.Equal(t, "hello", string(data))
}
func TestRedirectKeepsHeadersOnSameHost(t *testing.T) {
var got http.Header
var ts *httptest.Server
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/moved/") {
http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
return
}
got = r.Header.Clone()
_, _ = w.Write([]byte("hello"))
}))
defer ts.Close()
readRedirected(t, ts.URL)
require.NotNil(t, got)
for i := 0; i < len(headers); i += 2 {
assert.Equal(t, headers[i+1], got.Get(headers[i]))
}
}
func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
// A plaintext server to be redirected to
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("hello"))
}))
defer target.Close()
// An HTTPS server which redirects to plaintext HTTP
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
}))
defer ts.Close()
// open makes a remote pointing at ts with or without headers
// and opens file.txt on it
open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
configfile.Install()
m := configmap.Simple{
"type": "http",
"url": ts.URL + "/",
"no_head": "true",
}
if withHeaders {
m.Set("headers", strings.Join(headers, ","))
}
ctx, ci := fs.AddConfig(context.Background())
ci.InsecureSkipVerify = true
f, err := NewFs(ctx, remoteName, "", m)
require.NoError(t, err)
// no_head means NewObject doesn't make a request so use Open to make one
o, err := f.NewObject(ctx, "file.txt")
require.NoError(t, err)
return o.Open(ctx)
}
t.Run("WithHeaders", func(t *testing.T) {
// Headers could leak so the downgrade must be refused
_, err := open(t, true)
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
})
t.Run("WithoutHeaders", func(t *testing.T) {
// Nothing to leak so the downgrade is followed
fd, err := open(t, false)
require.NoError(t, err)
data, err := io.ReadAll(fd)
require.NoError(t, err)
require.NoError(t, fd.Close())
assert.Equal(t, "hello", string(data))
})
}
func TestRedirectKeepsHeadersOnSameHostDifferentCase(t *testing.T) {
// The redirect spells the host differently but it is the same
// host so the headers must be kept
var got http.Header
var ts *httptest.Server
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/moved/") {
http.Redirect(w, r, "http://LOCALHOST:"+port(t, ts)+"/moved"+r.URL.Path, http.StatusFound)
return
}
got = r.Header.Clone()
_, _ = w.Write([]byte("hello"))
}))
defer ts.Close()
readRedirected(t, "http://localhost:"+port(t, ts))
require.NotNil(t, got)
for i := 0; i < len(headers); i += 2 {
assert.Equal(t, headers[i+1], got.Get(headers[i]))
}
}
// port returns the port ts is listening on
func port(t *testing.T, ts *httptest.Server) string {
u, err := url.Parse(ts.URL)
require.NoError(t, err)
return u.Port()
}
func TestRedirectStripsHeadersAfterHostChange(t *testing.T) {
// The configured server redirects to a second server which
// redirects back to the configured server. The headers must not
// be sent on the way back as the other host chose the path.
var got http.Header
var ts *httptest.Server
bouncer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
}))
defer bouncer.Close()
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, "/moved/") {
http.Redirect(w, r, bouncer.URL+r.URL.Path, http.StatusFound)
return
}
got = r.Header.Clone()
_, _ = w.Write([]byte("hello"))
}))
defer ts.Close()
readRedirected(t, ts.URL)
require.NotNil(t, got)
for i := 0; i < len(headers); i += 2 {
assert.Empty(t, got.Values(headers[i]), "header %q sent after cross-host redirect", headers[i])
}
}
func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
// Serve the final response from a second server. httptest servers
// listen on 127.0.0.1:port so the two servers have different hosts
// for the purposes of the redirect check.
var got http.Header
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.Header.Clone()
_, _ = w.Write([]byte("hello"))
}))
defer target.Close()
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
}))
defer redirector.Close()
readRedirected(t, redirector.URL)
require.NotNil(t, got)
for i := 0; i < len(headers); i += 2 {
assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
}
}
Loaded 100 of 753 files, more files were not shown because too many files have changed in this diff. Show more