Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13a1d23bb8 |
No files matched your search
@@ -1,78 +0,0 @@
|
||||
# CODEOWNERS for rclone
|
||||
#
|
||||
# Maps paths to the maintainers auto-requested for review when a pull request
|
||||
# touches matching files. Patterns are evaluated top to bottom and the LAST
|
||||
# matching pattern wins, so the catch-all below is overridden by the more
|
||||
# specific subsystem/backend lines that follow.
|
||||
#
|
||||
# Every owner listed here has write access to the repo (GitHub ignores owners
|
||||
# without it). Assignments are derived from `git log` history per directory:
|
||||
# each area lists its dominant contributor(s) alongside @ncw, the project lead
|
||||
# and global fallback.
|
||||
|
||||
# Global fallback - everything not matched below
|
||||
* @ncw
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Core subsystems
|
||||
# -----------------------------------------------------------------------------
|
||||
/fs/ @ncw
|
||||
/vfs/ @ncw
|
||||
/lib/ @ncw
|
||||
/cmd/ @ncw
|
||||
/librclone/ @ncw
|
||||
/fstest/ @ncw
|
||||
|
||||
/lib/encoder/ @ncw @klauspost
|
||||
/fs/sync/ @ncw @nielash
|
||||
/fs/operations/ @ncw @nielash
|
||||
/cmd/bisync/ @nielash @ivandeex @ncw
|
||||
/cmd/serve/ @albertony @ncw
|
||||
/cmd/selfupdate/ @ivandeex @ncw
|
||||
/cmd/mount2/ @ncw @darthShadow
|
||||
/cmd/gitannex/ @dmcardle @ncw
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Backends - keyed to the dominant / specialist maintainer per `git log`
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# albertony
|
||||
/backend/jottacloud/ @albertony @buengese @ncw
|
||||
/backend/local/ @ncw @albertony @nielash
|
||||
/backend/netstorage/ @albertony @ncw
|
||||
/backend/hdfs/ @albertony @ncw
|
||||
|
||||
# buengese
|
||||
/backend/yandex/ @ncw @buengese
|
||||
/backend/zoho/ @buengese @ncw
|
||||
/backend/fichier/ @ncw @buengese
|
||||
/backend/compress/ @ncw @buengese
|
||||
|
||||
# ivandeex
|
||||
/backend/chunker/ @ncw @ivandeex
|
||||
/backend/mailru/ @ncw @ivandeex
|
||||
/backend/ftp/ @ncw @ivandeex
|
||||
/backend/sia/ @ncw @ivandeex
|
||||
|
||||
# Other maintainer-owned backends (see MAINTAINERS.md)
|
||||
/backend/cache/ @ncw @remusb
|
||||
/backend/union/ @ncw @Max-Sum @albertony
|
||||
/backend/filescom/ @ncw @childish-sambino
|
||||
|
||||
# Microsoft Azure
|
||||
/backend/azureblob/ @ncw @sandeepkru @darthShadow
|
||||
|
||||
# Single-maintainer specialist backends
|
||||
/backend/onedrive/ @ncw @Cnly @nielash
|
||||
/backend/iclouddrive/ @lostb1t @ncw
|
||||
/backend/pikpak/ @wiserain @ncw
|
||||
/backend/protondrive/ @henrybear327 @ncw
|
||||
/backend/seafile/ @ncw @creativeprojects
|
||||
/backend/filen/ @Enduriel @ncw
|
||||
/backend/storj/ @ncw @calebcase
|
||||
/backend/combine/ @ncw @nielash
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Repo plumbing
|
||||
# -----------------------------------------------------------------------------
|
||||
/.github/ @ncw
|
||||
@@ -0,0 +1,31 @@
|
||||
<!--
|
||||
|
||||
Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
|
||||
|
||||
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
|
||||
|
||||
https://forum.rclone.org/
|
||||
|
||||
instead of filing an issue for a quick response.
|
||||
|
||||
If you are reporting a bug or asking for a new feature then please use one of the templates here:
|
||||
|
||||
https://github.com/rclone/rclone/issues/new
|
||||
|
||||
otherwise fill in the form below.
|
||||
|
||||
Thank you
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
|
||||
#### Output of `rclone version`
|
||||
|
||||
|
||||
|
||||
#### Describe the issue
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Report a problem with rclone
|
||||
---
|
||||
|
||||
<!--
|
||||
|
||||
We understand you are having a problem with rclone; we want to help you with that!
|
||||
|
||||
**STOP and READ**
|
||||
**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
|
||||
Please show the effort you've put into solving the problem and please be specific.
|
||||
People are volunteering their time to help! Low effort posts are not likely to get good answers!
|
||||
|
||||
If you think you might have found a bug, try to replicate it with the latest beta (or stable).
|
||||
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
|
||||
|
||||
If you can still replicate it or just got a question then please use the rclone forum:
|
||||
|
||||
https://forum.rclone.org/
|
||||
|
||||
for a quick response instead of filing an issue on this repo.
|
||||
|
||||
If nothing else helps, then please fill in the info below which helps us help you.
|
||||
|
||||
**DO NOT REDACT** any information except passwords/keys/personal info.
|
||||
|
||||
You should use 3 backticks to begin and end your paste to make it readable.
|
||||
|
||||
Make sure to include a log obtained with '-vv'.
|
||||
|
||||
You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
|
||||
|
||||
Thank you
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
|
||||
#### What is the problem you are having with rclone?
|
||||
|
||||
|
||||
|
||||
#### What is your rclone version (output from `rclone version`)
|
||||
|
||||
|
||||
|
||||
#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
|
||||
|
||||
|
||||
|
||||
#### Which cloud storage system are you using? (e.g. Google Drive)
|
||||
|
||||
|
||||
|
||||
#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
|
||||
|
||||
|
||||
|
||||
#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your bug report. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest a new feature or enhancement for rclone
|
||||
---
|
||||
|
||||
<!--
|
||||
|
||||
Welcome :-)
|
||||
|
||||
So you've got an idea to improve rclone? We love that!
|
||||
You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
|
||||
|
||||
Probably the latest beta (or stable) release has your feature, so try to update your rclone.
|
||||
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
|
||||
|
||||
If it still isn't there, here is a checklist of things to do:
|
||||
|
||||
1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
|
||||
2. Discuss on the forum: https://forum.rclone.org/
|
||||
3. Make a feature request issue (this is the right place!).
|
||||
4. Be prepared to get involved making the feature :-)
|
||||
|
||||
Looking forward to your great idea!
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
|
||||
#### What is your current rclone version (output from `rclone version`)?
|
||||
|
||||
|
||||
|
||||
#### What problem are you are trying to solve?
|
||||
|
||||
|
||||
|
||||
#### How do you think rclone should be changed to solve that?
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your feature request. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -1,124 +0,0 @@
|
||||
name: Bug report
|
||||
description: Report a reproducible problem with rclone
|
||||
labels: ["bug"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
We want to help you with rclone - thank you for taking the time to fill this in!
|
||||
|
||||
**This tracker is for reproducible bugs only.** If you have a question, are
|
||||
not sure whether something is a bug, or want help with a configuration, please
|
||||
use the [rclone forum](https://forum.rclone.org/) instead - you will get a much
|
||||
quicker answer there.
|
||||
|
||||
Before filing, please **test with the [latest beta or stable](https://rclone.org/commands/rclone_selfupdate/)**
|
||||
(downloads at <https://beta.rclone.org/>) as your problem may already be fixed.
|
||||
|
||||
**Do not redact** any information except passwords, keys and personal info.
|
||||
- type: checkboxes
|
||||
id: acknowledgements
|
||||
attributes:
|
||||
label: Before you start
|
||||
description: Please confirm the following. Issues that skip these steps may be closed.
|
||||
options:
|
||||
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues for this problem.
|
||||
required: true
|
||||
- label: I have tested with the latest beta or stable release and can still reproduce the problem.
|
||||
required: true
|
||||
- label: This is a reproducible bug, not a usage question (questions belong on the forum).
|
||||
required: true
|
||||
- type: input
|
||||
id: forum-url
|
||||
attributes:
|
||||
label: Associated forum post URL
|
||||
description: If you discussed this on the forum first (encouraged), paste the link here.
|
||||
placeholder: https://forum.rclone.org/t/...
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: What is the problem you are having with rclone?
|
||||
description: Describe the problem and the exact steps to reproduce it.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: rclone version
|
||||
description: The full output of `rclone version`.
|
||||
placeholder: |
|
||||
rclone v1.75.0
|
||||
- os/version: ubuntu 24.04 (64 bit)
|
||||
- os/kernel: 6.8.0-111-generic (x86_64)
|
||||
- os/type: linux
|
||||
- os/arch: amd64
|
||||
- go/version: go1.26.4
|
||||
- go/linking: dynamic
|
||||
- go/tags: none
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating system
|
||||
description: Choose the operating system you are using with rclone.
|
||||
options:
|
||||
- Windows
|
||||
- macOS
|
||||
- Linux
|
||||
- FreeBSD
|
||||
- NetBSD
|
||||
- OpenBSD
|
||||
- Plan 9
|
||||
- Solaris
|
||||
- Android
|
||||
- iOS
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: backend
|
||||
attributes:
|
||||
label: Which cloud storage system are you using?
|
||||
description: e.g. Google Drive, S3, Dropbox, local disk. List all involved.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: config
|
||||
attributes:
|
||||
label: Output of `rclone config redacted`
|
||||
description: |
|
||||
Run `rclone config redacted` (or `rclone config redacted <remote>` for a
|
||||
single remote). This command automatically replaces passwords and tokens
|
||||
with `XXX`, so it should be safe to paste but please double-check before posting.
|
||||
render: ini
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: command
|
||||
attributes:
|
||||
label: The command you were trying to run
|
||||
description: e.g. `rclone copy /tmp remote:tmp`
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: log
|
||||
attributes:
|
||||
label: A log from the command with the `-vv` flag
|
||||
description: |
|
||||
e.g. output from `rclone -vv copy /tmp remote:tmp`. For long logs (> 100 lines) use
|
||||
`-vv --log-file bug.log` and attach it here or send to a service such as
|
||||
<https://gist.github.com/> or <https://pastebin.com/> and link it here.
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: extra
|
||||
attributes:
|
||||
label: Anything else?
|
||||
description: Anything else that might help us, e.g. a minimal way to replicate the problem.
|
||||
validations:
|
||||
required: false
|
||||
@@ -1,8 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Rclone Forum - questions & support
|
||||
- name: Rclone Forum Community Support
|
||||
url: https://forum.rclone.org/
|
||||
about: Have a question, need help, or not sure if it's a bug? Please ask here for a quick answer.
|
||||
- name: Rclone Documentation
|
||||
url: https://rclone.org/docs/
|
||||
about: The docs answer many "how do I..." questions, including per-backend guides.
|
||||
about: Please ask and answer questions here.
|
||||
@@ -1,68 +0,0 @@
|
||||
name: Feature request
|
||||
description: Suggest a new feature or enhancement for rclone
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
So you've got an idea to improve rclone? We love that - we've already
|
||||
incorporated hundreds of ideas from contributors!
|
||||
|
||||
Please **search first**: your idea may already exist in the latest
|
||||
[beta or stable](https://rclone.org/commands/rclone_selfupdate/), or already
|
||||
be requested. It is often worth [discussing on the forum](https://forum.rclone.org/)
|
||||
to refine the idea or checking to see if rclone can do it already before filing it here.
|
||||
- type: checkboxes
|
||||
id: acknowledgements
|
||||
attributes:
|
||||
label: Before you start
|
||||
options:
|
||||
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues and this hasn't already been requested.
|
||||
required: true
|
||||
- label: I have checked the latest beta and this feature doesn't already exist.
|
||||
required: true
|
||||
- type: input
|
||||
id: forum-url
|
||||
attributes:
|
||||
label: Associated forum post URL
|
||||
description: If you discussed this on the forum first (encouraged), paste the link here.
|
||||
placeholder: https://forum.rclone.org/t/...
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: rclone version
|
||||
description: The full output of `rclone version`, not just the version number.
|
||||
placeholder: |
|
||||
rclone v1.75.0
|
||||
- os/version: ubuntu 24.04 (64 bit)
|
||||
- os/kernel: 6.8.0-111-generic (x86_64)
|
||||
- os/type: linux
|
||||
- os/arch: amd64
|
||||
- go/version: go1.26.4
|
||||
- go/linking: dynamic
|
||||
- go/tags: none
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: What problem are you trying to solve?
|
||||
description: Describe the use case and what is difficult or impossible today.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: How do you think rclone should be changed to solve that?
|
||||
validations:
|
||||
required: true
|
||||
- type: checkboxes
|
||||
id: involvement
|
||||
attributes:
|
||||
label: Getting involved
|
||||
description: Features are much more likely to happen if someone helps make them.
|
||||
options:
|
||||
- label: I'm willing to help implement, test or fund this feature.
|
||||
required: false
|
||||
@@ -1,54 +1,29 @@
|
||||
<!--
|
||||
Thank you for contributing to rclone!
|
||||
Thank you very much for contributing code or documentation to rclone! Please
|
||||
fill out the following questions to make it easier for us to review your
|
||||
changes.
|
||||
|
||||
Please discuss anything more than a trivial fix in an issue FIRST - see the
|
||||
"Linked issue" section below. Then fill in the questions to help us review.
|
||||
You do not need to check all the boxes below all at once, feel free to take
|
||||
your time and add more commits. If you're done and ready for review, please
|
||||
check the last box.
|
||||
-->
|
||||
|
||||
#### What does this change do?
|
||||
|
||||
<!-- Describe the change here. -->
|
||||
|
||||
#### Linked issue
|
||||
#### What is the purpose of this change?
|
||||
|
||||
<!--
|
||||
Put `Fixes #1234` here if this closes an issue, or `#1234` to link without closing.
|
||||
|
||||
IMPORTANT: Anything beyond a trivial fix (typo, doc tweak, small obvious bug fix)
|
||||
should be discussed and agreed in an issue BEFORE you open a pull request.
|
||||
Pull requests for larger changes that have not been discussed in an issue first
|
||||
may be closed. This saves everyone's time if the change needs a different approach
|
||||
or isn't a good fit.
|
||||
Describe the changes here
|
||||
-->
|
||||
|
||||
Fixes #
|
||||
|
||||
#### For new or changed backends
|
||||
#### Was the change discussed in an issue or in the forum before?
|
||||
|
||||
<!--
|
||||
Backend pull requests are often sent without the integration tests having been run.
|
||||
We cannot merge a backend change until it passes the integration tests.
|
||||
|
||||
To be merged, a backend change REQUIRES:
|
||||
1. A clean run of `go run ./fstest/test_all -backends <remote>` (summarise the
|
||||
result below or paste a screenshot of the test_all output in the web browser).
|
||||
2. A test account so the maintainers can run the integration tests daily against
|
||||
the backend on the integration test server.
|
||||
|
||||
If the tests aren't quite passing yet and you need help getting them
|
||||
to pass, then submit the PR and we can help getting you over the line.
|
||||
|
||||
See https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#writing-a-new-backend
|
||||
and https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests
|
||||
Link issues and relevant forum posts here.
|
||||
-->
|
||||
|
||||
#### Checklist
|
||||
|
||||
- [ ] This change is trivial **OR** it has been discussed and agreed in the linked issue.
|
||||
- [ ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
|
||||
- [ ] **(If I used AI tools to help write this code)** I have read and understood the [AI-assisted contributions guidance](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#ai-assisted-contributions), and I have tested and take ownership of this change myself.
|
||||
- [ ] I have added tests for all changes in this PR if appropriate.
|
||||
- [ ] I have added documentation for the changes if appropriate.
|
||||
- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
|
||||
- [ ] **(Backend changes only)** `test_all` passes for this backend and if submitting a new backend can provide a test account for the integration tester - see [CONTRIBUTING.md](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests).
|
||||
- [ ] This Pull Request is ready for review.
|
||||
- [ ] I'm done, this Pull Request is ready for review :-)
|
||||
@@ -4,5 +4,3 @@ updates:
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
+19
-139
@@ -19,9 +19,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
@@ -32,12 +29,12 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.0'
|
||||
go: '~1.26.0'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +45,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.0'
|
||||
go: '~1.26.0'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.27.0'
|
||||
go: '~1.26.0'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +61,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.27.0'
|
||||
go: '~1.26.0'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.27.0'
|
||||
go: '~1.26.0'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,14 +78,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.0'
|
||||
go: '~1.26.0'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.26
|
||||
- job_name: go1.25
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.6'
|
||||
go: '~1.25.7'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -97,58 +94,16 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
check-latest: true
|
||||
# Caching is handled explicitly below. setup-go's built-in cache
|
||||
# keys only on go.sum, so all matrix jobs collide on one key and
|
||||
# only the first to finish saves - which loses the cross-compile
|
||||
# build cache.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
# The module cache holds downloaded source and is identical across
|
||||
# jobs (it depends only on go.sum, not on OS, arch or Go version), so
|
||||
# it is shared via a single key to avoid storing the same ~600 MiB in
|
||||
# every job's cache and overflowing the repo cache limit.
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
# The build cache holds compiled artifacts which are specific to the
|
||||
# OS, arch and Go version, so it is kept per job.
|
||||
- name: Build cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
@@ -165,11 +120,6 @@ jobs:
|
||||
sudo chown root:$USER /etc/fuse.conf
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
|
||||
# The podman static bundle in runner image ubuntu24/20260726.254 and
|
||||
# later ships a non-setuid fusermount3 in /usr/local/bin which
|
||||
# shadows the setuid one from the fuse3 package and breaks FUSE
|
||||
# mounts with "Operation not permitted", so remove it.
|
||||
sudo rm -f /usr/local/bin/fusermount /usr/local/bin/fusermount3
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
|
||||
- name: Install Libraries on macOS
|
||||
@@ -179,7 +129,6 @@ jobs:
|
||||
unset HOMEBREW_NO_INSTALL_FROM_API
|
||||
brew untap --force homebrew/core
|
||||
brew untap --force homebrew/cask
|
||||
brew untap --force aws/tap
|
||||
brew update
|
||||
brew install --cask macfuse
|
||||
brew install git-annex git-annex-remote-rclone
|
||||
@@ -253,15 +202,6 @@ jobs:
|
||||
# Deploy binaries if enabled in config && not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
lint:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -276,31 +216,23 @@ jobs:
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.27.0'
|
||||
go-version: '~1.26.0'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Cache
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
~/.cache/golangci-lint
|
||||
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
@@ -367,15 +299,6 @@ jobs:
|
||||
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
android:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -383,50 +306,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Upgrade together with NDK version
|
||||
- name: Set up Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.27.0'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Build cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
go-version: '~1.26.0'
|
||||
|
||||
- name: Set global environment variables
|
||||
run: |
|
||||
@@ -502,12 +391,3 @@ jobs:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# Upload artifacts if not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
@@ -18,10 +18,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build-image:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
|
||||
@@ -56,7 +52,7 @@ jobs:
|
||||
df -h .
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -96,7 +92,7 @@ jobs:
|
||||
# There's no way around this, because "ImageOS" is only available to
|
||||
# processes, but the setup-go action uses it in its key.
|
||||
id: imageos
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
@@ -104,7 +100,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
|
||||
with:
|
||||
@@ -126,14 +122,14 @@ jobs:
|
||||
type=raw,value=beta,enable={{is_default_branch}}
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Load Go Build Cache for Docker
|
||||
id: go-cache
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
@@ -152,7 +148,7 @@ jobs:
|
||||
skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
@@ -163,7 +159,7 @@ jobs:
|
||||
|
||||
- name: Build and Publish Image Digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
file: Dockerfile
|
||||
context: .
|
||||
@@ -214,7 +210,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
|
||||
with:
|
||||
@@ -265,16 +261,16 @@ jobs:
|
||||
env.write(f"ANNOTATIONS={annotations_string}\n")
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
|
||||
@@ -14,9 +14,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build_docker_volume_plugin:
|
||||
if: inputs.manual || github.repository == 'rclone/rclone'
|
||||
@@ -33,7 +30,7 @@ jobs:
|
||||
sudo rm -rf /usr/share/dotnet || true
|
||||
df -h .
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and publish docker plugin
|
||||
|
||||
@@ -4,9 +4,6 @@ on:
|
||||
issues:
|
||||
types: [labeled]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -3,9 +3,6 @@ on:
|
||||
release:
|
||||
types: [released]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -46,11 +46,6 @@ linters:
|
||||
- -ST1022
|
||||
# Disable quickfix checks
|
||||
- -QF*
|
||||
# Disabled because its dataflow analysis (new in staticcheck
|
||||
# 0.8.0) makes linting large packages more than 10x slower,
|
||||
# which takes the CI lint job past its timeout, and it
|
||||
# produces false positives (eg on operations.Delete).
|
||||
- -SA4023
|
||||
gocritic:
|
||||
# With gocritic there are different settings, but since enabled-checks
|
||||
# and disabled-checks cannot both be set, for full customization the
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
default: true
|
||||
|
||||
# Use specific styles, to be consistent across all documents.
|
||||
# Use specific styles, to be consistent accross all documents.
|
||||
# Default is to accept any as long as it is consistent within the same document.
|
||||
heading-style: # MD003
|
||||
style: atx
|
||||
@@ -48,12 +48,12 @@ link-fragments: false # MD051
|
||||
# https://gohugo.io/content-management/syntax-highlighting/#languages
|
||||
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
|
||||
# In addition, we only want to allow identifiers (aliases) that correspond to
|
||||
# the same language in Hugo and GitHub, and preferably also VSCode and other
|
||||
# the same language in Hugo and GitHub, and preferrably also VSCode and other
|
||||
# commonly used tools, to avoid confusion. An example of this is that "shell"
|
||||
# by some are considered an identifier for shell scripts, i.e. an alias for
|
||||
# "sh", while others consider it an identifier for shell sessions, i.e. an
|
||||
# alias for "console". Although Hugo and GitHub in this case are consistent and
|
||||
# have chosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# have choosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# avoids the confusion entirely.
|
||||
fenced-code-language: # MD040
|
||||
allowed_languages:
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
This file provides guidance to AI coding agents (e.g. Claude Code, Codex, Cursor, Gemini CLI, and similar tools) when working with code in this repository.
|
||||
|
||||
Rclone welcomes AI-assisted contributions, but the expectation is that you, the human submitter, understand every line you propose and have compiled and tested it against real rclone code - not just generated it. See the "AI-assisted contributions" section of [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request.
|
||||
|
||||
## Project Overview
|
||||
|
||||
Rclone is a command-line program to sync files and directories to and from cloud storage providers. It's written in Go and supports 70+ backends (cloud storage systems). Think "rsync for cloud storage".
|
||||
|
||||
## General Notes
|
||||
|
||||
**We take backwards compatibility very seriously.** PRs should not change the observable behaviour of existing commands, flags or rc API without very good reason. Rclone does not try to preserve a stable Go API but try not to change it gratuitously.
|
||||
|
||||
Rclone operates with a lot of different backends, so **compatibility is key**. It is the backend integration tests which guarantee that compatibility. Changes should consider both known and unknown backends and should take care not to break functionality of existing installations.
|
||||
|
||||
The core parts of rclone under `fs` and `vfs` need to work with all backends and **backend specific hacks won't be merged**. Fixes likely need to go in the relevant backend or if new behaviour is really needed, a new Feature flag needs to be added.
|
||||
|
||||
**Changes should be kept to the minimum.** Work hard to make the most elegant, smallest change you can. Do not refactor or re-order code unless necessary as this makes review more challenging. Re-use existing test scaffolding, existing or library routines (e.g. `lib`) where possible.
|
||||
|
||||
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
|
||||
|
||||
## Build and Test Commands
|
||||
|
||||
```bash
|
||||
# Build rclone (simple)
|
||||
go build
|
||||
|
||||
# Build with version info (preferred)
|
||||
make
|
||||
|
||||
# Run all unit tests (no cloud credentials needed)
|
||||
make quicktest
|
||||
# or equivalently:
|
||||
RCLONE_CONFIG="/notfound" go test ./...
|
||||
|
||||
# Run tests for a specific package
|
||||
cd backend/memory && go test -v
|
||||
# or from root:
|
||||
go test -v ./backend/memory/
|
||||
|
||||
# Run a single test
|
||||
go test -v -run TestIntegration/FsCheckWrap ./backend/memory/
|
||||
|
||||
# Run tests with race detector
|
||||
make racequicktest
|
||||
|
||||
# Lint (requires golangci-lint)
|
||||
golangci-lint run ./...
|
||||
|
||||
# Run backend integration tests (requires configured TestRemote remote)
|
||||
cd backend/drive && go test -v
|
||||
# Run sync/operations integration tests against a remote
|
||||
cd fs/sync && go test -v -remote TestDrive:
|
||||
cd fs/operations && go test -v -remote TestDrive:
|
||||
|
||||
# Run integration tests via test framework
|
||||
go run ./fstest/test_all -backends drive
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Entry Point and Plugin Registration
|
||||
|
||||
`rclone.go` is the main entry point. It imports `backend/all` and `cmd/all` which use Go's `init()` pattern to register all backends and commands. Each backend calls `fs.Register()` with a `fs.RegInfo` struct during init.
|
||||
|
||||
### Core Interfaces (`fs/`)
|
||||
|
||||
The `fs` package defines the core abstractions:
|
||||
- **`fs.Fs`** (`fs/types.go`): The filesystem interface every backend must implement (List, NewObject, Put, Mkdir, Rmdir).
|
||||
- **`fs.Object`** (`fs/types.go`): Interface for a file/object (Open, Update, Remove, SetModTime).
|
||||
- **`fs.Features`** (`fs/features.go`): Optional capabilities a backend can declare (Purge, Copy, Move, DirMove, etc.). Backends set function pointers for operations they support; nil means not supported.
|
||||
- **`fs.RegInfo`** (`fs/registry.go`): Registration metadata for a backend including its name, config options, and NewFs constructor.
|
||||
|
||||
### Backend Structure (`backend/`)
|
||||
|
||||
Each backend is a single Go package (e.g., `backend/s3/`, `backend/drive/`). Key conventions:
|
||||
- Main implementation in a single file (e.g., `s3.go`) - **do not** split into `fs.go`/`object.go`.
|
||||
- API types go in a separate `api/types.go` file.
|
||||
- Test file (e.g., `s3_test.go`) uses `fstests.Run()` from `fstest/fstests` for standardized integration tests.
|
||||
- Register in `backend/all/all.go` via blank import.
|
||||
- HTTP-based backends should use `lib/rest` for HTTP calls and `fs/fshttp` for the HTTP client.
|
||||
- Use `lib/dircache` for directory-ID-based remotes, `lib/oauthutil` for OAuth, `lib/pacer` for rate limiting.
|
||||
|
||||
### Command Structure (`cmd/`)
|
||||
|
||||
Each command is a package under `cmd/` registered in `cmd/all/all.go` via blank import. Commands use cobra via `cmd.Main()`.
|
||||
|
||||
### Key Subsystems
|
||||
|
||||
- **`fs/operations/`**: Core file operations (Copy, Move, Delete, etc.)
|
||||
- **`fs/sync/`**: Directory sync logic
|
||||
- **`fs/march/`**: Parallel directory tree walker used by sync
|
||||
- **`fs/filter/`**: Include/exclude filtering
|
||||
- **`fs/accounting/`**: Transfer statistics and bandwidth limiting
|
||||
- **`fs/config/`**: Configuration file management
|
||||
- **`vfs/`**: Virtual filesystem layer (used by mount, serve)
|
||||
- **`librclone/`**: C-compatible library interface for embedding rclone
|
||||
- **`fstest/`**: Integration test framework; `fstest/fstests/` has the generic backend test suite
|
||||
|
||||
## Commit Message Convention
|
||||
|
||||
Prefix with the directory of the change, then a colon: `drive: add team drive support - fixes #885`. For cross-cutting changes use a broader prefix like `fs` or `operations`.
|
||||
|
||||
Make the first line of your commit message a summary of the change that a user (not a developer) of rclone would like to read. So write `drive: fix server side copy of big files` instead of `drive: no longer set the MimeType in Move or Copy`. This is important because these lines go into the change log which is read by users.
|
||||
|
||||
## Code Commenting Style
|
||||
|
||||
Comments describe the code as it is now, for a future reader who has no knowledge of the change that introduced it.
|
||||
|
||||
Every exported type, function, field, and constant has a **godoc comment** that starts with its name and is phrased as a present-tense statement of what it is or does (`// Mkdir makes the directory (container, bucket)`)
|
||||
|
||||
**Document the contract** callers need - preconditions, what's returned, which sentinel errors are returned and when, and any "shouldn't return an error if it already exists" style caveats - rather than the implementation.
|
||||
|
||||
**Keep comments terse**: a single line for most things, with extra paragraphs (separated by blank `//` lines) reserved for genuine subtlety.
|
||||
|
||||
Inline comments inside function bodies should **explain why** - a non-obvious API quirk, a workaround, a gotcha, or an ordering constraint - and may cite an external reference (forum thread, vendor docs, RFC) when that's what makes the behaviour non-obvious; skip comments that merely restate what the code plainly does.
|
||||
|
||||
Use `FIXME` and `TODO` for known shortcomings.
|
||||
|
||||
**Do not write comments that narrate the change itself** or compare against the previous behaviour (no "now we also handle...", "changed to...", "previously this returned...", or references to bug/PR numbers in the code) - that context belongs in the commit message, not in source that will outlive the change.
|
||||
|
||||
Code comments should only refer to **current** behaviour - and shouldn't describe behaviour that is trivially deducible from reading the code.
|
||||
|
||||
## Linting Configuration
|
||||
|
||||
Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck, govet, ineffassign, staticcheck, unused, gocritic, misspell, revive, unconvert. The `goimports` formatter is also enabled.
|
||||
|
||||
## Documentation
|
||||
|
||||
- Backend option docs come from `Help:` fields in the Go source Options structs, not from markdown files.
|
||||
- Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
|
||||
- Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
|
||||
- Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
|
||||
@@ -1,6 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
Guidance for this repository lives in AGENTS.md, shared with all AI coding
|
||||
agents. Claude Code reads it via the import below.
|
||||
|
||||
@AGENTS.md
|
||||
+4
-69
@@ -113,36 +113,6 @@ GitHub.
|
||||
You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master)
|
||||
or [squash your commits](#squashing-your-commits).
|
||||
|
||||
## AI-assisted contributions
|
||||
|
||||
You are welcome to use AI coding assistants (Claude Code, Codex, Cursor, Gemini
|
||||
CLI, and similar) to help write your contribution. Rclone has an
|
||||
[AGENTS.md](AGENTS.md) file at the top of the repository describing the project's
|
||||
conventions; point your tool at it so the code it produces matches rclone's
|
||||
style.
|
||||
|
||||
However, the same standard applies to every pull request whether or not a tool
|
||||
was involved: **you are responsible for the code you submit.** Before you open a
|
||||
pull request please make sure that:
|
||||
|
||||
- You understand every line of the change and can explain why it is correct. If
|
||||
a reviewer asks a question about it, you should be able to answer without
|
||||
going back to the tool.
|
||||
- You have actually built and run it. At a minimum `go build` and
|
||||
`make quicktest` must pass, and for a backend change you should run the
|
||||
[backend tests](#backend-testing) against a real remote where you can.
|
||||
- The change is a genuine fix or feature that you have verified solves the
|
||||
problem, not a plausible-looking guess. Unverified, AI-generated pull requests
|
||||
that do not compile, do not pass the tests, invent APIs that do not exist, or
|
||||
do not actually do what the description claims waste maintainer time and are
|
||||
likely to be closed.
|
||||
- You have trimmed the comments. AI tools tend to add verbose comments that
|
||||
restate what the code plainly does or narrate the change being made; please
|
||||
cut these down to match the [code commenting style](AGENTS.md#code-commenting-style).
|
||||
|
||||
In short: an AI assistant is a tool to help *you* contribute, not a substitute
|
||||
for understanding and testing your own work.
|
||||
|
||||
## Using Git and GitHub
|
||||
|
||||
### Committing your changes
|
||||
@@ -630,37 +600,10 @@ remote or an fs.
|
||||
- **Remember** we have >50 backends to maintain so keeping them as similar as
|
||||
possible to each other is a high priority!
|
||||
|
||||
### Managing memory
|
||||
|
||||
Use rclone's memory pool for upload buffers rather than allocating them
|
||||
yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
|
||||
rclone's memory usage smaller.
|
||||
|
||||
- If the provider supports multipart or parallel chunk uploads, implement
|
||||
[fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
|
||||
and drive it with
|
||||
[multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
|
||||
This gives you pooled chunk buffers, concurrency, retries and
|
||||
`--multi-thread-streams` support for free. See s3/b2 backends for examples.
|
||||
- If you must buffer data yourself take the buffer from the global pool with
|
||||
[multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
|
||||
and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
|
||||
be rewound for a retry.
|
||||
- When using a pooled buffer `Close()` it on all paths so its pages go back to
|
||||
the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
|
||||
attempt, set the request's `ContentLength` explicitly, and if the buffer
|
||||
reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
|
||||
in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
|
||||
so the transport can't close it and free its pages before the retry.
|
||||
|
||||
### Unit tests
|
||||
|
||||
- Create a config entry called `TestRemote` for the unit tests to use
|
||||
- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
|
||||
- If your backend does chunked uploads, set `ChunkedUpload` in the
|
||||
`fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
|
||||
on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
|
||||
`FsPutRetry` tests can run the chunked path with small chunks.
|
||||
- Make sure all tests pass with `go test -v`
|
||||
|
||||
### Integration tests
|
||||
@@ -680,22 +623,14 @@ Or if you want to run the integration tests manually:
|
||||
- If your remote defines `ListR` check with this also
|
||||
- `go test -v -remote TestRemote: -fast-list`
|
||||
|
||||
Before a new or changed backend can be merged we require:
|
||||
|
||||
- A clean run of `go run ./fstest/test_all -backends remote` (please include the
|
||||
result in your pull request).
|
||||
- A test account for the backend so the maintainers can add it to the
|
||||
[integration test server](https://integration.rclone.org) and keep the backend
|
||||
working as rclone evolves. A backend that we cannot test is likely to break and
|
||||
may be removed.
|
||||
|
||||
See the [testing](#testing) section for more information on integration tests.
|
||||
|
||||
### Backend documentation
|
||||
|
||||
Add your backend to the docs. Keep lists of remotes in alphabetical
|
||||
order of full name of remote (e.g. `drive` is ordered as `Google
|
||||
Drive`) but with the local file system last.
|
||||
Add your backend to the docs - you'll need to pick an icon for it from
|
||||
[fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in
|
||||
alphabetical order of full name of remote (e.g. `drive` is ordered as
|
||||
`Google Drive`) but with the local file system last.
|
||||
|
||||
First add a data file about your backend in
|
||||
`docs/data/backends/remote.yaml` - this is used to build the overview
|
||||
|
||||
+1
-4
@@ -35,9 +35,6 @@ RUN echo "**** Print Version Binary ****" && \
|
||||
# Begin final image
|
||||
FROM alpine:latest
|
||||
|
||||
ARG RCLONE_UID=1009
|
||||
ARG RCLONE_GID=1009
|
||||
|
||||
RUN echo "**** Install Dependencies ****" && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
@@ -48,7 +45,7 @@ RUN echo "**** Install Dependencies ****" && \
|
||||
|
||||
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
|
||||
|
||||
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
|
||||
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
|
||||
|
||||
ENTRYPOINT [ "rclone" ]
|
||||
|
||||
|
||||
+21
-28
@@ -2,34 +2,27 @@
|
||||
|
||||
Current active maintainers of rclone are:
|
||||
|
||||
| Name | GitHub ID |
|
||||
| :--------------- | :---------------- |
|
||||
| Nick Craig-Wood | @ncw |
|
||||
| Stefan Breunig | @breunigs |
|
||||
| Ishuah Kariuki | @ishuah |
|
||||
| Remus Bunduc | @remusb |
|
||||
| Fabian Möller | @B4dM4n |
|
||||
| Alex Chen | @Cnly |
|
||||
| Sandeep Ummadi | @sandeepkru |
|
||||
| Sebastian Bünger | @buengese |
|
||||
| Ivan Andreev | @ivandeex |
|
||||
| Max Sum | @Max-Sum |
|
||||
| Fred | @creativeprojects |
|
||||
| Caleb Case | @calebcase |
|
||||
| wiserain | @wiserain |
|
||||
| albertony | @albertony |
|
||||
| Chun-Hung Tseng | @henrybear327 |
|
||||
| Hideo Aoyama | @boukendesho |
|
||||
| nielash | @nielash |
|
||||
| Dan McArdle | @dmcardle |
|
||||
| Sam Harrison | @childish-sambino |
|
||||
| Enduriel | @Enduriel |
|
||||
|
||||
Per-area ownership of code (which backend or command each maintainer looks
|
||||
after) is defined in [.github/CODEOWNERS](.github/CODEOWNERS), which GitHub uses
|
||||
to auto-request reviews. Two responsibilities don't map to a code path: @ncw
|
||||
looks after overall project health, and @boukendesho maintains the snap
|
||||
packaging.
|
||||
| Name | GitHub ID | Specific Responsibilities |
|
||||
| :--------------- | :---------------- | :-------------------------- |
|
||||
| Nick Craig-Wood | @ncw | overall project health |
|
||||
| Stefan Breunig | @breunigs | |
|
||||
| Ishuah Kariuki | @ishuah | |
|
||||
| Remus Bunduc | @remusb | cache backend |
|
||||
| Fabian Möller | @B4dM4n | |
|
||||
| Alex Chen | @Cnly | onedrive backend |
|
||||
| Sandeep Ummadi | @sandeepkru | azureblob backend |
|
||||
| Sebastian Bünger | @buengese | jottacloud, yandex & compress backends |
|
||||
| Ivan Andreev | @ivandeex | chunker & mailru backends |
|
||||
| Max Sum | @Max-Sum | union backend |
|
||||
| Fred | @creativeprojects | seafile backend |
|
||||
| Caleb Case | @calebcase | storj backend |
|
||||
| wiserain | @wiserain | pikpak backend |
|
||||
| albertony | @albertony | |
|
||||
| Chun-Hung Tseng | @henrybear327 | Proton Drive Backend |
|
||||
| Hideo Aoyama | @boukendesho | snap packaging |
|
||||
| nielash | @nielash | bisync |
|
||||
| Dan McArdle | @dmcardle | gitannex |
|
||||
| Sam Harrison | @childish-sambino | filescom |
|
||||
|
||||
## This is a work in progress draft
|
||||
|
||||
|
||||
Generated
+2577
-7670
File diff suppressed because it is too large.
Load diff
Generated
+918
-5426
File diff suppressed because it is too large.
Load diff
@@ -45,7 +45,7 @@ LINTTAGS=--build-tags "$(GOTAGS)"
|
||||
endif
|
||||
LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
|
||||
|
||||
.PHONY: rclone test_all vars version fetch-gui fetch-gui-and-commit
|
||||
.PHONY: rclone test_all vars version
|
||||
|
||||
rclone:
|
||||
ifeq ($(GO_OS),windows)
|
||||
@@ -59,12 +59,6 @@ endif
|
||||
cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
|
||||
mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
|
||||
|
||||
fetch-gui:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh
|
||||
|
||||
fetch-gui-and-commit:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh --commit
|
||||
|
||||
test_all:
|
||||
go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
|
||||
|
||||
@@ -93,15 +87,11 @@ test: rclone test_all
|
||||
@echo "Written logs in test_all.log"
|
||||
|
||||
# Quick test
|
||||
#
|
||||
# The timeout is raised above the go test default of 10m as the
|
||||
# cmd/gitannex end to end tests can take longer than that on slow CI
|
||||
# runners.
|
||||
quicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
|
||||
|
||||
racequicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
|
||||
|
||||
compiletest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
|
||||
@@ -131,25 +121,6 @@ updatedirect:
|
||||
go get $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
go mod tidy
|
||||
|
||||
# Update direct dependencies only but won't update the `go` line in go.mod
|
||||
updatedirectnoupgrade:
|
||||
@GO_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
for mod in $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all); do \
|
||||
cp go.mod go.mod.bak && \
|
||||
cp go.sum go.sum.bak && \
|
||||
go get $$mod && \
|
||||
NEW_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
if [ "$$NEW_VERSION" != "$$GO_VERSION" ]; then \
|
||||
echo "SKIPPING $$mod (requires go $$NEW_VERSION)"; \
|
||||
cp go.mod.bak go.mod; \
|
||||
cp go.sum.bak go.sum; \
|
||||
else \
|
||||
echo "updated $$mod"; \
|
||||
fi; \
|
||||
done && \
|
||||
rm -f go.mod.bak go.sum.bak && \
|
||||
go mod tidy
|
||||
|
||||
# Update direct and indirect dependencies and test dependencies
|
||||
update:
|
||||
go get -u -t ./...
|
||||
@@ -175,7 +146,6 @@ MANUAL.txt: MANUAL.md
|
||||
|
||||
commanddocs: rclone
|
||||
go generate ./lib/transform
|
||||
go generate ./cmd/bisync
|
||||
-@rmdir -p '$$HOME/.config/rclone'
|
||||
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
|
||||
@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
|
||||
|
||||
@@ -59,17 +59,14 @@ directories to and from different cloud storage providers.
|
||||
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
|
||||
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
|
||||
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
|
||||
- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp)
|
||||
- HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
- Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
|
||||
- Huawei Drive [:page_facing_up:](https://rclone.org/huaweidrive/)
|
||||
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
|
||||
- ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
|
||||
- Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
|
||||
- Internxt [:page_facing_up:](https://rclone.org/internxt/)
|
||||
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
|
||||
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
|
||||
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
|
||||
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
|
||||
@@ -112,7 +109,6 @@ directories to and from different cloud storage providers.
|
||||
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
|
||||
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
|
||||
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
|
||||
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
|
||||
- Seafile [:page_facing_up:](https://rclone.org/seafile/)
|
||||
- Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
|
||||
- SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
|
||||
@@ -127,12 +123,10 @@ directories to and from different cloud storage providers.
|
||||
- Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
|
||||
- Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
|
||||
- Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
|
||||
- US3 Object Storage [:page_facing_up:](https://rclone.org/s3/#us3)
|
||||
- Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
|
||||
- WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
- Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
|
||||
- Zadara Object Storage [:page_facing_up:](https://rclone.org/s3/#zadara)
|
||||
- Zero Services (ZERO-Z3) [:page_facing_up:](https://rclone.org/s3/#zero-z3)
|
||||
- Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
|
||||
- Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
|
||||
- The local filesystem [:page_facing_up:](https://rclone.org/local/)
|
||||
|
||||
+27
-31
@@ -11,12 +11,11 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
- git checkout master # see below for stable branch
|
||||
- git pull # IMPORTANT
|
||||
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
|
||||
- git status - make sure everything is checked in
|
||||
- Check GitHub actions build for master is Green
|
||||
- make test # see integration test server or run locally
|
||||
- make tag
|
||||
- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
|
||||
- edit docs/content/changelog.md # make sure to remove duplicate logs from point
|
||||
releases
|
||||
- make tidy
|
||||
- make doc
|
||||
@@ -33,20 +32,16 @@ This file describes how to make the various kinds of releases
|
||||
- make sign_upload
|
||||
- make check_sign
|
||||
- make upload
|
||||
- make upload_test_website # check sponsors are correct/unchanged
|
||||
- make upload_website
|
||||
- make upload_github
|
||||
- make startdev # make startstable for stable branch
|
||||
- \# announce with forum post, twitter post, and post to annoucements list
|
||||
- \# announce with forum post, twitter post, patreon post
|
||||
|
||||
## Update dependencies
|
||||
|
||||
Early in the next release cycle update the dependencies.
|
||||
|
||||
- Active Pins
|
||||
- `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
|
||||
- See: [#9539](https://github.com/rclone/rclone/issues/9539)
|
||||
- Review Active Pins in go.mod and remove if possible
|
||||
- Review any pinned packages in go.mod and remove if possible
|
||||
- `make updatedirect`
|
||||
- `make GOTAGS=cmount`
|
||||
- `make compiletest`
|
||||
@@ -59,18 +54,32 @@ If the `make updatedirect` upgrades the version of go in the `go.mod`
|
||||
go 1.22.0
|
||||
```
|
||||
|
||||
then so this instead `make updatedirectnoupgrade`. This will try each
|
||||
upgrade individually. Consider making an upstream bug report for any
|
||||
`SKIPPING` lines as these only support the latest Go version not the
|
||||
previous version too.
|
||||
then go to manual mode. `go1.22` here is the lowest supported version
|
||||
in the `go.mod`.
|
||||
|
||||
If `make updatedirect` or `make updatedirectnoupgrade` added a
|
||||
`toolchain` directive then remove it. We don't want to force a
|
||||
toolchain on our users. Linux packagers are often using a version of
|
||||
Go that is a few versions out of date.
|
||||
If `make updatedirect` added a `toolchain` directive then remove it.
|
||||
We don't want to force a toolchain on our users. Linux packagers are
|
||||
often using a version of Go that is a few versions out of date.
|
||||
|
||||
```console
|
||||
go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
|
||||
go get -d $(cat /tmp/potential-upgrades)
|
||||
go mod tidy -go=1.22 -compat=1.22
|
||||
```
|
||||
|
||||
If the `go mod tidy` fails use the output from it to remove the
|
||||
package which can't be upgraded from `/tmp/potential-upgrades` when
|
||||
done
|
||||
|
||||
```console
|
||||
git co go.mod go.sum
|
||||
```
|
||||
|
||||
And try again.
|
||||
|
||||
Optionally upgrade the direct and indirect dependencies. This is very
|
||||
likely to fail if `make updatedirectnoupgrade` was used.
|
||||
likely to fail if the manual method was used abve - in that case
|
||||
ignore it as it is too time consuming to fix.
|
||||
|
||||
- `make update`
|
||||
- `make GOTAGS=cmount`
|
||||
@@ -88,19 +97,6 @@ build.
|
||||
Once it compiles locally, push it on a test branch and commit fixes
|
||||
until the tests pass.
|
||||
|
||||
### Pseudo versions
|
||||
|
||||
Go makes pseudo versions for untagged repos and repos not at a tag.
|
||||
The pseudo versions on repos that have been tagged before do not get
|
||||
updated automatically so need manually checking. These can be found with
|
||||
|
||||
```console
|
||||
grep -E '[0-9]{14}-[0-9a-f]{12}' go.mod | grep -v indirect | grep -v 'v0\.0\.0'
|
||||
```
|
||||
|
||||
These will need to be updated manually using the `go get ...@branch`
|
||||
syntax.
|
||||
|
||||
### Major versions
|
||||
|
||||
The above procedure will not upgrade major versions, so v2 to v3.
|
||||
@@ -195,7 +191,7 @@ this will be done already.
|
||||
Now
|
||||
|
||||
- git co ${BASE_TAG}-stable
|
||||
- `git cherry-pick -x` any fixes
|
||||
- git cherry-pick any fixes
|
||||
- make startstable
|
||||
- Do the steps as above
|
||||
- git co master
|
||||
|
||||
@@ -32,7 +32,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/hdfs"
|
||||
_ "github.com/rclone/rclone/backend/hidrive"
|
||||
_ "github.com/rclone/rclone/backend/http"
|
||||
_ "github.com/rclone/rclone/backend/huaweidrive"
|
||||
_ "github.com/rclone/rclone/backend/iclouddrive"
|
||||
_ "github.com/rclone/rclone/backend/imagekit"
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -186,12 +185,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
foundArchive := subArchive(remote)
|
||||
if foundArchive != nil {
|
||||
fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
|
||||
// Archive path, in canonical form so that it compares equal
|
||||
// to the cleaned entry names inside the archive
|
||||
foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
|
||||
if foundArchive.root == "." {
|
||||
foundArchive.root = ""
|
||||
}
|
||||
// Archive path
|
||||
foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
|
||||
// Path to the archive
|
||||
archiveRemote := remote[:len(foundArchive.remote)]
|
||||
// Remote is archive leaf name
|
||||
@@ -496,50 +491,24 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
subEntries, err := subFs.List(ctx, dir)
|
||||
entries, err = subFs.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = subEntries[:0]
|
||||
skipped := 0
|
||||
for _, entry := range subEntries {
|
||||
remote := entry.Remote()
|
||||
// Only pass on direct children of dir - anything else
|
||||
// could escape the archive's namespace
|
||||
if !isDirectChild(dir, remote) {
|
||||
fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
for i, entry := range entries {
|
||||
// Can only unarchive files
|
||||
if o, ok := entry.(fs.Object); ok {
|
||||
remote := o.Remote()
|
||||
archive := f.findArchive(remote)
|
||||
if archive != nil {
|
||||
// Overwrite entry with directory
|
||||
entry = fs.NewDir(remote, o.ModTime(ctx))
|
||||
entries[i] = fs.NewDir(remote, o.ModTime(ctx))
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// isDirectChild reports whether remote names an entry directly in dir
|
||||
// ("" being the root), with no ".." or other components in between.
|
||||
func isDirectChild(dir, remote string) bool {
|
||||
if dir != "" {
|
||||
var ok bool
|
||||
remote, ok = strings.CutPrefix(remote, dir+"/")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return sanitize.Leaf(remote) == nil
|
||||
}
|
||||
|
||||
// NewObject creates a new remote archive file object
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
@@ -557,11 +526,6 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Don't trust the archiver to have returned the object asked for
|
||||
if o.Remote() != remote {
|
||||
fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,11 +3,9 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
@@ -16,7 +14,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
@@ -222,184 +219,3 @@ func TestArchiveSquashfs(t *testing.T) {
|
||||
run(t, "mksquashfs", input, output)
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveSquashfsIssue9004 lists and reads squashfs images that exercise
|
||||
// two layouts go-diskfs used to choke on (fixed in go-diskfs v1.9.4):
|
||||
//
|
||||
// - 1.sqfs: a single empty directory, so the image has no fragment table
|
||||
// (its fragment-table start holds the "not present" sentinel).
|
||||
// - 2.sqfs: a small tree whose superblock has the NO_XATTRS flag set while
|
||||
// inodes still carry a (non-sentinel) xattr index - the shape squashfs-
|
||||
// tools-ng can emit. Built by packing a two-file tree with xattrs via
|
||||
// `gensquashfs -x`, then setting the NO_XATTRS superblock flag; the tree
|
||||
// content is trivial placeholder data.
|
||||
//
|
||||
// Both images used to fail to list. Regression test for #9004.
|
||||
func TestArchiveSquashfsIssue9004(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
testdata, err := filepath.Abs(filepath.Join("squashfs", "testdata"))
|
||||
require.NoError(t, err)
|
||||
|
||||
archiveFor := func(t *testing.T, name string) fs.Fs {
|
||||
f, err := cache.Get(ctx, ":archive:"+filepath.Join(testdata, name))
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
|
||||
t.Run("EmptyDir", func(t *testing.T) {
|
||||
// 1.sqfs is a single empty directory - it must list without error.
|
||||
entries, err := archiveFor(t, "1.sqfs").List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, len(entries))
|
||||
})
|
||||
|
||||
t.Run("NoXattrTree", func(t *testing.T) {
|
||||
f := archiveFor(t, "2.sqfs")
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, path.Base(e.Remote()))
|
||||
}
|
||||
assert.Contains(t, names, "alpha")
|
||||
assert.Contains(t, names, "beta")
|
||||
|
||||
// A file in the tree must be readable with its real content.
|
||||
obj, err := f.NewObject(ctx, "beta/sample.xml")
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, obj.Size(), int64(0))
|
||||
rc, err := obj.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, int(obj.Size()), len(data))
|
||||
assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveUncleanRoot checks that a path into an archive which isn't
|
||||
// in canonical form (with "./" or doubled slashes) still finds its
|
||||
// directory.
|
||||
func TestArchiveUncleanRoot(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create("sub/dir/a.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
zipPath := filepath.Join(t.TempDir(), "test.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
|
||||
for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
|
||||
t.Run(root, func(t *testing.T) {
|
||||
f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
|
||||
require.NoError(t, err)
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "a.txt", entries[0].Remote())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// escapingObject is an object whose remote is not where it was asked for.
|
||||
type escapingObject struct {
|
||||
fs.Object
|
||||
remote string
|
||||
}
|
||||
|
||||
func (o *escapingObject) Remote() string { return o.remote }
|
||||
func (o *escapingObject) String() string { return o.remote }
|
||||
|
||||
// escapingFs stands in for a badly behaved archiver which exposes entry
|
||||
// names outside the directory being listed.
|
||||
type escapingFs struct {
|
||||
fs.Fs
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (f *escapingFs) Name() string { return "escaping" }
|
||||
func (f *escapingFs) Root() string { return "" }
|
||||
func (f *escapingFs) String() string { return "escaping" }
|
||||
func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
|
||||
|
||||
func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return fs.DirEntries{
|
||||
&escapingObject{remote: path.Join(dir, "good.txt")},
|
||||
fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
&escapingObject{remote: path.Join(dir, "../escape.txt")},
|
||||
&escapingObject{remote: "../../escape.txt"},
|
||||
&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
|
||||
fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
return &escapingObject{remote: "../escape.txt"}, nil
|
||||
}
|
||||
|
||||
// TestArchiveEscapingArchiver checks that the archive backend does not
|
||||
// pass on entries from an archiver which escape the directory being
|
||||
// listed, whatever the archiver does.
|
||||
func TestArchiveEscapingArchiver(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
return &escapingFs{prefix: prefix}, nil
|
||||
},
|
||||
Extension: ".escaping",
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
|
||||
f, err := cache.Get(ctx, ":archive:"+dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Archives are discovered when their parent directory is listed
|
||||
_, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "test.escaping")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
|
||||
|
||||
_, err = f.NewObject(ctx, "test.escaping/file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
|
||||
// TestIsDirectChild checks the guard which decides whether an entry
|
||||
// returned by an archiver belongs directly in the directory listed.
|
||||
func TestIsDirectChild(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
dir, remote string
|
||||
want bool
|
||||
}{
|
||||
{"", "a.txt", true},
|
||||
{"", "/a.txt", false},
|
||||
{"", "a.txt/", false},
|
||||
{"", "../a.txt", false},
|
||||
{"", "sub/a.txt", false},
|
||||
{"d", "d/a.txt", true},
|
||||
{"d", "d", false},
|
||||
{"d", "d/", false},
|
||||
{"d", "d//a.txt", false},
|
||||
{"d", "d/../a.txt", false},
|
||||
{"d", "dd/a.txt", false},
|
||||
{"d", "a.txt", false},
|
||||
} {
|
||||
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
|
||||
}
|
||||
}
|
||||
@@ -8,11 +8,6 @@ import (
|
||||
)
|
||||
|
||||
// Archiver describes an archive package
|
||||
//
|
||||
// Entry names inside an archive are attacker controlled. An archiver
|
||||
// must not expose an entry whose remote escapes the archive's own
|
||||
// namespace (for example one with a ".." component), typically by
|
||||
// validating names with lib/sanitize.
|
||||
type Archiver struct {
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
|
||||
@@ -36,7 +36,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
VFS := vfs.New(ctx, wrappedFs, nil)
|
||||
VFS := vfs.New(wrappedFs, nil)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
|
||||
@@ -161,10 +161,5 @@ func (c *cache) Writable() (backend.WritableFile, error) {
|
||||
return nil, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Path returns the path to the underlying storage - not applicable here
|
||||
func (c *cache) Path() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// check interfaces
|
||||
var _ backend.Storage = (*cache)(nil)
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -16,7 +15,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
@@ -44,38 +42,15 @@ type Fs struct {
|
||||
root string // position to read from within the archive
|
||||
}
|
||||
|
||||
// recoverParsePanic converts a panic raised while parsing a squashfs image
|
||||
// into an error assigned through err.
|
||||
func recoverParsePanic(err *error) {
|
||||
if r := recover(); r != nil {
|
||||
*err = fmt.Errorf("invalid or corrupt squashfs image: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// recoveringReadCloser wraps a reader returned by the go-diskfs parser so
|
||||
// panics raised while reading become errors.
|
||||
//
|
||||
// The parser reads file data lazily, so an image which opens cleanly can
|
||||
// still panic on the first Read.
|
||||
type recoveringReadCloser struct {
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
func (r recoveringReadCloser) Read(p []byte) (n int, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
return r.ReadCloser.Read(p)
|
||||
}
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ fs.Fs, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
@@ -181,17 +156,6 @@ func (f *Fs) toNative(remote string) (string, error) {
|
||||
return native, nil
|
||||
}
|
||||
|
||||
// This turns a native path with a leading / into a path suitable for
|
||||
// the squashfs library which requires io/fs.ValidPath paths with no
|
||||
// leading / and "." for the root directory.
|
||||
func toIOFS(native string) string {
|
||||
native = strings.Trim(native, "/")
|
||||
if native == "" {
|
||||
return "."
|
||||
}
|
||||
return native
|
||||
}
|
||||
|
||||
// Turn a (nativeDir, leaf) into a remote
|
||||
func (f *Fs) fromNative(nativeDir string, leaf string) string {
|
||||
// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
|
||||
@@ -205,12 +169,13 @@ func (f *Fs) fromNative(nativeDir string, leaf string) string {
|
||||
}
|
||||
|
||||
// Convert a FileInfo into an Object from native dir
|
||||
func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
|
||||
func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()),
|
||||
size: item.Size(),
|
||||
modTime: item.ModTime(),
|
||||
item: item,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,69 +190,55 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
||||
defer recoverParsePanic(&err)
|
||||
|
||||
nativeDir, err := f.toNative(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items, err := f.sqfs.ReadDir(toIOFS(nativeDir))
|
||||
items, err := f.sqfs.ReadDir(nativeDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
|
||||
}
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
skipped := 0
|
||||
for _, item := range items {
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
// Skip entry names that aren't safe
|
||||
if err := sanitize.Leaf(item.Name()); err != nil {
|
||||
fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
|
||||
skipped++
|
||||
continue
|
||||
for _, fi := range items {
|
||||
item, ok := fi.(squashfs.FileStat)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
|
||||
}
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
var entry fs.DirEntry
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)
|
||||
}
|
||||
if item.IsDir() {
|
||||
remote := f.fromNative(nativeDir, item.Name())
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = fs.NewDir(remote, info.ModTime())
|
||||
} else if item.Type().IsRegular() {
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = f.objectFromFileInfo(nativeDir, info)
|
||||
var remote = f.fromNative(nativeDir, item.Name())
|
||||
entry = fs.NewDir(remote, item.ModTime())
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
if item.Mode().IsRegular() {
|
||||
entry = f.objectFromFileInfo(nativeDir, item)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
// Log once per listing rather than per entry so a crafted image
|
||||
// can't flood the log
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// newObjectNative finds the object at the native path passed in
|
||||
func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
// get the path and filename
|
||||
dir, leaf := path.Split(nativePath)
|
||||
dir = strings.TrimRight(dir, "/")
|
||||
leaf = strings.Trim(leaf, "/")
|
||||
|
||||
// FIXME need to detect directory not found
|
||||
fis, err := f.sqfs.ReadDir(toIOFS(dir))
|
||||
fis, err := f.sqfs.ReadDir(dir)
|
||||
if err != nil {
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -298,11 +249,11 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
if fi.IsDir() {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
info, err := fi.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
|
||||
item, ok := fi.(squashfs.FileStat)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
|
||||
}
|
||||
o = f.objectFromFileInfo(dir, info)
|
||||
o = f.objectFromFileInfo(dir, item)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -377,6 +328,7 @@ type Object struct {
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
item squashfs.FileStat
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
@@ -433,7 +385,6 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
@@ -454,7 +405,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
}
|
||||
|
||||
fs.Debugf(o, "Opening %q", remote)
|
||||
fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)
|
||||
//fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)
|
||||
fh, err := o.item.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -466,15 +418,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rc = recoveringReadCloser{fh}
|
||||
|
||||
// If limited then don't return everything
|
||||
if limit >= 0 {
|
||||
fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
|
||||
return readers.NewLimitedReadCloser(rc, limit), nil
|
||||
return readers.NewLimitedReadCloser(fh, limit), nil
|
||||
}
|
||||
|
||||
return rc, nil
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
fscache "github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A backslash is an ordinary character in a file name on the systems
|
||||
// squashfs images are made on, so an entry containing one must be
|
||||
// listed and openable like any other file.
|
||||
func TestBackslashInName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localFs, err := fscache.Get(ctx, "testdata")
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, "backslash.sqfs", "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, `back\slash.txt`, entries[0].Remote())
|
||||
|
||||
o, err := f.NewObject(ctx, `back\slash.txt`)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(6), o.Size())
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// panickingReader panics on Read, standing in for the go-diskfs parser
|
||||
// hitting an unchecked slice while reading a malformed image.
|
||||
type panickingReader struct{}
|
||||
|
||||
func (panickingReader) Read([]byte) (int, error) { panic("boom") }
|
||||
func (panickingReader) Close() error { return nil }
|
||||
|
||||
// Test that a panic raised while reading file data becomes an error. The
|
||||
// parser reads lazily, so an image can open cleanly and only panic here.
|
||||
func TestRecoveringReadCloser(t *testing.T) {
|
||||
var rc io.ReadCloser = recoveringReadCloser{panickingReader{}}
|
||||
|
||||
_, err := rc.Read(make([]byte, 16))
|
||||
assert.ErrorContains(t, err, "invalid or corrupt squashfs image")
|
||||
assert.ErrorContains(t, err, "boom")
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -1,89 +0,0 @@
|
||||
//go:build !plan9
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// squashfsSuperblock builds a minimal 96-byte squashfs superblock with the
|
||||
// given attacker-controllable fields. It mirrors the on-disk layout the
|
||||
// go-diskfs parser reads, and is only used to construct deliberately malformed
|
||||
// images for the tests below.
|
||||
func squashfsSuperblock(blockSize uint32, blockLog, compression uint16, rootInode, inodeTableStart uint64) []byte {
|
||||
const u64max = ^uint64(0)
|
||||
b := make([]byte, 96)
|
||||
binary.LittleEndian.PutUint32(b[0:], 0x73717368) // "hsqs" magic
|
||||
binary.LittleEndian.PutUint32(b[4:], 1) // inode count
|
||||
binary.LittleEndian.PutUint32(b[8:], 0) // modtime
|
||||
binary.LittleEndian.PutUint32(b[12:], blockSize)
|
||||
binary.LittleEndian.PutUint32(b[16:], 0) // fragment count
|
||||
binary.LittleEndian.PutUint16(b[20:], compression)
|
||||
binary.LittleEndian.PutUint16(b[22:], blockLog)
|
||||
binary.LittleEndian.PutUint16(b[24:], 0x0210) // no fragments/xattrs
|
||||
binary.LittleEndian.PutUint16(b[26:], 0) // id count
|
||||
binary.LittleEndian.PutUint16(b[28:], 4) // version major
|
||||
binary.LittleEndian.PutUint16(b[30:], 0) // version minor
|
||||
binary.LittleEndian.PutUint64(b[32:], rootInode)
|
||||
binary.LittleEndian.PutUint64(b[40:], 96) // bytes used
|
||||
binary.LittleEndian.PutUint64(b[48:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[56:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[64:], inodeTableStart)
|
||||
binary.LittleEndian.PutUint64(b[72:], inodeTableStart)
|
||||
binary.LittleEndian.PutUint64(b[80:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[88:], u64max)
|
||||
return b
|
||||
}
|
||||
|
||||
// TestArchiveSquashfsMalformed feeds the archive backend squashfs images whose
|
||||
// superblock/metadata make the go-diskfs parser panic. See GHSA-6jcg-q3wp-x2f4.
|
||||
func TestArchiveSquashfsMalformed(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
// Variant 1: zero block size reaches an integer division in the parser.
|
||||
zeroBlock := squashfsSuperblock(0, 0, 1, 0, 96)
|
||||
|
||||
// Variant 2: an out-of-range root inode byte offset reaches an unchecked
|
||||
// slice against an empty, uncompressed metadata block.
|
||||
oob := append(squashfsSuperblock(65536, 16, 1, 12336, 96), 0x00, 0x80)
|
||||
binary.LittleEndian.PutUint64(oob[40:], uint64(len(oob)))
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
image []byte
|
||||
}{
|
||||
{"ZeroBlockSize", zeroBlock},
|
||||
{"OutOfRangeInodeOffset", oob},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// A directory per image: the fs cache keys on the parent
|
||||
// directory, so sharing one would hand the second subtest a
|
||||
// cached listing which does not contain its image.
|
||||
path := filepath.Join(t.TempDir(), tc.name+".sqfs")
|
||||
require.NoError(t, os.WriteFile(path, tc.image, 0o644))
|
||||
|
||||
// The whole point is that this returns rather than panicking and
|
||||
// killing the test binary (and, in production, the serve daemon).
|
||||
f, err := cache.Get(ctx, ":archive:"+path)
|
||||
if err == nil {
|
||||
// The image parsed, so the panic must surface from a later
|
||||
// lazily-parsed operation instead.
|
||||
_, err = f.List(ctx, "")
|
||||
}
|
||||
require.Error(t, err, "a malformed squashfs image must error, not panic")
|
||||
assert.Contains(t, err.Error(), "invalid or corrupt squashfs image",
|
||||
"error should come from the recover, not an unrelated failure")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -19,7 +18,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
@@ -54,7 +52,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
@@ -137,27 +135,15 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
return singleObject, fmt.Errorf("failed to read zip file: %w", err)
|
||||
}
|
||||
dt := dirtree.New()
|
||||
skipped := 0
|
||||
for _, file := range zr.File {
|
||||
// Skip entries whose name escapes the archive's own namespace
|
||||
remote, err := sanitize.Path(file.Name)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
isDir := strings.HasSuffix(file.Name, "/")
|
||||
// A file entry whose last component is "", "." or ".." names
|
||||
// a directory rather than a file, so skip
|
||||
if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
|
||||
skipped++
|
||||
continue
|
||||
remote := strings.Trim(path.Clean(file.Name), "/")
|
||||
if remote == "." {
|
||||
remote = ""
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root, requiring a path
|
||||
// boundary so that root "foo" does not also match a
|
||||
// sibling entry such as "foobar"
|
||||
if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
|
||||
// Ignore all files outside the root
|
||||
if !strings.HasPrefix(remote, f.root) {
|
||||
continue
|
||||
}
|
||||
if remote == f.root {
|
||||
@@ -166,12 +152,11 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
remote = strings.TrimPrefix(remote, f.root+"/")
|
||||
}
|
||||
}
|
||||
if isDir {
|
||||
if strings.HasSuffix(file.Name, "/") {
|
||||
dir := fs.NewDir(remote, file.Modified)
|
||||
dt.AddDir(dir)
|
||||
} else {
|
||||
if remote == "" {
|
||||
// A file at the root is the archive member f.root points at
|
||||
remote = path.Base(f.root)
|
||||
singleObject = true
|
||||
dt = dirtree.New()
|
||||
@@ -188,9 +173,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
@@ -218,8 +200,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
// Return a copy as callers may filter the listing in place
|
||||
return slices.Clone(entries), nil
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
package zip
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// writeZip builds a zip file with the given entry names (a name ending
|
||||
// in "/" is a directory) in dir and returns its leaf name.
|
||||
func writeZip(t *testing.T, dir, name string, names ...string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, n := range names {
|
||||
w, err := zw.Create(n)
|
||||
require.NoError(t, err)
|
||||
if !strings.HasSuffix(n, "/") {
|
||||
_, err = w.Write([]byte("data for " + n))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// allRemotes returns every Object remote in the mounted archive's dirtree.
|
||||
func allRemotes(t *testing.T, f fs.Fs) []string {
|
||||
zf, ok := f.(*Fs)
|
||||
require.True(t, ok)
|
||||
var remotes []string
|
||||
for _, entries := range zf.dt {
|
||||
for _, entry := range entries {
|
||||
if o, ok := entry.(*Object); ok {
|
||||
remotes = append(remotes, o.Remote())
|
||||
}
|
||||
}
|
||||
}
|
||||
// dt iteration order is nondeterministic, so sort for stable comparison.
|
||||
sort.Strings(remotes)
|
||||
return remotes
|
||||
}
|
||||
|
||||
// A malicious zip whose entry names escape the archive's own namespace
|
||||
// must not be exposed by the zip backend (Zip Slip).
|
||||
func TestReadZipSlip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "evil.zip",
|
||||
"good.txt",
|
||||
"../../etc/cron.d/evil",
|
||||
"../escape.txt",
|
||||
"sub/../../above.txt",
|
||||
`..\..\evil.exe`,
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
|
||||
// The one benign entry must survive.
|
||||
assert.Contains(t, remotes, "good.txt")
|
||||
|
||||
// No escaping entry may be exposed.
|
||||
for _, remote := range remotes {
|
||||
assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
|
||||
"escaping remote exposed: %q", remote)
|
||||
}
|
||||
assert.Equal(t, []string{"good.txt"}, remotes)
|
||||
}
|
||||
|
||||
// Mounting with a non-empty root must only expose entries within that
|
||||
// root directory, not sibling directories that merely share a name
|
||||
// prefix (root "foo" must not match "foobar").
|
||||
func TestReadZipRootBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip",
|
||||
"foo/a.txt",
|
||||
"foobar/b.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "foo")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
assert.Equal(t, []string{"a.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A file entry whose name refers to a directory (".", "/", "./", "" or
|
||||
// "sub/.") must be skipped, not turn the archive or the directory
|
||||
// mounted as its root into a single file which hides every other
|
||||
// entry, or become an object named for the archive itself when the
|
||||
// archive is found by listing its directory.
|
||||
func TestReadZipDirNamedFileEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "dot.zip",
|
||||
".",
|
||||
"/",
|
||||
"./",
|
||||
"",
|
||||
"good.txt",
|
||||
"sub/",
|
||||
"sub/.",
|
||||
"sub/good.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, test := range []struct {
|
||||
prefix, root string
|
||||
want []string
|
||||
}{
|
||||
{"", "", []string{"good.txt", "sub/good.txt"}},
|
||||
{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
|
||||
{"", "sub", []string{"good.txt"}},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
|
||||
f, err := New(ctx, localFs, name, test.prefix, test.root)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, allRemotes(t, f))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Listings are served from a cache which must survive callers
|
||||
// filtering the returned slice in place, as fs/list does.
|
||||
func TestListCacheNotAliased(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 3)
|
||||
// Compact in place, dropping the first entry
|
||||
copy(entries, entries[1:])
|
||||
|
||||
entries, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
|
||||
}
|
||||
@@ -1,169 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Parallel listing support for azureblob, built on the Apache Arrow
|
||||
// listing format.
|
||||
//
|
||||
// The Arrow listing exposes the server-side startFrom (inclusive) and endBefore
|
||||
// (exclusive) name-range parameters, which let a listing be sharded by
|
||||
// blob-name range and listed concurrently.
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// listArrowParallel lists (containerName, directory) by splitting the blob name
|
||||
// keyspace into shards and listing them concurrently. opts must already request
|
||||
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
|
||||
// returns the number of raw items seen. directory/prefix are already normalised
|
||||
// by the caller.
|
||||
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
|
||||
// Parallel listing needs the arrowlist client (server-side endBefore is
|
||||
// only honoured on the Arrow path) - without it fall back to sequential.
|
||||
if _, err := f.arrowCntSVC(containerName); err != nil {
|
||||
fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
|
||||
// Split the keyspace by a single-case character ladder (see
|
||||
// arrowLadderBoundaries for why single-case matters).
|
||||
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
|
||||
if len(points) < 1 {
|
||||
// Couldn't usefully split - list sequentially.
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
|
||||
// Build half-open [startFrom, endBefore) shards from the sorted boundary
|
||||
// points p0..pk: ["", p0), [p0, p1), ..., [pk, "").
|
||||
type shard struct{ startFrom, endBefore string }
|
||||
shards := make([]shard, 0, len(points)+1)
|
||||
prev := ""
|
||||
for _, p := range points {
|
||||
shards = append(shards, shard{prev, p})
|
||||
prev = p
|
||||
}
|
||||
shards = append(shards, shard{prev, ""})
|
||||
|
||||
// fn (and the list.Helper behind it) is not concurrency-safe, so serialise
|
||||
// delivery. Only the cheap callback is locked, not the HTTP/decode work.
|
||||
var fnMu sync.Mutex
|
||||
safeFn := func(remote string, item *container.BlobItem, isDir bool) error {
|
||||
fnMu.Lock()
|
||||
defer fnMu.Unlock()
|
||||
return fn(remote, item, isDir)
|
||||
}
|
||||
|
||||
// Schedule every shard; SetLimit bounds concurrency, so more shards than
|
||||
// workers gives free load balancing as workers pick up the next shard.
|
||||
var total atomic.Int64
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(f.opt.ListParallelism)
|
||||
for _, sh := range shards {
|
||||
if gCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
// Clone opts per shard so concurrent shards don't share StartFrom/EndBefore.
|
||||
shardOpts := *opts
|
||||
if sh.startFrom != "" {
|
||||
startFrom := sh.startFrom
|
||||
shardOpts.StartFrom = &startFrom
|
||||
} else {
|
||||
shardOpts.StartFrom = nil
|
||||
}
|
||||
if sh.endBefore != "" {
|
||||
endBefore := sh.endBefore
|
||||
shardOpts.EndBefore = &endBefore
|
||||
} else {
|
||||
shardOpts.EndBefore = nil
|
||||
}
|
||||
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
|
||||
total.Add(int64(n))
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := g.Wait()
|
||||
found := int(total.Load())
|
||||
if isEndBeforeUnsupported(err) {
|
||||
// The account can't honour the shards' endBefore bounds, so parallel
|
||||
// listing can't be used on it.
|
||||
f.arrowXMLFallback.Store(true)
|
||||
if found == 0 {
|
||||
// No shard delivered anything yet (the normal case - Arrow listing
|
||||
// enablement is per account so every shard fails on its first
|
||||
// page) so it is safe to relist sequentially.
|
||||
fs.Debugf(f, "Parallel Arrow listing not supported by this account (%v) - listing sequentially", err)
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
// Some entries were already delivered so a rerun would duplicate them.
|
||||
return found, fmt.Errorf("parallel listing aborted after partial results (retry, or unset list_parallelism): %w", err)
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
|
||||
// isEndBeforeUnsupported reports whether err means the service can't honour
|
||||
// the endBefore listing bound, so sharded parallel listing must not be used.
|
||||
//
|
||||
// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
|
||||
// server answered XML with endBefore set), or the service rejecting the
|
||||
// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
|
||||
// requested operation is not allowed as EndBefore parameter support is
|
||||
// missing on this account" - endBefore is only supported on the Arrow
|
||||
// listing path of accounts with Blob Listing with Apache Arrow enabled.
|
||||
func isEndBeforeUnsupported(err error) bool {
|
||||
if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
|
||||
return true
|
||||
}
|
||||
var respErr *azcore.ResponseError
|
||||
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
|
||||
}
|
||||
|
||||
// arrowShardTarget is the number of ladder boundaries to use for the given
|
||||
// parallelism (a few per worker for load balancing).
|
||||
func arrowShardTarget(parallelism int) int {
|
||||
return parallelism * 4
|
||||
}
|
||||
|
||||
// arrowLadderBoundaries returns up to target sorted ENCODED-space boundaries of
|
||||
// the form directory+c, used to split the keyspace into shards.
|
||||
//
|
||||
// The partition is always correct (no dup/gap) regardless of the alphabet: keys
|
||||
// whose next byte is outside the alphabet simply fall into an adjacent shard;
|
||||
// the alphabet only affects balance. A single-case (digit+lowercase) alphabet
|
||||
// is deliberate: the boundaries are then monotonic under BOTH byte order (the
|
||||
// listing order) AND the service's case-insensitive startFrom/endBefore
|
||||
// validation, so server-side endBefore is accepted. (A mixed-case ladder gives
|
||||
// pairs like [".../Z", ".../a") which are byte-ascending but rejected 400 as
|
||||
// "endbefore precedes startfrom".) directory is already encoded; the alphabet
|
||||
// is ASCII (encoder-invariant) and ascending, so the result is sorted.
|
||||
func arrowLadderBoundaries(directory string, target int) []string {
|
||||
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
if target < 1 {
|
||||
return nil
|
||||
}
|
||||
if target > len(alphabet) {
|
||||
target = len(alphabet)
|
||||
}
|
||||
points := make([]string, 0, target)
|
||||
last := -1
|
||||
for i := 0; i < target; i++ {
|
||||
idx := i * len(alphabet) / target
|
||||
if idx == last {
|
||||
continue
|
||||
}
|
||||
last = idx
|
||||
points = append(points, directory+string(alphabet[idx]))
|
||||
}
|
||||
return points
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testArrowList exercises the experimental Apache Arrow listing.
|
||||
//
|
||||
// It is skipped automatically if Arrow listing is not enabled on the test
|
||||
// account (the server returns XML, or 409 on HNS accounts).
|
||||
func (f *Fs) testArrowList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
containerName, _ := f.split("probe")
|
||||
if containerName == "" {
|
||||
t.Skip("Arrow list test needs a container in the remote root")
|
||||
}
|
||||
|
||||
// withArrow runs fn with UseArrowList temporarily set.
|
||||
withArrow := func(on bool, fn func()) {
|
||||
old := f.opt.UseArrowList
|
||||
f.opt.UseArrowList = on
|
||||
defer func() { f.opt.UseArrowList = old }()
|
||||
fn()
|
||||
}
|
||||
|
||||
// withParallel runs fn with ListParallelism temporarily set.
|
||||
withParallel := func(n int, fn func()) {
|
||||
old := f.opt.ListParallelism
|
||||
f.opt.ListParallelism = n
|
||||
defer func() { f.opt.ListParallelism = old }()
|
||||
fn()
|
||||
}
|
||||
|
||||
// Create a set of known objects under a unique directory: flat files plus
|
||||
// one in a subdirectory (so we exercise both the flat and delimited paths).
|
||||
dir := "arrow-list-" + random.String(8)
|
||||
// Includes upper-case and digit first-characters: these byte-sort between
|
||||
// the digit and lowercase ladder boundaries, so they verify the server's
|
||||
// endBefore filtering is byte-ordered (not case-insensitive, which would
|
||||
// drop them from a shard and make parallel != sequential).
|
||||
objects := []string{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt", "sub/x.txt", "Foo.txt", "Zoo.txt", "0num.txt", "9end.txt"}
|
||||
wantSizes := map[string]int64{}
|
||||
created := make([]string, 0, len(objects))
|
||||
for i, n := range objects {
|
||||
contents := random.String(10 + i)
|
||||
remote := dir + "/" + n
|
||||
item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
|
||||
_ = fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
created = append(created, remote)
|
||||
wantSizes[remote] = int64(len(contents))
|
||||
}
|
||||
defer func() {
|
||||
for _, remote := range created {
|
||||
if o, err := f.NewObject(ctx, remote); err == nil {
|
||||
_ = o.Remove(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// listing returns remote->size for dir (dirs reported as size -1).
|
||||
listing := func(recurse bool) (map[string]int64, error) {
|
||||
got := map[string]int64{}
|
||||
add := func(entries fs.DirEntries) error {
|
||||
for _, e := range entries {
|
||||
if o, ok := e.(fs.Object); ok {
|
||||
got[e.Remote()] = o.Size()
|
||||
} else {
|
||||
got[e.Remote()] = -1
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if recurse {
|
||||
return got, f.ListR(ctx, dir, add)
|
||||
}
|
||||
entries, err := f.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = add(entries)
|
||||
return got, nil
|
||||
}
|
||||
|
||||
// Probe: is Arrow available? If not, skip cleanly.
|
||||
var probeErr error
|
||||
withArrow(true, func() { _, probeErr = listing(true) })
|
||||
if probeErr != nil {
|
||||
if strings.Contains(probeErr.Error(), "OperationNotSupportedWithFeatureMissing") || strings.Contains(probeErr.Error(), "409") {
|
||||
t.Skipf("Arrow listing not available on this account: %v", probeErr)
|
||||
}
|
||||
require.NoError(t, probeErr)
|
||||
}
|
||||
|
||||
// Arrow listing must equal XML listing, recursively and per-directory
|
||||
// (including the "sub" subdirectory).
|
||||
t.Run("EqualsXML", func(t *testing.T) {
|
||||
for _, recurse := range []bool{true, false} {
|
||||
var xml, arrow map[string]int64
|
||||
var err error
|
||||
withArrow(false, func() { xml, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
withArrow(true, func() { arrow, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, xml, arrow, "recurse=%v: Arrow listing differs from XML", recurse)
|
||||
}
|
||||
|
||||
// Recursive Arrow listing contains every object with the right size.
|
||||
var arrow map[string]int64
|
||||
withArrow(true, func() { arrow, _ = listing(true) })
|
||||
for remote, size := range wantSizes {
|
||||
assert.Equal(t, size, arrow[remote], remote)
|
||||
}
|
||||
|
||||
// Single-directory Arrow listing reports "sub" as a directory.
|
||||
var entries fs.DirEntries
|
||||
withArrow(true, func() {
|
||||
var err error
|
||||
entries, err = f.List(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
foundSubDir := false
|
||||
for _, e := range entries {
|
||||
if e.Remote() == dir+"/sub" {
|
||||
_, isObj := e.(fs.Object)
|
||||
foundSubDir = !isObj
|
||||
}
|
||||
}
|
||||
assert.True(t, foundSubDir, "expected %q to be listed as a directory", dir+"/sub")
|
||||
})
|
||||
|
||||
// Parallel listing must equal sequential listing, recursively and
|
||||
// per-directory, including the "sub" subdirectory.
|
||||
t.Run("ParallelEqualsSequential", func(t *testing.T) {
|
||||
for _, recurse := range []bool{true, false} {
|
||||
var seq, par map[string]int64
|
||||
var err error
|
||||
withArrow(true, func() {
|
||||
withParallel(0, func() { seq, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
withParallel(8, func() { par, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
})
|
||||
assert.Equal(t, seq, par, "recurse=%v: parallel listing differs from sequential", recurse)
|
||||
}
|
||||
})
|
||||
|
||||
// A tiny list_chunk forces each shard to span multiple Arrow pages, so this
|
||||
// exercises pagination (startFrom+endBefore+marker resent together) inside a
|
||||
// shard. The "sub/" boundary puts a.txt..e.txt in one shard => >1 page at
|
||||
// chunk size 2.
|
||||
t.Run("ParallelMultiPage", func(t *testing.T) {
|
||||
oldChunk := f.opt.ListChunkSize
|
||||
f.opt.ListChunkSize = 2
|
||||
defer func() { f.opt.ListChunkSize = oldChunk }()
|
||||
var seq, par map[string]int64
|
||||
var err error
|
||||
withArrow(true, func() {
|
||||
withParallel(0, func() { seq, err = listing(true) })
|
||||
require.NoError(t, err)
|
||||
withParallel(8, func() { par, err = listing(true) })
|
||||
require.NoError(t, err)
|
||||
})
|
||||
assert.Equal(t, seq, par)
|
||||
for remote, size := range wantSizes {
|
||||
assert.Equal(t, size, par[remote], remote)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestArrowLadderBoundaries(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
directory string
|
||||
target int
|
||||
}{
|
||||
{"", 8},
|
||||
{"dir/", 16},
|
||||
{"dir/", 100}, // capped at alphabet size
|
||||
{"dir/", 1},
|
||||
} {
|
||||
points := arrowLadderBoundaries(tc.directory, tc.target)
|
||||
require.NotEmpty(t, points)
|
||||
assert.LessOrEqual(t, len(points), 62)
|
||||
for i, p := range points {
|
||||
assert.True(t, strings.HasPrefix(p, tc.directory), "boundary %q lacks prefix %q", p, tc.directory)
|
||||
if i > 0 {
|
||||
assert.Less(t, points[i-1], p, "boundaries not strictly ascending")
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Nil(t, arrowLadderBoundaries("dir/", 0))
|
||||
// alphabet is single-case (digit+lowercase) so boundaries are monotonic
|
||||
// under both byte order and case-insensitive order
|
||||
pts := arrowLadderBoundaries("dir/", 36)
|
||||
for _, p := range pts {
|
||||
assert.Equal(t, p, strings.ToLower(p), "boundary must be lowercase: %q", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEndBeforeUnsupported(t *testing.T) {
|
||||
assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
|
||||
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
|
||||
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
|
||||
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
|
||||
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
|
||||
assert.False(t, isEndBeforeUnsupported(nil))
|
||||
}
|
||||
@@ -1,421 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
|
||||
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
|
||||
// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
|
||||
// public type aliases so it compiles against the released azblob module.
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
"github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/apache/arrow-go/v18/arrow/ipc"
|
||||
"github.com/apache/arrow-go/v18/arrow/memory"
|
||||
)
|
||||
|
||||
const (
|
||||
// ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
|
||||
ArrowContentType = "application/vnd.apache.arrow.stream"
|
||||
|
||||
// ArrowAcceptHeader is the Accept header value to request Arrow format.
|
||||
ArrowAcceptHeader = ArrowContentType
|
||||
|
||||
// resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
|
||||
// in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
|
||||
// in Arrow, all rows share the same schema and are distinguished by this field value.
|
||||
resourceTypeBlobPrefix = "blobprefix"
|
||||
)
|
||||
|
||||
// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
|
||||
func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
|
||||
result := container.ListBlobsFlatResponse{}
|
||||
|
||||
// Extract response headers
|
||||
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
|
||||
|
||||
// Parse Arrow IPC stream
|
||||
items, nextMarker, err := parseArrowStream(resp.Body)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
|
||||
}
|
||||
|
||||
result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
|
||||
Segment: &container.BlobFlatListSegment{
|
||||
BlobItems: items,
|
||||
},
|
||||
NextMarker: nextMarker,
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
|
||||
func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
|
||||
result := container.ListBlobsHierarchyResponse{}
|
||||
|
||||
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
|
||||
|
||||
items, nextMarker, err := parseArrowStream(resp.Body)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
|
||||
}
|
||||
|
||||
// Separate blob items from blob prefixes based on ResourceType field.
|
||||
var blobItems []*container.BlobItem
|
||||
var blobPrefixes []*container.BlobPrefix
|
||||
for _, item := range items {
|
||||
if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
|
||||
blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
|
||||
Name: item.Name,
|
||||
Properties: item.Properties,
|
||||
})
|
||||
} else {
|
||||
blobItems = append(blobItems, item)
|
||||
}
|
||||
}
|
||||
|
||||
result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
|
||||
Segment: &container.BlobHierarchyListSegment{
|
||||
BlobItems: blobItems,
|
||||
BlobPrefixes: blobPrefixes,
|
||||
},
|
||||
NextMarker: nextMarker,
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
|
||||
if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
|
||||
*clientRequestID = &val
|
||||
}
|
||||
if val := resp.Header.Get("Content-Type"); val != "" {
|
||||
*contentType = &val
|
||||
}
|
||||
if val := resp.Header.Get("Date"); val != "" {
|
||||
if t, err := time.Parse(time.RFC1123, val); err == nil {
|
||||
*date = &t
|
||||
}
|
||||
}
|
||||
if val := resp.Header.Get("x-ms-request-id"); val != "" {
|
||||
*requestID = &val
|
||||
}
|
||||
if val := resp.Header.Get("x-ms-version"); val != "" {
|
||||
*version = &val
|
||||
}
|
||||
}
|
||||
|
||||
// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
|
||||
func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
|
||||
reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
|
||||
}
|
||||
defer reader.Release()
|
||||
|
||||
// Extract NextMarker from schema metadata
|
||||
var nextMarker *string
|
||||
var numRecords int
|
||||
if md := reader.Schema().Metadata(); md.Len() > 0 {
|
||||
if idx := md.FindKey("NextMarker"); idx >= 0 {
|
||||
val := md.Values()[idx]
|
||||
if val != "" {
|
||||
nextMarker = &val
|
||||
}
|
||||
}
|
||||
if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
|
||||
if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
|
||||
numRecords = n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build column index for the schema
|
||||
schema := reader.Schema()
|
||||
colIndex := buildColumnIndex(schema)
|
||||
|
||||
// Pre-allocate if we know the count
|
||||
items := make([]*container.BlobItem, 0, numRecords)
|
||||
|
||||
// Iterate over record batches
|
||||
for reader.Next() {
|
||||
rec := reader.RecordBatch()
|
||||
rows := int(rec.NumRows())
|
||||
|
||||
for row := range rows {
|
||||
item := &container.BlobItem{
|
||||
Properties: &container.BlobProperties{},
|
||||
}
|
||||
populateBlobItem(item, rec, row, colIndex)
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
|
||||
if err := reader.Err(); err != nil {
|
||||
return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
|
||||
}
|
||||
|
||||
return items, nextMarker, nil
|
||||
}
|
||||
|
||||
// columnIndex maps column names to their indices in the record batch for O(1) lookup.
|
||||
type columnIndex map[string]int
|
||||
|
||||
func buildColumnIndex(schema *arrow.Schema) columnIndex {
|
||||
idx := make(columnIndex, len(schema.Fields()))
|
||||
for i, f := range schema.Fields() {
|
||||
idx[f.Name] = i
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// populateBlobItem fills a BlobItem from a single row across all columns.
|
||||
func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
|
||||
// Top-level BlobItem fields
|
||||
item.Name = getStringField(rec, row, colIdx, "Name")
|
||||
item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
|
||||
item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
|
||||
item.VersionID = getStringField(rec, row, colIdx, "VersionId")
|
||||
item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
|
||||
item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
|
||||
|
||||
// Metadata maps
|
||||
item.Metadata = getMapField(rec, row, colIdx, "Metadata")
|
||||
item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
|
||||
|
||||
// Tags
|
||||
if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
|
||||
var blobTags []*container.BlobTag
|
||||
for k, v := range tags {
|
||||
key := k
|
||||
blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
|
||||
}
|
||||
item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
|
||||
}
|
||||
|
||||
// BlobProperties fields
|
||||
p := item.Properties
|
||||
p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
|
||||
p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
|
||||
p.ETag = getETagField(rec, row, colIdx, "Etag")
|
||||
p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
|
||||
p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
|
||||
p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
|
||||
p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
|
||||
p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
|
||||
p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
|
||||
p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
|
||||
p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
|
||||
p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
|
||||
p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
|
||||
p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
|
||||
p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
|
||||
p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
|
||||
p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
|
||||
p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
|
||||
p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
|
||||
p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
|
||||
p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
|
||||
p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
|
||||
p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
|
||||
p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
|
||||
p.CopyID = getStringField(rec, row, colIdx, "CopyId")
|
||||
p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
|
||||
p.CopySource = getStringField(rec, row, colIdx, "CopySource")
|
||||
p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
|
||||
p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
|
||||
p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
|
||||
p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
|
||||
p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
|
||||
p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
|
||||
p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
|
||||
p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
|
||||
p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
|
||||
p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
|
||||
p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
|
||||
p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
|
||||
p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
|
||||
|
||||
// Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
|
||||
// The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
|
||||
// It is silently ignored for forward compatibility.
|
||||
}
|
||||
|
||||
// Field accessor helpers that safely handle missing columns and null values.
|
||||
|
||||
func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.String); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Boolean); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Timestamp); ok {
|
||||
val := arr.Value(row)
|
||||
// Arrow timestamps are stored as int64 with a unit; convert to time.Time
|
||||
dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
|
||||
return &dt
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Uint64); ok {
|
||||
val := int64(arr.Value(row))
|
||||
return &val
|
||||
}
|
||||
if arr, ok := col.(*array.Int64); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
if arr, ok := col.(*array.Uint64); ok {
|
||||
val := int32(arr.Value(row))
|
||||
return &val
|
||||
}
|
||||
if arr, ok := col.(*array.Int32); ok {
|
||||
val := arr.Value(row)
|
||||
return &val
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
|
||||
s := getStringField(rec, row, colIdx, name)
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
etag := azcore.ETag(*s)
|
||||
return &etag
|
||||
}
|
||||
|
||||
func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
|
||||
s := getStringField(rec, row, colIdx, name)
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(*s)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return decoded
|
||||
}
|
||||
|
||||
// getEnumField returns a pointer to a string-typed enum value.
|
||||
type stringEnum interface {
|
||||
~string
|
||||
}
|
||||
|
||||
func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
|
||||
s := getStringField(rec, row, colIdx, name)
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
val := T(*s)
|
||||
return &val
|
||||
}
|
||||
|
||||
func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
|
||||
idx, ok := colIdx[name]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
col := rec.Column(idx)
|
||||
if col.IsNull(row) {
|
||||
return nil
|
||||
}
|
||||
mapArr, ok := col.(*array.Map)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get the offsets for this row's map entries
|
||||
start := int(mapArr.Offsets()[row])
|
||||
end := int(mapArr.Offsets()[row+1])
|
||||
if start == end {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := mapArr.Keys().(*array.String)
|
||||
values := mapArr.Items().(*array.String)
|
||||
|
||||
result := make(map[string]*string, end-start)
|
||||
for i := start; i < end; i++ {
|
||||
k := keys.Value(i)
|
||||
if values.IsNull(i) {
|
||||
result[k] = nil
|
||||
} else {
|
||||
v := values.Value(i)
|
||||
result[k] = &v
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,374 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
|
||||
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
|
||||
// c6fa341ca22b (branch feature/storage/bifrost).
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/apache/arrow-go/v18/arrow/ipc"
|
||||
"github.com/apache/arrow-go/v18/arrow/memory"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// coreSchema returns a basic Arrow schema for testing with common blob fields.
|
||||
func coreSchema(md *arrow.Metadata) *arrow.Schema {
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
|
||||
{Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
|
||||
{Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
|
||||
{Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
{Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
}
|
||||
return arrow.NewSchema(fields, md)
|
||||
}
|
||||
|
||||
// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
|
||||
func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
|
||||
t.Helper()
|
||||
alloc := memory.DefaultAllocator
|
||||
|
||||
var buf bytes.Buffer
|
||||
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
|
||||
|
||||
builder := arrowArray.NewRecordBuilder(alloc, schema)
|
||||
defer builder.Release()
|
||||
|
||||
populate(builder)
|
||||
|
||||
rec := builder.NewRecordBatch()
|
||||
defer rec.Release()
|
||||
|
||||
require.NoError(t, w.Write(rec))
|
||||
require.NoError(t, w.Close())
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
|
||||
h := http.Header{}
|
||||
h.Set("Content-Type", ArrowContentType)
|
||||
for k, v := range headers {
|
||||
h.Set(k, v)
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: h,
|
||||
Body: io.NopCloser(bytes.NewReader(data)),
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
|
||||
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
|
||||
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
|
||||
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
|
||||
b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
|
||||
b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
|
||||
b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
|
||||
b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
|
||||
b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
|
||||
b.Field(9).(*arrowArray.StringBuilder).Append("available")
|
||||
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
|
||||
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
|
||||
b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
|
||||
b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
|
||||
b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
|
||||
b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
|
||||
b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
|
||||
b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
|
||||
b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
|
||||
b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
|
||||
b.Field(9).(*arrowArray.StringBuilder).Append("available")
|
||||
b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, result.NextMarker)
|
||||
require.Equal(t, "marker123", *result.NextMarker)
|
||||
require.Len(t, result.Segment.BlobItems, 2)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.Equal(t, "blob1.txt", *item.Name)
|
||||
require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
|
||||
require.Equal(t, int64(1024), *item.Properties.ContentLength)
|
||||
require.Equal(t, "Hot", string(*item.Properties.AccessTier))
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
// No rows added
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, result.NextMarker)
|
||||
require.Empty(t, result.Segment.BlobItems)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_NullFields(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
|
||||
b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
|
||||
b.Field(3).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(4).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
|
||||
b.Field(6).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
|
||||
b.Field(8).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(9).(*arrowArray.StringBuilder).AppendNull()
|
||||
b.Field(10).(*arrowArray.StringBuilder).AppendNull()
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.Equal(t, "blob.txt", *item.Name)
|
||||
require.Nil(t, item.Properties.CreationTime)
|
||||
require.Nil(t, item.Properties.BlobType)
|
||||
require.Nil(t, item.Properties.ETag)
|
||||
require.Nil(t, item.Properties.ContentLength)
|
||||
require.Nil(t, item.Properties.ContentType)
|
||||
require.Nil(t, item.Properties.ServerEncrypted)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
|
||||
alloc := memory.DefaultAllocator
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
|
||||
schema := coreSchema(&md)
|
||||
|
||||
var buf bytes.Buffer
|
||||
w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
|
||||
|
||||
// First batch: 2 rows
|
||||
builder := arrowArray.NewRecordBuilder(alloc, schema)
|
||||
for range 2 {
|
||||
builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
|
||||
for j := 1; j < 11; j++ {
|
||||
builder.Field(j).AppendNull()
|
||||
}
|
||||
}
|
||||
rec := builder.NewRecordBatch()
|
||||
require.NoError(t, w.Write(rec))
|
||||
rec.Release()
|
||||
builder.Release()
|
||||
|
||||
// Second batch: 1 row
|
||||
builder = arrowArray.NewRecordBuilder(alloc, schema)
|
||||
builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
|
||||
for j := 1; j < 11; j++ {
|
||||
builder.Field(j).AppendNull()
|
||||
}
|
||||
rec = builder.NewRecordBatch()
|
||||
require.NoError(t, w.Write(rec))
|
||||
rec.Release()
|
||||
builder.Release()
|
||||
|
||||
require.NoError(t, w.Close())
|
||||
|
||||
resp := makeHTTPResponse(buf.Bytes(), nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 3)
|
||||
}
|
||||
|
||||
func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
// A regular blob
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
|
||||
b.Field(1).(*arrowArray.StringBuilder).AppendNull()
|
||||
|
||||
// A blob prefix (virtual directory)
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
|
||||
b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleHierarchyListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
|
||||
|
||||
require.Len(t, result.Segment.BlobPrefixes, 1)
|
||||
require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, map[string]string{
|
||||
"x-ms-request-id": "req-123",
|
||||
"x-ms-version": "2026-10-06",
|
||||
"x-ms-client-request-id": "client-req-456",
|
||||
"Date": "Mon, 01 Jan 2024 00:00:00 GMT",
|
||||
})
|
||||
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "req-123", *result.RequestID)
|
||||
require.Equal(t, "2026-10-06", *result.Version)
|
||||
require.Equal(t, "client-req-456", *result.ClientRequestID)
|
||||
require.NotNil(t, result.Date)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_MapFields(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
|
||||
{Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
|
||||
metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
|
||||
metaBuilder.Append(true)
|
||||
metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
|
||||
metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
|
||||
|
||||
tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
|
||||
tagBuilder.Append(true)
|
||||
tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
|
||||
tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.Segment.BlobItems, 1)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.NotNil(t, item.Metadata)
|
||||
require.Equal(t, "value1", *item.Metadata["key1"])
|
||||
|
||||
require.NotNil(t, item.BlobTags)
|
||||
require.Len(t, item.BlobTags.BlobTagSet, 1)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
|
||||
b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
|
||||
b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
|
||||
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.True(t, *item.Deleted)
|
||||
require.False(t, *item.IsCurrentVersion)
|
||||
require.True(t, *item.Properties.ServerEncrypted)
|
||||
}
|
||||
|
||||
func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
|
||||
md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
|
||||
data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
|
||||
b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
|
||||
b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
|
||||
b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
|
||||
})
|
||||
|
||||
resp := makeHTTPResponse(data, nil)
|
||||
result, err := HandleFlatListResponse(resp)
|
||||
require.NoError(t, err)
|
||||
|
||||
item := result.Segment.BlobItems[0]
|
||||
require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
|
||||
require.Equal(t, "snap-123", *item.Snapshot)
|
||||
require.True(t, *item.IsCurrentVersion)
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Package arrowlist implements the experimental "Blob Listing with Apache
|
||||
// Arrow" feature on top of the released Azure azblob SDK.
|
||||
//
|
||||
// The Azure SDK for Go has support for Arrow listing on the unreleased
|
||||
// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
|
||||
// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
|
||||
// response types. Until that ships in a tagged azblob release this package
|
||||
// fills the gap: it exposes the same options and pager interface as the
|
||||
// experimental SDK, built only on the public API of the released SDK plus
|
||||
// copies of two small internal auth policies (see sharedkey.go and
|
||||
// challenge_policy.go).
|
||||
//
|
||||
// This package is temporary. When Arrow listing ships in a released azblob
|
||||
// the whole package should be deleted and its callers pointed back at
|
||||
// container.Client.NewListBlobsHierarchyPager - the option and response
|
||||
// types here are deliberately source compatible with the SDK's to make that
|
||||
// a mechanical change.
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
)
|
||||
|
||||
const (
|
||||
// moduleName and moduleVersion identify this package in the azcore
|
||||
// telemetry policy's User-Agent fragment.
|
||||
moduleName = "github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
moduleVersion = "v0.0.1"
|
||||
|
||||
// serviceVersion is the x-ms-version sent with listing requests. Arrow
|
||||
// responses and the startFrom/endBefore parameters need at least
|
||||
// 2026-06-06.
|
||||
serviceVersion = "2026-06-06"
|
||||
|
||||
// tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
|
||||
// own clients.
|
||||
tokenScope = "https://storage.azure.com/.default"
|
||||
)
|
||||
|
||||
// ErrEndBeforeXMLFallback is returned by the pager when the server answered
|
||||
// with XML instead of Arrow while EndBefore was set. The service only honours
|
||||
// endBefore on the Arrow listing path, so an XML page could silently contain
|
||||
// blobs outside the requested range and cannot be used.
|
||||
var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
|
||||
|
||||
// ListBlobsHierarchyOptions provides the configuration for
|
||||
// Client.NewListBlobsHierarchyPager.
|
||||
//
|
||||
// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
|
||||
// the released SDK's options are embedded (providing Include, Marker,
|
||||
// MaxResults, Prefix and StartFrom by promotion) with the two fields the
|
||||
// released SDK lacks added alongside.
|
||||
type ListBlobsHierarchyOptions struct {
|
||||
container.ListBlobsHierarchyOptions
|
||||
// EndBefore limits listing to blobs whose full container path is
|
||||
// lexically before it (exclusive). Only honoured on Arrow responses.
|
||||
EndBefore *string
|
||||
// UseArrowFormat requests the response as an Apache Arrow IPC stream.
|
||||
UseArrowFormat *bool
|
||||
}
|
||||
|
||||
// ClientOptions contains the optional parameters for client creation.
|
||||
type ClientOptions struct {
|
||||
azcore.ClientOptions
|
||||
}
|
||||
|
||||
// Client issues Arrow listing requests to a single container.
|
||||
type Client struct {
|
||||
endpoint string
|
||||
azClient *azcore.Client
|
||||
}
|
||||
|
||||
// newClient creates a Client for the container at containerURL sending
|
||||
// requests through a pipeline with the given auth policy (which may be nil
|
||||
// for anonymous or SAS access).
|
||||
func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
|
||||
if options == nil {
|
||||
options = &ClientOptions{}
|
||||
}
|
||||
plOpts := runtime.PipelineOptions{}
|
||||
if authPolicy != nil {
|
||||
plOpts.PerRetry = []policy.Policy{authPolicy}
|
||||
}
|
||||
azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{
|
||||
endpoint: containerURL,
|
||||
azClient: azClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewClient creates a Client authenticating with a token credential.
|
||||
func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
|
||||
return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
|
||||
}
|
||||
|
||||
// NewClientWithNoCredential creates a Client with no authentication, for
|
||||
// anonymous access or a containerURL carrying a SAS token in its query.
|
||||
func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
|
||||
return newClient(containerURL, nil, options)
|
||||
}
|
||||
|
||||
// NewClientWithSharedKeyCredential creates a Client signing requests with the
|
||||
// account's shared key.
|
||||
func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
|
||||
return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
|
||||
}
|
||||
|
||||
// NewListBlobsHierarchyPager returns a pager over the container's blobs,
|
||||
// requesting Arrow responses when o.UseArrowFormat is set and returning the
|
||||
// same response type as container.Client.NewListBlobsHierarchyPager.
|
||||
//
|
||||
// If the service does not support Arrow listing (the feature not enabled on
|
||||
// the account, or an HNS account) it answers with XML, which is decoded
|
||||
// transparently - unless EndBefore is set, in which case the fetcher returns
|
||||
// ErrEndBeforeXMLFallback (see there).
|
||||
func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
|
||||
// Copy the options so advancing the Marker doesn't mutate the caller's struct.
|
||||
opts := ListBlobsHierarchyOptions{}
|
||||
if o != nil {
|
||||
opts = *o
|
||||
}
|
||||
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
|
||||
return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
|
||||
More: func(page container.ListBlobsHierarchyResponse) bool {
|
||||
return page.NextMarker != nil && len(*page.NextMarker) > 0
|
||||
},
|
||||
Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
|
||||
if page != nil {
|
||||
opts.Marker = page.NextMarker
|
||||
}
|
||||
req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
|
||||
if err != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, err
|
||||
}
|
||||
resp, err := c.azClient.Pipeline().Do(req)
|
||||
if err != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, err
|
||||
}
|
||||
if !runtime.HasStatusCode(resp, http.StatusOK) {
|
||||
return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
|
||||
}
|
||||
if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
|
||||
return HandleHierarchyListResponse(resp)
|
||||
}
|
||||
if opts.EndBefore != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
|
||||
}
|
||||
return handleXMLResponse(resp)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// listCreateRequest builds the ListBlobs request, mirroring the generated
|
||||
// request builders in the SDK.
|
||||
func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
|
||||
req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reading the existing query first preserves any SAS token in the endpoint.
|
||||
reqQP := req.Raw().URL.Query()
|
||||
reqQP.Set("comp", "list")
|
||||
reqQP.Set("restype", "container")
|
||||
reqQP.Set("delimiter", delimiter)
|
||||
if include := formatInclude(opts.Include); include != "" {
|
||||
reqQP.Set("include", include)
|
||||
}
|
||||
if opts.Marker != nil {
|
||||
reqQP.Set("marker", *opts.Marker)
|
||||
}
|
||||
if opts.MaxResults != nil {
|
||||
reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
|
||||
}
|
||||
if opts.Prefix != nil {
|
||||
reqQP.Set("prefix", *opts.Prefix)
|
||||
}
|
||||
if opts.StartFrom != nil {
|
||||
reqQP.Set("startFrom", *opts.StartFrom)
|
||||
}
|
||||
if useArrow && opts.EndBefore != nil {
|
||||
reqQP.Set("endBefore", *opts.EndBefore)
|
||||
}
|
||||
req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
|
||||
if useArrow {
|
||||
// Stream the Arrow body rather than buffering it in the pipeline.
|
||||
runtime.SkipBodyDownload(req)
|
||||
req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
|
||||
} else {
|
||||
req.Raw().Header["Accept"] = []string{"application/xml"}
|
||||
}
|
||||
req.Raw().Header["x-ms-version"] = []string{serviceVersion}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
|
||||
// generated ListBlobHierarchySegmentHandleResponse.
|
||||
func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
|
||||
result := container.ListBlobsHierarchyResponse{}
|
||||
extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
|
||||
if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
|
||||
return container.ListBlobsHierarchyResponse{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// formatInclude renders the include datasets as the comma separated query
|
||||
// parameter value, in the same order as the SDK's unexported
|
||||
// ListBlobsInclude.format.
|
||||
func formatInclude(l container.ListBlobsInclude) string {
|
||||
var include []string
|
||||
if l.Copy {
|
||||
include = append(include, "copy")
|
||||
}
|
||||
if l.Deleted {
|
||||
include = append(include, "deleted")
|
||||
}
|
||||
if l.DeletedWithVersions {
|
||||
include = append(include, "deletedwithversions")
|
||||
}
|
||||
if l.ImmutabilityPolicy {
|
||||
include = append(include, "immutabilitypolicy")
|
||||
}
|
||||
if l.LegalHold {
|
||||
include = append(include, "legalhold")
|
||||
}
|
||||
if l.Metadata {
|
||||
include = append(include, "metadata")
|
||||
}
|
||||
if l.Snapshots {
|
||||
include = append(include, "snapshots")
|
||||
}
|
||||
if l.Tags {
|
||||
include = append(include, "tags")
|
||||
}
|
||||
if l.UncommittedBlobs {
|
||||
include = append(include, "uncommittedblobs")
|
||||
}
|
||||
if l.Versions {
|
||||
include = append(include, "versions")
|
||||
}
|
||||
if l.Permissions {
|
||||
include = append(include, "permissions")
|
||||
}
|
||||
return strings.Join(include, ",")
|
||||
}
|
||||
@@ -1,198 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/apache/arrow-go/v18/arrow"
|
||||
arrowArray "github.com/apache/arrow-go/v18/arrow/array"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// arrowPage builds a one row Arrow IPC listing page with the given blob name
|
||||
// and NextMarker.
|
||||
func arrowPage(t *testing.T, name, nextMarker string) []byte {
|
||||
md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
|
||||
fields := []arrow.Field{
|
||||
{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
|
||||
{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
|
||||
}
|
||||
schema := arrow.NewSchema(fields, &md)
|
||||
return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
|
||||
b.Field(0).(*arrowArray.StringBuilder).Append(name)
|
||||
b.Field(1).(*arrowArray.Uint64Builder).Append(42)
|
||||
})
|
||||
}
|
||||
|
||||
func TestPagerArrow(t *testing.T) {
|
||||
var requests []*http.Request
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests = append(requests, r.Clone(context.Background()))
|
||||
w.Header().Set("Content-Type", ArrowContentType)
|
||||
if r.URL.Query().Get("marker") == "" {
|
||||
_, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
|
||||
} else {
|
||||
_, _ = w.Write(arrowPage(t, "blob2.txt", ""))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// SAS style query on the endpoint must be preserved in requests.
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
opts := &ListBlobsHierarchyOptions{
|
||||
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
|
||||
Include: container.ListBlobsInclude{Metadata: true, Tags: true},
|
||||
Prefix: new("dir/"),
|
||||
MaxResults: new(int32(1000)),
|
||||
StartFrom: new("testcontainer/dir/a"),
|
||||
},
|
||||
EndBefore: new("testcontainer/dir/n"),
|
||||
UseArrowFormat: new(true),
|
||||
}
|
||||
pager := client.NewListBlobsHierarchyPager("/", opts)
|
||||
|
||||
var names []string
|
||||
for pager.More() {
|
||||
page, err := pager.NextPage(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ArrowContentType, *page.ContentType)
|
||||
for _, item := range page.Segment.BlobItems {
|
||||
names = append(names, *item.Name)
|
||||
assert.Equal(t, int64(42), *item.Properties.ContentLength)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
|
||||
|
||||
require.Len(t, requests, 2)
|
||||
q := requests[0].URL.Query()
|
||||
assert.Equal(t, "list", q.Get("comp"))
|
||||
assert.Equal(t, "container", q.Get("restype"))
|
||||
assert.Equal(t, "/", q.Get("delimiter"))
|
||||
assert.Equal(t, "metadata,tags", q.Get("include"))
|
||||
assert.Equal(t, "dir/", q.Get("prefix"))
|
||||
assert.Equal(t, "1000", q.Get("maxresults"))
|
||||
assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
|
||||
assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
|
||||
assert.Equal(t, "fakesas", q.Get("sv"))
|
||||
assert.Empty(t, q.Get("marker"))
|
||||
assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
|
||||
assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
|
||||
assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
|
||||
|
||||
// The caller's options must not have been mutated by paging.
|
||||
assert.Nil(t, opts.Marker)
|
||||
}
|
||||
|
||||
const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
|
||||
<Blobs>
|
||||
<Blob>
|
||||
<Name>file.txt</Name>
|
||||
<Properties>
|
||||
<Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
|
||||
<Etag>0x8D1</Etag>
|
||||
<Content-Length>7</Content-Length>
|
||||
<BlobType>BlockBlob</BlobType>
|
||||
</Properties>
|
||||
</Blob>
|
||||
<BlobPrefix>
|
||||
<Name>subdir/</Name>
|
||||
</BlobPrefix>
|
||||
</Blobs>
|
||||
<NextMarker />
|
||||
</EnumerationResults>`
|
||||
|
||||
func TestPagerXMLFallback(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(xmlListResponse))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
UseArrowFormat: new(true),
|
||||
})
|
||||
page, err := pager.NextPage(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.False(t, pager.More())
|
||||
assert.Equal(t, "application/xml", *page.ContentType)
|
||||
require.Len(t, page.Segment.BlobItems, 1)
|
||||
assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
|
||||
assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
|
||||
require.Len(t, page.Segment.BlobPrefixes, 1)
|
||||
assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
|
||||
}
|
||||
|
||||
func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(xmlListResponse))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
EndBefore: new("testcontainer/n"),
|
||||
UseArrowFormat: new(true),
|
||||
})
|
||||
_, err = pager.NextPage(context.Background())
|
||||
require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
|
||||
}
|
||||
|
||||
func TestPagerSharedKeyAuth(t *testing.T) {
|
||||
var gotAuth, gotDate string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
gotDate = r.Header.Get("x-ms-date")
|
||||
w.Header().Set("Content-Type", ArrowContentType)
|
||||
_, _ = w.Write(arrowPage(t, "blob.txt", ""))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
|
||||
require.NoError(t, err)
|
||||
client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
UseArrowFormat: new(true),
|
||||
})
|
||||
_, err = pager.NextPage(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
|
||||
assert.NotEmpty(t, gotDate)
|
||||
}
|
||||
|
||||
func TestPagerResponseError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
|
||||
UseArrowFormat: new(true),
|
||||
})
|
||||
_, err = pager.NextPage(context.Background())
|
||||
require.Error(t, err)
|
||||
assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
|
||||
// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
|
||||
// released SDK does not export its storage bearer-challenge policy, which is
|
||||
// needed for token authentication with the same semantics as the SDK's own
|
||||
// clients.
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
|
||||
)
|
||||
|
||||
type storageAuthorizer struct {
|
||||
scopes []string
|
||||
tenantID string
|
||||
}
|
||||
|
||||
// NewStorageChallengePolicy returns a bearer token policy for cred which
|
||||
// handles the storage service's authentication challenges.
|
||||
func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
|
||||
s := storageAuthorizer{scopes: []string{audience}}
|
||||
return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
|
||||
AuthorizationHandler: policy.AuthorizationHandler{
|
||||
OnRequest: s.onRequest,
|
||||
OnChallenge: s.onChallenge,
|
||||
},
|
||||
InsecureAllowCredentialWithHTTP: allowHTTP,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
|
||||
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
|
||||
}
|
||||
|
||||
func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
|
||||
// parse the challenge
|
||||
err := s.parseChallenge(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
|
||||
return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
|
||||
}
|
||||
|
||||
type challengePolicyError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *challengePolicyError) Error() string {
|
||||
return c.err.Error()
|
||||
}
|
||||
|
||||
func (*challengePolicyError) NonRetriable() {
|
||||
// marker method
|
||||
}
|
||||
|
||||
func (c *challengePolicyError) Unwrap() error {
|
||||
return c.err
|
||||
}
|
||||
|
||||
// parses Tenant ID from auth challenge
|
||||
// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
|
||||
func parseTenant(url string) string {
|
||||
if url == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(url, "/")
|
||||
if len(parts) >= 3 {
|
||||
tenant := parts[3]
|
||||
tenant = strings.ReplaceAll(tenant, ",", "")
|
||||
return tenant
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
|
||||
authHeader := resp.Header.Get("WWW-Authenticate")
|
||||
if authHeader == "" {
|
||||
return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
|
||||
}
|
||||
|
||||
// Strip down to auth and resource
|
||||
// Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
|
||||
authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
|
||||
|
||||
parts := strings.Split(authHeader, " ")
|
||||
|
||||
vals := map[string]string{}
|
||||
for _, part := range parts {
|
||||
subParts := strings.Split(part, "=")
|
||||
if len(subParts) == 2 {
|
||||
stripped := strings.ReplaceAll(subParts[1], "\"", "")
|
||||
stripped = strings.TrimSuffix(stripped, ",")
|
||||
vals[subParts[0]] = stripped
|
||||
}
|
||||
}
|
||||
|
||||
s.tenantID = parseTenant(vals["authorization_uri"])
|
||||
|
||||
scope := vals["resource_id"]
|
||||
if scope == "" {
|
||||
return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(scope, "/.default") {
|
||||
scope += "/.default"
|
||||
}
|
||||
s.scopes = []string{scope}
|
||||
return nil
|
||||
}
|
||||
@@ -1,318 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
//
|
||||
// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
|
||||
// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
|
||||
// with the internal header constants inlined. The released SDK does not
|
||||
// export its shared key signing policy, so a copy is needed to sign the
|
||||
// Arrow listing requests this package sends through its own pipeline.
|
||||
|
||||
package arrowlist
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
|
||||
// storage account's name and either its primary or secondary key.
|
||||
func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
|
||||
c := SharedKeyCredential{accountName: accountName}
|
||||
if err := c.SetAccountKey(accountKey); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// SharedKeyCredential contains an account's name and its primary or secondary key.
|
||||
type SharedKeyCredential struct {
|
||||
// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
|
||||
accountName string
|
||||
accountKey atomic.Value // []byte
|
||||
}
|
||||
|
||||
// AccountName returns the Storage account's name.
|
||||
func (c *SharedKeyCredential) AccountName() string {
|
||||
return c.accountName
|
||||
}
|
||||
|
||||
// SetAccountKey replaces the existing account key with the specified account key.
|
||||
func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
|
||||
_bytes, err := base64.StdEncoding.DecodeString(accountKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode account key: %w", err)
|
||||
}
|
||||
c.accountKey.Store(_bytes)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
|
||||
func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
|
||||
h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
|
||||
_, err := h.Write([]byte(message))
|
||||
return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
|
||||
}
|
||||
|
||||
func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
|
||||
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
|
||||
headers := req.Header
|
||||
contentLength := getHeader("Content-Length", headers)
|
||||
if contentLength == "0" {
|
||||
contentLength = ""
|
||||
}
|
||||
|
||||
canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
stringToSign := strings.Join([]string{
|
||||
req.Method,
|
||||
getHeader("Content-Encoding", headers),
|
||||
getHeader("Content-Language", headers),
|
||||
contentLength,
|
||||
getHeader("Content-MD5", headers),
|
||||
getHeader("Content-Type", headers),
|
||||
"", // Empty date because x-ms-date is expected (as per web page above)
|
||||
getHeader("If-Modified-Since", headers),
|
||||
getHeader("If-Match", headers),
|
||||
getHeader("If-None-Match", headers),
|
||||
getHeader("If-Unmodified-Since", headers),
|
||||
getHeader("Range", headers),
|
||||
c.buildCanonicalizedHeader(headers),
|
||||
canonicalizedResource,
|
||||
}, "\n")
|
||||
return stringToSign, nil
|
||||
}
|
||||
|
||||
func getHeader(key string, headers map[string][]string) string {
|
||||
if headers == nil {
|
||||
return ""
|
||||
}
|
||||
if v, ok := headers[key]; ok {
|
||||
if len(v) > 0 {
|
||||
return v[0]
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func getWeightTables() [][]int {
|
||||
tableLv0 := [...]int{
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
|
||||
0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
|
||||
0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
|
||||
0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
|
||||
0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
|
||||
0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
|
||||
0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
|
||||
}
|
||||
tableLv2 := [...]int{
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
|
||||
}
|
||||
tables := [][]int{tableLv0[:], tableLv2[:]}
|
||||
return tables
|
||||
}
|
||||
|
||||
// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
|
||||
func compareHeaders(lhs, rhs string, tables [][]int) int {
|
||||
currLevel, i, j := 0, 0, 0
|
||||
n := len(tables)
|
||||
lhsLen := len(lhs)
|
||||
rhsLen := len(rhs)
|
||||
|
||||
for currLevel < n {
|
||||
if currLevel == (n-1) && i != j {
|
||||
if i > j {
|
||||
return -1
|
||||
}
|
||||
if i < j {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var w1, w2 int
|
||||
|
||||
// Check bounds before accessing lhs[i]
|
||||
if i < lhsLen {
|
||||
w1 = tables[currLevel][lhs[i]]
|
||||
} else {
|
||||
w1 = 0x1
|
||||
}
|
||||
|
||||
// Check bounds before accessing rhs[j]
|
||||
if j < rhsLen {
|
||||
w2 = tables[currLevel][rhs[j]]
|
||||
} else {
|
||||
w2 = 0x1
|
||||
}
|
||||
|
||||
if w1 == 0x1 && w2 == 0x1 {
|
||||
i = 0
|
||||
j = 0
|
||||
currLevel++
|
||||
} else if w1 == w2 {
|
||||
i++
|
||||
j++
|
||||
} else if w1 == 0 {
|
||||
i++
|
||||
} else if w2 == 0 {
|
||||
j++
|
||||
} else {
|
||||
if w1 < w2 {
|
||||
return -1
|
||||
}
|
||||
if w1 > w2 {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
|
||||
cm := map[string][]string{}
|
||||
for k, v := range headers {
|
||||
headerName := strings.TrimSpace(strings.ToLower(k))
|
||||
if strings.HasPrefix(headerName, "x-ms-") {
|
||||
cm[headerName] = v // NOTE: the value must not have any whitespace around it.
|
||||
}
|
||||
}
|
||||
if len(cm) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(cm))
|
||||
for key := range cm {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
tables := getWeightTables()
|
||||
// Sort the keys using the custom comparator
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
return compareHeaders(keys[i], keys[j], tables) < 0
|
||||
})
|
||||
ch := bytes.NewBufferString("")
|
||||
for i, key := range keys {
|
||||
if i > 0 {
|
||||
ch.WriteRune('\n')
|
||||
}
|
||||
ch.WriteString(key)
|
||||
ch.WriteRune(':')
|
||||
ch.WriteString(strings.Join(cm[key], ","))
|
||||
}
|
||||
return ch.String()
|
||||
}
|
||||
|
||||
func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
|
||||
// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
|
||||
cr := bytes.NewBufferString("/")
|
||||
cr.WriteString(c.accountName)
|
||||
|
||||
if len(u.Path) > 0 {
|
||||
// Any portion of the CanonicalizedResource string that is derived from
|
||||
// the resource's URI should be encoded exactly as it is in the URI.
|
||||
// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
|
||||
cr.WriteString(u.EscapedPath())
|
||||
} else {
|
||||
// a slash is required to indicate the root path
|
||||
cr.WriteString("/")
|
||||
}
|
||||
|
||||
// params is a map[string][]string; param name is key; params values is []string
|
||||
params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse query params: %w", err)
|
||||
}
|
||||
|
||||
if len(params) > 0 { // There is at least 1 query parameter
|
||||
var paramNames []string // We use this to sort the parameter key names
|
||||
for paramName := range params {
|
||||
paramNames = append(paramNames, paramName) // paramNames must be lowercase
|
||||
}
|
||||
sort.Strings(paramNames)
|
||||
|
||||
for _, paramName := range paramNames {
|
||||
paramValues := params[paramName]
|
||||
sort.Strings(paramValues)
|
||||
|
||||
// Join the sorted key values separated by ','
|
||||
// Then prepend "keyName:"; then add this string to the buffer
|
||||
cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
|
||||
}
|
||||
}
|
||||
return cr.String(), nil
|
||||
}
|
||||
|
||||
// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
|
||||
func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
|
||||
return cred.computeHMACSHA256(message)
|
||||
}
|
||||
|
||||
// SharedKeyCredPolicy is a pipeline policy that signs each request with the
|
||||
// account's shared key.
|
||||
type SharedKeyCredPolicy struct {
|
||||
cred *SharedKeyCredential
|
||||
}
|
||||
|
||||
// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
|
||||
func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
|
||||
return &SharedKeyCredPolicy{cred: cred}
|
||||
}
|
||||
|
||||
// Do signs the request and sends it via the next policy in the pipeline.
|
||||
func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
|
||||
// skip adding the authorization header if no SharedKeyCredential was provided.
|
||||
// this prevents a panic that might be hard to diagnose and allows testing
|
||||
// against http endpoints that don't require authentication.
|
||||
if s.cred == nil {
|
||||
return req.Next()
|
||||
}
|
||||
|
||||
if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
|
||||
req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
|
||||
}
|
||||
stringToSign, err := s.cred.buildStringToSign(req.Raw())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signature, err := s.cred.computeHMACSHA256(stringToSign)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
|
||||
req.Raw().Header.Set("Authorization", authHeader)
|
||||
|
||||
response, err := req.Next()
|
||||
if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
|
||||
// Service failed to authenticate request, log it
|
||||
fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
|
||||
}
|
||||
return response, err
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
@@ -284,9 +283,6 @@ type transporter struct {
|
||||
|
||||
// Make a new transporter
|
||||
func newTransporter(ctx context.Context) transporter {
|
||||
// Set the User-Agent to include the Microsoft Partner Network prefix
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.UserAgent = "APN/1.0 rclone/1.0 rclone/" + strings.TrimPrefix(fs.Version, "v")
|
||||
return transporter{
|
||||
RoundTripper: fshttp.NewTransport(ctx),
|
||||
}
|
||||
@@ -297,13 +293,6 @@ func (tr transporter) Do(req *http.Request) (*http.Response, error) {
|
||||
return tr.RoundTripper.RoundTrip(req)
|
||||
}
|
||||
|
||||
// Transporter returns the policy.Transporter rclone uses for Azure SDK
|
||||
// clients (an fshttp based transport with the APN user agent), for callers
|
||||
// which build their own azcore pipelines.
|
||||
func Transporter(ctx context.Context) policy.Transporter {
|
||||
return newTransporter(ctx)
|
||||
}
|
||||
|
||||
// NewClientOpts should be passed to configure NewClient
|
||||
type NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct {
|
||||
DefaultBaseURL string // Base URL, eg blob.core.windows.net
|
||||
|
||||
+85
-314
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -19,21 +19,19 @@ import (
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
|
||||
"github.com/rclone/rclone/backend/azureblob/arrowlist"
|
||||
"github.com/rclone/rclone/backend/azureblob/auth"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
@@ -49,8 +47,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/transferaccounter"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -69,19 +65,8 @@ const (
|
||||
defaultChunkSize = 4 * fs.Mebi
|
||||
defaultAccessTier = blob.AccessTier("") // FIXME AccessTierNone
|
||||
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
|
||||
sasCopyStartSkew = 15 * time.Minute // how far back SAS should start to allow for clock skew
|
||||
)
|
||||
|
||||
// setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
|
||||
// on every request. This prevents the Go HTTP transport from automatically
|
||||
// decompressing gzip-encoded blobs on download.
|
||||
type setAcceptEncodingGzip struct{}
|
||||
|
||||
func (p setAcceptEncodingGzip) Do(req *policy.Request) (*http.Response, error) {
|
||||
req.Raw().Header.Set("Accept-Encoding", "gzip")
|
||||
return req.Next()
|
||||
}
|
||||
|
||||
var (
|
||||
errCantUpdateArchiveTierBlobs = fserrors.NoRetryError(errors.New("can't update archive tier blob without --azureblob-archive-tier-delete"))
|
||||
|
||||
@@ -239,42 +224,6 @@ minutes per megabyte on average, it will time out (
|
||||
avoid the time out.`,
|
||||
Default: maxListChunkSize,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_arrow_list",
|
||||
Help: `Use the experimental Apache Arrow listing format.
|
||||
|
||||
If set, directory listings are fetched using the ListBlobs Apache
|
||||
Arrow response format instead of XML. This can be faster for very
|
||||
large containers.
|
||||
|
||||
This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
|
||||
preview feature to be enabled on the storage account. It is NOT
|
||||
supported on accounts with a hierarchical namespace (ADLS Gen2) -
|
||||
those return a 409 error. If the feature is not enabled the server
|
||||
returns XML and the listing transparently falls back to the normal XML
|
||||
path (logged at debug level).
|
||||
|
||||
Not supported with connection_string auth - falls back to normal
|
||||
listing.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: "list_parallelism",
|
||||
Help: `Number of parallel shards to list a directory with.
|
||||
|
||||
EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
|
||||
directory is split into this many ranges which are listed concurrently
|
||||
using the Arrow startFrom/endBefore range parameters. This can
|
||||
dramatically speed up listing containers with millions of objects, for
|
||||
both recursive (ListR) and single directory listings.
|
||||
|
||||
This has no effect unless "use_arrow_list" is also set, as Arrow is the
|
||||
only listing path that supports server-side name ranges. The default of
|
||||
0 (or 1) lists sequentially.`,
|
||||
Default: 0,
|
||||
Advanced: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: "access_tier",
|
||||
Help: `Access tier of blob: hot, cool, cold or archive.
|
||||
@@ -401,19 +350,6 @@ rclone does if you know the container exists already.
|
||||
Default: "",
|
||||
Exclusive: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "decompress",
|
||||
Help: `If set this will decompress gzip encoded objects.
|
||||
|
||||
It is possible to upload objects to Azure Blob Storage with "Content-Encoding: gzip"
|
||||
set. Normally rclone will download these files as compressed objects.
|
||||
|
||||
If this flag is set then rclone will decompress these files with
|
||||
"Content-Encoding: gzip" as they are received. This means that rclone
|
||||
can't check the size and hash but the file contents will be decompressed.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}}),
|
||||
})
|
||||
}
|
||||
@@ -428,8 +364,6 @@ type Options struct {
|
||||
UseCopyBlob bool `config:"use_copy_blob"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
ListChunkSize uint `config:"list_chunk"`
|
||||
UseArrowList bool `config:"use_arrow_list"`
|
||||
ListParallelism int `config:"list_parallelism"`
|
||||
AccessTier string `config:"access_tier"`
|
||||
ArchiveTierDelete bool `config:"archive_tier_delete"`
|
||||
DisableCheckSum bool `config:"disable_checksum"`
|
||||
@@ -439,7 +373,6 @@ type Options struct {
|
||||
NoCheckContainer bool `config:"no_check_container"`
|
||||
NoHeadObject bool `config:"no_head_object"`
|
||||
DeleteSnapshots string `config:"delete_snapshots"`
|
||||
Decompress bool `config:"decompress"`
|
||||
}
|
||||
|
||||
// Fs represents a remote azure server
|
||||
@@ -449,11 +382,8 @@ type Fs struct {
|
||||
opt Options // parsed config options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache and arrowCntSVCcache
|
||||
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache
|
||||
cntSVCcache map[string]*container.Client // reference to containerClient per container
|
||||
arrowCntSVCcache map[string]*arrowlist.Client // reference to arrowlist client per container
|
||||
arrowClientOpts *arrowlist.ClientOptions // client options for the arrowlist clients
|
||||
arrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retried
|
||||
svc *service.Client // client to access azblob
|
||||
cred azcore.TokenCredential // how to generate tokens (may be nil)
|
||||
usingSharedKeyCred bool // set if using shared key credentials
|
||||
@@ -467,8 +397,6 @@ type Fs struct {
|
||||
copyToken *pacer.TokenDispenser // global multipart copy concurrency limiter
|
||||
publicAccess container.PublicAccessType // Container Public Access Level
|
||||
|
||||
warnCompressed sync.Once // warn once about compressed files
|
||||
|
||||
// user delegation cache
|
||||
userDelegationMu sync.Mutex
|
||||
userDelegation *service.UserDelegationCredential
|
||||
@@ -477,16 +405,15 @@ type Fs struct {
|
||||
|
||||
// Object describes an azure object
|
||||
type Object struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
accessTier blob.AccessTier // Blob Access Tier
|
||||
meta map[string]string // blob metadata - take metadataMu when accessing
|
||||
tags map[string]string // blob tags
|
||||
contentEncoding *string // Content-Encoding of the object
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
accessTier blob.AccessTier // Blob Access Tier
|
||||
meta map[string]string // blob metadata - take metadataMu when accessing
|
||||
tags map[string]string // blob tags
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -575,7 +502,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
// General errors from:
|
||||
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
|
||||
// Blob specific errors from:
|
||||
@@ -652,9 +580,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if opt.ListChunkSize > maxListChunkSize {
|
||||
return nil, fmt.Errorf("blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
|
||||
}
|
||||
if opt.ListParallelism > 1 && !opt.UseArrowList {
|
||||
fs.Logf(nil, "azureblob: list_parallelism has no effect without use_arrow_list - listing sequentially")
|
||||
}
|
||||
|
||||
if opt.AccessTier == "" {
|
||||
opt.AccessTier = string(defaultAccessTier)
|
||||
@@ -670,15 +595,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
ci: ci,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
|
||||
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
|
||||
cache: bucket.NewCache(),
|
||||
cntSVCcache: make(map[string]*container.Client, 1),
|
||||
arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
|
||||
name: name,
|
||||
opt: *opt,
|
||||
ci: ci,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
uploadToken: pacer.NewTokenDispenser(ci.Transfers),
|
||||
copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
|
||||
cache: bucket.NewCache(),
|
||||
cntSVCcache: make(map[string]*container.Client, 1),
|
||||
}
|
||||
f.publicAccess = container.PublicAccessType(opt.PublicAccess)
|
||||
f.setRoot(root)
|
||||
@@ -710,9 +634,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential,
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
// Override the automatic decompression in the transport
|
||||
// to download compressed files as-is
|
||||
policyClientOptions.PerCallPolicies = append(policyClientOptions.PerCallPolicies, setAcceptEncodingGzip{})
|
||||
options.ClientOptions = policyClientOptions
|
||||
},
|
||||
}
|
||||
@@ -725,15 +646,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.usingSharedKeyCred = res.UsingSharedKeyCred
|
||||
f.anonymous = res.Anonymous
|
||||
|
||||
// Client options for the arrowlist clients, using the same transport and
|
||||
// gzip policy as the SDK clients built above.
|
||||
f.arrowClientOpts = &arrowlist.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Transport: auth.Transporter(ctx),
|
||||
PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
|
||||
},
|
||||
}
|
||||
|
||||
// if using Container level SAS put the container client into the cache
|
||||
if opt.SASURL != "" && res.Container != "" {
|
||||
_ = f.cntSVC(res.Container)
|
||||
@@ -772,44 +684,6 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
|
||||
return containerClient
|
||||
}
|
||||
|
||||
// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
|
||||
// credentials can't be used for Arrow listing.
|
||||
var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
|
||||
|
||||
// return the arrowlist client for the container passed in
|
||||
//
|
||||
// Returns errArrowAuthUnsupported if the configured credentials can't be
|
||||
// reused for the arrowlist client's pipeline.
|
||||
func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
|
||||
// The container URL includes any SAS token in its query
|
||||
url := f.cntSVC(containerName).URL()
|
||||
f.cntSVCcacheMu.Lock()
|
||||
defer f.cntSVCcacheMu.Unlock()
|
||||
if client, ok := f.arrowCntSVCcache[containerName]; ok {
|
||||
return client, nil
|
||||
}
|
||||
switch {
|
||||
case f.usingSharedKeyCred:
|
||||
// Covers account+key and the emulator (auth fills in Account/Key)
|
||||
var cred *arrowlist.SharedKeyCredential
|
||||
cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
|
||||
if err == nil {
|
||||
client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
|
||||
}
|
||||
case f.cred != nil:
|
||||
client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
|
||||
case f.anonymous || f.opt.SASURL != "":
|
||||
client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
|
||||
default:
|
||||
return nil, errArrowAuthUnsupported
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.arrowCntSVCcache[containerName] = client
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Return an Object from a path
|
||||
//
|
||||
// If it can't be found it returns the error fs.ErrorObjectNotFound.
|
||||
@@ -904,15 +778,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
|
||||
lowerKey := strings.ToLower(k)
|
||||
switch lowerKey {
|
||||
case "cache-control":
|
||||
headers.BlobCacheControl = new(v)
|
||||
headers.BlobCacheControl = pString(v)
|
||||
case "content-disposition":
|
||||
headers.BlobContentDisposition = new(v)
|
||||
headers.BlobContentDisposition = pString(v)
|
||||
case "content-encoding":
|
||||
headers.BlobContentEncoding = new(v)
|
||||
headers.BlobContentEncoding = pString(v)
|
||||
case "content-language":
|
||||
headers.BlobContentLanguage = new(v)
|
||||
headers.BlobContentLanguage = pString(v)
|
||||
case "content-type":
|
||||
headers.BlobContentType = new(v)
|
||||
headers.BlobContentType = pString(v)
|
||||
case "x-ms-tags":
|
||||
parsed, perr := parseXMsTags(v)
|
||||
if perr != nil {
|
||||
@@ -1186,77 +1060,19 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
delimiter = "/"
|
||||
}
|
||||
|
||||
opts := &arrowlist.ListBlobsHierarchyOptions{
|
||||
ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
|
||||
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
|
||||
Include: container.ListBlobsInclude{
|
||||
Copy: false,
|
||||
Metadata: true,
|
||||
Snapshots: false,
|
||||
UncommittedBlobs: false,
|
||||
Deleted: false,
|
||||
},
|
||||
Prefix: &directory,
|
||||
MaxResults: &maxResults,
|
||||
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &container.ListBlobsHierarchyOptions{
|
||||
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
|
||||
Include: container.ListBlobsInclude{
|
||||
Copy: false,
|
||||
Metadata: true,
|
||||
Snapshots: false,
|
||||
UncommittedBlobs: false,
|
||||
Deleted: false,
|
||||
},
|
||||
}
|
||||
// Experimental: request the Apache Arrow listing format. The arrowlist
|
||||
// pager requests an Arrow IPC stream and decodes it, falling back to XML
|
||||
// if the account doesn't have Arrow listing enabled. Skip the
|
||||
// maxResults==1 probe (isEmpty) which doesn't benefit.
|
||||
useArrow := f.opt.UseArrowList && maxResults != 1
|
||||
if useArrow {
|
||||
opts.UseArrowFormat = new(true)
|
||||
}
|
||||
|
||||
var foundItems int
|
||||
var err error
|
||||
if useArrow && f.opt.ListParallelism > 1 && !f.arrowXMLFallback.Load() {
|
||||
// Arrow exposes server-side startFrom/endBefore name ranges, so the
|
||||
// keyspace can be sharded and listed concurrently.
|
||||
foundItems, err = f.listArrowParallel(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
} else {
|
||||
foundItems, err = f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
|
||||
// Determine whether the directory exists or not by whether it has a marker
|
||||
_, err := f.readMetaData(ctx, containerName, directory)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// listBlobsPager runs the hierarchy listing described by opts, calling fn for
|
||||
// each blob and subdirectory, and returns the number of raw items seen.
|
||||
// delimiter selects flat (recurse) vs hierarchical listing. If opts requests
|
||||
// the Apache Arrow format the listing goes through the arrowlist pager
|
||||
// (falling back to the SDK pager if the credentials don't support it); if the
|
||||
// service then answers with XML (Arrow listing not enabled) a debug message
|
||||
// is logged.
|
||||
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
|
||||
useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
|
||||
var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
|
||||
if useArrow {
|
||||
arrowSVC, err := f.arrowCntSVC(containerName)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Not using Arrow listing: %v", err)
|
||||
useArrow = false
|
||||
} else {
|
||||
pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
|
||||
}
|
||||
}
|
||||
if pager == nil {
|
||||
pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
|
||||
}
|
||||
checkedArrow := false
|
||||
Prefix: &directory,
|
||||
MaxResults: &maxResults,
|
||||
})
|
||||
foundItems := 0
|
||||
for pager.More() {
|
||||
var response container.ListBlobsHierarchyResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -1266,22 +1082,12 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
// If Arrow was requested but the service answered with XML, Arrow
|
||||
// listing is not enabled on this account; the listing is still correct
|
||||
// but not accelerated.
|
||||
if useArrow && !checkedArrow && err == nil {
|
||||
checkedArrow = true
|
||||
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
|
||||
fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
|
||||
if storageErr, ok := err.(*azcore.ResponseError); ok && (storageErr.ErrorCode == string(bloberror.ContainerNotFound) || storageErr.StatusCode == http.StatusNotFound) {
|
||||
return foundItems, fs.ErrorDirNotFound
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return foundItems, err
|
||||
return err
|
||||
}
|
||||
// Advance marker to next
|
||||
// marker = response.NextMarker
|
||||
@@ -1301,12 +1107,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
fs.Debugf(f, "Odd name received %q", remote)
|
||||
continue
|
||||
}
|
||||
// ContentLength is documented as nullable in the Arrow listing schema
|
||||
size := int64(-1)
|
||||
if file.Properties.ContentLength != nil {
|
||||
size = *file.Properties.ContentLength
|
||||
}
|
||||
isDirectory := isDirectoryMarker(size, file.Metadata, remote)
|
||||
isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
|
||||
if isDirectory {
|
||||
// Don't insert the root directory
|
||||
if remote == f.opt.Enc.ToStandardPath(directory) {
|
||||
@@ -1322,7 +1123,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
// Send object
|
||||
err = fn(remote, file, isDirectory)
|
||||
if err != nil {
|
||||
return foundItems, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Send the subdirectories
|
||||
@@ -1337,13 +1138,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
fs.Debugf(f, "Odd directory name received %q", remote)
|
||||
continue
|
||||
}
|
||||
// Don't insert the root directory. The Arrow listing returns a
|
||||
// directory marker blob with the same name as the listed directory
|
||||
// as a BlobPrefix, whereas XML returns it as a blob (handled
|
||||
// above).
|
||||
if remote == f.opt.Enc.ToStandardPath(directory) {
|
||||
continue
|
||||
}
|
||||
remote = remote[len(prefix):]
|
||||
// Trim one slash off the remote name
|
||||
remote, _ = strings.CutSuffix(remote, "/")
|
||||
@@ -1356,11 +1150,21 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
// Send object
|
||||
err = fn(remote, nil, true)
|
||||
if err != nil {
|
||||
return foundItems, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return foundItems, nil
|
||||
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
|
||||
// Determine whether the directory exists or not by whether it has a marker
|
||||
_, err := f.readMetaData(ctx, containerName, directory)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert a list item into a DirEntry
|
||||
@@ -1826,7 +1630,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
|
||||
}
|
||||
|
||||
// Validity window
|
||||
start := time.Now().UTC().Add(-sasCopyStartSkew)
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(2 * sasCopyValidity)
|
||||
startStr := start.Format(time.RFC3339)
|
||||
expiryStr := expiry.Format(time.RFC3339)
|
||||
@@ -1875,7 +1679,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
|
||||
// Build the SAS values
|
||||
perms := sas.BlobPermissions{Read: true}
|
||||
container, containerPath := o.split()
|
||||
start := time.Now().UTC().Add(-sasCopyStartSkew)
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(sasCopyValidity)
|
||||
vals := sas.BlobSignatureValues{
|
||||
StartTime: start,
|
||||
@@ -2197,10 +2001,6 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.MD5 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// If decompressing, erase the hash
|
||||
if o.size < 0 {
|
||||
return "", nil
|
||||
}
|
||||
// Convert base64 encoded md5 into lower case hex
|
||||
if o.md5 == "" {
|
||||
return "", nil
|
||||
@@ -2326,13 +2126,6 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *blob.GetPropertiesRe
|
||||
o.accessTier = blob.AccessTier(*info.AccessTier)
|
||||
}
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2345,31 +2138,12 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
|
||||
} else {
|
||||
size = *info.ContentLength
|
||||
}
|
||||
// On a range request Content-Length is the length of the range, not the object, so take the
|
||||
// object's size from the total in Content-Range instead.
|
||||
if info.ContentRange != nil {
|
||||
contentRange, err := rest.ParseContentRange(*info.ContentRange)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
|
||||
} else if contentRange.Size >= 0 {
|
||||
size = contentRange.Size
|
||||
}
|
||||
}
|
||||
if isDirectoryMarker(size, metadata, o.remote) {
|
||||
return fs.ErrorNotAFile
|
||||
}
|
||||
// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
|
||||
// this as base64 encoded string.
|
||||
//
|
||||
// On a range request the service returns the whole-blob MD5 in the
|
||||
// x-ms-blob-content-md5 header (BlobContentMD5) and leaves Content-MD5
|
||||
// (ContentMD5) empty, so prefer BlobContentMD5. Never overwrite a known
|
||||
// hash with an empty one, or a ranged read would drop the object's MD5.
|
||||
if md5 := info.BlobContentMD5; len(md5) > 0 {
|
||||
o.md5 = base64.StdEncoding.EncodeToString(md5)
|
||||
} else if len(info.ContentMD5) > 0 {
|
||||
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
|
||||
}
|
||||
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
|
||||
if info.ContentType == nil {
|
||||
o.mimeType = ""
|
||||
} else {
|
||||
@@ -2388,12 +2162,21 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
|
||||
// o.accessTier = blob.AccessTier(*info.AccessTier)
|
||||
// }
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
// If it was a Range request, the size is wrong, so correct it
|
||||
if info.ContentRange != nil {
|
||||
contentRange := *info.ContentRange
|
||||
slash := strings.IndexRune(contentRange, '/')
|
||||
if slash >= 0 {
|
||||
i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
|
||||
if err == nil {
|
||||
o.size = i
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find length in %q", contentRange)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -2433,13 +2216,6 @@ func (o *Object) decodeMetaDataFromBlob(info *container.BlobItem) (err error) {
|
||||
o.accessTier = *info.Properties.AccessTier
|
||||
}
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.Properties.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2615,20 +2391,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode metadata for download: %w", err)
|
||||
}
|
||||
|
||||
// Decompress body if necessary
|
||||
if downloadResponse.ContentEncoding != nil && *downloadResponse.ContentEncoding == "gzip" {
|
||||
if o.fs.opt.Decompress {
|
||||
return readers.NewGzipReader(downloadResponse.Body)
|
||||
}
|
||||
o.fs.warnCompressed.Do(func() {
|
||||
fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --azureblob-decompress to override")
|
||||
})
|
||||
}
|
||||
|
||||
return downloadResponse.Body, nil
|
||||
}
|
||||
|
||||
// Converts a string into a pointer to a string
|
||||
func pString(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
|
||||
type readSeekCloser struct {
|
||||
io.Reader
|
||||
@@ -2783,7 +2553,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
// isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
|
||||
// returning true if it is found
|
||||
func isInvalidBlockOrBlob(err error) bool {
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
|
||||
}
|
||||
return false
|
||||
@@ -3155,7 +2926,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
|
||||
// Start with default content-type based on source
|
||||
ui.httpHeaders = blob.HTTPHeaders{
|
||||
BlobContentType: new(fs.MimeType(ctx, src)),
|
||||
BlobContentType: pString(fs.MimeType(ctx, src)),
|
||||
}
|
||||
|
||||
// Apply mapped metadata/headers/tags if requested
|
||||
@@ -3201,15 +2972,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
o.tags[parts[0]] = parts[1]
|
||||
}
|
||||
case "cache-control":
|
||||
ui.httpHeaders.BlobCacheControl = new(value)
|
||||
ui.httpHeaders.BlobCacheControl = pString(value)
|
||||
case "content-disposition":
|
||||
ui.httpHeaders.BlobContentDisposition = new(value)
|
||||
ui.httpHeaders.BlobContentDisposition = pString(value)
|
||||
case "content-encoding":
|
||||
ui.httpHeaders.BlobContentEncoding = new(value)
|
||||
ui.httpHeaders.BlobContentEncoding = pString(value)
|
||||
case "content-language":
|
||||
ui.httpHeaders.BlobContentLanguage = new(value)
|
||||
ui.httpHeaders.BlobContentLanguage = pString(value)
|
||||
case "content-type":
|
||||
ui.httpHeaders.BlobContentType = new(value)
|
||||
ui.httpHeaders.BlobContentType = pString(value)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -17,7 +14,6 @@ import (
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
@@ -54,58 +50,6 @@ func TestBlockIDCreator(t *testing.T) {
|
||||
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
|
||||
}
|
||||
|
||||
func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
|
||||
pInt64 := func(i int64) *int64 { return &i }
|
||||
pString := func(s string) *string { return &s }
|
||||
newTestObject := func() *Object {
|
||||
return &Object{
|
||||
fs: &Fs{},
|
||||
remote: "test.bin",
|
||||
size: -2, // sentinel to check it gets set
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("WholeBlob", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(12345)
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(12345), o.size)
|
||||
})
|
||||
|
||||
t.Run("RangeRequest", func(t *testing.T) {
|
||||
// On a ranged download Content-Length is the chunk length so the
|
||||
// size must come from the total in Content-Range
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("bytes 67108864-134217727/169721004032")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(169721004032), o.size)
|
||||
})
|
||||
|
||||
t.Run("BadContentRange", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("potato")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(67108864), o.size)
|
||||
})
|
||||
|
||||
t.Run("UnknownLength", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(-1), o.size)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCopySASTimingConstants(t *testing.T) {
|
||||
require.Greater(t, sasCopyStartSkew, time.Duration(0))
|
||||
require.Greater(t, sasCopyValidity, sasCopyStartSkew)
|
||||
}
|
||||
|
||||
func (f *Fs) testFeatures(t *testing.T) {
|
||||
// Check first feature flags are set on this remote
|
||||
enabled := f.Features().SetTier
|
||||
@@ -206,66 +150,10 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
|
||||
require.NoError(t, dst.Remove(ctx))
|
||||
}
|
||||
|
||||
func gz(t *testing.T, s string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
_, err := zw.Write([]byte(s))
|
||||
require.NoError(t, err)
|
||||
err = zw.Close()
|
||||
require.NoError(t, err)
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func md5sum(t *testing.T, s string) string {
|
||||
hash := md5.Sum([]byte(s))
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
func (f *Fs) testGzipEncoding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
original := random.String(1000)
|
||||
contents := gz(t, original)
|
||||
|
||||
item := fstest.NewItem("test-gzip", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
metadata := fs.Metadata{
|
||||
"content-encoding": "gzip",
|
||||
"content-type": "text/plain",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", metadata)
|
||||
defer func() {
|
||||
assert.NoError(t, obj.Remove(ctx))
|
||||
}()
|
||||
o := obj.(*Object)
|
||||
|
||||
// Test that the gzipped file we uploaded can be
|
||||
// downloaded with and without decompression
|
||||
checkDownload := func(wantContents string, wantSize int64, wantHash string) {
|
||||
gotContents := fstests.ReadObject(ctx, t, o, -1)
|
||||
assert.Equal(t, wantContents, gotContents)
|
||||
assert.Equal(t, wantSize, o.Size())
|
||||
gotHash, err := o.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantHash, gotHash)
|
||||
}
|
||||
|
||||
t.Run("NoDecompress", func(t *testing.T) {
|
||||
checkDownload(contents, int64(len(contents)), md5sum(t, contents))
|
||||
})
|
||||
t.Run("Decompress", func(t *testing.T) {
|
||||
f.opt.Decompress = true
|
||||
defer func() {
|
||||
f.opt.Decompress = false
|
||||
}()
|
||||
checkDownload(original, -1, "")
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Features", f.testFeatures)
|
||||
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
||||
t.Run("Metadata", f.testMetadataPaths)
|
||||
t.Run("GzipEncoding", f.testGzipEncoding)
|
||||
t.Run("ArrowList", f.testArrowList)
|
||||
}
|
||||
|
||||
// helper to read blob properties for an object
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || solaris || js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -58,9 +58,6 @@ const (
|
||||
maxFileSize = 4 * fs.Tebi
|
||||
defaultChunkSize = 4 * fs.Mebi
|
||||
storageDefaultBaseURL = "file.core.windows.net"
|
||||
// smbTimePrecision is the precision the server stores
|
||||
// modtimes with (100 ns FILETIME ticks)
|
||||
smbTimePrecision = 100 * time.Nanosecond
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -176,10 +173,6 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
options.ClientOptions = policyClientOptions
|
||||
// FileRequestIntent is required for TokenCredential (OAuth)
|
||||
// auth; the SDK omits it for shared-key / SAS.
|
||||
backup := service.ShareTokenIntentBackup
|
||||
options.FileRequestIntent = &backup
|
||||
},
|
||||
}
|
||||
res, err := auth.NewClient(ctx, conf, &opt.Options)
|
||||
@@ -253,10 +246,10 @@ func (f *Fs) Features() *fs.Features {
|
||||
|
||||
// Precision return the precision of this Fs
|
||||
//
|
||||
// The SMB last write time properties are ISO 8601 with 100 ns (FILETIME) precision
|
||||
// One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds
|
||||
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return smbTimePrecision
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
@@ -601,10 +594,7 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to set modTime: %w", err)
|
||||
}
|
||||
// Truncate to the precision the server stores the modtime with
|
||||
// to keep the in-memory modtime identical to the one a fresh
|
||||
// listing returns.
|
||||
o.modTime = t.Truncate(smbTimePrecision)
|
||||
o.modTime = t
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -650,6 +640,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
|
||||
@@ -691,9 +686,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
// Count the bytes read so short reads from the source can be detected
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
// Measure the size if it is unknown
|
||||
if sizeUnknown {
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
}
|
||||
|
||||
// Check we have a source MD5 hash...
|
||||
if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
|
||||
@@ -728,21 +725,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if sizeUnknown {
|
||||
// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
|
||||
size = int64(counter.BytesRead())
|
||||
} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
|
||||
// The file was created at the declared size so a short source
|
||||
// leaves the remainder padded with zeroes
|
||||
if isNewlyCreated {
|
||||
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
|
||||
fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
|
||||
}
|
||||
} else if bytesRead < size {
|
||||
// Truncate the file to the bytes actually received so it
|
||||
// is not left padded to the declared size with zeroes
|
||||
if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
|
||||
fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if hashUnknown {
|
||||
md5Hash = hasher.Sum(nil)
|
||||
@@ -783,12 +765,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return fmt.Errorf("update: failed to set properties: %w", err)
|
||||
}
|
||||
|
||||
// Make sure Object is in sync, truncating the modtime to the
|
||||
// precision the server stores it with so it is identical to the
|
||||
// one a fresh listing returns
|
||||
// Make sure Object is in sync
|
||||
o.size = size
|
||||
o.md5 = md5Hash
|
||||
o.modTime = modTime.Truncate(smbTimePrecision)
|
||||
o.modTime = modTime
|
||||
o.contentType = contentType
|
||||
return nil
|
||||
}
|
||||
@@ -813,8 +793,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
|
||||
}
|
||||
opt := file.RenameOptions{
|
||||
IgnoreReadOnly: new(true),
|
||||
ReplaceIfExists: new(true),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
ReplaceIfExists: ptr(true),
|
||||
}
|
||||
dstAbsPath := f.absPath(remote)
|
||||
fc := srcObj.fileClient()
|
||||
@@ -859,8 +839,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
opt := directory.RenameOptions{
|
||||
IgnoreReadOnly: new(false),
|
||||
ReplaceIfExists: new(false),
|
||||
IgnoreReadOnly: ptr(false),
|
||||
ReplaceIfExists: ptr(false),
|
||||
}
|
||||
dstAbsPath := dstFs.absPath(dstRemote)
|
||||
dirClient := srcFs.dirClient(srcRemote)
|
||||
@@ -899,8 +879,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
ChangeTime: file.SourceCopyFileChangeTime{},
|
||||
CreationTime: file.SourceCopyFileCreationTime{},
|
||||
LastWriteTime: file.SourceCopyFileLastWriteTime{},
|
||||
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: new(true),
|
||||
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
},
|
||||
}
|
||||
srcURL := srcObj.fileClient().URL()
|
||||
|
||||
+4
-11
@@ -45,7 +45,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/youmark/pkcs8"
|
||||
)
|
||||
@@ -1461,7 +1460,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
// Item in the cache has the same or newer SequenceID than
|
||||
// this event. Ignore this event, it must be old.
|
||||
f.itemMetaCacheMu.Unlock()
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1692,7 +1691,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// upload does a single non-multipart upload
|
||||
//
|
||||
// This is recommended for less than 50 MiB of content
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
upload := api.UploadFile{
|
||||
Name: o.fs.opt.Enc.FromStandardName(leaf),
|
||||
ContentModifiedAt: api.Time(modTime),
|
||||
@@ -1704,10 +1703,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
|
||||
var resp *http.Response
|
||||
var result api.FolderItems
|
||||
counter := readers.NewCountingReader(in)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Body: counter,
|
||||
Body: in,
|
||||
MultipartMetadataName: "attributes",
|
||||
MultipartContentName: "contents",
|
||||
MultipartFileName: upload.Name,
|
||||
@@ -1727,11 +1725,6 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload.
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
if result.TotalCount != 1 || len(result.Entries) != 1 {
|
||||
return fmt.Errorf("failed to upload %v - not sure why", o)
|
||||
}
|
||||
@@ -1761,7 +1754,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Upload with simple or multipart
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
|
||||
} else {
|
||||
err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
}
|
||||
|
||||
+15
-23
@@ -3,6 +3,7 @@
|
||||
package box
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
@@ -19,7 +20,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
|
||||
}
|
||||
|
||||
// uploadPart uploads a part in an upload session
|
||||
//
|
||||
// chunk must contain exactly the part's data and is read from the start
|
||||
// on each attempt.
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
hasher := sha1.New()
|
||||
chunkSize, err := io.Copy(hasher, chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
chunkSize := int64(len(chunk))
|
||||
sha1sum := sha1.Sum(chunk)
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/files/upload_sessions/" + SessionID,
|
||||
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
|
||||
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Digest": sha1Digest(hasher.Sum(nil)),
|
||||
"Digest": sha1Digest(sha1sum[:]),
|
||||
},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = chunk.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = wrap(chunk)
|
||||
opts.Body = wrap(bytes.NewReader(chunk))
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -223,25 +213,27 @@ outer:
|
||||
|
||||
reqSize := min(remaining, chunkSize)
|
||||
|
||||
// Read the chunk into memory from the global pool, hashing
|
||||
// it for the whole file hash (must be done sequentially)
|
||||
rw := multipart.NewRW().Reserve(reqSize)
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
|
||||
// Make a block of memory
|
||||
buf := make([]byte, reqSize)
|
||||
|
||||
// Read the chunk
|
||||
_, err = io.ReadFull(in, buf)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
err = fmt.Errorf("multipart upload failed to read source: %w", err)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Make the global hash (must be done sequentially)
|
||||
_, _ = hash.Write(buf)
|
||||
|
||||
// Transfer the chunk
|
||||
wg.Add(1)
|
||||
o.fs.uploadToken.Get()
|
||||
go func(part int, position int64) {
|
||||
defer wg.Done()
|
||||
defer o.fs.uploadToken.Put()
|
||||
defer func() { _ = rw.Close() }()
|
||||
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
|
||||
select {
|
||||
|
||||
Vendored
+5
-10
@@ -503,19 +503,15 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
|
||||
}
|
||||
|
||||
go func() {
|
||||
// Wait on the timer and the stop channel together so that a stop
|
||||
// signalled by StopBackgroundRunners is honoured immediately.
|
||||
timer := time.NewTimer(time.Duration(f.opt.ChunkCleanInterval))
|
||||
defer timer.Stop()
|
||||
for {
|
||||
time.Sleep(time.Duration(f.opt.ChunkCleanInterval))
|
||||
select {
|
||||
case <-f.cleanupChan:
|
||||
fs.Infof(f, "stopping cleanup")
|
||||
return
|
||||
case <-timer.C:
|
||||
default:
|
||||
fs.Debugf(f, "starting cleanup")
|
||||
f.CleanUpCache(false)
|
||||
timer.Reset(time.Duration(f.opt.ChunkCleanInterval))
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -567,10 +563,9 @@ Eg
|
||||
})
|
||||
|
||||
rc.Add(rc.Call{
|
||||
Path: "cache/stats",
|
||||
NoAuth: true,
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Path: "cache/stats",
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Help: `
|
||||
Show statistics for the cache remote.
|
||||
`,
|
||||
|
||||
@@ -43,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
|
||||
@@ -34,13 +34,6 @@ import (
|
||||
"github.com/zeebo/blake3"
|
||||
)
|
||||
|
||||
// Pacer settings - tuned to match other rclone backends
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Cloudinary shouldn't have a trailing dot if there is no path
|
||||
func cldPathDir(somePath string) string {
|
||||
if somePath == "" || somePath == "." {
|
||||
@@ -196,7 +189,7 @@ func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (f
|
||||
root: root,
|
||||
opt: *opt,
|
||||
cld: cld,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
|
||||
srv: rest.NewClient(client),
|
||||
}
|
||||
|
||||
@@ -472,8 +465,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
if updateOptions, ok := option.(*api.UpdateOptions); ok {
|
||||
if updateOptions.PublicID != "" {
|
||||
updateObject = true
|
||||
params.Overwrite = new(true)
|
||||
params.Invalidate = new(true)
|
||||
params.Overwrite = SDKApi.Bool(true)
|
||||
params.Invalidate = SDKApi.Bool(true)
|
||||
params.PublicID = updateOptions.PublicID
|
||||
params.ResourceType = updateOptions.ResourceType
|
||||
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
|
||||
@@ -593,18 +586,21 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
// Cloudinary returns rate-limit windows as "Try again on YYYY-MM-DD HH:MM:SS UTC"
|
||||
// embedded in the error message - honour it as a retry-after.
|
||||
if err != nil {
|
||||
tryAgain := "Try again on "
|
||||
if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
|
||||
layout := "2006-01-02 15:04:05 UTC"
|
||||
dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
|
||||
if timestamp, err2 := time.Parse(layout, dateStr); err2 == nil {
|
||||
timestamp, err2 := time.Parse(layout, dateStr)
|
||||
if err2 == nil {
|
||||
return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
|
||||
}
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "Retrying API error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
|
||||
@@ -1104,7 +1104,7 @@ func (o *Object) Remote() string {
|
||||
newPath, err := o.u.pathAdjustment.do(o.Object.String())
|
||||
if err != nil {
|
||||
fs.Errorf(o.Object, "Bad object: %v", err)
|
||||
return ""
|
||||
return err.Error()
|
||||
}
|
||||
return newPath
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -564,19 +563,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
|
||||
|
||||
// cache small files in memory and do normal upload
|
||||
rw := multipart.NewRW()
|
||||
defer fs.CheckClose(rw, &err)
|
||||
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
|
||||
if err == io.EOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, rw, src, options...)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
buf := make([]byte, f.opt.RAMCacheLimit)
|
||||
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
|
||||
}
|
||||
|
||||
// Need to include what we already read
|
||||
in = &ReadCloserWrapper{
|
||||
Reader: io.MultiReader(rw, in),
|
||||
Reader: io.MultiReader(bytes.NewReader(buf), in),
|
||||
Closer: in,
|
||||
}
|
||||
|
||||
@@ -703,16 +698,6 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared -
|
||||
// compressed data is stored under a name containing the
|
||||
// original size so a mismatch makes the object unreadable.
|
||||
if size := src.Size(); size >= 0 && meta.Size != size {
|
||||
if removeErr := dataObject.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
|
||||
|
||||
// meta data upload may fail. in this case we try to remove the original object
|
||||
@@ -1085,35 +1070,6 @@ type ObjectMetadata struct {
|
||||
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
|
||||
}
|
||||
|
||||
// validate checks metadata read from the wrapped remote before it is used to
|
||||
// construct a compressed reader.
|
||||
func (meta *ObjectMetadata) validate() error {
|
||||
if meta.Mode != Gzip {
|
||||
return nil
|
||||
}
|
||||
if meta.CompressionMetadataGzip == nil {
|
||||
return errors.New("missing gzip metadata")
|
||||
}
|
||||
if meta.CompressionMetadataGzip.BlockSize <= 0 {
|
||||
return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
|
||||
}
|
||||
if meta.Size != meta.CompressionMetadataGzip.Size {
|
||||
return errors.New("gzip metadata size does not match object size")
|
||||
}
|
||||
if meta.Size < 0 {
|
||||
return errors.New("invalid gzip object size")
|
||||
}
|
||||
blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
|
||||
blocks := meta.Size / blockSize
|
||||
if meta.Size%blockSize != 0 {
|
||||
blocks++
|
||||
}
|
||||
if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
|
||||
return errors.New("gzip block data is incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Object with external metadata
|
||||
type Object struct {
|
||||
fs.Object // Wraps around data object for this object
|
||||
@@ -1137,9 +1093,6 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
|
||||
if err = jr.Decode(meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = meta.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
@@ -1157,15 +1110,6 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return objErr
|
||||
}
|
||||
|
||||
// countingDiscard is an io.Writer which discards its input and counts
|
||||
// the bytes written
|
||||
type countingDiscard int64
|
||||
|
||||
func (c *countingDiscard) Write(p []byte) (int, error) {
|
||||
*c += countingDiscard(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
|
||||
type ReadCloserWrapper struct {
|
||||
io.Reader
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/buengese/sgzip"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
@@ -16,93 +14,6 @@ import (
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
func TestObjectMetadataValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
meta ObjectMetadata
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "nil gzip metadata",
|
||||
meta: ObjectMetadata{Mode: Gzip},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "zero block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 0,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "negative block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: -1,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "mismatched sizes",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 2048,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "short block data",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1, 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := test.meta.validate()
|
||||
if (err != nil) != test.want {
|
||||
t.Fatalf("validate() error = %v, want error: %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var defaultOpt = fstests.Opt{
|
||||
RemoteName: "TestCompress:",
|
||||
NilObject: (*Object)(nil),
|
||||
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type gzipModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
|
||||
// PutStream. It reads each body twice when it is seekable to model a
|
||||
// backend retrying an upload.
|
||||
type putRecorder struct {
|
||||
fs.Fs
|
||||
t *testing.T
|
||||
stream bool // expose PutStream from the wrapped Fs
|
||||
size int64
|
||||
seekable bool
|
||||
body []byte
|
||||
streamed bool
|
||||
}
|
||||
|
||||
func (r *putRecorder) Features() *fs.Features {
|
||||
features := *r.Fs.Features()
|
||||
features.PutStream = nil
|
||||
if r.stream {
|
||||
features.PutStream = r.PutStream
|
||||
}
|
||||
return &features
|
||||
}
|
||||
|
||||
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
|
||||
r.size = src.Size()
|
||||
body, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
r.seekable = true
|
||||
_, err = seeker.Seek(0, io.SeekStart)
|
||||
require.NoError(r.t, err)
|
||||
again, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
assert.Equal(r.t, body, again, "retried body should be identical")
|
||||
}
|
||||
r.body = body
|
||||
return body
|
||||
}
|
||||
|
||||
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
body := r.record(in, src)
|
||||
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
|
||||
}
|
||||
|
||||
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
r.streamed = true
|
||||
return r.Put(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// TestRcat checks the three upload paths rcat takes: small files buffered
|
||||
// in memory and uploaded with a known size and a seekable body, large
|
||||
// files streamed when the wrapped remote supports it, and large files
|
||||
// spooled to a temporary file when it doesn't. Pool pages taken for the
|
||||
// buffer must be returned on every path.
|
||||
func TestRcat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const limit = 1024
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
stream bool
|
||||
wantSize int64
|
||||
}{
|
||||
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
|
||||
{name: "Empty", size: 0, stream: true, wantSize: 0},
|
||||
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
|
||||
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
|
||||
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
|
||||
require.NoError(t, err)
|
||||
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
|
||||
f := &Fs{Fs: rec}
|
||||
f.opt.RAMCacheLimit = limit
|
||||
content := []byte(random.String(test.size))
|
||||
inUse := pool.Global().InUse()
|
||||
|
||||
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
|
||||
|
||||
assert.Equal(t, test.wantSize, rec.size)
|
||||
assert.Equal(t, content, rec.body)
|
||||
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
|
||||
if test.size < limit {
|
||||
assert.True(t, rec.seekable, "small file body should be seekable")
|
||||
}
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, content, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsCompressible checks the compressibility heuristic on data at
|
||||
// both ends of the scale for every compression mode which has one.
|
||||
func TestIsCompressible(t *testing.T) {
|
||||
compressible := bytes.Repeat([]byte("compress me "), 4096)
|
||||
incompressible := make([]byte, len(compressible))
|
||||
_, err := io.ReadFull(rand.Reader, incompressible)
|
||||
require.NoError(t, err)
|
||||
for name, handler := range map[string]compressionModeHandler{
|
||||
"gzip": &gzipModeHandler{},
|
||||
"zstd": &zstdModeHandler{},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type zstdModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
+8
-35
@@ -11,7 +11,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -487,12 +486,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
}
|
||||
|
||||
// encryptFileName encrypts a file path
|
||||
//
|
||||
// If stripVersion is set, a version string (as used by --b2-versions)
|
||||
// on the last segment is kept in plain text. Set it for file names
|
||||
// only: directory names must encrypt identically whether passed alone
|
||||
// or as the parent of a file name.
|
||||
func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
|
||||
func (c *Cipher) encryptFileName(in string) string {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
// Skip directory name encryption if the user chose to
|
||||
@@ -505,7 +499,7 @@ func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
|
||||
// of the file name gets encrypted/obfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -535,24 +529,19 @@ func (c *Cipher) EncryptFileName(in string) string {
|
||||
if c.mode == NameEncryptionOff {
|
||||
return in + c.encryptedSuffix
|
||||
}
|
||||
return c.encryptFileName(in, true)
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// EncryptDirName encrypts a directory path
|
||||
//
|
||||
// Unlike EncryptFileName, a version string on the last segment is
|
||||
// encrypted along with the rest of the name.
|
||||
func (c *Cipher) EncryptDirName(in string) string {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in
|
||||
}
|
||||
return c.encryptFileName(in, false)
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// decryptFileName decrypts a file path
|
||||
//
|
||||
// stripVersion is as for encryptFileName.
|
||||
func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
|
||||
func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
var err error
|
||||
@@ -566,7 +555,7 @@ func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
|
||||
// of the file name gets decrypted/deobfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -612,31 +601,15 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
|
||||
// Leave the version string on, if it was there
|
||||
return decrypted, nil
|
||||
}
|
||||
return c.decryptFileName(in, true)
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// DecryptDirName decrypts a directory path
|
||||
//
|
||||
// Unlike DecryptFileName, a version string on the last segment is
|
||||
// expected to be part of the encrypted name. Directory names created
|
||||
// by rclone before v1.76 had a version-like suffix left in plain
|
||||
// text; these are still decrypted so that they appear in listings,
|
||||
// but they can't be opened or removed until renamed on the
|
||||
// underlying remote.
|
||||
func (c *Cipher) DecryptDirName(in string) (string, error) {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in, nil
|
||||
}
|
||||
out, err := c.decryptFileName(in, false)
|
||||
if err != nil && version.Match(path.Base(in)) {
|
||||
var legacyErr error
|
||||
out, legacyErr = c.decryptFileName(in, true)
|
||||
if legacyErr == nil {
|
||||
fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// NameEncryptionMode returns the encryption mode in use for names
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -683,43 +682,6 @@ func TestNonStandardDecryptDirName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test directories whose name looks like it has a version string -
|
||||
// these encrypt verbatim, so that EncryptDirName agrees with the same
|
||||
// directory encrypted as the parent of a file name, whereas file names
|
||||
// keep the version string in plain text
|
||||
func TestVersionedDirName(t *testing.T) {
|
||||
const dir = "dir-v2001-02-03-040506-123"
|
||||
enc, err := NewNameEncoding("base32")
|
||||
require.NoError(t, err)
|
||||
for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
|
||||
c, err := newCipher(mode, "", "", true, enc)
|
||||
require.NoError(t, err)
|
||||
what := fmt.Sprintf("Testing mode=%v", mode)
|
||||
// Check EncryptDirName matches the parent of an encrypted file name
|
||||
encryptedDir := c.EncryptDirName(dir)
|
||||
encryptedFile := c.EncryptFileName(dir + "/file.txt")
|
||||
assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
|
||||
// Check the encrypted directory name round trips OK
|
||||
decryptedDir, err := c.DecryptDirName(encryptedDir)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// Check a file with the same name keeps its version string in plain text
|
||||
encryptedFile = c.EncryptFileName(dir)
|
||||
assert.NotEqual(t, encryptedDir, encryptedFile, what)
|
||||
assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
|
||||
decryptedFile, err := c.DecryptFileName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedFile, what)
|
||||
// Check a directory with a legacy encrypted name (the file form) still decrypts
|
||||
decryptedDir, err = c.DecryptDirName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// but a name which is bad in both forms is still an error
|
||||
_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
|
||||
assert.Error(t, err, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSize(t *testing.T) {
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
|
||||
for _, test := range []struct {
|
||||
|
||||
@@ -1179,11 +1179,6 @@ func (o *ObjectInfo) Size() int64 {
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
|
||||
// If the data is unchanged then the hash of the source is the
|
||||
// hash of the object which will be uploaded
|
||||
if o.f.opt.NoDataEncryption {
|
||||
return o.ObjectInfo.Hash(ctx, hash)
|
||||
}
|
||||
var srcObj fs.Object
|
||||
var ok bool
|
||||
// Get the underlying object if there is one
|
||||
|
||||
@@ -84,10 +84,6 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
|
||||
|
||||
// Test ObjectInfo.Hash
|
||||
wantHash := md5.Sum(outBuf.Bytes())
|
||||
if f.opt.NoDataEncryption {
|
||||
// If the data isn't encrypted, the hash should be that of the plaintext
|
||||
wantHash = md5.Sum([]byte(contents))
|
||||
}
|
||||
gotHash, err := src.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
|
||||
|
||||
@@ -21,7 +21,7 @@ type User struct {
|
||||
Avatar string `json:"avatar"`
|
||||
ModelType string `json:"model_type"`
|
||||
OwnsEntry bool `json:"owns_entry"`
|
||||
EntryPermissions any `json:"entry_permissions"`
|
||||
EntryPermissions []any `json:"entry_permissions"`
|
||||
DisplayName string `json:"display_name"`
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ type Item struct {
|
||||
FileHash any `json:"file_hash"`
|
||||
Public bool `json:"public"`
|
||||
Thumbnail bool `json:"thumbnail"`
|
||||
MuxStatus any `json:"mux_status"`
|
||||
ThumbnailURL any `json:"thumbnail_url"`
|
||||
WorkspaceID int `json:"workspace_id"`
|
||||
IsEncrypted int `json:"is_encrypted"`
|
||||
@@ -76,9 +77,9 @@ type Listing struct {
|
||||
Data []Item `json:"data"`
|
||||
From int `json:"from"`
|
||||
LastPage int `json:"last_page"`
|
||||
NextPageURL string `json:"next_page_url"`
|
||||
NextPage int `json:"next_page"`
|
||||
PerPage int `json:"per_page"`
|
||||
PrevPageURL string `json:"prev_page_url"`
|
||||
PrevPage int `json:"prev_page"`
|
||||
To int `json:"to"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
@@ -170,7 +171,7 @@ type MultiPartCreateRequest struct {
|
||||
Mime string `json:"mime"`
|
||||
Size int64 `json:"size"`
|
||||
Extension string `json:"extension"`
|
||||
ParentID json.Number `json:"parentId"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
@@ -221,7 +222,7 @@ type MultiPartEntriesRequest struct {
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
ClientExtension string `json:"clientExtension"`
|
||||
ParentID json.Number `json:"parentId"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
|
||||
+19
-90
@@ -54,7 +54,7 @@ const (
|
||||
rootURL = baseURL + "api/v1"
|
||||
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
|
||||
minChunkSize = fs.SizeSuffix(1024 * 1024 * 5)
|
||||
defaultUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024) // as per https://docs.drime.cloud/uploads-guide
|
||||
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -146,6 +146,14 @@ and these uploads do not fully utilize your bandwidth, then increasing
|
||||
this may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: `Cutoff for switching to chunked upload.
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -270,11 +278,6 @@ var retryErrorCodes = []int{
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
509, // Bandwidth Limit Exceeded
|
||||
520, // Cloudflare: Web server returns an unknown error
|
||||
521, // Cloudflare: Web server is down
|
||||
522, // Cloudflare: Connection timed out
|
||||
523, // Cloudflare: Origin is unreachable
|
||||
524, // Cloudflare: A timeout occurred
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
@@ -524,7 +527,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
if dirID != "" {
|
||||
opts.Parameters.Add("folderId", dirID)
|
||||
opts.Parameters.Add("parentIds", dirID)
|
||||
}
|
||||
if directoriesOnly {
|
||||
opts.Parameters.Add("type", api.ItemTypeFolder)
|
||||
@@ -562,10 +565,10 @@ OUTER:
|
||||
break OUTER
|
||||
}
|
||||
}
|
||||
if result.CurrentPage == result.LastPage {
|
||||
if result.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
page = result.CurrentPage + 1
|
||||
page = result.NextPage
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
@@ -706,11 +709,6 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
// If Drime returns 422 with invalid entry ID, the object is already gone.
|
||||
// Map this to a standard not-found error
|
||||
if strings.Contains(err.Error(), "entry ids is invalid") {
|
||||
return fs.ErrorObjectNotFound
|
||||
}
|
||||
return fmt.Errorf("failed to delete item: %w", err)
|
||||
}
|
||||
// Check the individual result codes also
|
||||
@@ -784,16 +782,10 @@ func (f *Fs) patch(ctx context.Context, id, attribute string, value string) (ite
|
||||
Name: value,
|
||||
}
|
||||
var result api.UpdateItemResponse
|
||||
// The drime origin returns a malformed response (seen by Cloudflare as
|
||||
// a 520) for a literal PUT to this endpoint, so use a POST with Laravel
|
||||
// method spoofing instead - the API routes it to the same handler.
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Method: "PUT",
|
||||
Path: "/file-entries/" + id,
|
||||
Parameters: url.Values{},
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-HTTP-Method-Override": "PUT",
|
||||
},
|
||||
}
|
||||
if f.opt.WorkspaceID != "" {
|
||||
opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
|
||||
@@ -927,48 +919,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return dstObj, nil
|
||||
}
|
||||
|
||||
// countDirChildren returns the number of direct children of dirID.
|
||||
func (f *Fs) countDirChildren(ctx context.Context, dirID string) (int, error) {
|
||||
n := 0
|
||||
_, err := f.listAll(ctx, dirID, false, false, "", func(*api.Item) bool {
|
||||
n++
|
||||
return false
|
||||
})
|
||||
return n, err
|
||||
}
|
||||
|
||||
// waitForDirChildren polls dirID until it lists at least want direct
|
||||
// children, or the timeout elapses. Drime's folder rename has visible
|
||||
// eventual-consistency on the children listing - the rename returns
|
||||
// success immediately but the renamed folder lists as empty for a
|
||||
// short window - so callers that rename a folder use this to wait for
|
||||
// the listing to settle before returning (see issue #9450).
|
||||
func (f *Fs) waitForDirChildren(ctx context.Context, dirID string, want int) error {
|
||||
const timeout = 30 * time.Second
|
||||
deadline := time.Now().Add(timeout)
|
||||
backoff := 100 * time.Millisecond
|
||||
for {
|
||||
got, err := f.countDirChildren(ctx, dirID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if got >= want {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("timed out waiting for directory listing to settle: got %d, want %d", got, want)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 2*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
// using server-side move operations.
|
||||
//
|
||||
@@ -989,25 +939,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
return err
|
||||
}
|
||||
|
||||
// Count children before the move so we can wait for the server's
|
||||
// children listing to reflect the rename - see waitForDirChildren.
|
||||
preCount, err := f.countDirChildren(ctx, srcID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DirMove: count source children: %w", err)
|
||||
}
|
||||
|
||||
// Do the move
|
||||
info, err := f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
_, err = f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if preCount > 0 {
|
||||
if err := f.waitForDirChildren(ctx, info.ID.String(), preCount); err != nil {
|
||||
fs.Logf(f, "DirMove: %v (continuing)", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
@@ -1160,11 +1096,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
// Send just the leaf as relativePath, matching the single-part /uploads
|
||||
// convention. The file is placed by parentId; sending an absolute path
|
||||
// here makes the server build folders from it and drop "0" path segments.
|
||||
relPath := f.opt.Enc.FromStandardName(leaf)
|
||||
|
||||
// Temporary Object under construction
|
||||
o := &Object{
|
||||
fs: f,
|
||||
@@ -1198,7 +1129,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
Size: createSize,
|
||||
Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
|
||||
ParentID: json.Number(directoryID),
|
||||
RelativePath: relPath,
|
||||
RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
@@ -1225,6 +1156,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
if ext == "" {
|
||||
ext = "bin"
|
||||
}
|
||||
rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
|
||||
|
||||
chunkWriter := &drimeChunkWriter{
|
||||
uploadID: resp.UploadID,
|
||||
key: resp.Key,
|
||||
@@ -1237,7 +1170,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
mime: mime,
|
||||
extension: ext,
|
||||
parentID: json.Number(directoryID),
|
||||
relativePath: relPath,
|
||||
relativePath: rel,
|
||||
}
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(chunkSize),
|
||||
@@ -1597,11 +1530,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Debugf(o, "Removing old object on successful upload")
|
||||
deleteErr := o.fs.deleteObject(ctx, id)
|
||||
if deleteErr != nil {
|
||||
if errors.Is(deleteErr, fs.ErrorObjectNotFound) {
|
||||
fs.Debugf(o, "Old object already deleted, safely ignoring")
|
||||
} else {
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
}
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
+15
-82
@@ -144,7 +144,6 @@ var (
|
||||
_importFormats map[string][]string // allowed import MIME type conversions
|
||||
templatesOnce sync.Once // parse link templates only once
|
||||
_linkTemplates map[string]*template.Template // available link types
|
||||
gdocsWarnOnce sync.Once // warn once about skipped non-exportable Google documents
|
||||
)
|
||||
|
||||
// rwChoices type for fs.Bits
|
||||
@@ -208,10 +207,7 @@ func driveOAuthOptions() []fs.Option {
|
||||
opts := []fs.Option{}
|
||||
for _, opt := range oauthutil.SharedOptions {
|
||||
if opt.Name == config.ConfigClientID {
|
||||
opt.Help = "Google Application Client Id\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nThe shared client_id is being retired and will stop working during 2026, so creating your own is now strongly recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own."
|
||||
}
|
||||
if opt.Name == config.ConfigClientSecret {
|
||||
opt.Help = "Google Application Client Secret\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nIf you created your own client_id then enter its client secret here."
|
||||
opt.Help = "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance."
|
||||
}
|
||||
opts = append(opts, opt)
|
||||
}
|
||||
@@ -225,7 +221,7 @@ func init() {
|
||||
Description: "Google Drive",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
@@ -233,7 +229,7 @@ func init() {
|
||||
return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
|
||||
}
|
||||
|
||||
switch configIn.State {
|
||||
switch config.State {
|
||||
case "":
|
||||
// Fill in the scopes
|
||||
driveConfig.Scopes = driveScopes(opt.Scope)
|
||||
@@ -244,43 +240,24 @@ func init() {
|
||||
}
|
||||
|
||||
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
|
||||
return fs.ConfigGoto("client_id")
|
||||
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
|
||||
OAuth2Config: driveConfig,
|
||||
})
|
||||
}
|
||||
return fs.ConfigGoto("teamdrive")
|
||||
case "client_id":
|
||||
if clientID, _ := m.Get(config.ConfigClientID); clientID != "" {
|
||||
return fs.ConfigGoto("oauth")
|
||||
}
|
||||
return oauthutil.SharedClientIDConfigConfirm("client_id_warning", "Google Drive", "https://rclone.org/drive/#making-your-own-client-id")
|
||||
case "client_id_warning":
|
||||
if configIn.Result == "true" {
|
||||
// Continue using the shared client_id
|
||||
return fs.ConfigGoto("oauth")
|
||||
}
|
||||
return fs.ConfigInput("client_id_set", config.ConfigClientID, "Google Application Client Id")
|
||||
case "client_id_set":
|
||||
m.Set(config.ConfigClientID, configIn.Result)
|
||||
return fs.ConfigInput("client_secret_set", config.ConfigClientSecret, "Google Application Client Secret")
|
||||
case "client_secret_set":
|
||||
m.Set(config.ConfigClientSecret, configIn.Result)
|
||||
return fs.ConfigGoto("oauth")
|
||||
case "oauth":
|
||||
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
|
||||
OAuth2Config: driveConfig,
|
||||
})
|
||||
case "teamdrive":
|
||||
if opt.TeamDriveID == "" {
|
||||
return fs.ConfigConfirm("teamdrive_ok", false, "config_change_team_drive", "Configure this as a Shared Drive (Team Drive)?\n")
|
||||
}
|
||||
return fs.ConfigConfirm("teamdrive_change", false, "config_change_team_drive", fmt.Sprintf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID))
|
||||
case "teamdrive_ok":
|
||||
if configIn.Result == "false" {
|
||||
if config.Result == "false" {
|
||||
m.Set("team_drive", "")
|
||||
return nil, nil
|
||||
}
|
||||
return fs.ConfigGoto("teamdrive_config")
|
||||
case "teamdrive_change":
|
||||
if configIn.Result == "false" {
|
||||
if config.Result == "false" {
|
||||
return nil, nil
|
||||
}
|
||||
return fs.ConfigGoto("teamdrive_config")
|
||||
@@ -301,14 +278,14 @@ func init() {
|
||||
return teamDrive.Id, teamDrive.Name
|
||||
})
|
||||
case "teamdrive_final":
|
||||
driveID := configIn.Result
|
||||
driveID := config.Result
|
||||
m.Set("team_drive", driveID)
|
||||
m.Set("root_folder_id", "")
|
||||
opt.TeamDriveID = driveID
|
||||
opt.RootFolderID = ""
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown state %q", configIn.State)
|
||||
return nil, fmt.Errorf("unknown state %q", config.State)
|
||||
},
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
System: systemMetadataInfo,
|
||||
@@ -1325,7 +1302,6 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
|
||||
return nil, fmt.Errorf("failed to create client from environment: %w", err)
|
||||
}
|
||||
} else {
|
||||
oauthutil.SharedClientIDWarning(name, "Google Drive", "https://rclone.org/drive/#making-your-own-client-id", m)
|
||||
oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create oauth client: %w", err)
|
||||
@@ -1710,9 +1686,6 @@ func (f *Fs) newObjectWithExportInfo(
|
||||
// If item MimeType is in the ExportFormats then it is a google doc
|
||||
if !isDocument {
|
||||
fs.Debugf(remote, "Ignoring unknown document type %q", info.MimeType)
|
||||
gdocsWarnOnce.Do(func() {
|
||||
fs.Logf(remote, "Skipping unexportable google document %q. Use --drive-show-all-gdocs to include them in server side copy and move", info.MimeType)
|
||||
})
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
if extension == "" {
|
||||
@@ -2439,40 +2412,12 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
|
||||
return newItem, nil
|
||||
}
|
||||
|
||||
// shortcutLoop reports whether a folder shortcut at remote pointing to
|
||||
// the directory targetID would create a recursive loop.
|
||||
//
|
||||
// This is the case when targetID is the ID of remote's parent or any
|
||||
// other ancestor directory, as descending into it would list an
|
||||
// ancestor again forever. The ancestor IDs are looked up in the
|
||||
// directory cache which is always populated for the parents of remote
|
||||
// as they are listed before their children.
|
||||
func (f *Fs) shortcutLoop(remote, targetID string) bool {
|
||||
for dir := path.Dir(remote); ; dir = path.Dir(dir) {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
if id, ok := f.dirCache.Get(dir); ok && actualID(id) == targetID {
|
||||
return true
|
||||
}
|
||||
if dir == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// itemToDirEntry converts a drive.File to an fs.DirEntry.
|
||||
// When the drive.File cannot be represented as an fs.DirEntry
|
||||
// (nil, nil) is returned.
|
||||
func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
|
||||
switch {
|
||||
case item.MimeType == driveFolderType:
|
||||
// A folder shortcut pointing at one of its own ancestors
|
||||
// would make rclone recurse forever, so drop it.
|
||||
if isShortcutID(item.Id) && f.shortcutLoop(remote, actualID(item.Id)) {
|
||||
fs.Errorf(remote, "Ignoring folder shortcut as it points to an ancestor directory creating a loop")
|
||||
return nil, nil
|
||||
}
|
||||
// cache the directory ID for later lookups
|
||||
f.dirCache.Put(remote, item.Id)
|
||||
// cache the resource key for later lookups
|
||||
@@ -2766,12 +2711,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
}
|
||||
trashedFiles := false
|
||||
if check {
|
||||
// trashedFiles is only consulted to decide between trashing and hard deleting
|
||||
// below, so when the directory is going to be trashed anyway there is no reason
|
||||
// to enumerate trashed children: let the server filter them out instead of
|
||||
// paging through them. Hard deletes still need to see them (#1040).
|
||||
includeAll := !f.opt.UseTrash || f.opt.TrashedOnly
|
||||
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, includeAll, func(item *drive.File) bool {
|
||||
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
|
||||
if !item.Trashed {
|
||||
fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
|
||||
return true
|
||||
@@ -3094,18 +3034,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
remote = remote[:len(remote)-len(ext)]
|
||||
}
|
||||
|
||||
// Find the ID of the parent to remove the file from.
|
||||
var srcParentID string
|
||||
if len(srcObj.parents) == 1 {
|
||||
srcParentID = srcObj.parents[0]
|
||||
} else {
|
||||
var err error
|
||||
_, srcParentID, err = srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcParentID = actualID(srcParentID)
|
||||
_, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcParentID = actualID(srcParentID)
|
||||
|
||||
// Temporary Object under construction
|
||||
dstInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
|
||||
|
||||
@@ -8,14 +8,10 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
stdsync "sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -28,114 +24,13 @@ import (
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
service, err := drive.NewService(
|
||||
context.Background(),
|
||||
option.WithHTTPClient(server.Client()),
|
||||
option.WithEndpoint(server.URL+"/"),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return &Fs{
|
||||
svc: service,
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
permissionsMu: new(stdsync.Mutex),
|
||||
permissions: make(map[string]*drive.Permission),
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionConcurrent(t *testing.T) {
|
||||
const requestCount = 2
|
||||
arrived := make(chan string, requestCount)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce stdsync.Once
|
||||
releaseRequests := func() {
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
}
|
||||
defer releaseRequests()
|
||||
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
arrived <- permissionID
|
||||
<-release
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
type permissionResult struct {
|
||||
permission *drive.Permission
|
||||
inherited bool
|
||||
err error
|
||||
}
|
||||
results := make(chan permissionResult, requestCount)
|
||||
permissionIDs := []string{"permission-1", "permission-2"}
|
||||
for _, permissionID := range permissionIDs {
|
||||
go func() {
|
||||
permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
|
||||
results <- permissionResult{permission: permission, inherited: inherited, err: err}
|
||||
}()
|
||||
}
|
||||
|
||||
arrivedIDs := make(map[string]bool, requestCount)
|
||||
timer := time.NewTimer(time.Second)
|
||||
defer timer.Stop()
|
||||
for len(arrivedIDs) < requestCount {
|
||||
select {
|
||||
case permissionID := <-arrived:
|
||||
arrivedIDs[permissionID] = true
|
||||
case <-timer.C:
|
||||
releaseRequests()
|
||||
t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
|
||||
}
|
||||
}
|
||||
releaseRequests()
|
||||
|
||||
assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
|
||||
returnedIDs := make(map[string]bool, requestCount)
|
||||
for range requestCount {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
require.NotNil(t, result.permission)
|
||||
assert.False(t, result.inherited)
|
||||
returnedIDs[result.permission.Id] = true
|
||||
}
|
||||
assert.Equal(t, arrivedIDs, returnedIDs)
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionCacheHit(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
|
||||
assert.Same(t, first, second)
|
||||
assert.Equal(t, int32(1), requests.Load())
|
||||
}
|
||||
|
||||
func TestDriveScopes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
@@ -250,34 +145,6 @@ func TestInternalFindExportFormat(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestShortcutLoop(t *testing.T) {
|
||||
f := &Fs{}
|
||||
f.dirCache = dircache.New("", "root", f)
|
||||
// root is FolderX with ID "X"
|
||||
f.dirCache.Put("", "X")
|
||||
// a real subfolder
|
||||
f.dirCache.Put("sub", "sub-id")
|
||||
// a nested subfolder reached via a shortcut, composite ID
|
||||
f.dirCache.Put("sub/nested", joinID("nested-id", "shortcut-id"))
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
remote string
|
||||
targetID string
|
||||
want bool
|
||||
}{
|
||||
{"shortcut to the root itself", "loop", "X", true},
|
||||
{"shortcut to the parent", "sub/loop", "sub-id", true},
|
||||
{"shortcut to a grandparent", "sub/nested/loop", "X", true},
|
||||
{"shortcut to a composite ancestor", "sub/nested/loop", "nested-id", true},
|
||||
{"shortcut to an unrelated folder", "sub/loop", "other-id", false},
|
||||
{"shortcut to a sibling", "sub/loop", "sub-id2", false},
|
||||
} {
|
||||
got := f.shortcutLoop(test.remote, test.targetID)
|
||||
assert.Equal(t, test.want, got, test.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMimeTypesToExtension(t *testing.T) {
|
||||
for mimeType, extension := range _mimeTypeToExtension {
|
||||
extensions, err := mime.ExtensionsByType(mimeType)
|
||||
@@ -814,83 +681,6 @@ func (f *Fs) InternalTestSingleQuoteFolder(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestIntegration/FsMkdir/FsPutFiles/Internal/MoveDuplicateParent
|
||||
//
|
||||
// Check that moving a file removes it from its real parent even when there
|
||||
// are two directories with the same name and the directory cache resolves the
|
||||
// path to the wrong duplicate. Otherwise the file ends up with two parents
|
||||
// which fails on shared drives with a teamDrivesParentLimit error (#9472).
|
||||
func (f *Fs) InternalTestMoveDuplicateParent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if f.Features().Move == nil {
|
||||
t.Skip("Move not supported")
|
||||
}
|
||||
|
||||
rootID, err := f.dirCache.RootID(ctx, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
const dupName = "dup9472"
|
||||
const dstName = "dst9472"
|
||||
const fileName = "michael.txt"
|
||||
|
||||
// Create the first "dup9472" directory - this is the one the directory
|
||||
// cache will resolve the path to.
|
||||
dirA, err := f.createDir(ctx, rootID, dupName, nil)
|
||||
require.NoError(t, err)
|
||||
f.dirCache.Put(dupName, dirA.Id)
|
||||
|
||||
// Create a second directory with the SAME name and parent: a duplicate
|
||||
// the directory cache does not know about.
|
||||
dirB, err := f.createDir(ctx, rootID, dupName, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Destination directory
|
||||
dirDst, err := f.createDir(ctx, rootID, dstName, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_ = f.delete(ctx, dirA.Id, false)
|
||||
_ = f.delete(ctx, dirB.Id, false)
|
||||
_ = f.delete(ctx, dirDst.Id, false)
|
||||
f.dirCache.Flush()
|
||||
}()
|
||||
|
||||
// Upload a file directly into dirB (the duplicate the cache doesn't know).
|
||||
createInfo := &drive.File{
|
||||
Name: fileName,
|
||||
Parents: []string{dirB.Id},
|
||||
}
|
||||
var fileInfo *drive.File
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
fileInfo, err = f.svc.Files.Create(createInfo).
|
||||
Media(strings.NewReader("9472 duplicate parent test")).
|
||||
Fields(f.getFileFields(ctx)).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{dirB.Id}, fileInfo.Parents)
|
||||
|
||||
// Sanity check: the directory cache resolves the path to dirA, not the
|
||||
// dirB the file actually lives in.
|
||||
gotDirID, err := f.dirCache.FindDir(ctx, dupName, false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, dirA.Id, gotDirID, "directory cache should resolve to the wrong duplicate")
|
||||
|
||||
// Build the source object pointing at the file in dirB and move it.
|
||||
srcObj, err := f.newObjectWithInfo(ctx, dupName+"/"+fileName, fileInfo)
|
||||
require.NoError(t, err)
|
||||
|
||||
dstObj, err := f.Move(ctx, srcObj, dstName+"/"+fileName)
|
||||
require.NoError(t, err)
|
||||
|
||||
// The moved file must have exactly one parent: the destination.
|
||||
movedInfo, err := f.getFile(ctx, dstObj.(fs.IDer).ID(), "id,parents")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{dirDst.Id}, movedInfo.Parents)
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
// These tests all depend on each other so run them as nested tests
|
||||
t.Run("DocumentImport", func(t *testing.T) {
|
||||
@@ -911,7 +701,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Query", f.InternalTestQuery)
|
||||
t.Run("AgeQuery", f.InternalTestAgeQuery)
|
||||
t.Run("SingleQuoteFolder", f.InternalTestSingleQuoteFolder)
|
||||
t.Run("MoveDuplicateParent", f.InternalTestMoveDuplicateParent)
|
||||
t.Run("ShouldRetry", f.InternalTestShouldRetry)
|
||||
}
|
||||
|
||||
|
||||
@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
|
||||
|
||||
// getPermission returns permissions for the fileID and permissionID passed in
|
||||
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
|
||||
f.permissionsMu.Lock()
|
||||
defer f.permissionsMu.Unlock()
|
||||
if useCache {
|
||||
f.permissionsMu.Lock()
|
||||
perm = f.permissions[permissionID]
|
||||
f.permissionsMu.Unlock()
|
||||
if perm != nil {
|
||||
return perm, false, nil
|
||||
}
|
||||
@@ -131,9 +131,7 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
|
||||
cleanPermission(perm)
|
||||
|
||||
// cache the permission
|
||||
f.permissionsMu.Lock()
|
||||
f.permissions[permissionID] = perm
|
||||
f.permissionsMu.Unlock()
|
||||
|
||||
return perm, inherited, err
|
||||
}
|
||||
|
||||
+18
-30
@@ -11,6 +11,7 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -21,7 +22,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
@@ -112,13 +112,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
|
||||
}
|
||||
|
||||
// Make an http.Request for the range passed in
|
||||
//
|
||||
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
|
||||
// to an io.Closer and close it after each attempt — the chunk buffer is
|
||||
// pool-backed, so an early close would free its pages and a retry would then
|
||||
// read a dead buffer. The upload loop owns the buffer's lifetime.
|
||||
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
|
||||
req.ContentLength = reqSize
|
||||
totalSize := "*"
|
||||
if rx.ContentLength >= 0 {
|
||||
@@ -173,40 +168,37 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
start := int64(0)
|
||||
var StatusCode int
|
||||
var err error
|
||||
chunkSize := int64(rx.f.opt.ChunkSize)
|
||||
buf := make([]byte, int(rx.f.opt.ChunkSize))
|
||||
for finished := false; !finished; {
|
||||
reqSize := chunkSize
|
||||
var reqSize int64
|
||||
var chunk io.ReadSeeker
|
||||
if rx.ContentLength >= 0 {
|
||||
// If size known use repeatable reader for smoother bwlimit
|
||||
if start >= rx.ContentLength {
|
||||
break
|
||||
}
|
||||
reqSize = min(rx.ContentLength-start, chunkSize)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool so reads are
|
||||
// repeatable for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, rx.Media, reqSize)
|
||||
if rx.ContentLength < 0 {
|
||||
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
|
||||
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
|
||||
} else {
|
||||
// If size unknown read into buffer
|
||||
var n int
|
||||
n, err = readers.ReadFill(rx.Media, buf)
|
||||
if err == io.EOF {
|
||||
// Send the last chunk with the correct ContentLength
|
||||
// otherwise Google doesn't know we've finished
|
||||
rx.ContentLength = start + n
|
||||
rx.ContentLength = start + int64(n)
|
||||
finished = true
|
||||
err = nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqSize = n
|
||||
}
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
reqSize = int64(n)
|
||||
chunk = bytes.NewReader(buf[:reqSize])
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
err = rx.f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
|
||||
again, err := rx.f.shouldRetry(ctx, err)
|
||||
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
|
||||
again = false
|
||||
@@ -214,13 +206,9 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
}
|
||||
return again, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
|
||||
start += reqSize
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
|
||||
// retried successfully. The chunk buffer is pool-backed and implements
|
||||
// io.Closer, so if it reaches the http transport unwrapped the transport
|
||||
// closes it after the failed attempt, returning its pages to the pool, and
|
||||
// the retry then reads a freed buffer.
|
||||
func TestResumableUploadRetry(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("resumable"), 512)
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
if attempts.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
client: server.Client(),
|
||||
}
|
||||
f.opt.ChunkSize = fs.SizeSuffix(len(content))
|
||||
rx := &resumableUpload{
|
||||
f: f,
|
||||
remote: "remote",
|
||||
URI: server.URL,
|
||||
Media: bytes.NewReader(content),
|
||||
MediaType: "application/octet-stream",
|
||||
ContentLength: int64(len(content)),
|
||||
}
|
||||
info, err := rx.Upload(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, info)
|
||||
assert.Equal(t, "fake-id", info.Id)
|
||||
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
|
||||
}
|
||||
@@ -19,7 +19,7 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
|
||||
Entries: items,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
complete, err = f.srv.UploadSessionFinishBatchV2Context(ctx, arg)
|
||||
complete, err = f.srv.UploadSessionFinishBatchV2(arg)
|
||||
if retry, err := shouldRetryExclude(ctx, err); !retry {
|
||||
return retry, err
|
||||
}
|
||||
|
||||
+115
-275
@@ -51,7 +51,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -151,20 +150,14 @@ var (
|
||||
|
||||
// Gets an oauth config with the right scopes
|
||||
func getOauthConfig(m configmap.Mapper) *oauthutil.Config {
|
||||
impersonate, _ := m.Get("impersonate")
|
||||
impersonateAdmin, _ := m.Get("impersonate_admin")
|
||||
|
||||
// If not impersonating, use standard scopes
|
||||
if impersonate == "" && impersonateAdmin == "" {
|
||||
if impersonate, _ := m.Get("impersonate"); impersonate == "" {
|
||||
return dropboxConfig
|
||||
}
|
||||
// Make a copy of the config
|
||||
config := *dropboxConfig
|
||||
// Make a copy of the scopes with extra scopes requires appended
|
||||
config.Scopes = append(config.Scopes, "team_data.member")
|
||||
if impersonate != "" {
|
||||
config.Scopes = append(config.Scopes, "members.read")
|
||||
}
|
||||
config.Scopes = append(config.Scopes, "members.read", "team_data.member")
|
||||
return &config
|
||||
}
|
||||
|
||||
@@ -213,29 +206,6 @@ You will have to use your own App (setting your own client_id and
|
||||
client_secret) to use this option as currently rclone's default set of
|
||||
permissions doesn't include "members.read". This can be added once
|
||||
v1.55 or later is in use everywhere.
|
||||
`,
|
||||
Default: "",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "impersonate_admin",
|
||||
Help: `Team admin ID to use when performing actions as a team administrator.
|
||||
|
||||
This sets the Dropbox-API-Select-Admin header with the given team
|
||||
member ID (for example "dbmid:...").
|
||||
|
||||
This takes a team member ID directly rather than an email address.
|
||||
|
||||
Note that if you want to use impersonate_admin, you should make sure this
|
||||
flag is set when running "rclone config" as this will cause rclone to
|
||||
request the "team_data.member" scope which it won't normally.
|
||||
|
||||
Using the "team_data.member" scope will require a Dropbox Team Admin
|
||||
to approve during the OAuth flow.
|
||||
|
||||
You will have to use your own App (setting your own client_id and
|
||||
client_secret) to use this option as currently rclone's default set of
|
||||
permissions doesn't include "team_data.member".
|
||||
`,
|
||||
Default: "",
|
||||
Advanced: true,
|
||||
@@ -268,36 +238,6 @@ See also --dropbox-root-namespace for an alternative way to work with shared
|
||||
folders.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_shared_folders",
|
||||
Help: `Instructs rclone to skip all shared folders.
|
||||
|
||||
When set, any folder that is a shared folder mount point will be
|
||||
excluded from directory listings, regardless of ownership.
|
||||
This is useful if you prefer to back up shared folders separately
|
||||
using a separate remote configured with the shared folder namespace.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_unowned_folders",
|
||||
Help: `Instructs rclone to skip shared folders not owned by the current user.
|
||||
|
||||
When set, any folder that is a shared folder mount point and not
|
||||
owned by the current user will be excluded from directory listings.
|
||||
This is useful when backing up multiple Dropbox accounts that share
|
||||
common folders, to avoid duplicating the shared data across accounts.
|
||||
|
||||
Note: In Dropbox Business, 'Team Folders' are owned by the Team.
|
||||
For standard team members, these folders evaluate as 'unowned'
|
||||
(editor/viewer access) and will be excluded by this flag. To back up
|
||||
Team Folders, do not use this flag or run the backup using a Team Admin
|
||||
account.
|
||||
|
||||
If --dropbox-skip-shared-folders is also enabled, this flag has no effect.
|
||||
|
||||
This makes an extra API call per shared folder mount point.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "pacer_min_sleep",
|
||||
Default: defaultMinSleep,
|
||||
@@ -362,41 +302,38 @@ will fail to download them.
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
Impersonate string `config:"impersonate"`
|
||||
ImpersonateAdmin string `config:"impersonate_admin"`
|
||||
SharedFiles bool `config:"shared_files"`
|
||||
SharedFolders bool `config:"shared_folders"`
|
||||
SkipSharedFolders bool `config:"skip_shared_folders"`
|
||||
SkipUnownedFolders bool `config:"skip_unowned_folders"`
|
||||
BatchMode string `config:"batch_mode"`
|
||||
BatchSize int `config:"batch_size"`
|
||||
BatchTimeout fs.Duration `config:"batch_timeout"`
|
||||
AsyncBatch bool `config:"async_batch"`
|
||||
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
RootNsid string `config:"root_namespace"`
|
||||
ExportFormats fs.CommaSepList `config:"export_formats"`
|
||||
SkipExports bool `config:"skip_exports"`
|
||||
ShowAllExports bool `config:"show_all_exports"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
Impersonate string `config:"impersonate"`
|
||||
SharedFiles bool `config:"shared_files"`
|
||||
SharedFolders bool `config:"shared_folders"`
|
||||
BatchMode string `config:"batch_mode"`
|
||||
BatchSize int `config:"batch_size"`
|
||||
BatchTimeout fs.Duration `config:"batch_timeout"`
|
||||
AsyncBatch bool `config:"async_batch"`
|
||||
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
RootNsid string `config:"root_namespace"`
|
||||
ExportFormats fs.CommaSepList `config:"export_formats"`
|
||||
SkipExports bool `config:"skip_exports"`
|
||||
ShowAllExports bool `config:"show_all_exports"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dropbox server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
srv files.ContextClient // the connection to the dropbox server
|
||||
svc files.ContextClient // the connection to the dropbox server (unauthorized)
|
||||
sharing sharing.ContextClient // as above, but for generating sharing links
|
||||
users users.ContextClient // as above, but for accessing user information
|
||||
team team.ContextClient // for the Teams API
|
||||
slashRoot string // root with "/" prefix
|
||||
slashRootSlash string // root with "/" prefix and postfix
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
ns string // The namespace we are using or "" for none
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
srv files.Client // the connection to the dropbox server
|
||||
svc files.Client // the connection to the dropbox server (unauthorized)
|
||||
sharing sharing.Client // as above, but for generating sharing links
|
||||
users users.Client // as above, but for accessing user information
|
||||
team team.Client // for the Teams API
|
||||
slashRoot string // root with "/" prefix, lowercase
|
||||
slashRootSlash string // root with "/" prefix and postfix, lowercase
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
ns string // The namespace we are using or "" for none
|
||||
batcher *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
|
||||
exportExts []exportExtension
|
||||
}
|
||||
@@ -578,7 +515,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
|
||||
// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
|
||||
f.team = team.NewContext(cfg)
|
||||
f.team = team.New(cfg)
|
||||
|
||||
if opt.Impersonate != "" {
|
||||
user := team.UserSelectorArg{
|
||||
@@ -589,7 +526,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
members := []*team.UserSelectorArg{&user}
|
||||
args := team.NewMembersGetInfoArgs(members)
|
||||
|
||||
memberIDs, err := f.team.MembersGetInfoContext(ctx, args)
|
||||
memberIDs, err := f.team.MembersGetInfo(args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
|
||||
}
|
||||
@@ -600,14 +537,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId
|
||||
}
|
||||
|
||||
if opt.ImpersonateAdmin != "" {
|
||||
cfg.AsAdminID = opt.ImpersonateAdmin
|
||||
}
|
||||
|
||||
f.srv = files.NewContext(cfg)
|
||||
f.svc = files.NewContext(ucfg)
|
||||
f.sharing = sharing.NewContext(cfg)
|
||||
f.users = users.NewContext(cfg)
|
||||
f.srv = files.New(cfg)
|
||||
f.svc = files.New(ucfg)
|
||||
f.sharing = sharing.New(cfg)
|
||||
f.users = users.New(cfg)
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: true,
|
||||
ReadMimeType: false,
|
||||
@@ -634,8 +567,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil // our root it empty so we probably want to list shared folders
|
||||
}
|
||||
|
||||
dir := path.Dir(f.root)
|
||||
if dir == "." {
|
||||
dir = f.root
|
||||
}
|
||||
|
||||
// root is not empty so we have find the right shared folder if it exists
|
||||
id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
|
||||
id, err := f.findSharedFolder(ctx, dir)
|
||||
if err != nil {
|
||||
// if we didn't find the specified shared folder we have to bail out here
|
||||
return nil, err
|
||||
@@ -669,7 +607,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// If root starts with / then use the actual root
|
||||
var acc *users.FullAccount
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
acc, err = f.users.GetCurrentAccountContext(ctx)
|
||||
acc, err = f.users.GetCurrentAccount()
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -689,7 +627,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
// See if the root is actually an object
|
||||
if f.root != "" {
|
||||
_, _, err = f.getFileMetadata(ctx, f.slashRoot)
|
||||
_, err = f.getFileMetadata(ctx, f.slashRoot)
|
||||
if err == nil {
|
||||
newRoot := path.Dir(f.root)
|
||||
if newRoot == "." {
|
||||
@@ -724,16 +662,15 @@ func (f *Fs) setRoot(root string) {
|
||||
}
|
||||
|
||||
type getMetadataResult struct {
|
||||
entry files.IsMetadata
|
||||
remoteIsExportPath bool
|
||||
notFound bool
|
||||
err error
|
||||
entry files.IsMetadata
|
||||
notFound bool
|
||||
err error
|
||||
}
|
||||
|
||||
// getMetadata gets the metadata for a file or directory
|
||||
func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
|
||||
res.err = f.pacer.Call(func() (bool, error) {
|
||||
res.entry, res.err = f.srv.GetMetadataContext(ctx, &files.GetMetadataArg{
|
||||
res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
|
||||
Path: f.opt.Enc.FromStandardPath(objPath),
|
||||
})
|
||||
return shouldRetry(ctx, res.err)
|
||||
@@ -776,7 +713,6 @@ func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportE
|
||||
ch <- getMetadataResult{notFound: true}
|
||||
return
|
||||
}
|
||||
res.remoteIsExportPath = true
|
||||
}
|
||||
|
||||
// Return our real result or error
|
||||
@@ -822,7 +758,7 @@ func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-ch
|
||||
}
|
||||
|
||||
// getFileMetadata gets the metadata for a file
|
||||
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, bool, error) {
|
||||
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
|
||||
var res getMetadataResult
|
||||
|
||||
// Try all possible metadatas
|
||||
@@ -831,7 +767,7 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
|
||||
res = <-ch
|
||||
|
||||
if res.err != nil {
|
||||
return nil, false, res.err
|
||||
return nil, res.err
|
||||
}
|
||||
if !res.notFound {
|
||||
break
|
||||
@@ -839,17 +775,17 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
|
||||
}
|
||||
|
||||
if res.notFound {
|
||||
return nil, false, fs.ErrorObjectNotFound
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
fileInfo, ok := res.entry.(*files.FileMetadata)
|
||||
if !ok {
|
||||
if _, ok = res.entry.(*files.FolderMetadata); ok {
|
||||
return nil, false, fs.ErrorIsDir
|
||||
return nil, fs.ErrorIsDir
|
||||
}
|
||||
return nil, false, fs.ErrorNotAFile
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
return fileInfo, res.remoteIsExportPath, nil
|
||||
return fileInfo, nil
|
||||
}
|
||||
|
||||
// getDirMetadata gets the metadata for a directory
|
||||
@@ -908,7 +844,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Limit: 100,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFoldersContext(ctx, &arg)
|
||||
res, err = f.sharing.ListFolders(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -920,7 +856,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFoldersContinueContext(ctx, &arg)
|
||||
res, err = f.sharing.ListFoldersContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -943,21 +879,13 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// sharedFolderName returns the shared folder name in root, which is its first
|
||||
// path component. root must be the trimmed root as produced by setRoot (no
|
||||
// leading slash).
|
||||
func sharedFolderName(root string) string {
|
||||
name, _, _ := strings.Cut(root, "/")
|
||||
return name
|
||||
}
|
||||
|
||||
// findSharedFolder find the id for a given shared folder name
|
||||
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
|
||||
// so our only option is to iterate over all shared folders
|
||||
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
|
||||
errFoundFile := errors.New("found file")
|
||||
err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
|
||||
if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
|
||||
if entry.(*fs.Dir).Remote() == name {
|
||||
id = entry.(*fs.Dir).ID()
|
||||
return errFoundFile
|
||||
}
|
||||
@@ -977,7 +905,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
|
||||
SharedFolderId: id,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.sharing.MountFolderContext(ctx, &arg)
|
||||
_, err := f.sharing.MountFolder(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -994,7 +922,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Limit: 100,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFilesContext(ctx, &arg)
|
||||
res, err = f.sharing.ListReceivedFiles(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1006,7 +934,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFilesContinueContext(ctx, &arg)
|
||||
res, err = f.sharing.ListReceivedFilesContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1014,12 +942,13 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
}
|
||||
}
|
||||
for _, entry := range res.Entries {
|
||||
entryPath := f.opt.Enc.ToStandardName(entry.Name)
|
||||
fmt.Printf("%+v\n", entry)
|
||||
entryPath := entry.Name
|
||||
o := &Object{
|
||||
fs: f,
|
||||
url: entry.PreviewUrl,
|
||||
remote: entryPath,
|
||||
modTime: time.Time(*entry.TimeInvited),
|
||||
modTime: *entry.TimeInvited,
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1039,7 +968,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
|
||||
errFoundFile := errors.New("found file")
|
||||
err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
|
||||
if strings.EqualFold(entry.(*Object).remote, name) {
|
||||
if entry.(*Object).remote == name {
|
||||
o = entry.(*Object)
|
||||
return errFoundFile
|
||||
}
|
||||
@@ -1113,7 +1042,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
arg.Path = "" // Specify root folder as empty string
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContext(ctx, arg)
|
||||
res, err = f.srv.ListFolder(arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1131,7 +1060,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
res, err = f.srv.ListFolderContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1159,26 +1088,6 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
|
||||
remote := path.Join(dir, leaf)
|
||||
if folderInfo != nil {
|
||||
if folderInfo.SharingInfo != nil && folderInfo.SharingInfo.SharedFolderId != "" {
|
||||
if f.opt.SkipSharedFolders {
|
||||
fs.Debugf(remote, "Skipping shared folder")
|
||||
continue
|
||||
}
|
||||
if f.opt.SkipUnownedFolders {
|
||||
var sfMeta *sharing.SharedFolderMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var apiErr error
|
||||
sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
|
||||
return shouldRetry(ctx, apiErr)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Errorf(remote, "Failed to get shared folder metadata (defaulting to include): %v", err)
|
||||
} else if sfMeta != nil && sfMeta.AccessType != nil && sfMeta.AccessType.Tag != sharing.AccessLevelOwner {
|
||||
fs.Debugf(remote, "Skipping unowned shared folder")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)
|
||||
err = list.Add(d)
|
||||
if err != nil {
|
||||
@@ -1255,7 +1164,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return cErr
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.CreateFolderV2Context(ctx, &arg2)
|
||||
_, err = f.srv.CreateFolderV2(&arg2)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -1273,41 +1182,34 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
encRoot := f.opt.Enc.FromStandardPath(root)
|
||||
|
||||
if check {
|
||||
// ListFolder reports a missing path and a path that is a file, so it
|
||||
// checks that the directory exists and is empty in a single request.
|
||||
// check directory exists
|
||||
_, err = f.getDirMetadata(ctx, root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Rmdir: %w", err)
|
||||
}
|
||||
|
||||
// check directory empty
|
||||
arg := files.NewListFolderArg(encRoot)
|
||||
arg.Recursive = false
|
||||
arg.Limit = 1
|
||||
if root == "/" {
|
||||
arg.Path = "" // Specify root folder as empty string
|
||||
}
|
||||
var res *files.ListFolderResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContext(ctx, arg)
|
||||
res, err = f.srv.ListFolder(arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
switch e := err.(type) {
|
||||
case files.ListFolderAPIError:
|
||||
if e.EndpointError != nil && e.EndpointError.Path != nil {
|
||||
switch e.EndpointError.Path.Tag {
|
||||
case files.LookupErrorNotFound:
|
||||
err = fs.ErrorDirNotFound
|
||||
case files.LookupErrorNotFolder:
|
||||
err = fs.ErrorIsFile
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Rmdir: %w", err)
|
||||
}
|
||||
if len(res.Entries) != 0 || res.HasMore {
|
||||
if len(res.Entries) != 0 {
|
||||
return errors.New("directory not empty")
|
||||
}
|
||||
}
|
||||
|
||||
// remove it
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.DeleteV2Context(ctx, &files.DeleteArg{Path: encRoot})
|
||||
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -1366,7 +1268,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
}
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.CopyV2Context(ctx, &arg)
|
||||
result, err = f.srv.CopyV2(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1427,7 +1329,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var err error
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.MoveV2Context(ctx, &arg)
|
||||
result, err = f.srv.MoveV2(&arg)
|
||||
switch e := err.(type) {
|
||||
case files.MoveV2APIError:
|
||||
// There seems to be a bit of eventual consistency here which causes this to
|
||||
@@ -1475,13 +1377,13 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
},
|
||||
}
|
||||
if expire < fs.DurationOff {
|
||||
expiryTime := dropbox.DBXTime(time.Now().Add(time.Duration(expire)).UTC().Round(time.Second))
|
||||
expiryTime := time.Now().Add(time.Duration(expire)).UTC().Round(time.Second)
|
||||
createArg.Settings.Expires = &expiryTime
|
||||
}
|
||||
|
||||
var linkRes sharing.IsSharedLinkMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -1490,7 +1392,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
fs.Debugf(absPath, "can't create link with expiry, trying without")
|
||||
createArg.Settings.Expires = nil
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
@@ -1504,7 +1406,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
}
|
||||
var listRes *sharing.ListSharedLinksResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
listRes, err = f.sharing.ListSharedLinksContext(ctx, &listArg)
|
||||
listRes, err = f.sharing.ListSharedLinks(&listArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1565,7 +1467,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
},
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.MoveV2Context(ctx, &arg)
|
||||
_, err = f.srv.MoveV2(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1579,7 +1481,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
var q *users.SpaceUsage
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
q, err = f.users.GetSpaceUsageContext(ctx)
|
||||
q, err = f.users.GetSpaceUsage()
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1666,7 +1568,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
|
||||
arg.Path = ""
|
||||
}
|
||||
|
||||
startCursor, err = f.srv.ListFolderGetLatestCursorContext(ctx, arg)
|
||||
startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
|
||||
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -1676,34 +1578,6 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
|
||||
return startCursor.Cursor, nil
|
||||
}
|
||||
|
||||
// trimPrefixFold returns s with the leading prefix removed, matching
|
||||
// case insensitively rune by rune so folded runes of differing UTF-8
|
||||
// length still match.
|
||||
//
|
||||
// If s is exactly the root that prefix names (prefix without its
|
||||
// trailing "/") it returns "". If prefix doesn't match it returns s
|
||||
// unchanged.
|
||||
func trimPrefixFold(s, prefix string) string {
|
||||
if strings.HasSuffix(prefix, "/") && strings.EqualFold(s, strings.TrimSuffix(prefix, "/")) {
|
||||
return ""
|
||||
}
|
||||
|
||||
offset := 0
|
||||
for prefix != "" {
|
||||
if offset >= len(s) {
|
||||
return s
|
||||
}
|
||||
sRune, sSize := utf8.DecodeRuneInString(s[offset:])
|
||||
prefixRune, prefixSize := utf8.DecodeRuneInString(prefix)
|
||||
if !strings.EqualFold(string(sRune), string(prefixRune)) {
|
||||
return s
|
||||
}
|
||||
offset += sSize
|
||||
prefix = prefix[prefixSize:]
|
||||
}
|
||||
return s[offset:]
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
|
||||
cursor := startCursor
|
||||
var res *files.ListFolderLongpollResult
|
||||
@@ -1727,7 +1601,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
Timeout: timeout,
|
||||
}
|
||||
|
||||
res, err = f.svc.ListFolderLongpollContext(ctx, &args)
|
||||
res, err = f.svc.ListFolderLongpoll(&args)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1750,7 +1624,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
Cursor: cursor,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
changeList, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
changeList, err = f.srv.ListFolderContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1763,18 +1637,17 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
switch info := entry.(type) {
|
||||
case *files.FolderMetadata:
|
||||
entryType = fs.EntryDirectory
|
||||
entryPath = info.PathDisplay
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
case *files.FileMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = info.PathDisplay
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
case *files.DeletedMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = info.PathDisplay
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
default:
|
||||
fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
|
||||
continue
|
||||
}
|
||||
entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
|
||||
|
||||
if entryPath != "" {
|
||||
notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
|
||||
@@ -1886,7 +1759,7 @@ func (o *Object) Size() int64 {
|
||||
return o.bytes
|
||||
}
|
||||
|
||||
func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPath bool) {
|
||||
func (o *Object) setMetadataForExport(info *files.FileMetadata) {
|
||||
o.bytes = -1
|
||||
o.hash = ""
|
||||
|
||||
@@ -1907,10 +1780,8 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
|
||||
o.exportType = exportExportable
|
||||
// get rid of any paper extension, if present
|
||||
o.remote = strings.TrimSuffix(o.remote, paperExtension)
|
||||
if !remoteIsExportPath {
|
||||
// add the export extension
|
||||
o.remote += "." + string(exportExt)
|
||||
}
|
||||
// add the export extension
|
||||
o.remote += "." + string(exportExt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1918,23 +1789,19 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
|
||||
//
|
||||
// This isn't a complete set of metadata and has an inaccurate date
|
||||
func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
|
||||
return o.setMetadataFromEntryWithExportPath(info, false)
|
||||
}
|
||||
|
||||
func (o *Object) setMetadataFromEntryWithExportPath(info *files.FileMetadata, remoteIsExportPath bool) error {
|
||||
o.id = info.Id
|
||||
o.bytes = int64(info.Size)
|
||||
o.modTime = time.Time(info.ClientModified)
|
||||
o.modTime = info.ClientModified
|
||||
o.hash = info.ContentHash
|
||||
|
||||
if !info.IsDownloadable {
|
||||
o.setMetadataForExport(info, remoteIsExportPath)
|
||||
o.setMetadataForExport(info)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reads the entry for a file from dropbox
|
||||
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, bool, error) {
|
||||
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
|
||||
return o.fs.getFileMetadata(ctx, o.remotePath())
|
||||
}
|
||||
|
||||
@@ -1944,11 +1811,11 @@ func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
|
||||
if !o.modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
entry, remoteIsExportPath, err := o.readEntry(ctx)
|
||||
entry, err := o.readEntry(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return o.setMetadataFromEntryWithExportPath(entry, remoteIsExportPath)
|
||||
return o.setMetadataFromEntry(entry)
|
||||
}
|
||||
|
||||
// Returns the remote path for the object
|
||||
@@ -2002,7 +1869,7 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
|
||||
arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
|
||||
var exportResult *files.ExportResult
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
exportResult, in, err = o.fs.srv.ExportContext(ctx, &arg)
|
||||
exportResult, in, err = o.fs.srv.Export(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2024,7 +1891,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
Url: o.url,
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.sharing.GetSharedLinkFileContext(ctx, &arg)
|
||||
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2044,7 +1911,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
ExtraHeaders: headers,
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.srv.DownloadContext(ctx, &arg)
|
||||
_, in, err = o.fs.srv.Download(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -2068,7 +1935,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
// start upload
|
||||
var res *files.UploadSessionStartResult
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
res, err = o.fs.srv.UploadSessionStartContext(ctx, &files.UploadSessionStartArg{}, nil)
|
||||
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2083,16 +1950,12 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
|
||||
// write chunks
|
||||
in := readers.NewCountingReader(in0)
|
||||
buf := make([]byte, int(chunkSize))
|
||||
cursor := files.UploadSessionCursor{
|
||||
SessionId: res.SessionId,
|
||||
Offset: 0,
|
||||
}
|
||||
appendArg := files.UploadSessionAppendArg{
|
||||
Cursor: &cursor,
|
||||
// A known-size upload which fits in a single chunk can close the
|
||||
// session with its only append, saving an empty append request
|
||||
Close: size >= 0 && size <= chunkSize,
|
||||
}
|
||||
appendArg := files.UploadSessionAppendArg{Cursor: &cursor}
|
||||
for currentChunk := 1; ; currentChunk++ {
|
||||
cursor.Offset = in.BytesRead()
|
||||
|
||||
@@ -2102,21 +1965,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
}
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
|
||||
skip := int64(0)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// seek to the start in case this is a retry
|
||||
if _, err = rw.Seek(skip, io.SeekStart); err != nil {
|
||||
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
|
||||
err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
|
||||
// after session is started, we retry everything
|
||||
if err != nil {
|
||||
// Check for incorrect offset error and retry with new offset
|
||||
@@ -2128,10 +1984,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
|
||||
if skip < 0 {
|
||||
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
|
||||
} else if skip == n {
|
||||
} else if skip == chunkSize {
|
||||
fs.Debugf(o, "%s: chunk received OK - continuing", what)
|
||||
return false, nil
|
||||
} else if skip > n {
|
||||
} else if skip > chunkSize {
|
||||
// This error should never happen
|
||||
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
|
||||
}
|
||||
@@ -2141,27 +1997,11 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
}
|
||||
}
|
||||
// Don't waste the low level retries if the context has
|
||||
// been cancelled
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
return err != nil, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if size >= 0 {
|
||||
// Check for sources which truncate early
|
||||
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
|
||||
if in.BytesRead() < expected {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
}
|
||||
if appendArg.Close {
|
||||
break
|
||||
}
|
||||
@@ -2191,7 +2031,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
entry, err = o.fs.srv.UploadSessionFinishContext(ctx, args, nil)
|
||||
entry, err = o.fs.srv.UploadSessionFinish(args, nil)
|
||||
if retry, err := shouldRetryExclude(ctx, err); !retry {
|
||||
return retry, err
|
||||
}
|
||||
@@ -2245,7 +2085,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
commitInfo := files.NewCommitInfo(o.fs.opt.Enc.FromStandardPath(o.remotePath()))
|
||||
commitInfo.Mode.Tag = "overwrite"
|
||||
// The Dropbox API only accepts timestamps in UTC with second precision.
|
||||
clientModified := dropbox.DBXTime(src.ModTime(ctx).UTC().Round(time.Second))
|
||||
clientModified := src.ModTime(ctx).UTC().Round(time.Second)
|
||||
commitInfo.ClientModified = &clientModified
|
||||
// Don't attempt to create filenames that are too long
|
||||
if cErr := checkPathLength(commitInfo.Path); cErr != nil {
|
||||
@@ -2259,7 +2099,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
|
||||
} else {
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
entry, err = o.fs.srv.UploadContext(ctx, &files.UploadArg{CommitInfo: *commitInfo}, in)
|
||||
entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
@@ -2271,7 +2111,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// This will only happen if we are uploading async batches
|
||||
if entry == nil {
|
||||
o.bytes = size
|
||||
o.modTime = time.Time(*commitInfo.ClientModified)
|
||||
o.modTime = *commitInfo.ClientModified
|
||||
o.hash = "" // we don't have this
|
||||
return nil
|
||||
}
|
||||
@@ -2284,7 +2124,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
return errNotSupportedInSharedMode
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = o.fs.srv.DeleteV2Context(ctx, &files.DeleteArg{
|
||||
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
|
||||
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
|
||||
})
|
||||
return shouldRetry(ctx, err)
|
||||
|
||||
@@ -2,155 +2,17 @@ package dropbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type paperMetadataClient struct {
|
||||
files.ContextClient
|
||||
info *files.FileMetadata
|
||||
}
|
||||
|
||||
func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
|
||||
if arg.Path == "document" {
|
||||
return c.info, nil
|
||||
}
|
||||
return nil, files.GetMetadataAPIError{
|
||||
APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
|
||||
EndpointError: &files.GetMetadataError{
|
||||
Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
|
||||
Path: &files.LookupError{
|
||||
Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetMetadataCancellation(t *testing.T) {
|
||||
requestStarted := make(chan struct{})
|
||||
releaseRequest := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(requestStarted)
|
||||
<-releaseRequest
|
||||
}))
|
||||
defer server.Close()
|
||||
defer close(releaseRequest)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
f := &Fs{
|
||||
srv: files.NewContext(dropbox.Config{
|
||||
Client: server.Client(),
|
||||
URLGenerator: func(hostType string, namespace string, route string) string {
|
||||
return server.URL
|
||||
},
|
||||
}),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
|
||||
}
|
||||
|
||||
result := make(chan getMetadataResult, 1)
|
||||
go func() {
|
||||
result <- f.getMetadata(ctx, "/file")
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-requestStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dropbox request did not start")
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case res := <-result:
|
||||
require.ErrorIs(t, res.err, context.Canceled)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dropbox request did not observe cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
type changeNotifyClient struct {
|
||||
files.ContextClient
|
||||
entries []files.IsMetadata
|
||||
}
|
||||
|
||||
func (c changeNotifyClient) ListFolderLongpollContext(context.Context, *files.ListFolderLongpollArg) (*files.ListFolderLongpollResult, error) {
|
||||
return files.NewListFolderLongpollResult(true), nil
|
||||
}
|
||||
|
||||
func (c changeNotifyClient) ListFolderContinueContext(context.Context, *files.ListFolderContinueArg) (*files.ListFolderResult, error) {
|
||||
return files.NewListFolderResult(c.entries, "next", false), nil
|
||||
}
|
||||
|
||||
func TestTrimPrefixFold(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path string
|
||||
prefix string
|
||||
want string
|
||||
}{
|
||||
{name: "exact casing", path: "/Docs/Sub", prefix: "/Docs/", want: "Sub"},
|
||||
{name: "different casing", path: "/docs/Sub", prefix: "/Docs/", want: "Sub"},
|
||||
{name: "different UTF-8 lengths", path: "/K/Sub", prefix: "/K/", want: "Sub"},
|
||||
{name: "exact root", path: "/docs", prefix: "/Docs/", want: ""},
|
||||
{name: "sibling boundary", path: "/Docs2/File", prefix: "/Docs/", want: "/Docs2/File"},
|
||||
{name: "root remote", path: "/File", prefix: "/", want: "File"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, trimPrefixFold(test.path, test.prefix))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeNotifyTrimsRootCaseInsensitively(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := changeNotifyClient{entries: []files.IsMetadata{
|
||||
&files.FolderMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub"}},
|
||||
&files.FileMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub/File.TXT"}},
|
||||
&files.DeletedMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Gone.md"}},
|
||||
}}
|
||||
f := &Fs{
|
||||
ci: fs.GetConfig(ctx),
|
||||
srv: client,
|
||||
svc: client,
|
||||
slashRootSlash: "/Docs/",
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
|
||||
}
|
||||
|
||||
type notification struct {
|
||||
path string
|
||||
entryType fs.EntryType
|
||||
}
|
||||
var notifications []notification
|
||||
cursor, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
|
||||
notifications = append(notifications, notification{path: path, entryType: entryType})
|
||||
}, "start")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "next", cursor)
|
||||
assert.Equal(t, []notification{
|
||||
{path: "Sub", entryType: fs.EntryDirectory},
|
||||
{path: "Sub/File.TXT", entryType: fs.EntryObject},
|
||||
{path: "Gone.md", entryType: fs.EntryObject},
|
||||
}, notifications)
|
||||
}
|
||||
|
||||
func TestInternalCheckPathLength(t *testing.T) {
|
||||
rep := func(n int, r rune) (out string) {
|
||||
rs := make([]rune, n)
|
||||
@@ -188,170 +50,6 @@ func TestInternalCheckPathLength(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalSharedFolderName(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
root string
|
||||
want string
|
||||
}{
|
||||
{root: "", want: ""},
|
||||
{root: "SharedFolder", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
|
||||
} {
|
||||
assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaperExportRemote(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
info := &files.FileMetadata{
|
||||
ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
|
||||
}
|
||||
f := &Fs{
|
||||
exportExts: []exportExtension{"md"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
|
||||
srv: paperMetadataClient{info: info},
|
||||
}
|
||||
|
||||
direct, err := f.NewObject(ctx, "document.md")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md", direct.Remote())
|
||||
|
||||
listed, err := f.newObjectWithInfo(ctx, "document.md", info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md.md", listed.Remote())
|
||||
|
||||
legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md", legacy.Remote())
|
||||
}
|
||||
|
||||
// uploadSessionClient is a mock files.ContextClient which records the
|
||||
// chunked upload calls made to it
|
||||
type uploadSessionClient struct {
|
||||
files.ContextClient
|
||||
appends int // number of UploadSessionAppendV2Context calls
|
||||
maxAppends int // fail the append after this many calls to stop runaway loops
|
||||
bytesWritten int64 // bytes received by UploadSessionAppendV2Context
|
||||
finishCalled bool // set if UploadSessionFinishContext was called
|
||||
appended func() // if set, called after each successful append
|
||||
}
|
||||
|
||||
var errTooManyAppends = errors.New("too many appends - upload looping?")
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
|
||||
return &files.UploadSessionStartResult{SessionId: "session"}, nil
|
||||
}
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
|
||||
// the real client fails the request if the context is cancelled
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
c.appends++
|
||||
if c.appends > c.maxAppends {
|
||||
return errTooManyAppends
|
||||
}
|
||||
n, err := io.Copy(io.Discard, content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.bytesWritten += n
|
||||
if c.appended != nil {
|
||||
c.appended()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
|
||||
c.finishCalled = true
|
||||
return &files.FileMetadata{}, nil
|
||||
}
|
||||
|
||||
// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
|
||||
func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
|
||||
srv: srv,
|
||||
}
|
||||
f.opt.ChunkSize = chunkSize
|
||||
batcherOptions := defaultBatcherOptions
|
||||
batcherOptions.Mode = "off"
|
||||
var err error
|
||||
f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
|
||||
// endlessReader supplies bytes forever
|
||||
type endlessReader struct{}
|
||||
|
||||
func (endlessReader) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = 'x'
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestUploadChunkedEarlyEOF(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("MultiChunk", func(t *testing.T) {
|
||||
// The declared size spans 4 chunks but the source ends after 1.5
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
||||
})
|
||||
|
||||
t.Run("SingleChunk", func(t *testing.T) {
|
||||
// The declared size fits in one chunk but the source ends early
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 500)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
||||
})
|
||||
|
||||
t.Run("Complete", func(t *testing.T) {
|
||||
// A source which supplies exactly the declared size uploads OK
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.True(t, client.finishCalled)
|
||||
assert.Equal(t, int64(250), client.bytesWritten)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUploadChunkedCancel(t *testing.T) {
|
||||
// Cancelling the context must stop the upload even though every
|
||||
// append is succeeding
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
client.appended = func() {
|
||||
if client.appends == 2 {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, endlessReader{}, files.NewCommitInfo("/test.bin"), -1)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.False(t, client.finishCalled)
|
||||
}
|
||||
|
||||
func (f *Fs) importPaperForTest(t *testing.T) {
|
||||
content := `# test doc
|
||||
|
||||
@@ -366,7 +64,7 @@ Lorem ipsum __dolor__ sit amet
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
reader := strings.NewReader(content)
|
||||
_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
|
||||
_, err = f.srv.PaperCreate(&arg, reader)
|
||||
return shouldRetry(context.Background(), err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -402,136 +100,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// newSharingTestFs builds a minimal *Fs whose sharing client talks to a
|
||||
// local mock server instead of the real Dropbox API.
|
||||
func newSharingTestFs(t *testing.T, handler http.HandlerFunc) *Fs {
|
||||
t.Helper()
|
||||
ts := httptest.NewServer(handler)
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
cfg := dropbox.Config{
|
||||
Client: &http.Client{Transport: http.DefaultTransport},
|
||||
URLGenerator: func(hostType, namespace, route string) string {
|
||||
return fmt.Sprintf("%s/2/%s/%s", ts.URL, namespace, route)
|
||||
},
|
||||
}
|
||||
|
||||
return &Fs{
|
||||
opt: Options{
|
||||
Enc: encoder.Base |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeDel |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8,
|
||||
},
|
||||
sharing: sharing.NewContext(cfg),
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond))),
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseDBXTime(t *testing.T, s string) *dropbox.DBXTime {
|
||||
t.Helper()
|
||||
tm, err := time.Parse(time.RFC3339, s)
|
||||
require.NoError(t, err)
|
||||
return (*dropbox.DBXTime)(&tm)
|
||||
}
|
||||
|
||||
func receivedFilesHandler(t *testing.T, rawName string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.URL.Path, "list_received_files") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
resp := sharing.ListFilesResult{
|
||||
Entries: []*sharing.SharedFileMetadata{{
|
||||
Id: "id:123",
|
||||
Name: rawName,
|
||||
PreviewUrl: "https://dropbox.com/preview/123",
|
||||
Policy: &sharing.FolderPolicy{},
|
||||
TimeInvited: mustParseDBXTime(t, "2024-01-01T00:00:00Z"),
|
||||
}},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListReceivedFilesDecodesName confirms received-file names are decoded
|
||||
// via Enc.ToStandardName before being returned, matching listSharedFolders'
|
||||
// handling of shared-folder names.
|
||||
func TestListReceivedFilesDecodesName(t *testing.T) {
|
||||
const rawName = "report.txt␠"
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
|
||||
|
||||
wantName := f.opt.Enc.ToStandardName(rawName)
|
||||
require.NotEqual(t, rawName, wantName)
|
||||
|
||||
var gotNames []string
|
||||
err := f.listReceivedFiles(context.Background(), func(entry fs.DirEntry) error {
|
||||
gotNames = append(gotNames, entry.Remote())
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, gotNames, 1)
|
||||
assert.Equal(t, wantName, gotNames[0])
|
||||
}
|
||||
|
||||
// TestFindSharedFileResolvesDecodedName confirms findSharedFile can look up
|
||||
// a received file by its standard, decoded rclone-visible name.
|
||||
func TestFindSharedFileResolvesDecodedName(t *testing.T) {
|
||||
const rawName = "report.txt␠"
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
|
||||
|
||||
encodedName := f.opt.Enc.ToStandardName(rawName)
|
||||
require.NotEqual(t, rawName, encodedName)
|
||||
|
||||
o, err := f.findSharedFile(context.Background(), encodedName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, encodedName, o.remote)
|
||||
}
|
||||
|
||||
// sharedFoldersHandler serves a single shared folder from list_folders.
|
||||
func sharedFoldersHandler(name, id string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.URL.Path, "list_folders") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
resp := sharing.ListFoldersResult{
|
||||
Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
}
|
||||
|
||||
// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
|
||||
// lookup must match regardless of the case of the requested name.
|
||||
func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
|
||||
f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))
|
||||
|
||||
for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
|
||||
id, err := f.findSharedFolder(context.Background(), name)
|
||||
require.NoError(t, err, name)
|
||||
assert.Equal(t, "folder-id", id, name)
|
||||
}
|
||||
|
||||
_, err := f.findSharedFolder(context.Background(), "no-such-folder")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
}
|
||||
|
||||
// The Dropbox backend advertises CaseInsensitive: true, so a received file
|
||||
// lookup must match regardless of the case of the requested name.
|
||||
func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))
|
||||
|
||||
for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
|
||||
o, err := f.findSharedFile(context.Background(), name)
|
||||
require.NoError(t, err, name)
|
||||
assert.Equal(t, "TestFile.txt", o.remote, name)
|
||||
}
|
||||
|
||||
_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
+17
-15
@@ -222,36 +222,38 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fldMap := map[string]bool{}
|
||||
for _, folder := range result.Result.Folders {
|
||||
fldMap[folder.FldID.String()] = true
|
||||
if f.root == "" && dir == "" && strings.Contains(folder.Path, "/") {
|
||||
continue
|
||||
}
|
||||
|
||||
folderPath := strings.Trim(folder.Path, "/")
|
||||
root := strings.Trim(f.root, "/")
|
||||
|
||||
remotePathWithoutRoot := folderPath
|
||||
|
||||
if root != "" {
|
||||
if remotePathWithoutRoot == root {
|
||||
continue
|
||||
}
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, root+"/")
|
||||
paths := strings.Split(folder.Path, fullPath+"/")
|
||||
remote := paths[0]
|
||||
if len(paths) > 1 {
|
||||
remote = paths[1]
|
||||
}
|
||||
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
|
||||
|
||||
if remotePathWithoutRoot == "" {
|
||||
if strings.Contains(remote, "/") {
|
||||
continue
|
||||
}
|
||||
|
||||
pathsWithoutRoot := strings.Split(folder.Path, "/"+f.root+"/")
|
||||
remotePathWithoutRoot := pathsWithoutRoot[0]
|
||||
if len(pathsWithoutRoot) > 1 {
|
||||
remotePathWithoutRoot = pathsWithoutRoot[1]
|
||||
}
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
|
||||
entries = append(entries, fs.NewDir(remotePathWithoutRoot, time.Now()))
|
||||
}
|
||||
|
||||
for _, file := range result.Result.Files {
|
||||
if _, ok := fldMap[file.FldID.String()]; ok {
|
||||
continue
|
||||
}
|
||||
remote := path.Join(dir, file.Name)
|
||||
// trim leading slashes
|
||||
remote = strings.TrimPrefix(remote, "/")
|
||||
|
||||
obj := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -10,11 +11,9 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
rclonemultipart "github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -26,9 +25,7 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
}
|
||||
|
||||
if dir != "" {
|
||||
if _, err := f.createFolder(ctx, dir); err != nil {
|
||||
return fmt.Errorf("failed to create multipart folder: %w", err)
|
||||
}
|
||||
_ = f.Mkdir(ctx, dir)
|
||||
}
|
||||
|
||||
folder := strings.Trim(dir, "/")
|
||||
@@ -48,32 +45,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
server := initResp.Result.Server
|
||||
objectPath := initResp.Result.ObjectPath
|
||||
|
||||
chunkSize := int64(f.opt.ChunkSize)
|
||||
if chunkSize <= 0 {
|
||||
return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)
|
||||
}
|
||||
for partNo := 1; ; partNo++ {
|
||||
// Buffer the part in memory from the global pool so it can be
|
||||
// re-sent on retry
|
||||
rw := rclonemultipart.NewRW()
|
||||
n, err := io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("read failed: %w", err)
|
||||
}
|
||||
chunkSize := int(f.opt.ChunkSize)
|
||||
buf := make([]byte, 0, chunkSize)
|
||||
tmp := make([]byte, 1024*1024)
|
||||
partNo := 1
|
||||
|
||||
for {
|
||||
n, errRead := in.Read(tmp)
|
||||
if n > 0 {
|
||||
uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)
|
||||
if uploadErr != nil {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)
|
||||
buf = append(buf, tmp[:n]...)
|
||||
|
||||
// If buffer reached chunkSize, upload a full part
|
||||
if len(buf) >= chunkSize {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
partNo++
|
||||
buf = buf[:0]
|
||||
}
|
||||
}
|
||||
if closeErr := rw.Close(); closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if err == io.EOF {
|
||||
|
||||
if errRead == io.EOF {
|
||||
break
|
||||
}
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read failed: %w", errRead)
|
||||
}
|
||||
}
|
||||
|
||||
if len(buf) > 0 {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
|
||||
@@ -84,40 +89,31 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadPart sends a single multipart chunk of size bytes to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: server,
|
||||
Parameters: url.Values{
|
||||
"partNumber": {strconv.Itoa(partNo)},
|
||||
"uploadId": {uploadID},
|
||||
},
|
||||
Body: r,
|
||||
ContentLength: &size,
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-RC-Upload-Id": uploadID,
|
||||
"X-RC-Part-No": strconv.Itoa(partNo),
|
||||
"X-Sess-ID": sessID,
|
||||
"X-Object-Path": objectPath,
|
||||
},
|
||||
NoResponse: true,
|
||||
// uploadPart sends a single multipart chunk to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
|
||||
url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
// rewind the pooled buffer so each attempt sends the whole chunk
|
||||
_, err := r.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)
|
||||
}
|
||||
return shouldRetry(err), err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
req.Header.Set("X-RC-Upload-Id", uploadID)
|
||||
req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
|
||||
req.Header.Set("X-Sess-ID", sessID)
|
||||
req.Header.Set("X-Object-Path", objectPath)
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("uploadPart failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFile uploads a file to FileLu
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -142,8 +141,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
// Update updates the object with new data
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
size := src.Size()
|
||||
counter := readers.NewCountingReader(in)
|
||||
in = counter
|
||||
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err := o.fs.uploadFile(ctx, in, o.remote)
|
||||
@@ -158,15 +155,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
if removeErr := o.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
o.size = size
|
||||
o.modTime = src.ModTime(ctx)
|
||||
return nil
|
||||
|
||||
@@ -12,8 +12,5 @@ func TestIntegration(t *testing.T) {
|
||||
RemoteName: "TestFileLu:",
|
||||
NilObject: nil,
|
||||
SkipInvalidUTF8: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: 1,
|
||||
},
|
||||
})
|
||||
}
|
||||
+4
-38
@@ -21,11 +21,9 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -49,7 +47,7 @@ func init() {
|
||||
},
|
||||
{
|
||||
Name: "api_key",
|
||||
Help: `API Key for your Filen account
|
||||
Help: `API Key for your Filen account
|
||||
|
||||
Get this using the Filen CLI export-api-key command
|
||||
You can download the Filen CLI from https://github.com/FilenCloudDienste/filen-cli`,
|
||||
@@ -163,7 +161,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
filen: filen,
|
||||
Enc: opt.Encoder,
|
||||
concurrency: opt.UploadConcurrency,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
|
||||
fileSystem.features = (&fs.Features{
|
||||
@@ -197,13 +194,6 @@ type Options struct {
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
}
|
||||
|
||||
// defaults copied from box
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Fs represents a virtual filesystem mounted on a specific root folder
|
||||
type Fs struct {
|
||||
name string
|
||||
@@ -212,7 +202,6 @@ type Fs struct {
|
||||
Enc encoder.MultiEncoder
|
||||
features *fs.Features
|
||||
concurrency int
|
||||
pacer *fs.Pacer
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
@@ -362,7 +351,6 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
type chunkWriter struct {
|
||||
sdk.FileUpload
|
||||
filen *sdk.Filen
|
||||
pacer *fs.Pacer
|
||||
bucketAndRegion chan client.V3UploadResponse
|
||||
chunkSize int64
|
||||
|
||||
@@ -374,13 +362,6 @@ type chunkWriter struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
|
||||
func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
|
||||
realChunkNumber := int64(chunkNumber) * (cw.chunkSize) / sdk.ChunkSize
|
||||
chunk := make([]byte, sdk.ChunkSize, sdk.ChunkSize+cw.EncryptionKey.Cipher.Overhead())
|
||||
@@ -433,15 +414,7 @@ func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader i
|
||||
if err != nil {
|
||||
return totalWritten, err
|
||||
}
|
||||
var resp *client.V3UploadResponse
|
||||
err = cw.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
resp, err := cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
|
||||
if err != nil {
|
||||
return totalWritten, err
|
||||
}
|
||||
@@ -521,7 +494,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, &chunkWriter{
|
||||
FileUpload: *fu,
|
||||
filen: f.filen,
|
||||
pacer: f.pacer,
|
||||
chunkSize: chunkSize,
|
||||
bucketAndRegion: make(chan client.V3UploadResponse, 1),
|
||||
knownChunks: make(map[int64][]byte),
|
||||
@@ -710,10 +682,7 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
// The server stores modtimes with millisecond precision so round
|
||||
// here too to keep the in-memory modtime identical to the one a
|
||||
// fresh listing returns.
|
||||
o.file.LastModified = t.Round(time.Millisecond)
|
||||
o.file.LastModified = t
|
||||
return o.fs.filen.UpdateMeta(ctx, o.file)
|
||||
}
|
||||
|
||||
@@ -755,10 +724,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Logf(option, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
// The server stores modtimes with millisecond precision so round
|
||||
// here too to keep the in-memory modtime identical to the one a
|
||||
// fresh listing returns.
|
||||
newModTime := src.ModTime(ctx).Round(time.Millisecond)
|
||||
newModTime := src.ModTime(ctx)
|
||||
newIncomplete, err := o.file.NewFromBase(o.fs.filen.FileEncryptionVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"github.com/Files-com/files-sdk-go/v3/file"
|
||||
file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
|
||||
"github.com/Files-com/files-sdk-go/v3/folder"
|
||||
files_sdk_lib "github.com/Files-com/files-sdk-go/v3/lib"
|
||||
"github.com/Files-com/files-sdk-go/v3/session"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
@@ -177,13 +176,6 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Errors from the upload storage servers are of this type
|
||||
var httpErr files_sdk_lib.ResponseError
|
||||
if errors.As(err, &httpErr) && slices.Contains(retryErrorCodes, httpErr.StatusCode) {
|
||||
fs.Debugf(nil, "Retrying HTTP error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
|
||||
@@ -353,7 +345,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
|
||||
for it.Next() {
|
||||
item := new(it.File())
|
||||
item := ptr(it.File())
|
||||
remote := f.opt.Enc.ToStandardPath(item.DisplayName)
|
||||
remote = path.Join(dir, remote)
|
||||
if remote == dir {
|
||||
@@ -424,7 +416,7 @@ func (f *Fs) mkdir(ctx context.Context, path string) error {
|
||||
|
||||
params := files_sdk.FolderCreateParams{
|
||||
Path: path,
|
||||
MkdirParents: new(true),
|
||||
MkdirParents: ptr(true),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -466,7 +458,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
|
||||
params := files_sdk.FileDeleteParams{
|
||||
Path: path,
|
||||
Recursive: new(!check),
|
||||
Recursive: ptr(!check),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -537,7 +529,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.
|
||||
params := files_sdk.FileCopyParams{
|
||||
Path: srcPath,
|
||||
Destination: dstPath,
|
||||
Overwrite: new(true),
|
||||
Overwrite: ptr(true),
|
||||
}
|
||||
|
||||
var action files_sdk.FileAction
|
||||
@@ -680,7 +672,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
Paths: []string{f.absPath(remote)},
|
||||
}
|
||||
if expire < fs.DurationOff {
|
||||
params.ExpiresAt = new(time.Now().Add(time.Duration(expire)))
|
||||
params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
|
||||
}
|
||||
|
||||
var bundle files_sdk.Bundle
|
||||
@@ -842,6 +834,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
func isFolderNotEmpty(err error) bool {
|
||||
var re files_sdk.ResponseError
|
||||
ok := errors.As(err, &re)
|
||||
@@ -861,7 +858,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
file.UploadWithProvidedMtime(src.ModTime(ctx)),
|
||||
}
|
||||
|
||||
err := o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
err := o.fs.fileClient.Upload(uploadOpts...)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -869,24 +866,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return err
|
||||
}
|
||||
|
||||
// The server computes the MD5 asynchronously after upload so
|
||||
// retry reading the metadata for a short time until it appears.
|
||||
const maxTries = 10
|
||||
for tries := 1; ; tries++ {
|
||||
err = o.readMetaData(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if o.md5 != "" || tries >= maxTries {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(tries) * 100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return o.readMetaData(ctx)
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
|
||||
+4
-23
@@ -637,18 +637,6 @@ func (f *Fs) drainPool(ctx context.Context) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
// commandEncoding hardens the user-configured filename encoding so that it
|
||||
// can never restore a raw CR or LF.
|
||||
//
|
||||
// The FTP control channel is line oriented and the ftp library writes command
|
||||
// arguments (paths) straight onto it without escaping, so a filename
|
||||
// containing CR/LF would otherwise be able to inject an independent FTP
|
||||
// command. CR/LF are therefore always encoded to safe symbols regardless of
|
||||
// the configured encoding, which is what the default encoding already does.
|
||||
func commandEncoding(enc encoder.MultiEncoder) encoder.MultiEncoder {
|
||||
return enc | encoder.EncodeCrLf
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
|
||||
// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
|
||||
@@ -658,7 +646,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opt.Enc = commandEncoding(opt.Enc)
|
||||
pass := ""
|
||||
if opt.AskPassword && opt.Pass == "" {
|
||||
pass = config.GetPassword("FTP server password")
|
||||
@@ -1248,10 +1235,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
path = o.fs.opt.Enc.FromStandardPath(path)
|
||||
err = c.SetTime(path, modTime.In(time.UTC))
|
||||
if err == nil && o.info != nil {
|
||||
// The server stores modtimes with second precision so
|
||||
// truncate here too to keep the in-memory modtime identical
|
||||
// to the one a fresh listing returns.
|
||||
o.info.ModTime = modTime.Truncate(time.Second)
|
||||
o.info.ModTime = modTime
|
||||
o.info.precise = true
|
||||
}
|
||||
o.fs.putFtpConnection(&c, err)
|
||||
@@ -1406,12 +1390,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
o.fs.putFtpConnection(&c, nil)
|
||||
if o.fs.opt.NoCheckUpload {
|
||||
o.info = &FileInfo{
|
||||
Name: o.remote,
|
||||
Size: uint64(src.Size()),
|
||||
// The server stores modtimes with second precision so
|
||||
// truncate here too to keep the in-memory modtime
|
||||
// identical to the one a fresh listing returns.
|
||||
ModTime: src.ModTime(ctx).Truncate(time.Second),
|
||||
Name: o.remote,
|
||||
Size: uint64(src.Size()),
|
||||
ModTime: src.ModTime(ctx),
|
||||
precise: true,
|
||||
IsDir: false,
|
||||
}
|
||||
|
||||
@@ -12,38 +12,15 @@ import (
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A filename containing raw CR/LF must never survive encoding into an FTP
|
||||
// command argument, whatever encoding the user configured, otherwise it could
|
||||
// inject an independent FTP command onto the line-oriented control channel.
|
||||
func TestCommandEncodingNeutralisesCRLF(t *testing.T) {
|
||||
// Internal Standard-encoded name as produced by e.g. a Unix local
|
||||
// source, carrying raw CR/LF.
|
||||
for _, std := range []string{
|
||||
"victim\r\nDELE other-secret\r\nNOOP",
|
||||
"victim␍␊DELE other-secret␍␊NOOP",
|
||||
} {
|
||||
for _, configured := range []encoder.MultiEncoder{
|
||||
encoder.EncodeSlash, // "Slash" - omits Ctl and CrLf
|
||||
encoder.EncodeZero, // "None"
|
||||
encoder.Display | encoder.EncodeRightSpace, // the FTP default
|
||||
} {
|
||||
got := commandEncoding(configured).FromStandardPath(std)
|
||||
assert.NotContains(t, got, "\r", "encoding %v leaked raw CR: %q", configured, got)
|
||||
assert.NotContains(t, got, "\n", "encoding %v leaked raw LF: %q", configured, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
@@ -75,21 +52,13 @@ func (f *Fs) testUploadTimeout(t *testing.T) {
|
||||
ci.Timeout = saveTimeout
|
||||
}()
|
||||
ci.LowLevelRetries = 1
|
||||
ci.Timeout = fs.Duration(idleTimeout)
|
||||
|
||||
upload := func(concurrency int, shutTimeout time.Duration) (obj fs.Object, err error) {
|
||||
// Use the saved (long) timeout while NewFs dials the FTP control
|
||||
// connection and reads the welcome banner. On slow CI servers this
|
||||
// can take longer than idleTimeout, which would cause a spurious
|
||||
// i/o timeout before the test can run. The connection's idle
|
||||
// deadline is captured at dial time, so the data connection
|
||||
// dialled later inside Put will still get idleTimeout - which is
|
||||
// what the test is actually trying to exercise.
|
||||
ci.Timeout = saveTimeout
|
||||
fixFs := deriveFs(ctx, t, f, settings{
|
||||
"concurrency": concurrency,
|
||||
"shut_timeout": shutTimeout,
|
||||
})
|
||||
ci.Timeout = fs.Duration(idleTimeout)
|
||||
|
||||
// Make test object
|
||||
fileTime := fstest.Time("2020-03-08T09:30:00.000000000Z")
|
||||
|
||||
@@ -194,7 +194,8 @@ var retryErrorCodes = []int{
|
||||
|
||||
// Return true if the api error has the status given
|
||||
func isAPIErr(err error, status string) bool {
|
||||
if apiErr, ok := errors.AsType[api.Error](err); ok {
|
||||
var apiErr api.Error
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.Status == status
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -94,23 +94,6 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
// gphotosOAuthOptions returns a copy of oauthutil.SharedOptions with
|
||||
// the client_id help tailored for Google Photos. It copies the shared
|
||||
// options rather than mutating the global slice.
|
||||
func gphotosOAuthOptions() []fs.Option {
|
||||
opts := []fs.Option{}
|
||||
for _, opt := range oauthutil.SharedOptions {
|
||||
if opt.Name == config.ConfigClientID {
|
||||
opt.Help = "OAuth Client Id.\n\nCreating your own is now strongly recommended.\nIf you leave this blank rclone uses a shared client_id which is being retired and will stop working during 2026.\nSee https://rclone.org/googlephotos/#making-your-own-client-id for how to create your own."
|
||||
}
|
||||
if opt.Name == config.ConfigClientSecret {
|
||||
opt.Help = "OAuth Client Secret.\n\nLeave blank to use rclone's shared client_id.\nIf you created your own client_id then enter its client secret here."
|
||||
}
|
||||
opts = append(opts, opt)
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
@@ -118,7 +101,7 @@ func init() {
|
||||
Prefix: "gphotos",
|
||||
Description: "Google Photos",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
@@ -126,7 +109,7 @@ func init() {
|
||||
return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
|
||||
}
|
||||
|
||||
switch configIn.State {
|
||||
switch config.State {
|
||||
case "":
|
||||
// Fill in the scopes
|
||||
if opt.ReadOnly {
|
||||
@@ -134,25 +117,6 @@ func init() {
|
||||
} else {
|
||||
oauthConfig.Scopes = scopesReadWrite
|
||||
}
|
||||
return fs.ConfigGoto("client_id")
|
||||
case "client_id":
|
||||
if clientID, _ := m.Get(config.ConfigClientID); clientID != "" {
|
||||
return fs.ConfigGoto("oauth")
|
||||
}
|
||||
return oauthutil.SharedClientIDConfigConfirm("client_id_warning", "Google Photos", "https://rclone.org/googlephotos/#making-your-own-client-id")
|
||||
case "client_id_warning":
|
||||
if configIn.Result == "true" {
|
||||
// Continue using the shared client_id
|
||||
return fs.ConfigGoto("oauth")
|
||||
}
|
||||
return fs.ConfigInput("client_id_set", config.ConfigClientID, "OAuth Client Id")
|
||||
case "client_id_set":
|
||||
m.Set(config.ConfigClientID, configIn.Result)
|
||||
return fs.ConfigInput("client_secret_set", config.ConfigClientSecret, "OAuth Client Secret")
|
||||
case "client_secret_set":
|
||||
m.Set(config.ConfigClientSecret, configIn.Result)
|
||||
return fs.ConfigGoto("oauth")
|
||||
case "oauth":
|
||||
return oauthutil.ConfigOut("warning1", &oauthutil.Options{
|
||||
OAuth2Config: oauthConfig,
|
||||
})
|
||||
@@ -172,9 +136,9 @@ IMPORTANT: Due to Google policy changes rclone can now only download photos it u
|
||||
case "warning_done":
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown state %q", configIn.State)
|
||||
return nil, fmt.Errorf("unknown state %q", config.State)
|
||||
},
|
||||
Options: append(append(gphotosOAuthOptions(), []fs.Option{{
|
||||
Options: append(append(oauthutil.SharedOptions, []fs.Option{{
|
||||
Name: "read_only",
|
||||
Default: false,
|
||||
Help: `Set to make the Google Photos backend read only.
|
||||
@@ -385,8 +349,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
oauthutil.SharedClientIDWarning(name, "Google Photos", "https://rclone.org/googlephotos/#making-your-own-client-id", m)
|
||||
|
||||
baseClient := fshttp.NewClient(ctx)
|
||||
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
|
||||
if err != nil {
|
||||
|
||||
@@ -69,66 +69,12 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {
|
||||
_ = operations.Purge(ctx, f, dirName)
|
||||
}
|
||||
|
||||
func (f *Fs) testUpdateStoresHash(t *testing.T) {
|
||||
// make a temporary local remote
|
||||
tempRoot, err := fstest.LocalRemote()
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
_ = os.RemoveAll(tempRoot)
|
||||
}()
|
||||
|
||||
// make a temporary crypt remote as source
|
||||
ctx := context.Background()
|
||||
pass := obscure.MustObscure("crypt")
|
||||
remote := fmt.Sprintf(`:crypt,remote="%s",password="%s":`, tempRoot, pass)
|
||||
cryptFs, err := fs.NewFs(ctx, remote)
|
||||
require.NoError(t, err)
|
||||
|
||||
const dirName = "update_hash_1"
|
||||
const fileName = dirName + "/file_update_1"
|
||||
const longTime = fs.ModTimeNotSupported
|
||||
hashType := f.keepHashes.GetOne()
|
||||
|
||||
// upload initial file to hasher via Put
|
||||
src1 := putFile(ctx, t, cryptFs, fileName, "initial content")
|
||||
in1, err := src1.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
dst, err := f.Put(ctx, in1, src1)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, dst)
|
||||
|
||||
// verify hash was stored after Put
|
||||
var hash1 string
|
||||
if f.opt.MaxAge > 0 {
|
||||
hash1, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, hash1)
|
||||
}
|
||||
|
||||
// update the file with new content via Update (this is what sync does for replacements)
|
||||
src2 := putFile(ctx, t, cryptFs, fileName, "updated content")
|
||||
in2, err := src2.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
err = dst.Update(ctx, in2, src2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// verify hash was stored after Update and is different from the original
|
||||
if f.opt.MaxAge > 0 {
|
||||
hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, hash2, "hash should be stored after Update")
|
||||
assert.NotEqual(t, hash1, hash2, "hash should change after Update with different content")
|
||||
}
|
||||
_ = operations.Purge(ctx, f, dirName)
|
||||
}
|
||||
|
||||
// InternalTest dispatches all internal tests
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
if !kv.Supported() {
|
||||
t.Skip("hasher is not supported on this OS")
|
||||
}
|
||||
t.Run("UploadFromCrypt", f.testUploadFromCrypt)
|
||||
t.Run("UpdateStoresHash", f.testUpdateStoresHash)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
+41
-57
@@ -131,62 +131,10 @@ func (o *Object) updateHashes(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// wrapInHashingReader wraps in with a hashing reader if needed and
|
||||
// returns the (possibly wrapped) reader. After the transfer completes
|
||||
// successfully, call the returned storeHashes function to store the
|
||||
// computed hashes.
|
||||
func (f *Fs) wrapInHashingReader(ctx context.Context, in io.Reader, src fs.ObjectInfo) (wrapIn io.Reader, storeHashes func(o *Object)) {
|
||||
var (
|
||||
common hash.Set
|
||||
rehash bool
|
||||
hashes hashMap
|
||||
)
|
||||
if fsrc := src.Fs(); fsrc != nil {
|
||||
common = fsrc.Hashes().Overlap(f.keepHashes)
|
||||
// Rehash if source does not have all required hashes or hashing is slow
|
||||
rehash = fsrc.Features().SlowHash || common != f.keepHashes
|
||||
}
|
||||
|
||||
wrapIn = in
|
||||
if rehash {
|
||||
r, err := f.newHashingReader(ctx, in, func(sums hashMap) {
|
||||
hashes = sums
|
||||
})
|
||||
fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err)
|
||||
if err == nil {
|
||||
wrapIn = r
|
||||
} else {
|
||||
rehash = false
|
||||
}
|
||||
}
|
||||
|
||||
storeHashes = func(o *Object) {
|
||||
if !rehash {
|
||||
hashes = hashMap{}
|
||||
for _, ht := range common.Array() {
|
||||
if h, e := src.Hash(ctx, ht); e == nil && h != "" {
|
||||
hashes[ht] = h
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(hashes) > 0 {
|
||||
err := o.putHashes(ctx, hashes)
|
||||
fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err)
|
||||
}
|
||||
}
|
||||
return wrapIn, storeHashes
|
||||
}
|
||||
|
||||
// Update the object with the given data, time and size.
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
wrapIn, storeHashes := o.f.wrapInHashingReader(ctx, in, src)
|
||||
_ = o.f.pruneHash(src.Remote())
|
||||
err := o.Object.Update(ctx, wrapIn, src, options...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
storeHashes(o)
|
||||
return nil
|
||||
return o.Object.Update(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// Remove an object.
|
||||
@@ -241,15 +189,51 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (r io.ReadC
|
||||
|
||||
// Put data into the remote path with given modTime and size
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
wrapIn, storeHashes := f.wrapInHashingReader(ctx, in, src)
|
||||
var (
|
||||
o fs.Object
|
||||
common hash.Set
|
||||
rehash bool
|
||||
hashes hashMap
|
||||
)
|
||||
if fsrc := src.Fs(); fsrc != nil {
|
||||
common = fsrc.Hashes().Overlap(f.keepHashes)
|
||||
// Rehash if source does not have all required hashes or hashing is slow
|
||||
rehash = fsrc.Features().SlowHash || common != f.keepHashes
|
||||
}
|
||||
|
||||
wrapIn := in
|
||||
if rehash {
|
||||
r, err := f.newHashingReader(ctx, in, func(sums hashMap) {
|
||||
hashes = sums
|
||||
})
|
||||
fs.Debugf(src, "Rehash in-fly due to incomplete or slow source set %v (err: %v)", common, err)
|
||||
if err == nil {
|
||||
wrapIn = r
|
||||
} else {
|
||||
rehash = false
|
||||
}
|
||||
}
|
||||
|
||||
_ = f.pruneHash(src.Remote())
|
||||
oResult, err := f.Fs.Put(ctx, wrapIn, src, options...)
|
||||
o, err := f.wrapObject(oResult, err)
|
||||
o, err = f.wrapObject(oResult, err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storeHashes(o.(*Object))
|
||||
return o, nil
|
||||
|
||||
if !rehash {
|
||||
hashes = hashMap{}
|
||||
for _, ht := range common.Array() {
|
||||
if h, e := src.Hash(ctx, ht); e == nil && h != "" {
|
||||
hashes[ht] = h
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(hashes) > 0 {
|
||||
err := o.(*Object).putHashes(ctx, hashes)
|
||||
fs.Debugf(o, "Applied %d source hashes, err: %v", len(hashes), err)
|
||||
}
|
||||
return o, err
|
||||
}
|
||||
|
||||
type hashingReader struct {
|
||||
|
||||
@@ -50,10 +50,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Chtimes sets the modtime with second precision so truncate
|
||||
// here too to keep the in-memory modtime identical to the one a
|
||||
// fresh listing returns.
|
||||
o.modTime = modTime.Truncate(time.Second)
|
||||
o.modTime = modTime
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+61
-73
@@ -10,6 +10,7 @@ package hidrive
|
||||
// to be resolved prior to execution with resolvePath(...).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -23,9 +24,8 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/ranges"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/sync/semaphore"
|
||||
@@ -454,17 +454,17 @@ func (f *Fs) deleteObject(ctx context.Context, path string) error {
|
||||
}
|
||||
|
||||
// createFile creates a file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// This guarantees that existing files will not be overwritten.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
// This returns ErrorFileExists if a file already exists at the specified path.
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
if onExist == AutoNameOnExist {
|
||||
@@ -479,14 +479,12 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "POST",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -512,16 +510,16 @@ func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modT
|
||||
}
|
||||
|
||||
// overwriteFile updates the content of the file at the given path
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the file does not exist it will be created.
|
||||
// The maximum size of the content is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
// This returns fs.ErrorDirNotFound
|
||||
// if the parent directory of the file is not found.
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetFileInDirectory(path)
|
||||
|
||||
@@ -533,14 +531,12 @@ func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, m
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
Method: "PUT",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
}
|
||||
|
||||
var result api.HiDriveObject
|
||||
@@ -638,18 +634,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
}
|
||||
|
||||
// Read a chunk of data.
|
||||
var (
|
||||
chunkReader *pool.RW
|
||||
bytesRead int64
|
||||
)
|
||||
chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize))
|
||||
if bytesRead < int64(chunkSize) {
|
||||
chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
|
||||
if bytesRead < chunkSize {
|
||||
startMoreTransfers = false
|
||||
}
|
||||
if readErr != nil || bytesRead <= 0 {
|
||||
if chunkReader != nil {
|
||||
_ = chunkReader.Close()
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
@@ -659,12 +648,11 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// After this upload is done,
|
||||
// signal that another transfer can be started.
|
||||
defer transferSemaphore.Release(1)
|
||||
defer func() { _ = chunkReader.Close() }()
|
||||
uploadErr := f.patchFile(gCtx, path, chunkReader, chunkOffset, zeroTime)
|
||||
uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
|
||||
if uploadErr == nil {
|
||||
// Remember successfully written chunks.
|
||||
okChunksMu.Lock()
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: bytesRead})
|
||||
okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
|
||||
okChunksMu.Unlock()
|
||||
fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
|
||||
} else {
|
||||
@@ -708,10 +696,10 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// starting at the given offset.
|
||||
//
|
||||
// Replaces the file contents starting from the given byte offset
|
||||
// with the content of the buffer.
|
||||
// with the content of the io.ReadSeeker.
|
||||
// If the offset is beyond the file end, the file is extended up to the offset.
|
||||
// The maximum size of the update is limited by MaximumUploadBytes.
|
||||
// The caller remains responsible for closing the buffer.
|
||||
// The io.ReadSeeker should be resettable by seeking to its start.
|
||||
// If modTime is not the zero time instant,
|
||||
// it will be set as the file's modification time after the operation.
|
||||
//
|
||||
@@ -723,7 +711,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
|
||||
// usually expand the holes into regions of 0-bytes.
|
||||
//
|
||||
// This returns fs.ErrorObjectNotFound if the object is not found.
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offset uint64, modTime time.Time) error {
|
||||
func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
|
||||
parameters := api.NewQueryParameters()
|
||||
parameters.SetPath(path)
|
||||
parameters.Set("offset", strconv.FormatUint(offset, 10))
|
||||
@@ -735,15 +723,13 @@ func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offse
|
||||
}
|
||||
}
|
||||
|
||||
contentLength := content.Size()
|
||||
opts := rest.Opts{
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
ContentLength: &contentLength,
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
Method: "PATCH",
|
||||
Path: "/file",
|
||||
Body: content,
|
||||
ContentType: "application/octet-stream",
|
||||
Parameters: parameters.Values,
|
||||
NoResponse: true,
|
||||
}
|
||||
|
||||
var resp *http.Response
|
||||
@@ -853,39 +839,41 @@ func createHiDriveScopes(role string, access string) []string {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
// unwrapAccounting splits any transfer accounting off reader,
|
||||
// returning the raw stream and the accounting (nil if there is none),
|
||||
// so the stream can be buffered and accounted when the buffer is uploaded.
|
||||
func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Account) {
|
||||
// Any kind of accounter re-wraps into the one type UnWrapAccounting
|
||||
// knows how to take the *Account out of.
|
||||
unwrapped, wrap := accounting.UnWrap(reader)
|
||||
_, acc = accounting.UnWrapAccounting(wrap(unwrapped))
|
||||
return unwrapped, acc
|
||||
}
|
||||
|
||||
// readerForChunk reads up to length bytes from reader into a buffer
|
||||
// from the global memory pool and returns it
|
||||
// along with the number of bytes that have been read from reader.
|
||||
// Reaching the end of reader early is not an error.
|
||||
//
|
||||
// Any accounting on reader is applied when the buffer is read instead,
|
||||
// so the upload is what gets accounted.
|
||||
// The caller must Close the buffer when done with it.
|
||||
func readerForChunk(reader io.Reader, length int64) (chunkReader *pool.RW, bytesRead int64, err error) {
|
||||
reader, acc := unwrapAccounting(reader)
|
||||
chunkReader = multipart.NewRW()
|
||||
if acc != nil {
|
||||
chunkReader.SetAccounting(acc.AccountRead)
|
||||
// cachedReader returns a version of the reader that caches its contents and
|
||||
// can therefore be reset using Seek.
|
||||
func cachedReader(reader io.Reader) io.ReadSeeker {
|
||||
bytesReader, ok := reader.(*bytes.Reader)
|
||||
if ok {
|
||||
return bytesReader
|
||||
}
|
||||
|
||||
bytesRead, err = io.CopyN(chunkReader, reader, length)
|
||||
if err == io.EOF {
|
||||
repeatableReader, ok := reader.(*readers.RepeatableReader)
|
||||
if ok {
|
||||
return repeatableReader
|
||||
}
|
||||
|
||||
return readers.NewRepeatableReader(reader)
|
||||
}
|
||||
|
||||
// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
|
||||
// Returns a new io.Reader (chunkReader) for that chunk
|
||||
// and the number of bytes that have been read from reader.
|
||||
func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
|
||||
// Unwrap any accounting from the input if present.
|
||||
reader, wrap := accounting.UnWrap(reader)
|
||||
|
||||
// Read a chunk of data.
|
||||
buffer := make([]byte, length)
|
||||
bytesRead, err = io.ReadFull(reader, buffer)
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
_ = chunkReader.Close()
|
||||
return nil, bytesRead, err
|
||||
}
|
||||
return chunkReader, bytesRead, nil
|
||||
// Truncate unused capacity.
|
||||
buffer = buffer[:bytesRead]
|
||||
|
||||
// Use wrap to put any accounting back for chunkReader.
|
||||
return wrap(bytes.NewReader(buffer)), bytesRead, nil
|
||||
}
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -515,14 +514,8 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
// (i.e. everything up to the cutoff) in the first request,
|
||||
// avoids files being created on upload failure for small files.
|
||||
// (As opposed to creating an empty file and then uploading the content.)
|
||||
//
|
||||
// Only buffer as much as the source declares it has,
|
||||
// so small files do not cost the whole cutoff in memory.
|
||||
prefixSize := int64(f.opt.UploadCutoff)
|
||||
if size := src.Size(); size >= 0 && size < prefixSize {
|
||||
prefixSize = size
|
||||
}
|
||||
cutoffReader, bytesRead, err := readerForChunk(in, prefixSize)
|
||||
tmpReader, bytesRead, err := readerForChunk(in, int(f.opt.UploadCutoff))
|
||||
cutoffReader := cachedReader(tmpReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -547,7 +540,6 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
return false, createErr
|
||||
})
|
||||
_ = cutoffReader.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -564,7 +556,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
}
|
||||
// If there is more left to write, o.Update needs to skip ahead.
|
||||
// Use a fs.SeekOption with the current offset to do this.
|
||||
options = append(options, &fs.SeekOption{Offset: bytesRead})
|
||||
options = append(options, &fs.SeekOption{Offset: int64(bytesRead)})
|
||||
err = o.Update(ctx, in, src, options...)
|
||||
|
||||
if err == nil {
|
||||
@@ -896,10 +888,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The server stores modtimes with second precision so truncate
|
||||
// here too to keep the in-memory modtime identical to the one a
|
||||
// fresh listing returns.
|
||||
o.modTime = modTime.Truncate(time.Second)
|
||||
o.modTime = modTime
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -979,12 +968,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Metadata should be updated even if the upload fails.
|
||||
info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
|
||||
} else {
|
||||
var content *pool.RW
|
||||
content, _, err = readerForChunk(in, src.Size())
|
||||
if err == nil {
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
|
||||
_ = content.Close()
|
||||
}
|
||||
info, err = o.fs.overwriteFile(ctx, resolvedPath, cachedReader(in), modTime)
|
||||
metaErr = err
|
||||
}
|
||||
|
||||
|
||||
+2
-65
@@ -57,13 +57,7 @@ The input format is comma separated list of key,value pairs. Standard
|
||||
|
||||
For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
|
||||
|
||||
You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
|
||||
|
||||
The headers are only sent to the host in the configured URL. If the
|
||||
server redirects to another host (including a subdomain or a different
|
||||
port) the headers are not sent to it, or to any further hop in that
|
||||
redirect chain. When headers are set, a redirect from https to http is
|
||||
refused as it would send them in cleartext.`,
|
||||
You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -162,7 +156,6 @@ type Fs struct {
|
||||
endpoint *url.URL
|
||||
endpointURL string // endpoint as a string
|
||||
httpClient *http.Client
|
||||
fileName string // set if we are pointing to a file
|
||||
}
|
||||
|
||||
// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
|
||||
@@ -289,11 +282,6 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
|
||||
}
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
// Without configured headers keep the default policy which
|
||||
// public mirrors that redirect from https to http rely on.
|
||||
if len(opt.Headers) > 0 {
|
||||
client.CheckRedirect = checkRedirect(opt)
|
||||
}
|
||||
|
||||
endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
|
||||
fs.Debugf(nil, "Root: %s", endpoint)
|
||||
@@ -309,7 +297,6 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
|
||||
|
||||
if isFile {
|
||||
// Correct root if definitely pointing to a file
|
||||
f.fileName = path.Base(f.root)
|
||||
f.root = path.Dir(f.root)
|
||||
if f.root == "." || f.root == "/" {
|
||||
f.root = ""
|
||||
@@ -521,45 +508,6 @@ func addHeaders(req *http.Request, opt *Options) {
|
||||
}
|
||||
}
|
||||
|
||||
// checkRedirect returns an http.Client.CheckRedirect function which
|
||||
// follows redirects but refuses an HTTPS to HTTP downgrade with
|
||||
// rest.ErrHTTPSDowngrade and strips the configured headers when the
|
||||
// redirect chain has left the originally requested host at any point.
|
||||
func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
if err := rest.RefuseHTTPSDowngradeRedirectFn(req, via); err != nil {
|
||||
if errors.Is(err, rest.ErrHTTPSDowngrade) {
|
||||
err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if redirectLeavesHost(req, via) {
|
||||
for i := 0; i < len(opt.Headers); i += 2 {
|
||||
req.Header.Del(opt.Headers[i])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// redirectLeavesHost reports whether any hop in the redirect chain
|
||||
// via plus the pending request req is to a different host from the
|
||||
// original request via[0].
|
||||
//
|
||||
// net/http copies the headers afresh from the original request for
|
||||
// every hop, so once the chain has visited another host the headers
|
||||
// must be stripped from every subsequent hop, even one back to the
|
||||
// original host, as the other host chose the URL.
|
||||
func redirectLeavesHost(req *http.Request, via []*http.Request) bool {
|
||||
origin := via[0].URL
|
||||
for _, hop := range via {
|
||||
if !rest.SameHost(hop.URL, origin) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return !rest.SameHost(req.URL, origin)
|
||||
}
|
||||
|
||||
// Adds the configured headers to the request if any
|
||||
func (f *Fs) addHeaders(req *http.Request) {
|
||||
addHeaders(req, &f.opt)
|
||||
@@ -593,7 +541,7 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
|
||||
return nil, fmt.Errorf("failed to readDir: %w", err)
|
||||
}
|
||||
|
||||
contentType, _, _ := strings.Cut(res.Header.Get("Content-Type"), ";")
|
||||
contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
|
||||
switch contentType {
|
||||
case "text/html":
|
||||
names, err = parse(u, res.Body)
|
||||
@@ -616,17 +564,6 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
// pointed at a single file: only that file is visible
|
||||
if f.fileName != "" {
|
||||
if dir != "" {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
obj, err := f.NewObject(ctx, f.fileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fs.DirEntries{obj}, nil
|
||||
}
|
||||
if !strings.HasSuffix(dir, "/") && dir != "" {
|
||||
dir += "/"
|
||||
}
|
||||
|
||||
@@ -331,19 +331,7 @@ func TestIsAFileRoot(t *testing.T) {
|
||||
f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
|
||||
assert.Equal(t, err, fs.ErrorIsFile)
|
||||
|
||||
entries, err := f.List(context.Background(), "")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, 1, len(entries))
|
||||
|
||||
e := entries[0]
|
||||
assert.Equal(t, "one%.txt", e.Remote())
|
||||
assert.Equal(t, int64(5+lineEndSize), e.Size())
|
||||
_, ok := e.(*Object)
|
||||
assert.True(t, ok)
|
||||
|
||||
_, err = f.List(context.Background(), "anysub")
|
||||
assert.Equal(t, fs.ErrorDirNotFound, err)
|
||||
testListRoot(t, f, false)
|
||||
}
|
||||
|
||||
func TestIsAFileSubDir(t *testing.T) {
|
||||
@@ -581,182 +569,3 @@ func TestFsNoSlashRoots(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readRedirected makes an http remote with the configured headers
|
||||
// pointing at url and reads file.txt from it, checking the content
|
||||
func readRedirected(t *testing.T, url string) {
|
||||
configfile.Install()
|
||||
m := configmap.Simple{
|
||||
"type": "http",
|
||||
"url": url + "/",
|
||||
"headers": strings.Join(headers, ","),
|
||||
}
|
||||
f, err := NewFs(context.Background(), remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
o, err := f.NewObject(context.Background(), "file.txt")
|
||||
require.NoError(t, err)
|
||||
fd, err := o.Open(context.Background())
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(fd)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fd.Close())
|
||||
assert.Equal(t, "hello", string(data))
|
||||
}
|
||||
|
||||
func TestRedirectKeepsHeadersOnSameHost(t *testing.T) {
|
||||
var got http.Header
|
||||
var ts *httptest.Server
|
||||
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/moved/") {
|
||||
http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
readRedirected(t, ts.URL)
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Equal(t, headers[i+1], got.Get(headers[i]))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
|
||||
// A plaintext server to be redirected to
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
// An HTTPS server which redirects to plaintext HTTP
|
||||
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
// open makes a remote pointing at ts with or without headers
|
||||
// and opens file.txt on it
|
||||
open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
|
||||
configfile.Install()
|
||||
m := configmap.Simple{
|
||||
"type": "http",
|
||||
"url": ts.URL + "/",
|
||||
"no_head": "true",
|
||||
}
|
||||
if withHeaders {
|
||||
m.Set("headers", strings.Join(headers, ","))
|
||||
}
|
||||
ctx, ci := fs.AddConfig(context.Background())
|
||||
ci.InsecureSkipVerify = true
|
||||
f, err := NewFs(ctx, remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
// no_head means NewObject doesn't make a request so use Open to make one
|
||||
o, err := f.NewObject(ctx, "file.txt")
|
||||
require.NoError(t, err)
|
||||
return o.Open(ctx)
|
||||
}
|
||||
|
||||
t.Run("WithHeaders", func(t *testing.T) {
|
||||
// Headers could leak so the downgrade must be refused
|
||||
_, err := open(t, true)
|
||||
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
|
||||
})
|
||||
|
||||
t.Run("WithoutHeaders", func(t *testing.T) {
|
||||
// Nothing to leak so the downgrade is followed
|
||||
fd, err := open(t, false)
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(fd)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fd.Close())
|
||||
assert.Equal(t, "hello", string(data))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRedirectKeepsHeadersOnSameHostDifferentCase(t *testing.T) {
|
||||
// The redirect spells the host differently but it is the same
|
||||
// host so the headers must be kept
|
||||
var got http.Header
|
||||
var ts *httptest.Server
|
||||
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/moved/") {
|
||||
http.Redirect(w, r, "http://LOCALHOST:"+port(t, ts)+"/moved"+r.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
readRedirected(t, "http://localhost:"+port(t, ts))
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Equal(t, headers[i+1], got.Get(headers[i]))
|
||||
}
|
||||
}
|
||||
|
||||
// port returns the port ts is listening on
|
||||
func port(t *testing.T, ts *httptest.Server) string {
|
||||
u, err := url.Parse(ts.URL)
|
||||
require.NoError(t, err)
|
||||
return u.Port()
|
||||
}
|
||||
|
||||
func TestRedirectStripsHeadersAfterHostChange(t *testing.T) {
|
||||
// The configured server redirects to a second server which
|
||||
// redirects back to the configured server. The headers must not
|
||||
// be sent on the way back as the other host chose the path.
|
||||
var got http.Header
|
||||
var ts *httptest.Server
|
||||
bouncer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
|
||||
}))
|
||||
defer bouncer.Close()
|
||||
|
||||
ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/moved/") {
|
||||
http.Redirect(w, r, bouncer.URL+r.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
readRedirected(t, ts.URL)
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Empty(t, got.Values(headers[i]), "header %q sent after cross-host redirect", headers[i])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
|
||||
// Serve the final response from a second server. httptest servers
|
||||
// listen on 127.0.0.1:port so the two servers have different hosts
|
||||
// for the purposes of the redirect check.
|
||||
var got http.Header
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got = r.Header.Clone()
|
||||
_, _ = w.Write([]byte("hello"))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
|
||||
}))
|
||||
defer redirector.Close()
|
||||
|
||||
readRedirected(t, redirector.URL)
|
||||
|
||||
require.NotNil(t, got)
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
|
||||
}
|
||||
}
|
||||
Loaded 100 of 753 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user