Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13a1d23bb8 |
No files matched your search
@@ -2,9 +2,6 @@
|
||||
go.mod text eol=lf
|
||||
go.sum text eol=lf
|
||||
|
||||
# Embedded templates must render the same on every OS
|
||||
lib/http/templates/** text eol=lf
|
||||
|
||||
# Ignore generated files in GitHub language statistics and diffs
|
||||
/MANUAL.* linguist-generated=true
|
||||
/rclone.1 linguist-generated=true
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
# CODEOWNERS for rclone
|
||||
#
|
||||
# Maps paths to the maintainers auto-requested for review when a pull request
|
||||
# touches matching files. Patterns are evaluated top to bottom and the LAST
|
||||
# matching pattern wins, so the catch-all below is overridden by the more
|
||||
# specific subsystem/backend lines that follow.
|
||||
#
|
||||
# Every owner listed here has write access to the repo (GitHub ignores owners
|
||||
# without it). Assignments are derived from `git log` history per directory:
|
||||
# each area lists its dominant contributor(s) alongside @ncw, the project lead
|
||||
# and global fallback.
|
||||
|
||||
# Global fallback - everything not matched below
|
||||
* @ncw
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Core subsystems
|
||||
# -----------------------------------------------------------------------------
|
||||
/fs/ @ncw
|
||||
/vfs/ @ncw
|
||||
/lib/ @ncw
|
||||
/cmd/ @ncw
|
||||
/librclone/ @ncw
|
||||
/fstest/ @ncw
|
||||
|
||||
/lib/encoder/ @ncw @klauspost
|
||||
/fs/sync/ @ncw @nielash
|
||||
/fs/operations/ @ncw @nielash
|
||||
/cmd/bisync/ @nielash @ivandeex @ncw
|
||||
/cmd/serve/ @albertony @ncw
|
||||
/cmd/selfupdate/ @ivandeex @ncw
|
||||
/cmd/mount2/ @ncw @darthShadow
|
||||
/cmd/gitannex/ @dmcardle @ncw
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Backends - keyed to the dominant / specialist maintainer per `git log`
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# albertony
|
||||
/backend/jottacloud/ @albertony @buengese @ncw
|
||||
/backend/local/ @ncw @albertony @nielash
|
||||
/backend/netstorage/ @albertony @ncw
|
||||
/backend/hdfs/ @albertony @ncw
|
||||
|
||||
# buengese
|
||||
/backend/yandex/ @ncw @buengese
|
||||
/backend/zoho/ @buengese @ncw
|
||||
/backend/fichier/ @ncw @buengese
|
||||
/backend/compress/ @ncw @buengese
|
||||
|
||||
# ivandeex
|
||||
/backend/chunker/ @ncw @ivandeex
|
||||
/backend/mailru/ @ncw @ivandeex
|
||||
/backend/ftp/ @ncw @ivandeex
|
||||
/backend/sia/ @ncw @ivandeex
|
||||
|
||||
# Other maintainer-owned backends (see MAINTAINERS.md)
|
||||
/backend/cache/ @ncw @remusb
|
||||
/backend/union/ @ncw @Max-Sum @albertony
|
||||
/backend/filescom/ @ncw @childish-sambino
|
||||
|
||||
# Microsoft Azure
|
||||
/backend/azureblob/ @ncw @sandeepkru @darthShadow
|
||||
|
||||
# Single-maintainer specialist backends
|
||||
/backend/onedrive/ @ncw @Cnly @nielash
|
||||
/backend/iclouddrive/ @lostb1t @ncw
|
||||
/backend/pikpak/ @wiserain @ncw
|
||||
/backend/protondrive/ @henrybear327 @ncw
|
||||
/backend/seafile/ @ncw @creativeprojects
|
||||
/backend/filen/ @Enduriel @ncw
|
||||
/backend/storj/ @ncw @calebcase
|
||||
/backend/combine/ @ncw @nielash
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Repo plumbing
|
||||
# -----------------------------------------------------------------------------
|
||||
/.github/ @ncw
|
||||
@@ -0,0 +1,31 @@
|
||||
<!--
|
||||
|
||||
Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
|
||||
|
||||
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
|
||||
|
||||
https://forum.rclone.org/
|
||||
|
||||
instead of filing an issue for a quick response.
|
||||
|
||||
If you are reporting a bug or asking for a new feature then please use one of the templates here:
|
||||
|
||||
https://github.com/rclone/rclone/issues/new
|
||||
|
||||
otherwise fill in the form below.
|
||||
|
||||
Thank you
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
|
||||
#### Output of `rclone version`
|
||||
|
||||
|
||||
|
||||
#### Describe the issue
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Report a problem with rclone
|
||||
---
|
||||
|
||||
<!--
|
||||
|
||||
We understand you are having a problem with rclone; we want to help you with that!
|
||||
|
||||
**STOP and READ**
|
||||
**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
|
||||
Please show the effort you've put into solving the problem and please be specific.
|
||||
People are volunteering their time to help! Low effort posts are not likely to get good answers!
|
||||
|
||||
If you think you might have found a bug, try to replicate it with the latest beta (or stable).
|
||||
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
|
||||
|
||||
If you can still replicate it or just got a question then please use the rclone forum:
|
||||
|
||||
https://forum.rclone.org/
|
||||
|
||||
for a quick response instead of filing an issue on this repo.
|
||||
|
||||
If nothing else helps, then please fill in the info below which helps us help you.
|
||||
|
||||
**DO NOT REDACT** any information except passwords/keys/personal info.
|
||||
|
||||
You should use 3 backticks to begin and end your paste to make it readable.
|
||||
|
||||
Make sure to include a log obtained with '-vv'.
|
||||
|
||||
You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
|
||||
|
||||
Thank you
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
|
||||
#### What is the problem you are having with rclone?
|
||||
|
||||
|
||||
|
||||
#### What is your rclone version (output from `rclone version`)
|
||||
|
||||
|
||||
|
||||
#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
|
||||
|
||||
|
||||
|
||||
#### Which cloud storage system are you using? (e.g. Google Drive)
|
||||
|
||||
|
||||
|
||||
#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
|
||||
|
||||
|
||||
|
||||
#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your bug report. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest a new feature or enhancement for rclone
|
||||
---
|
||||
|
||||
<!--
|
||||
|
||||
Welcome :-)
|
||||
|
||||
So you've got an idea to improve rclone? We love that!
|
||||
You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
|
||||
|
||||
Probably the latest beta (or stable) release has your feature, so try to update your rclone.
|
||||
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
|
||||
|
||||
If it still isn't there, here is a checklist of things to do:
|
||||
|
||||
1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
|
||||
2. Discuss on the forum: https://forum.rclone.org/
|
||||
3. Make a feature request issue (this is the right place!).
|
||||
4. Be prepared to get involved making the feature :-)
|
||||
|
||||
Looking forward to your great idea!
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
|
||||
#### What is your current rclone version (output from `rclone version`)?
|
||||
|
||||
|
||||
|
||||
#### What problem are you are trying to solve?
|
||||
|
||||
|
||||
|
||||
#### How do you think rclone should be changed to solve that?
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your feature request. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -1,124 +0,0 @@
|
||||
name: Bug report
|
||||
description: Report a reproducible problem with rclone
|
||||
labels: ["bug"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
We want to help you with rclone - thank you for taking the time to fill this in!
|
||||
|
||||
**This tracker is for reproducible bugs only.** If you have a question, are
|
||||
not sure whether something is a bug, or want help with a configuration, please
|
||||
use the [rclone forum](https://forum.rclone.org/) instead - you will get a much
|
||||
quicker answer there.
|
||||
|
||||
Before filing, please **test with the [latest beta or stable](https://rclone.org/commands/rclone_selfupdate/)**
|
||||
(downloads at <https://beta.rclone.org/>) as your problem may already be fixed.
|
||||
|
||||
**Do not redact** any information except passwords, keys and personal info.
|
||||
- type: checkboxes
|
||||
id: acknowledgements
|
||||
attributes:
|
||||
label: Before you start
|
||||
description: Please confirm the following. Issues that skip these steps may be closed.
|
||||
options:
|
||||
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues for this problem.
|
||||
required: true
|
||||
- label: I have tested with the latest beta or stable release and can still reproduce the problem.
|
||||
required: true
|
||||
- label: This is a reproducible bug, not a usage question (questions belong on the forum).
|
||||
required: true
|
||||
- type: input
|
||||
id: forum-url
|
||||
attributes:
|
||||
label: Associated forum post URL
|
||||
description: If you discussed this on the forum first (encouraged), paste the link here.
|
||||
placeholder: https://forum.rclone.org/t/...
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: What is the problem you are having with rclone?
|
||||
description: Describe the problem and the exact steps to reproduce it.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: rclone version
|
||||
description: The full output of `rclone version`.
|
||||
placeholder: |
|
||||
rclone v1.75.0
|
||||
- os/version: ubuntu 24.04 (64 bit)
|
||||
- os/kernel: 6.8.0-111-generic (x86_64)
|
||||
- os/type: linux
|
||||
- os/arch: amd64
|
||||
- go/version: go1.26.4
|
||||
- go/linking: dynamic
|
||||
- go/tags: none
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating system
|
||||
description: Choose the operating system you are using with rclone.
|
||||
options:
|
||||
- Windows
|
||||
- macOS
|
||||
- Linux
|
||||
- FreeBSD
|
||||
- NetBSD
|
||||
- OpenBSD
|
||||
- Plan 9
|
||||
- Solaris
|
||||
- Android
|
||||
- iOS
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: backend
|
||||
attributes:
|
||||
label: Which cloud storage system are you using?
|
||||
description: e.g. Google Drive, S3, Dropbox, local disk. List all involved.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: config
|
||||
attributes:
|
||||
label: Output of `rclone config redacted`
|
||||
description: |
|
||||
Run `rclone config redacted` (or `rclone config redacted <remote>` for a
|
||||
single remote). This command automatically replaces passwords and tokens
|
||||
with `XXX`, so it should be safe to paste but please double-check before posting.
|
||||
render: ini
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: command
|
||||
attributes:
|
||||
label: The command you were trying to run
|
||||
description: e.g. `rclone copy /tmp remote:tmp`
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: log
|
||||
attributes:
|
||||
label: A log from the command with the `-vv` flag
|
||||
description: |
|
||||
e.g. output from `rclone -vv copy /tmp remote:tmp`. For long logs (> 100 lines) use
|
||||
`-vv --log-file bug.log` and attach it here or send to a service such as
|
||||
<https://gist.github.com/> or <https://pastebin.com/> and link it here.
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: extra
|
||||
attributes:
|
||||
label: Anything else?
|
||||
description: Anything else that might help us, e.g. a minimal way to replicate the problem.
|
||||
validations:
|
||||
required: false
|
||||
@@ -1,8 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Rclone Forum - questions & support
|
||||
- name: Rclone Forum Community Support
|
||||
url: https://forum.rclone.org/
|
||||
about: Have a question, need help, or not sure if it's a bug? Please ask here for a quick answer.
|
||||
- name: Rclone Documentation
|
||||
url: https://rclone.org/docs/
|
||||
about: The docs answer many "how do I..." questions, including per-backend guides.
|
||||
about: Please ask and answer questions here.
|
||||
@@ -1,68 +0,0 @@
|
||||
name: Feature request
|
||||
description: Suggest a new feature or enhancement for rclone
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
So you've got an idea to improve rclone? We love that - we've already
|
||||
incorporated hundreds of ideas from contributors!
|
||||
|
||||
Please **search first**: your idea may already exist in the latest
|
||||
[beta or stable](https://rclone.org/commands/rclone_selfupdate/), or already
|
||||
be requested. It is often worth [discussing on the forum](https://forum.rclone.org/)
|
||||
to refine the idea or checking to see if rclone can do it already before filing it here.
|
||||
- type: checkboxes
|
||||
id: acknowledgements
|
||||
attributes:
|
||||
label: Before you start
|
||||
options:
|
||||
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues and this hasn't already been requested.
|
||||
required: true
|
||||
- label: I have checked the latest beta and this feature doesn't already exist.
|
||||
required: true
|
||||
- type: input
|
||||
id: forum-url
|
||||
attributes:
|
||||
label: Associated forum post URL
|
||||
description: If you discussed this on the forum first (encouraged), paste the link here.
|
||||
placeholder: https://forum.rclone.org/t/...
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: rclone version
|
||||
description: The full output of `rclone version`, not just the version number.
|
||||
placeholder: |
|
||||
rclone v1.75.0
|
||||
- os/version: ubuntu 24.04 (64 bit)
|
||||
- os/kernel: 6.8.0-111-generic (x86_64)
|
||||
- os/type: linux
|
||||
- os/arch: amd64
|
||||
- go/version: go1.26.4
|
||||
- go/linking: dynamic
|
||||
- go/tags: none
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: What problem are you trying to solve?
|
||||
description: Describe the use case and what is difficult or impossible today.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: How do you think rclone should be changed to solve that?
|
||||
validations:
|
||||
required: true
|
||||
- type: checkboxes
|
||||
id: involvement
|
||||
attributes:
|
||||
label: Getting involved
|
||||
description: Features are much more likely to happen if someone helps make them.
|
||||
options:
|
||||
- label: I'm willing to help implement, test or fund this feature.
|
||||
required: false
|
||||
@@ -1,54 +1,29 @@
|
||||
<!--
|
||||
Thank you for contributing to rclone!
|
||||
Thank you very much for contributing code or documentation to rclone! Please
|
||||
fill out the following questions to make it easier for us to review your
|
||||
changes.
|
||||
|
||||
Please discuss anything more than a trivial fix in an issue FIRST - see the
|
||||
"Linked issue" section below. Then fill in the questions to help us review.
|
||||
You do not need to check all the boxes below all at once, feel free to take
|
||||
your time and add more commits. If you're done and ready for review, please
|
||||
check the last box.
|
||||
-->
|
||||
|
||||
#### What does this change do?
|
||||
|
||||
<!-- Describe the change here. -->
|
||||
|
||||
#### Linked issue
|
||||
#### What is the purpose of this change?
|
||||
|
||||
<!--
|
||||
Put `Fixes #1234` here if this closes an issue, or `#1234` to link without closing.
|
||||
|
||||
IMPORTANT: Anything beyond a trivial fix (typo, doc tweak, small obvious bug fix)
|
||||
should be discussed and agreed in an issue BEFORE you open a pull request.
|
||||
Pull requests for larger changes that have not been discussed in an issue first
|
||||
may be closed. This saves everyone's time if the change needs a different approach
|
||||
or isn't a good fit.
|
||||
Describe the changes here
|
||||
-->
|
||||
|
||||
Fixes #
|
||||
|
||||
#### For new or changed backends
|
||||
#### Was the change discussed in an issue or in the forum before?
|
||||
|
||||
<!--
|
||||
Backend pull requests are often sent without the integration tests having been run.
|
||||
We cannot merge a backend change until it passes the integration tests.
|
||||
|
||||
To be merged, a backend change REQUIRES:
|
||||
1. A clean run of `go run ./fstest/test_all -backends <remote>` (summarise the
|
||||
result below or paste a screenshot of the test_all output in the web browser).
|
||||
2. A test account so the maintainers can run the integration tests daily against
|
||||
the backend on the integration test server.
|
||||
|
||||
If the tests aren't quite passing yet and you need help getting them
|
||||
to pass, then submit the PR and we can help getting you over the line.
|
||||
|
||||
See https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#writing-a-new-backend
|
||||
and https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests
|
||||
Link issues and relevant forum posts here.
|
||||
-->
|
||||
|
||||
#### Checklist
|
||||
|
||||
- [ ] This change is trivial **OR** it has been discussed and agreed in the linked issue.
|
||||
- [ ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
|
||||
- [ ] **(If I used AI tools to help write this code)** I have read and understood the [AI-assisted contributions guidance](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#ai-assisted-contributions), and I have tested and take ownership of this change myself.
|
||||
- [ ] I have added tests for all changes in this PR if appropriate.
|
||||
- [ ] I have added documentation for the changes if appropriate.
|
||||
- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
|
||||
- [ ] **(Backend changes only)** `test_all` passes for this backend and if submitting a new backend can provide a test account for the integration tester - see [CONTRIBUTING.md](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests).
|
||||
- [ ] This Pull Request is ready for review.
|
||||
- [ ] I'm done, this Pull Request is ready for review :-)
|
||||
@@ -4,5 +4,3 @@ updates:
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
+19
-139
@@ -19,9 +19,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
@@ -32,12 +29,12 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.0'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +45,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.0'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.0'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +61,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.0'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.0'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,14 +78,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '~1.26.0'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.26
|
||||
- job_name: go1.25
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.9'
|
||||
go: '~1.25.7'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -97,58 +94,16 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
check-latest: true
|
||||
# Caching is handled explicitly below. setup-go's built-in cache
|
||||
# keys only on go.sum, so all matrix jobs collide on one key and
|
||||
# only the first to finish saves - which loses the cross-compile
|
||||
# build cache.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
# The module cache holds downloaded source and is identical across
|
||||
# jobs (it depends only on go.sum, not on OS, arch or Go version), so
|
||||
# it is shared via a single key to avoid storing the same ~600 MiB in
|
||||
# every job's cache and overflowing the repo cache limit.
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
# The build cache holds compiled artifacts which are specific to the
|
||||
# OS, arch and Go version, so it is kept per job.
|
||||
- name: Build cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
@@ -165,11 +120,6 @@ jobs:
|
||||
sudo chown root:$USER /etc/fuse.conf
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
|
||||
# The podman static bundle in runner image ubuntu24/20260726.254 and
|
||||
# later ships a non-setuid fusermount3 in /usr/local/bin which
|
||||
# shadows the setuid one from the fuse3 package and breaks FUSE
|
||||
# mounts with "Operation not permitted", so remove it.
|
||||
sudo rm -f /usr/local/bin/fusermount /usr/local/bin/fusermount3
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
|
||||
- name: Install Libraries on macOS
|
||||
@@ -179,7 +129,6 @@ jobs:
|
||||
unset HOMEBREW_NO_INSTALL_FROM_API
|
||||
brew untap --force homebrew/core
|
||||
brew untap --force homebrew/cask
|
||||
brew untap --force aws/tap
|
||||
brew update
|
||||
brew install --cask macfuse
|
||||
brew install git-annex git-annex-remote-rclone
|
||||
@@ -253,15 +202,6 @@ jobs:
|
||||
# Deploy binaries if enabled in config && not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
lint:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -276,31 +216,23 @@ jobs:
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
go-version: '~1.26.0'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Cache
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
~/.cache/golangci-lint
|
||||
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
@@ -367,15 +299,6 @@ jobs:
|
||||
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
android:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -383,50 +306,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Upgrade together with NDK version
|
||||
- name: Set up Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Build cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
go-version: '~1.26.0'
|
||||
|
||||
- name: Set global environment variables
|
||||
run: |
|
||||
@@ -502,12 +391,3 @@ jobs:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# Upload artifacts if not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
@@ -18,10 +18,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build-image:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
|
||||
@@ -56,7 +52,7 @@ jobs:
|
||||
df -h .
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -96,7 +92,7 @@ jobs:
|
||||
# There's no way around this, because "ImageOS" is only available to
|
||||
# processes, but the setup-go action uses it in its key.
|
||||
id: imageos
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
@@ -104,7 +100,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
|
||||
with:
|
||||
@@ -126,14 +122,14 @@ jobs:
|
||||
type=raw,value=beta,enable={{is_default_branch}}
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Load Go Build Cache for Docker
|
||||
id: go-cache
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
@@ -152,7 +148,7 @@ jobs:
|
||||
skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
@@ -163,7 +159,7 @@ jobs:
|
||||
|
||||
- name: Build and Publish Image Digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
file: Dockerfile
|
||||
context: .
|
||||
@@ -214,7 +210,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
|
||||
with:
|
||||
@@ -265,16 +261,16 @@ jobs:
|
||||
env.write(f"ANNOTATIONS={annotations_string}\n")
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
|
||||
@@ -14,9 +14,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build_docker_volume_plugin:
|
||||
if: inputs.manual || github.repository == 'rclone/rclone'
|
||||
@@ -33,7 +30,7 @@ jobs:
|
||||
sudo rm -rf /usr/share/dotnet || true
|
||||
df -h .
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and publish docker plugin
|
||||
|
||||
@@ -4,9 +4,6 @@ on:
|
||||
issues:
|
||||
types: [labeled]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -3,9 +3,6 @@ on:
|
||||
release:
|
||||
types: [released]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -46,11 +46,6 @@ linters:
|
||||
- -ST1022
|
||||
# Disable quickfix checks
|
||||
- -QF*
|
||||
# Disabled because its dataflow analysis (new in staticcheck
|
||||
# 0.8.0) makes linting large packages more than 10x slower,
|
||||
# which takes the CI lint job past its timeout, and it
|
||||
# produces false positives (eg on operations.Delete).
|
||||
- -SA4023
|
||||
gocritic:
|
||||
# With gocritic there are different settings, but since enabled-checks
|
||||
# and disabled-checks cannot both be set, for full customization the
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
default: true
|
||||
|
||||
# Use specific styles, to be consistent across all documents.
|
||||
# Use specific styles, to be consistent accross all documents.
|
||||
# Default is to accept any as long as it is consistent within the same document.
|
||||
heading-style: # MD003
|
||||
style: atx
|
||||
@@ -48,12 +48,12 @@ link-fragments: false # MD051
|
||||
# https://gohugo.io/content-management/syntax-highlighting/#languages
|
||||
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
|
||||
# In addition, we only want to allow identifiers (aliases) that correspond to
|
||||
# the same language in Hugo and GitHub, and preferably also VSCode and other
|
||||
# the same language in Hugo and GitHub, and preferrably also VSCode and other
|
||||
# commonly used tools, to avoid confusion. An example of this is that "shell"
|
||||
# by some are considered an identifier for shell scripts, i.e. an alias for
|
||||
# "sh", while others consider it an identifier for shell sessions, i.e. an
|
||||
# alias for "console". Although Hugo and GitHub in this case are consistent and
|
||||
# have chosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# have choosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# avoids the confusion entirely.
|
||||
fenced-code-language: # MD040
|
||||
allowed_languages:
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
This file provides guidance to AI coding agents (e.g. Claude Code, Codex, Cursor, Gemini CLI, and similar tools) when working with code in this repository.
|
||||
|
||||
Rclone welcomes AI-assisted contributions, but the expectation is that you, the human submitter, understand every line you propose and have compiled and tested it against real rclone code - not just generated it. See the "AI-assisted contributions" section of [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request.
|
||||
|
||||
## Project Overview
|
||||
|
||||
Rclone is a command-line program to sync files and directories to and from cloud storage providers. It's written in Go and supports 70+ backends (cloud storage systems). Think "rsync for cloud storage".
|
||||
|
||||
## General Notes
|
||||
|
||||
**We take backwards compatibility very seriously.** PRs should not change the observable behaviour of existing commands, flags or rc API without very good reason. Rclone does not try to preserve a stable Go API but try not to change it gratuitously.
|
||||
|
||||
Rclone operates with a lot of different backends, so **compatibility is key**. It is the backend integration tests which guarantee that compatibility. Changes should consider both known and unknown backends and should take care not to break functionality of existing installations.
|
||||
|
||||
The core parts of rclone under `fs` and `vfs` need to work with all backends and **backend specific hacks won't be merged**. Fixes likely need to go in the relevant backend or if new behaviour is really needed, a new Feature flag needs to be added.
|
||||
|
||||
**Changes should be kept to the minimum.** Work hard to make the most elegant, smallest change you can. Do not refactor or re-order code unless necessary as this makes review more challenging. Re-use existing test scaffolding, existing or library routines (e.g. `lib`) where possible.
|
||||
|
||||
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
|
||||
|
||||
Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
|
||||
|
||||
## Build and Test Commands
|
||||
|
||||
```bash
|
||||
# Build rclone (simple)
|
||||
go build
|
||||
|
||||
# Build with version info (preferred)
|
||||
make
|
||||
|
||||
# Run all unit tests (no cloud credentials needed)
|
||||
make quicktest
|
||||
# or equivalently:
|
||||
RCLONE_CONFIG="/notfound" go test ./...
|
||||
|
||||
# Run tests for a specific package
|
||||
cd backend/memory && go test -v
|
||||
# or from root:
|
||||
go test -v ./backend/memory/
|
||||
|
||||
# Run a single test
|
||||
go test -v -run TestIntegration/FsCheckWrap ./backend/memory/
|
||||
|
||||
# Run tests with race detector
|
||||
make racequicktest
|
||||
|
||||
# Lint (requires golangci-lint)
|
||||
golangci-lint run ./...
|
||||
|
||||
# Run backend integration tests (requires configured TestRemote remote)
|
||||
cd backend/drive && go test -v
|
||||
# Run sync/operations integration tests against a remote
|
||||
cd fs/sync && go test -v -remote TestDrive:
|
||||
cd fs/operations && go test -v -remote TestDrive:
|
||||
|
||||
# Run integration tests via test framework
|
||||
go run ./fstest/test_all -backends drive
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Entry Point and Plugin Registration
|
||||
|
||||
`rclone.go` is the main entry point. It imports `backend/all` and `cmd/all` which use Go's `init()` pattern to register all backends and commands. Each backend calls `fs.Register()` with a `fs.RegInfo` struct during init.
|
||||
|
||||
### Core Interfaces (`fs/`)
|
||||
|
||||
The `fs` package defines the core abstractions:
|
||||
- **`fs.Fs`** (`fs/types.go`): The filesystem interface every backend must implement (List, NewObject, Put, Mkdir, Rmdir).
|
||||
- **`fs.Object`** (`fs/types.go`): Interface for a file/object (Open, Update, Remove, SetModTime).
|
||||
- **`fs.Features`** (`fs/features.go`): Optional capabilities a backend can declare (Purge, Copy, Move, DirMove, etc.). Backends set function pointers for operations they support; nil means not supported.
|
||||
- **`fs.RegInfo`** (`fs/registry.go`): Registration metadata for a backend including its name, config options, and NewFs constructor.
|
||||
|
||||
### Backend Structure (`backend/`)
|
||||
|
||||
Each backend is a single Go package (e.g., `backend/s3/`, `backend/drive/`). Key conventions:
|
||||
- Main implementation in a single file (e.g., `s3.go`) - **do not** split into `fs.go`/`object.go`.
|
||||
- API types go in a separate `api/types.go` file.
|
||||
- Test file (e.g., `s3_test.go`) uses `fstests.Run()` from `fstest/fstests` for standardized integration tests.
|
||||
- Register in `backend/all/all.go` via blank import.
|
||||
- HTTP-based backends should use `lib/rest` for HTTP calls and `fs/fshttp` for the HTTP client.
|
||||
- Use `lib/dircache` for directory-ID-based remotes, `lib/oauthutil` for OAuth, `lib/pacer` for rate limiting.
|
||||
|
||||
### Command Structure (`cmd/`)
|
||||
|
||||
Each command is a package under `cmd/` registered in `cmd/all/all.go` via blank import. Commands use cobra via `cmd.Main()`.
|
||||
|
||||
### Key Subsystems
|
||||
|
||||
- **`fs/operations/`**: Core file operations (Copy, Move, Delete, etc.)
|
||||
- **`fs/sync/`**: Directory sync logic
|
||||
- **`fs/march/`**: Parallel directory tree walker used by sync
|
||||
- **`fs/filter/`**: Include/exclude filtering
|
||||
- **`fs/accounting/`**: Transfer statistics and bandwidth limiting
|
||||
- **`fs/config/`**: Configuration file management
|
||||
- **`vfs/`**: Virtual filesystem layer (used by mount, serve)
|
||||
- **`librclone/`**: C-compatible library interface for embedding rclone
|
||||
- **`fstest/`**: Integration test framework; `fstest/fstests/` has the generic backend test suite
|
||||
|
||||
## Commit Message Convention
|
||||
|
||||
Prefix with the directory of the change, then a colon: `drive: add team drive support - fixes #885`. For cross-cutting changes use a broader prefix like `fs` or `operations`.
|
||||
|
||||
Make the first line of your commit message a summary of the change that a user (not a developer) of rclone would like to read. So write `drive: fix server side copy of big files` instead of `drive: no longer set the MimeType in Move or Copy`. This is important because these lines go into the change log which is read by users.
|
||||
|
||||
## Code Commenting Style
|
||||
|
||||
Comments describe the code as it is now, for a future reader who has no knowledge of the change that introduced it.
|
||||
|
||||
Every exported type, function, field, and constant has a **godoc comment** that starts with its name and is phrased as a present-tense statement of what it is or does (`// Mkdir makes the directory (container, bucket)`)
|
||||
|
||||
**Document the contract** callers need - preconditions, what's returned, which sentinel errors are returned and when, and any "shouldn't return an error if it already exists" style caveats - rather than the implementation.
|
||||
|
||||
**Keep comments terse**: a single line for most things, with extra paragraphs (separated by blank `//` lines) reserved for genuine subtlety.
|
||||
|
||||
Inline comments inside function bodies should **explain why** - a non-obvious API quirk, a workaround, a gotcha, or an ordering constraint - and may cite an external reference (forum thread, vendor docs, RFC) when that's what makes the behaviour non-obvious; skip comments that merely restate what the code plainly does.
|
||||
|
||||
Use `FIXME` and `TODO` for known shortcomings.
|
||||
|
||||
**Do not write comments that narrate the change itself** or compare against the previous behaviour (no "now we also handle...", "changed to...", "previously this returned...", or references to bug/PR numbers in the code) - that context belongs in the commit message, not in source that will outlive the change.
|
||||
|
||||
Code comments should only refer to **current** behaviour - and shouldn't describe behaviour that is trivially deducible from reading the code.
|
||||
|
||||
## Linting Configuration
|
||||
|
||||
Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck, govet, ineffassign, staticcheck, unused, gocritic, misspell, revive, unconvert. The `goimports` formatter is also enabled.
|
||||
|
||||
## Documentation
|
||||
|
||||
- Backend option docs come from `Help:` fields in the Go source Options structs, not from markdown files.
|
||||
- Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
|
||||
- Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
|
||||
- Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
|
||||
@@ -1,6 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
Guidance for this repository lives in AGENTS.md, shared with all AI coding
|
||||
agents. Claude Code reads it via the import below.
|
||||
|
||||
@AGENTS.md
|
||||
+4
-69
@@ -113,36 +113,6 @@ GitHub.
|
||||
You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master)
|
||||
or [squash your commits](#squashing-your-commits).
|
||||
|
||||
## AI-assisted contributions
|
||||
|
||||
You are welcome to use AI coding assistants (Claude Code, Codex, Cursor, Gemini
|
||||
CLI, and similar) to help write your contribution. Rclone has an
|
||||
[AGENTS.md](AGENTS.md) file at the top of the repository describing the project's
|
||||
conventions; point your tool at it so the code it produces matches rclone's
|
||||
style.
|
||||
|
||||
However, the same standard applies to every pull request whether or not a tool
|
||||
was involved: **you are responsible for the code you submit.** Before you open a
|
||||
pull request please make sure that:
|
||||
|
||||
- You understand every line of the change and can explain why it is correct. If
|
||||
a reviewer asks a question about it, you should be able to answer without
|
||||
going back to the tool.
|
||||
- You have actually built and run it. At a minimum `go build` and
|
||||
`make quicktest` must pass, and for a backend change you should run the
|
||||
[backend tests](#backend-testing) against a real remote where you can.
|
||||
- The change is a genuine fix or feature that you have verified solves the
|
||||
problem, not a plausible-looking guess. Unverified, AI-generated pull requests
|
||||
that do not compile, do not pass the tests, invent APIs that do not exist, or
|
||||
do not actually do what the description claims waste maintainer time and are
|
||||
likely to be closed.
|
||||
- You have trimmed the comments. AI tools tend to add verbose comments that
|
||||
restate what the code plainly does or narrate the change being made; please
|
||||
cut these down to match the [code commenting style](AGENTS.md#code-commenting-style).
|
||||
|
||||
In short: an AI assistant is a tool to help *you* contribute, not a substitute
|
||||
for understanding and testing your own work.
|
||||
|
||||
## Using Git and GitHub
|
||||
|
||||
### Committing your changes
|
||||
@@ -630,37 +600,10 @@ remote or an fs.
|
||||
- **Remember** we have >50 backends to maintain so keeping them as similar as
|
||||
possible to each other is a high priority!
|
||||
|
||||
### Managing memory
|
||||
|
||||
Use rclone's memory pool for upload buffers rather than allocating them
|
||||
yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
|
||||
rclone's memory usage smaller.
|
||||
|
||||
- If the provider supports multipart or parallel chunk uploads, implement
|
||||
[fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
|
||||
and drive it with
|
||||
[multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
|
||||
This gives you pooled chunk buffers, concurrency, retries and
|
||||
`--multi-thread-streams` support for free. See s3/b2 backends for examples.
|
||||
- If you must buffer data yourself take the buffer from the global pool with
|
||||
[multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
|
||||
and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
|
||||
be rewound for a retry.
|
||||
- When using a pooled buffer `Close()` it on all paths so its pages go back to
|
||||
the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
|
||||
attempt, set the request's `ContentLength` explicitly, and if the buffer
|
||||
reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
|
||||
in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
|
||||
so the transport can't close it and free its pages before the retry.
|
||||
|
||||
### Unit tests
|
||||
|
||||
- Create a config entry called `TestRemote` for the unit tests to use
|
||||
- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
|
||||
- If your backend does chunked uploads, set `ChunkedUpload` in the
|
||||
`fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
|
||||
on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
|
||||
`FsPutRetry` tests can run the chunked path with small chunks.
|
||||
- Make sure all tests pass with `go test -v`
|
||||
|
||||
### Integration tests
|
||||
@@ -680,22 +623,14 @@ Or if you want to run the integration tests manually:
|
||||
- If your remote defines `ListR` check with this also
|
||||
- `go test -v -remote TestRemote: -fast-list`
|
||||
|
||||
Before a new or changed backend can be merged we require:
|
||||
|
||||
- A clean run of `go run ./fstest/test_all -backends remote` (please include the
|
||||
result in your pull request).
|
||||
- A test account for the backend so the maintainers can add it to the
|
||||
[integration test server](https://integration.rclone.org) and keep the backend
|
||||
working as rclone evolves. A backend that we cannot test is likely to break and
|
||||
may be removed.
|
||||
|
||||
See the [testing](#testing) section for more information on integration tests.
|
||||
|
||||
### Backend documentation
|
||||
|
||||
Add your backend to the docs. Keep lists of remotes in alphabetical
|
||||
order of full name of remote (e.g. `drive` is ordered as `Google
|
||||
Drive`) but with the local file system last.
|
||||
Add your backend to the docs - you'll need to pick an icon for it from
|
||||
[fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in
|
||||
alphabetical order of full name of remote (e.g. `drive` is ordered as
|
||||
`Google Drive`) but with the local file system last.
|
||||
|
||||
First add a data file about your backend in
|
||||
`docs/data/backends/remote.yaml` - this is used to build the overview
|
||||
|
||||
+1
-5
@@ -35,21 +35,17 @@ RUN echo "**** Print Version Binary ****" && \
|
||||
# Begin final image
|
||||
FROM alpine:latest
|
||||
|
||||
ARG RCLONE_UID=1009
|
||||
ARG RCLONE_GID=1009
|
||||
|
||||
RUN echo "**** Install Dependencies ****" && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
fuse3 \
|
||||
mailcap \
|
||||
tzdata && \
|
||||
echo "Enable user_allow_other in fuse" && \
|
||||
echo "user_allow_other" >> /etc/fuse.conf
|
||||
|
||||
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
|
||||
|
||||
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
|
||||
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
|
||||
|
||||
ENTRYPOINT [ "rclone" ]
|
||||
|
||||
|
||||
+21
-28
@@ -2,34 +2,27 @@
|
||||
|
||||
Current active maintainers of rclone are:
|
||||
|
||||
| Name | GitHub ID |
|
||||
| :--------------- | :---------------- |
|
||||
| Nick Craig-Wood | @ncw |
|
||||
| Stefan Breunig | @breunigs |
|
||||
| Ishuah Kariuki | @ishuah |
|
||||
| Remus Bunduc | @remusb |
|
||||
| Fabian Möller | @B4dM4n |
|
||||
| Alex Chen | @Cnly |
|
||||
| Sandeep Ummadi | @sandeepkru |
|
||||
| Sebastian Bünger | @buengese |
|
||||
| Ivan Andreev | @ivandeex |
|
||||
| Max Sum | @Max-Sum |
|
||||
| Fred | @creativeprojects |
|
||||
| Caleb Case | @calebcase |
|
||||
| wiserain | @wiserain |
|
||||
| albertony | @albertony |
|
||||
| Chun-Hung Tseng | @henrybear327 |
|
||||
| Hideo Aoyama | @boukendesho |
|
||||
| nielash | @nielash |
|
||||
| Dan McArdle | @dmcardle |
|
||||
| Sam Harrison | @childish-sambino |
|
||||
| Enduriel | @Enduriel |
|
||||
|
||||
Per-area ownership of code (which backend or command each maintainer looks
|
||||
after) is defined in [.github/CODEOWNERS](.github/CODEOWNERS), which GitHub uses
|
||||
to auto-request reviews. Two responsibilities don't map to a code path: @ncw
|
||||
looks after overall project health, and @boukendesho maintains the snap
|
||||
packaging.
|
||||
| Name | GitHub ID | Specific Responsibilities |
|
||||
| :--------------- | :---------------- | :-------------------------- |
|
||||
| Nick Craig-Wood | @ncw | overall project health |
|
||||
| Stefan Breunig | @breunigs | |
|
||||
| Ishuah Kariuki | @ishuah | |
|
||||
| Remus Bunduc | @remusb | cache backend |
|
||||
| Fabian Möller | @B4dM4n | |
|
||||
| Alex Chen | @Cnly | onedrive backend |
|
||||
| Sandeep Ummadi | @sandeepkru | azureblob backend |
|
||||
| Sebastian Bünger | @buengese | jottacloud, yandex & compress backends |
|
||||
| Ivan Andreev | @ivandeex | chunker & mailru backends |
|
||||
| Max Sum | @Max-Sum | union backend |
|
||||
| Fred | @creativeprojects | seafile backend |
|
||||
| Caleb Case | @calebcase | storj backend |
|
||||
| wiserain | @wiserain | pikpak backend |
|
||||
| albertony | @albertony | |
|
||||
| Chun-Hung Tseng | @henrybear327 | Proton Drive Backend |
|
||||
| Hideo Aoyama | @boukendesho | snap packaging |
|
||||
| nielash | @nielash | bisync |
|
||||
| Dan McArdle | @dmcardle | gitannex |
|
||||
| Sam Harrison | @childish-sambino | filescom |
|
||||
|
||||
## This is a work in progress draft
|
||||
|
||||
|
||||
Generated
+2577
-7670
File diff suppressed because it is too large.
Load diff
Generated
+918
-5426
File diff suppressed because it is too large.
Load diff
@@ -38,9 +38,6 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
|
||||
BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
|
||||
BETA_UPLOAD_ROOT := beta.rclone.org:
|
||||
BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
|
||||
# Update the beta site's directory listings after an upload - add
|
||||
# --changed for the uploaded directories
|
||||
BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
# Pass in GOTAGS=xyz on the make command line to set build tags
|
||||
ifdef GOTAGS
|
||||
BUILDTAGS=-tags "$(GOTAGS)"
|
||||
@@ -48,7 +45,7 @@ LINTTAGS=--build-tags "$(GOTAGS)"
|
||||
endif
|
||||
LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
|
||||
|
||||
.PHONY: rclone test_all vars version fetch-gui fetch-gui-and-commit
|
||||
.PHONY: rclone test_all vars version
|
||||
|
||||
rclone:
|
||||
ifeq ($(GO_OS),windows)
|
||||
@@ -62,12 +59,6 @@ endif
|
||||
cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
|
||||
mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
|
||||
|
||||
fetch-gui:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh
|
||||
|
||||
fetch-gui-and-commit:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh --commit
|
||||
|
||||
test_all:
|
||||
go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
|
||||
|
||||
@@ -96,15 +87,11 @@ test: rclone test_all
|
||||
@echo "Written logs in test_all.log"
|
||||
|
||||
# Quick test
|
||||
#
|
||||
# The timeout is raised above the go test default of 10m as the
|
||||
# cmd/gitannex end to end tests can take longer than that on slow CI
|
||||
# runners.
|
||||
quicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
|
||||
|
||||
racequicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
|
||||
|
||||
compiletest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
|
||||
@@ -134,25 +121,6 @@ updatedirect:
|
||||
go get $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
go mod tidy
|
||||
|
||||
# Update direct dependencies only but won't update the `go` line in go.mod
|
||||
updatedirectnoupgrade:
|
||||
@GO_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
for mod in $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all); do \
|
||||
cp go.mod go.mod.bak && \
|
||||
cp go.sum go.sum.bak && \
|
||||
go get $$mod && \
|
||||
NEW_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
if [ "$$NEW_VERSION" != "$$GO_VERSION" ]; then \
|
||||
echo "SKIPPING $$mod (requires go $$NEW_VERSION)"; \
|
||||
cp go.mod.bak go.mod; \
|
||||
cp go.sum.bak go.sum; \
|
||||
else \
|
||||
echo "updated $$mod"; \
|
||||
fi; \
|
||||
done && \
|
||||
rm -f go.mod.bak go.sum.bak && \
|
||||
go mod tidy
|
||||
|
||||
# Update direct and indirect dependencies and test dependencies
|
||||
update:
|
||||
go get -u -t ./...
|
||||
@@ -178,7 +146,6 @@ MANUAL.txt: MANUAL.md
|
||||
|
||||
commanddocs: rclone
|
||||
go generate ./lib/transform
|
||||
go generate ./cmd/bisync
|
||||
-@rmdir -p '$$HOME/.config/rclone'
|
||||
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
|
||||
@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
|
||||
@@ -237,7 +204,6 @@ check_sign:
|
||||
upload:
|
||||
rclone -P copy build/ downloads.rclone.org:/$(TAG)
|
||||
rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
|
||||
rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
|
||||
upload_github:
|
||||
./bin/upload-github $(TAG)
|
||||
@@ -270,7 +236,6 @@ ci_upload:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
|
||||
endif
|
||||
./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
|
||||
@echo Beta release ready at $(BETA_URL)/testbuilds
|
||||
|
||||
ci_beta:
|
||||
@@ -280,7 +245,6 @@ ci_beta:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
|
||||
endif
|
||||
rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
|
||||
@echo Beta release ready at $(BETA_URL)
|
||||
|
||||
# Fetch the binary builds from GitHub actions
|
||||
@@ -302,26 +266,21 @@ retag:
|
||||
@echo "Version is $(VERSION)"
|
||||
git tag -f -s -m "Version $(VERSION)" $(VERSION)
|
||||
|
||||
# Start development of the next version after a release: the next
|
||||
# minor version on master or the next patch version on a stable branch.
|
||||
startdev:
|
||||
@branch=$$(git rev-parse --abbrev-ref HEAD); \
|
||||
case "$$branch" in \
|
||||
master) next="$(NEXT_VERSION)" ;; \
|
||||
v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
|
||||
*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
|
||||
echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
|
||||
fi; \
|
||||
echo "Version is $(VERSION)"; \
|
||||
echo "Next version is $$next"; \
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
|
||||
echo -n "$$next" > docs/layouts/partials/version.html; \
|
||||
echo "$$next" > VERSION; \
|
||||
git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next version is $(NEXT_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
startstable: startdev
|
||||
startstable:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_PATCH_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
winzip:
|
||||
zip -9 rclone-$(TAG).zip rclone.exe
|
||||
|
||||
@@ -39,7 +39,6 @@ directories to and from different cloud storage providers.
|
||||
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
|
||||
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
|
||||
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
|
||||
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
|
||||
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
@@ -60,17 +59,14 @@ directories to and from different cloud storage providers.
|
||||
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
|
||||
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
|
||||
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
|
||||
- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp)
|
||||
- HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
- Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
|
||||
- Huawei Drive [:page_facing_up:](https://rclone.org/huaweidrive/)
|
||||
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
|
||||
- ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
|
||||
- Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
|
||||
- Internxt [:page_facing_up:](https://rclone.org/internxt/)
|
||||
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
|
||||
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
|
||||
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
|
||||
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
|
||||
@@ -111,10 +107,8 @@ directories to and from different cloud storage providers.
|
||||
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
|
||||
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
|
||||
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
|
||||
- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
|
||||
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
|
||||
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
|
||||
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
|
||||
- Seafile [:page_facing_up:](https://rclone.org/seafile/)
|
||||
- Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
|
||||
- SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
|
||||
@@ -129,12 +123,10 @@ directories to and from different cloud storage providers.
|
||||
- Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
|
||||
- Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
|
||||
- Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
|
||||
- US3 Object Storage [:page_facing_up:](https://rclone.org/s3/#us3)
|
||||
- Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
|
||||
- WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
- Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
|
||||
- Zadara Object Storage [:page_facing_up:](https://rclone.org/s3/#zadara)
|
||||
- Zero Services (ZERO-Z3) [:page_facing_up:](https://rclone.org/s3/#zero-z3)
|
||||
- Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
|
||||
- Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
|
||||
- The local filesystem [:page_facing_up:](https://rclone.org/local/)
|
||||
|
||||
+42
-97
@@ -6,65 +6,42 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
- [gh the github cli](https://github.com/cli/cli) for uploading packages
|
||||
- pandoc for making the html and man pages
|
||||
- hugo for building the website
|
||||
- golangci-lint and docker for `make check` (markdownlint runs in docker)
|
||||
- a gpg key for signing the tag and the checksums
|
||||
- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
|
||||
`www.rclone.org:` and `test-rclone-org:`
|
||||
- the sponsor logos - see [Sponsor logos](#sponsor-logos)
|
||||
|
||||
## Making a release
|
||||
|
||||
- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
|
||||
- git checkout master # see below for stable branch
|
||||
- git pull # IMPORTANT
|
||||
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
|
||||
- git status - make sure everything is checked in
|
||||
- Check GitHub actions build for master is Green
|
||||
- Check the integration tests at <https://integration.rclone.org>
|
||||
- cat VERSION # check this is the version being released
|
||||
- make tag # signed tag on HEAD + new changelog entry + docs
|
||||
- edit docs/content/changelog.md # entries already in point releases are
|
||||
removed automatically
|
||||
- make test # see integration test server or run locally
|
||||
- make tag
|
||||
- edit docs/content/changelog.md # make sure to remove duplicate logs from point
|
||||
releases
|
||||
- make tidy
|
||||
- make doc # again to pick up the changelog edits
|
||||
- git status - to check for new command docs in docs/content/commands - git add them
|
||||
- make doc
|
||||
- git status - to check for new man pages - git add them
|
||||
- git commit -a -v -m "Version v1.XX.0"
|
||||
- make check # if this fails, fix, git commit --amend and run it again
|
||||
- make retag # moves the tag onto the Version commit
|
||||
- make check
|
||||
- make retag
|
||||
- git push origin # without --follow-tags so it doesn't push the tag if it fails
|
||||
- git push --follow-tags origin
|
||||
- \# Wait for the GitHub builds to complete then...
|
||||
- \# Don't commit anything until after make upload_github as these steps
|
||||
use the tag on HEAD to find the version
|
||||
- make fetch_binaries
|
||||
- make tarball
|
||||
- make vendorball
|
||||
- make sign_upload
|
||||
- make check_sign
|
||||
- make upload
|
||||
- make upload_test_website # check sponsors are correct/unchanged
|
||||
- make upload_website
|
||||
- make upload_github # this triggers the docker plugin and winget workflows
|
||||
- Check the docker image, docker plugin and winget GitHub actions succeeded
|
||||
- make startdev # works out the next version for master or a stable branch
|
||||
- git push
|
||||
- Disable the release freeze rule
|
||||
- Release any security advisories
|
||||
- request a CVE for each advisory if not done already
|
||||
- set the patched version on each advisory and publish it
|
||||
- when the CVEs are assigned replace `CVE-PENDING` in the changelog on
|
||||
master and the stable branch then `make upload_website`
|
||||
- \# announce with forum post, twitter post, and post to announcements list
|
||||
- make upload_github
|
||||
- make startdev # make startstable for stable branch
|
||||
- \# announce with forum post, twitter post, patreon post
|
||||
|
||||
## Update dependencies
|
||||
|
||||
Early in the next release cycle update the dependencies.
|
||||
|
||||
- Active Pins
|
||||
- `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
|
||||
- See: [#9539](https://github.com/rclone/rclone/issues/9539)
|
||||
- Review Active Pins in go.mod and remove if possible
|
||||
- Review any pinned packages in go.mod and remove if possible
|
||||
- `make updatedirect`
|
||||
- `make GOTAGS=cmount`
|
||||
- `make compiletest`
|
||||
@@ -77,18 +54,32 @@ If the `make updatedirect` upgrades the version of go in the `go.mod`
|
||||
go 1.22.0
|
||||
```
|
||||
|
||||
then so this instead `make updatedirectnoupgrade`. This will try each
|
||||
upgrade individually. Consider making an upstream bug report for any
|
||||
`SKIPPING` lines as these only support the latest Go version not the
|
||||
previous version too.
|
||||
then go to manual mode. `go1.22` here is the lowest supported version
|
||||
in the `go.mod`.
|
||||
|
||||
If `make updatedirect` or `make updatedirectnoupgrade` added a
|
||||
`toolchain` directive then remove it. We don't want to force a
|
||||
toolchain on our users. Linux packagers are often using a version of
|
||||
Go that is a few versions out of date.
|
||||
If `make updatedirect` added a `toolchain` directive then remove it.
|
||||
We don't want to force a toolchain on our users. Linux packagers are
|
||||
often using a version of Go that is a few versions out of date.
|
||||
|
||||
```console
|
||||
go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
|
||||
go get -d $(cat /tmp/potential-upgrades)
|
||||
go mod tidy -go=1.22 -compat=1.22
|
||||
```
|
||||
|
||||
If the `go mod tidy` fails use the output from it to remove the
|
||||
package which can't be upgraded from `/tmp/potential-upgrades` when
|
||||
done
|
||||
|
||||
```console
|
||||
git co go.mod go.sum
|
||||
```
|
||||
|
||||
And try again.
|
||||
|
||||
Optionally upgrade the direct and indirect dependencies. This is very
|
||||
likely to fail if `make updatedirectnoupgrade` was used.
|
||||
likely to fail if the manual method was used abve - in that case
|
||||
ignore it as it is too time consuming to fix.
|
||||
|
||||
- `make update`
|
||||
- `make GOTAGS=cmount`
|
||||
@@ -106,19 +97,6 @@ build.
|
||||
Once it compiles locally, push it on a test branch and commit fixes
|
||||
until the tests pass.
|
||||
|
||||
### Pseudo versions
|
||||
|
||||
Go makes pseudo versions for untagged repos and repos not at a tag.
|
||||
The pseudo versions on repos that have been tagged before do not get
|
||||
updated automatically so need manually checking. These can be found with
|
||||
|
||||
```console
|
||||
grep -E '[0-9]{14}-[0-9a-f]{12}' go.mod | grep -v indirect | grep -v 'v0\.0\.0'
|
||||
```
|
||||
|
||||
These will need to be updated manually using the `go get ...@branch`
|
||||
syntax.
|
||||
|
||||
### Major versions
|
||||
|
||||
The above procedure will not upgrade major versions, so v2 to v3.
|
||||
@@ -208,53 +186,20 @@ First make the release branch. If this is a second point release then
|
||||
this will be done already.
|
||||
|
||||
- git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
|
||||
- make startdev # sets the version to ${BASE_TAG}.1
|
||||
- git push -u origin ${BASE_TAG}-stable
|
||||
- make startstable
|
||||
|
||||
Now
|
||||
|
||||
- Enable release freeze rule
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- make fetch-gui-and-commit # update the GUI on master first
|
||||
- git push
|
||||
- git co ${BASE_TAG}-stable
|
||||
- git pull
|
||||
- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
|
||||
- `git cherry-pick -x` any fixes, including the GUI update if there was one
|
||||
- for security releases consider updating the Go patch version in
|
||||
`.github/workflows/build.yml` and dependencies with CVE fixes
|
||||
- fixes may depend on commits which weren't picked so push to CI
|
||||
and check it is green before tagging
|
||||
- Do the steps in "Making a release" above, except
|
||||
- stay on ${BASE_TAG}-stable - don't checkout master or run
|
||||
make fetch-gui-and-commit
|
||||
- check the GitHub actions build for ${BASE_TAG}-stable not master
|
||||
- run the integration tests on ${BASE_TAG}-stable as described below
|
||||
- the release freeze rule is already enabled - leave it until the end
|
||||
- git cherry-pick any fixes
|
||||
- make startstable
|
||||
- Do the steps as above
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- `#` cherry pick the changes to the changelog - check the diff to make sure it
|
||||
is correct
|
||||
- git checkout ${BASE_TAG}-stable docs/content/changelog.md
|
||||
- git diff --cached # check only the ${NEW_TAG} entry has changed
|
||||
- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
|
||||
- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
|
||||
- git push
|
||||
- Disable release freeze rule
|
||||
|
||||
Point releases are only made for the latest release. Once v1.(XX+1).0
|
||||
is out the ${BASE_TAG}-stable branch is abandoned.
|
||||
|
||||
### Running the integration tests on the stable branch
|
||||
|
||||
The integration tests at <https://integration.rclone.org> test master.
|
||||
To test the stable branch push it to the testing server and run the
|
||||
tests there in `screen` as they take a while.
|
||||
|
||||
```console
|
||||
git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
|
||||
ssh rclone@rclone-testing
|
||||
screen
|
||||
cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
|
||||
```
|
||||
|
||||
## Sponsor logos
|
||||
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/compress"
|
||||
_ "github.com/rclone/rclone/backend/crypt"
|
||||
_ "github.com/rclone/rclone/backend/doi"
|
||||
_ "github.com/rclone/rclone/backend/dosya"
|
||||
_ "github.com/rclone/rclone/backend/drime"
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/dropbox"
|
||||
@@ -33,7 +32,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/hdfs"
|
||||
_ "github.com/rclone/rclone/backend/hidrive"
|
||||
_ "github.com/rclone/rclone/backend/http"
|
||||
_ "github.com/rclone/rclone/backend/huaweidrive"
|
||||
_ "github.com/rclone/rclone/backend/iclouddrive"
|
||||
_ "github.com/rclone/rclone/backend/imagekit"
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
|
||||
@@ -36,7 +36,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -186,12 +185,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
foundArchive := subArchive(remote)
|
||||
if foundArchive != nil {
|
||||
fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
|
||||
// Archive path, in canonical form so that it compares equal
|
||||
// to the cleaned entry names inside the archive
|
||||
foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
|
||||
if foundArchive.root == "." {
|
||||
foundArchive.root = ""
|
||||
}
|
||||
// Archive path
|
||||
foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
|
||||
// Path to the archive
|
||||
archiveRemote := remote[:len(foundArchive.remote)]
|
||||
// Remote is archive leaf name
|
||||
@@ -496,50 +491,24 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
subEntries, err := subFs.List(ctx, dir)
|
||||
entries, err = subFs.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = subEntries[:0]
|
||||
skipped := 0
|
||||
for _, entry := range subEntries {
|
||||
remote := entry.Remote()
|
||||
// Only pass on direct children of dir - anything else
|
||||
// could escape the archive's namespace
|
||||
if !isDirectChild(dir, remote) {
|
||||
fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
for i, entry := range entries {
|
||||
// Can only unarchive files
|
||||
if o, ok := entry.(fs.Object); ok {
|
||||
remote := o.Remote()
|
||||
archive := f.findArchive(remote)
|
||||
if archive != nil {
|
||||
// Overwrite entry with directory
|
||||
entry = fs.NewDir(remote, o.ModTime(ctx))
|
||||
entries[i] = fs.NewDir(remote, o.ModTime(ctx))
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// isDirectChild reports whether remote names an entry directly in dir
|
||||
// ("" being the root), with no ".." or other components in between.
|
||||
func isDirectChild(dir, remote string) bool {
|
||||
if dir != "" {
|
||||
var ok bool
|
||||
remote, ok = strings.CutPrefix(remote, dir+"/")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return sanitize.Leaf(remote) == nil
|
||||
}
|
||||
|
||||
// NewObject creates a new remote archive file object
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
@@ -557,11 +526,6 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Don't trust the archiver to have returned the object asked for
|
||||
if o.Remote() != remote {
|
||||
fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,27 +3,22 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -177,18 +172,6 @@ func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T,
|
||||
checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
|
||||
|
||||
// Now check a single file
|
||||
//
|
||||
// The filter below applies to both sides, so check first that the
|
||||
// archive exposes only that file without it
|
||||
Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
|
||||
require.NoError(t, err)
|
||||
entries, err := Fsingle.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
_, err = Fsingle.List(ctx, subDir)
|
||||
assert.Equal(t, fs.ErrorDirNotFound, err)
|
||||
_, err = Fsingle.NewObject(ctx, subDir)
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
fiCtx, fi := filter.AddConfig(ctx)
|
||||
require.NoError(t, fi.AddRule("+ "+aFile))
|
||||
require.NoError(t, fi.AddRule("- *"))
|
||||
@@ -236,284 +219,3 @@ func TestArchiveSquashfs(t *testing.T) {
|
||||
run(t, "mksquashfs", input, output)
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveSquashfsIssue9004 lists and reads squashfs images that exercise
|
||||
// two layouts go-diskfs used to choke on (fixed in go-diskfs v1.9.4):
|
||||
//
|
||||
// - 1.sqfs: a single empty directory, so the image has no fragment table
|
||||
// (its fragment-table start holds the "not present" sentinel).
|
||||
// - 2.sqfs: a small tree whose superblock has the NO_XATTRS flag set while
|
||||
// inodes still carry a (non-sentinel) xattr index - the shape squashfs-
|
||||
// tools-ng can emit. Built by packing a two-file tree with xattrs via
|
||||
// `gensquashfs -x`, then setting the NO_XATTRS superblock flag; the tree
|
||||
// content is trivial placeholder data.
|
||||
//
|
||||
// Both images used to fail to list. Regression test for #9004.
|
||||
func TestArchiveSquashfsIssue9004(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
testdata, err := filepath.Abs(filepath.Join("squashfs", "testdata"))
|
||||
require.NoError(t, err)
|
||||
|
||||
archiveFor := func(t *testing.T, name string) fs.Fs {
|
||||
f, err := cache.Get(ctx, ":archive:"+filepath.Join(testdata, name))
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
|
||||
t.Run("EmptyDir", func(t *testing.T) {
|
||||
// 1.sqfs is a single empty directory - it must list without error.
|
||||
entries, err := archiveFor(t, "1.sqfs").List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, len(entries))
|
||||
})
|
||||
|
||||
t.Run("NoXattrTree", func(t *testing.T) {
|
||||
f := archiveFor(t, "2.sqfs")
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, path.Base(e.Remote()))
|
||||
}
|
||||
assert.Contains(t, names, "alpha")
|
||||
assert.Contains(t, names, "beta")
|
||||
|
||||
// A file in the tree must be readable with its real content.
|
||||
obj, err := f.NewObject(ctx, "beta/sample.xml")
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, obj.Size(), int64(0))
|
||||
rc, err := obj.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, int(obj.Size()), len(data))
|
||||
assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveUncleanRoot checks that a path into an archive which isn't
|
||||
// in canonical form (with "./" or doubled slashes) still finds its
|
||||
// directory.
|
||||
func TestArchiveUncleanRoot(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create("sub/dir/a.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
zipPath := filepath.Join(t.TempDir(), "test.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
|
||||
for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
|
||||
t.Run(root, func(t *testing.T) {
|
||||
f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
|
||||
require.NoError(t, err)
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "a.txt", entries[0].Remote())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// escapingObject is an object whose remote is not where it was asked for.
|
||||
type escapingObject struct {
|
||||
fs.Object
|
||||
remote string
|
||||
}
|
||||
|
||||
func (o *escapingObject) Remote() string { return o.remote }
|
||||
func (o *escapingObject) String() string { return o.remote }
|
||||
|
||||
// escapingFs stands in for a badly behaved archiver which exposes entry
|
||||
// names outside the directory being listed.
|
||||
type escapingFs struct {
|
||||
fs.Fs
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (f *escapingFs) Name() string { return "escaping" }
|
||||
func (f *escapingFs) Root() string { return "" }
|
||||
func (f *escapingFs) String() string { return "escaping" }
|
||||
func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
|
||||
|
||||
func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return fs.DirEntries{
|
||||
&escapingObject{remote: path.Join(dir, "good.txt")},
|
||||
fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
&escapingObject{remote: path.Join(dir, "../escape.txt")},
|
||||
&escapingObject{remote: "../../escape.txt"},
|
||||
&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
|
||||
fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
return &escapingObject{remote: "../escape.txt"}, nil
|
||||
}
|
||||
|
||||
// TestArchiveEscapingArchiver checks that the archive backend does not
|
||||
// pass on entries from an archiver which escape the directory being
|
||||
// listed, whatever the archiver does.
|
||||
func TestArchiveEscapingArchiver(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
return &escapingFs{prefix: prefix}, nil
|
||||
},
|
||||
Extension: ".escaping",
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
|
||||
f, err := cache.Get(ctx, ":archive:"+dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Archives are discovered when their parent directory is listed
|
||||
_, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "test.escaping")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
|
||||
|
||||
_, err = f.NewObject(ctx, "test.escaping/file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
|
||||
// TestIsDirectChild checks the guard which decides whether an entry
|
||||
// returned by an archiver belongs directly in the directory listed.
|
||||
func TestIsDirectChild(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
dir, remote string
|
||||
want bool
|
||||
}{
|
||||
{"", "a.txt", true},
|
||||
{"", "/a.txt", false},
|
||||
{"", "a.txt/", false},
|
||||
{"", "../a.txt", false},
|
||||
{"", "sub/a.txt", false},
|
||||
{"d", "d/a.txt", true},
|
||||
{"d", "d", false},
|
||||
{"d", "d/", false},
|
||||
{"d", "d//a.txt", false},
|
||||
{"d", "d/../a.txt", false},
|
||||
{"d", "dd/a.txt", false},
|
||||
{"d", "a.txt", false},
|
||||
} {
|
||||
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
|
||||
}
|
||||
}
|
||||
|
||||
// A symlink in a squashfs image is skipped unless -l/--links is in use,
|
||||
// as it is on the other backends. With the flag it is exposed under its
|
||||
// name with the link suffix and reads back as its target.
|
||||
//
|
||||
// Note that this uses mksquashfs as an external binary.
|
||||
func TestArchiveSquashfsSymlink(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "mksquashfs")
|
||||
ctx := context.Background()
|
||||
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "input")
|
||||
require.NoError(t, os.Mkdir(input, 0777))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
|
||||
require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
|
||||
// a regular file which happens to carry the link suffix must stay
|
||||
// readable under the flag, not be taken for a symlink
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
|
||||
image := filepath.Join(dir, "test.sqfs")
|
||||
run(t, "mksquashfs", input, image)
|
||||
|
||||
remotes := func(t *testing.T, f fs.Fs) []string {
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
names = append(names, entry.Remote())
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
read := func(t *testing.T, f fs.Fs, remote string) string {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
require.NoError(t, err)
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
contents, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, o.Size(), int64(len(contents)))
|
||||
return string(contents)
|
||||
}
|
||||
|
||||
f, err := fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
|
||||
assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
|
||||
}
|
||||
|
||||
// A hostile archive must not be able to write outside the destination.
|
||||
// A symlink pointing above the root with a member inside it is the
|
||||
// classic way to try it, and the refusal comes from the local backend,
|
||||
// so pin it from end to end here rather than trusting it stays.
|
||||
func TestArchiveZipSymlinkEscape(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
hdr := &zip.FileHeader{Name: "a"}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err := zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("../outside"))
|
||||
require.NoError(t, err)
|
||||
w, err = zw.Create("a/pwned.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("pwned"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
zipPath := filepath.Join(dir, "evil.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
dst := filepath.Join(dir, "dst")
|
||||
require.NoError(t, os.Mkdir(dst, 0777))
|
||||
|
||||
fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
|
||||
require.NoError(t, err)
|
||||
fdst, err := fs.NewFs(ctx, dst)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
|
||||
|
||||
_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
|
||||
assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
|
||||
}
|
||||
@@ -8,11 +8,6 @@ import (
|
||||
)
|
||||
|
||||
// Archiver describes an archive package
|
||||
//
|
||||
// Entry names inside an archive are attacker controlled. An archiver
|
||||
// must not expose an entry whose remote escapes the archive's own
|
||||
// namespace (for example one with a ".." component), typically by
|
||||
// validating names with lib/sanitize.
|
||||
type Archiver struct {
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
|
||||
@@ -36,7 +36,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
VFS := vfs.New(ctx, wrappedFs, nil)
|
||||
VFS := vfs.New(wrappedFs, nil)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
|
||||
@@ -161,10 +161,5 @@ func (c *cache) Writable() (backend.WritableFile, error) {
|
||||
return nil, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Path returns the path to the underlying storage - not applicable here
|
||||
func (c *cache) Path() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// check interfaces
|
||||
var _ backend.Storage = (*cache)(nil)
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -16,7 +15,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
@@ -42,42 +40,17 @@ type Fs struct {
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
file string // remote of the single file root points at, if any
|
||||
links bool // read symlinks as .rclonelink files
|
||||
}
|
||||
|
||||
// recoverParsePanic converts a panic raised while parsing a squashfs image
|
||||
// into an error assigned through err.
|
||||
func recoverParsePanic(err *error) {
|
||||
if r := recover(); r != nil {
|
||||
*err = fmt.Errorf("invalid or corrupt squashfs image: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// recoveringReadCloser wraps a reader returned by the go-diskfs parser so
|
||||
// panics raised while reading become errors.
|
||||
//
|
||||
// The parser reads file data lazily, so an image which opens cleanly can
|
||||
// still panic on the first Read.
|
||||
type recoveringReadCloser struct {
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
func (r recoveringReadCloser) Read(p []byte) (n int, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
return r.ReadCloser.Read(p)
|
||||
}
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ fs.Fs, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
@@ -100,7 +73,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
|
||||
c: c,
|
||||
remote: remote,
|
||||
root: strings.Trim(root, "/"),
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -118,13 +90,10 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
|
||||
_, err := f.newObjectNative(native)
|
||||
if err == nil {
|
||||
// If it pointed to a file, find the directory above
|
||||
// and remember the file so that only it is listed
|
||||
f.root = path.Dir(f.root)
|
||||
if f.root == "." || f.root == "/" {
|
||||
f.root = ""
|
||||
}
|
||||
f.file = f.prefixSlash + path.Base(native)
|
||||
singleObject = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -156,9 +125,7 @@ func (f *Fs) Name() string {
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
// Include the file so the fs cache can tell this Fs apart from
|
||||
// the one for the directory containing it
|
||||
return path.Join(f.root, f.file)
|
||||
return f.root
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
@@ -189,17 +156,6 @@ func (f *Fs) toNative(remote string) (string, error) {
|
||||
return native, nil
|
||||
}
|
||||
|
||||
// This turns a native path with a leading / into a path suitable for
|
||||
// the squashfs library which requires io/fs.ValidPath paths with no
|
||||
// leading / and "." for the root directory.
|
||||
func toIOFS(native string) string {
|
||||
native = strings.Trim(native, "/")
|
||||
if native == "" {
|
||||
return "."
|
||||
}
|
||||
return native
|
||||
}
|
||||
|
||||
// Turn a (nativeDir, leaf) into a remote
|
||||
func (f *Fs) fromNative(nativeDir string, leaf string) string {
|
||||
// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
|
||||
@@ -213,39 +169,16 @@ func (f *Fs) fromNative(nativeDir string, leaf string) string {
|
||||
}
|
||||
|
||||
// Convert a FileInfo into an Object from native dir
|
||||
func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
|
||||
func (f *Fs) objectFromFileInfo(nativeDir string, item squashfs.FileStat) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()),
|
||||
size: item.Size(),
|
||||
modTime: item.ModTime(),
|
||||
item: item,
|
||||
}
|
||||
}
|
||||
|
||||
// Convert a symlink entry into an Object holding its target
|
||||
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
|
||||
size: int64(len(target)),
|
||||
modTime: item.ModTime(),
|
||||
isLink: true,
|
||||
link: target,
|
||||
}
|
||||
}
|
||||
|
||||
// readlink returns the target of the symlink the info describes.
|
||||
//
|
||||
// A symlink may carry an empty target, so the caller decides what is a
|
||||
// symlink from the mode rather than from the target being set.
|
||||
func readlink(info os.FileInfo) (string, error) {
|
||||
st, ok := info.Sys().(*squashfs.StatT)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
|
||||
}
|
||||
return st.LinkTarget, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
@@ -257,127 +190,70 @@ func readlink(info os.FileInfo) (string, error) {
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
||||
defer recoverParsePanic(&err)
|
||||
|
||||
if f.file != "" && dir != "" {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
nativeDir, err := f.toNative(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items, err := f.sqfs.ReadDir(toIOFS(nativeDir))
|
||||
items, err := f.sqfs.ReadDir(nativeDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
|
||||
}
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, item := range items {
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
// Skip entry names that aren't safe
|
||||
if err := sanitize.Leaf(item.Name()); err != nil {
|
||||
fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
|
||||
skipped++
|
||||
continue
|
||||
for _, fi := range items {
|
||||
item, ok := fi.(squashfs.FileStat)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
|
||||
}
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
var entry fs.DirEntry
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %q", item.Name(), err)
|
||||
}
|
||||
if item.IsDir() {
|
||||
remote := f.fromNative(nativeDir, item.Name())
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = fs.NewDir(remote, info.ModTime())
|
||||
} else if item.Type().IsRegular() {
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = f.objectFromFileInfo(nativeDir, info)
|
||||
} else if item.Type()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
var remote = f.fromNative(nativeDir, item.Name())
|
||||
entry = fs.NewDir(remote, item.ModTime())
|
||||
} else {
|
||||
if item.Mode().IsRegular() {
|
||||
entry = f.objectFromFileInfo(nativeDir, item)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
}
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry = f.objectFromLink(nativeDir, info, target)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
}
|
||||
if f.file != "" && entry.Remote() != f.file {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
// Log once per listing rather than per entry so a crafted image
|
||||
// can't flood the log
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// newObjectNative finds the object at the native path passed in
|
||||
func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
// get the path and filename
|
||||
dir, leaf := path.Split(nativePath)
|
||||
dir = strings.TrimRight(dir, "/")
|
||||
leaf = strings.Trim(leaf, "/")
|
||||
|
||||
// FIXME need to detect directory not found
|
||||
fis, err := f.sqfs.ReadDir(toIOFS(dir))
|
||||
fis, err := f.sqfs.ReadDir(dir)
|
||||
if err != nil {
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
// Name the entry as List does, so that the two agree on
|
||||
// which names exist
|
||||
isLink := fi.Type()&os.ModeSymlink != 0
|
||||
if isLink && !f.links {
|
||||
continue
|
||||
}
|
||||
name := fi.Name()
|
||||
if isLink {
|
||||
name += fs.LinkSuffix
|
||||
}
|
||||
if name == leaf {
|
||||
if fi.Name() == leaf {
|
||||
if fi.IsDir() {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
info, err := fi.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
|
||||
item, ok := fi.(squashfs.FileStat)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("internal error: unexpected type for %q: %T", fi.Name(), fi)
|
||||
}
|
||||
if isLink {
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o = f.objectFromLink(dir, info, target)
|
||||
break
|
||||
}
|
||||
o = f.objectFromFileInfo(dir, info)
|
||||
o = f.objectFromFileInfo(dir, item)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -391,9 +267,6 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
|
||||
|
||||
if f.file != "" && remote != f.file {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
nativePath, err := f.toNative(remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -455,8 +328,7 @@ type Object struct {
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
isLink bool // whether this is a symlink
|
||||
link string // target of the symlink
|
||||
item squashfs.FileStat
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
@@ -513,7 +385,6 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
@@ -528,24 +399,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
}
|
||||
}
|
||||
|
||||
if o.isLink {
|
||||
// Clamp offset into range to avoid panic
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset > int64(len(o.link)) {
|
||||
offset = int64(len(o.link))
|
||||
}
|
||||
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
|
||||
}
|
||||
|
||||
remote, err := o.fs.toNative(o.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fs.Debugf(o, "Opening %q", remote)
|
||||
fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)
|
||||
//fh, err := o.fs.sqfs.OpenFile(remote, os.O_RDONLY)
|
||||
fh, err := o.item.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -557,15 +418,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rc = recoveringReadCloser{fh}
|
||||
|
||||
// If limited then don't return everything
|
||||
if limit >= 0 {
|
||||
fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
|
||||
return readers.NewLimitedReadCloser(rc, limit), nil
|
||||
return readers.NewLimitedReadCloser(fh, limit), nil
|
||||
}
|
||||
|
||||
return rc, nil
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
fscache "github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A backslash is an ordinary character in a file name on the systems
|
||||
// squashfs images are made on, so an entry containing one must be
|
||||
// listed and openable like any other file.
|
||||
func TestBackslashInName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localFs, err := fscache.Get(ctx, "testdata")
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, "backslash.sqfs", "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, `back\slash.txt`, entries[0].Remote())
|
||||
|
||||
o, err := f.NewObject(ctx, `back\slash.txt`)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(6), o.Size())
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// panickingReader panics on Read, standing in for the go-diskfs parser
|
||||
// hitting an unchecked slice while reading a malformed image.
|
||||
type panickingReader struct{}
|
||||
|
||||
func (panickingReader) Read([]byte) (int, error) { panic("boom") }
|
||||
func (panickingReader) Close() error { return nil }
|
||||
|
||||
// Test that a panic raised while reading file data becomes an error. The
|
||||
// parser reads lazily, so an image can open cleanly and only panic here.
|
||||
func TestRecoveringReadCloser(t *testing.T) {
|
||||
var rc io.ReadCloser = recoveringReadCloser{panickingReader{}}
|
||||
|
||||
_, err := rc.Read(make([]byte, 16))
|
||||
assert.ErrorContains(t, err, "invalid or corrupt squashfs image")
|
||||
assert.ErrorContains(t, err, "boom")
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -1,89 +0,0 @@
|
||||
//go:build !plan9
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// squashfsSuperblock builds a minimal 96-byte squashfs superblock with the
|
||||
// given attacker-controllable fields. It mirrors the on-disk layout the
|
||||
// go-diskfs parser reads, and is only used to construct deliberately malformed
|
||||
// images for the tests below.
|
||||
func squashfsSuperblock(blockSize uint32, blockLog, compression uint16, rootInode, inodeTableStart uint64) []byte {
|
||||
const u64max = ^uint64(0)
|
||||
b := make([]byte, 96)
|
||||
binary.LittleEndian.PutUint32(b[0:], 0x73717368) // "hsqs" magic
|
||||
binary.LittleEndian.PutUint32(b[4:], 1) // inode count
|
||||
binary.LittleEndian.PutUint32(b[8:], 0) // modtime
|
||||
binary.LittleEndian.PutUint32(b[12:], blockSize)
|
||||
binary.LittleEndian.PutUint32(b[16:], 0) // fragment count
|
||||
binary.LittleEndian.PutUint16(b[20:], compression)
|
||||
binary.LittleEndian.PutUint16(b[22:], blockLog)
|
||||
binary.LittleEndian.PutUint16(b[24:], 0x0210) // no fragments/xattrs
|
||||
binary.LittleEndian.PutUint16(b[26:], 0) // id count
|
||||
binary.LittleEndian.PutUint16(b[28:], 4) // version major
|
||||
binary.LittleEndian.PutUint16(b[30:], 0) // version minor
|
||||
binary.LittleEndian.PutUint64(b[32:], rootInode)
|
||||
binary.LittleEndian.PutUint64(b[40:], 96) // bytes used
|
||||
binary.LittleEndian.PutUint64(b[48:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[56:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[64:], inodeTableStart)
|
||||
binary.LittleEndian.PutUint64(b[72:], inodeTableStart)
|
||||
binary.LittleEndian.PutUint64(b[80:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[88:], u64max)
|
||||
return b
|
||||
}
|
||||
|
||||
// TestArchiveSquashfsMalformed feeds the archive backend squashfs images whose
|
||||
// superblock/metadata make the go-diskfs parser panic. See GHSA-6jcg-q3wp-x2f4.
|
||||
func TestArchiveSquashfsMalformed(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
// Variant 1: zero block size reaches an integer division in the parser.
|
||||
zeroBlock := squashfsSuperblock(0, 0, 1, 0, 96)
|
||||
|
||||
// Variant 2: an out-of-range root inode byte offset reaches an unchecked
|
||||
// slice against an empty, uncompressed metadata block.
|
||||
oob := append(squashfsSuperblock(65536, 16, 1, 12336, 96), 0x00, 0x80)
|
||||
binary.LittleEndian.PutUint64(oob[40:], uint64(len(oob)))
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
image []byte
|
||||
}{
|
||||
{"ZeroBlockSize", zeroBlock},
|
||||
{"OutOfRangeInodeOffset", oob},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// A directory per image: the fs cache keys on the parent
|
||||
// directory, so sharing one would hand the second subtest a
|
||||
// cached listing which does not contain its image.
|
||||
path := filepath.Join(t.TempDir(), tc.name+".sqfs")
|
||||
require.NoError(t, os.WriteFile(path, tc.image, 0o644))
|
||||
|
||||
// The whole point is that this returns rather than panicking and
|
||||
// killing the test binary (and, in production, the serve daemon).
|
||||
f, err := cache.Get(ctx, ":archive:"+path)
|
||||
if err == nil {
|
||||
// The image parsed, so the panic must surface from a later
|
||||
// lazily-parsed operation instead.
|
||||
_, err = f.List(ctx, "")
|
||||
}
|
||||
require.Error(t, err, "a malformed squashfs image must error, not panic")
|
||||
assert.Contains(t, err.Error(), "invalid or corrupt squashfs image",
|
||||
"error should come from the recover, not an unrelated failure")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -19,7 +18,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
@@ -43,7 +41,6 @@ type Fs struct {
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
links bool // read symlinks as .rclonelink files
|
||||
dt dirtree.DirTree // read from zipfile
|
||||
}
|
||||
|
||||
@@ -55,7 +52,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
@@ -68,7 +65,6 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
@@ -139,37 +135,15 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
return singleObject, fmt.Errorf("failed to read zip file: %w", err)
|
||||
}
|
||||
dt := dirtree.New()
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, file := range zr.File {
|
||||
// Skip entries whose name escapes the archive's own namespace
|
||||
remote, err := sanitize.Path(file.Name)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
isDir := strings.HasSuffix(file.Name, "/")
|
||||
// A file entry whose last component is "", "." or ".." names
|
||||
// a directory rather than a file, so skip
|
||||
if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !isDir && file.Mode()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
// The member holds the link target, so it reads back
|
||||
// like a .rclonelink file
|
||||
remote += fs.LinkSuffix
|
||||
remote := strings.Trim(path.Clean(file.Name), "/")
|
||||
if remote == "." {
|
||||
remote = ""
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root, requiring a path
|
||||
// boundary so that root "foo" does not also match a
|
||||
// sibling entry such as "foobar"
|
||||
if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
|
||||
// Ignore all files outside the root
|
||||
if !strings.HasPrefix(remote, f.root) {
|
||||
continue
|
||||
}
|
||||
if remote == f.root {
|
||||
@@ -178,12 +152,11 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
remote = strings.TrimPrefix(remote, f.root+"/")
|
||||
}
|
||||
}
|
||||
if isDir {
|
||||
if strings.HasSuffix(file.Name, "/") {
|
||||
dir := fs.NewDir(remote, file.Modified)
|
||||
dt.AddDir(dir)
|
||||
} else {
|
||||
if remote == "" {
|
||||
// A file at the root is the archive member f.root points at
|
||||
remote = path.Base(f.root)
|
||||
singleObject = true
|
||||
dt = dirtree.New()
|
||||
@@ -200,12 +173,6 @@ func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
@@ -233,8 +200,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
// Return a copy as callers may filter the listing in place
|
||||
return slices.Clone(entries), nil
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
package zip
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// writeZip builds a zip file with the given entry names (a name ending
|
||||
// in "/" is a directory) in dir and returns its leaf name.
|
||||
func writeZip(t *testing.T, dir, name string, names ...string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, n := range names {
|
||||
w, err := zw.Create(n)
|
||||
require.NoError(t, err)
|
||||
if !strings.HasSuffix(n, "/") {
|
||||
_, err = w.Write([]byte("data for " + n))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// writeZipSymlink builds a zip holding file and a symlink entry link
|
||||
// pointing at target, and returns its leaf name.
|
||||
func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create(file)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data for " + file))
|
||||
require.NoError(t, err)
|
||||
hdr := &zip.FileHeader{Name: link}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err = zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(target))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// allRemotes returns every Object remote in the mounted archive's dirtree.
|
||||
func allRemotes(t *testing.T, f fs.Fs) []string {
|
||||
zf, ok := f.(*Fs)
|
||||
require.True(t, ok)
|
||||
var remotes []string
|
||||
for _, entries := range zf.dt {
|
||||
for _, entry := range entries {
|
||||
if o, ok := entry.(*Object); ok {
|
||||
remotes = append(remotes, o.Remote())
|
||||
}
|
||||
}
|
||||
}
|
||||
// dt iteration order is nondeterministic, so sort for stable comparison.
|
||||
sort.Strings(remotes)
|
||||
return remotes
|
||||
}
|
||||
|
||||
// A malicious zip whose entry names escape the archive's own namespace
|
||||
// must not be exposed by the zip backend (Zip Slip).
|
||||
func TestReadZipSlip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "evil.zip",
|
||||
"good.txt",
|
||||
"../../etc/cron.d/evil",
|
||||
"../escape.txt",
|
||||
"sub/../../above.txt",
|
||||
`..\..\evil.exe`,
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
|
||||
// The one benign entry must survive.
|
||||
assert.Contains(t, remotes, "good.txt")
|
||||
|
||||
// No escaping entry may be exposed.
|
||||
for _, remote := range remotes {
|
||||
assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
|
||||
"escaping remote exposed: %q", remote)
|
||||
}
|
||||
assert.Equal(t, []string{"good.txt"}, remotes)
|
||||
}
|
||||
|
||||
// Mounting with a non-empty root must only expose entries within that
|
||||
// root directory, not sibling directories that merely share a name
|
||||
// prefix (root "foo" must not match "foobar").
|
||||
func TestReadZipRootBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip",
|
||||
"foo/a.txt",
|
||||
"foobar/b.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "foo")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
assert.Equal(t, []string{"a.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A file entry whose name refers to a directory (".", "/", "./", "" or
|
||||
// "sub/.") must be skipped, not turn the archive or the directory
|
||||
// mounted as its root into a single file which hides every other
|
||||
// entry, or become an object named for the archive itself when the
|
||||
// archive is found by listing its directory.
|
||||
func TestReadZipDirNamedFileEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "dot.zip",
|
||||
".",
|
||||
"/",
|
||||
"./",
|
||||
"",
|
||||
"good.txt",
|
||||
"sub/",
|
||||
"sub/.",
|
||||
"sub/good.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, test := range []struct {
|
||||
prefix, root string
|
||||
want []string
|
||||
}{
|
||||
{"", "", []string{"good.txt", "sub/good.txt"}},
|
||||
{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
|
||||
{"", "sub", []string{"good.txt"}},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
|
||||
f, err := New(ctx, localFs, name, test.prefix, test.root)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, allRemotes(t, f))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Listings are served from a cache which must survive callers
|
||||
// filtering the returned slice in place, as fs/list does.
|
||||
func TestListCacheNotAliased(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 3)
|
||||
// Compact in place, dropping the first entry
|
||||
copy(entries, entries[1:])
|
||||
|
||||
entries, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A symlink stored in the archive is skipped unless -l/--links is in
|
||||
// use, as it is on the other backends. With the flag it is exposed
|
||||
// under its name with the link suffix and reads back as its target.
|
||||
func TestReadZipSymlink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
|
||||
|
||||
o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len("file.txt")), o.Size())
|
||||
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
target, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, "file.txt", string(target))
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Parallel listing support for azureblob, built on the Apache Arrow
|
||||
// listing format.
|
||||
//
|
||||
// The Arrow listing exposes the server-side startFrom (inclusive) and endBefore
|
||||
// (exclusive) name-range parameters, which let a listing be sharded by
|
||||
// blob-name range and listed concurrently.
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
|
||||
// response, used to tell whether the service honoured an Arrow request.
|
||||
const arrowContentType = "application/vnd.apache.arrow.stream"
|
||||
|
||||
// listArrowParallel lists (containerName, directory) by splitting the blob name
|
||||
// keyspace into shards and listing them concurrently. opts must already request
|
||||
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
|
||||
// returns the number of raw items seen. directory/prefix are already normalised
|
||||
// by the caller.
|
||||
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
|
||||
// Split the keyspace by a single-case character ladder (see
|
||||
// arrowLadderBoundaries for why single-case matters).
|
||||
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
|
||||
if len(points) < 1 {
|
||||
// Couldn't usefully split - list sequentially.
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
|
||||
// Build half-open [startFrom, endBefore) shards from the sorted boundary
|
||||
// points p0..pk: ["", p0), [p0, p1), ..., [pk, "").
|
||||
type shard struct{ startFrom, endBefore string }
|
||||
shards := make([]shard, 0, len(points)+1)
|
||||
prev := ""
|
||||
for _, p := range points {
|
||||
shards = append(shards, shard{prev, p})
|
||||
prev = p
|
||||
}
|
||||
shards = append(shards, shard{prev, ""})
|
||||
|
||||
// fn (and the list.Helper behind it) is not concurrency-safe, so serialise
|
||||
// delivery. Only the cheap callback is locked, not the HTTP/decode work.
|
||||
var fnMu sync.Mutex
|
||||
safeFn := func(remote string, item *container.BlobItem, isDir bool) error {
|
||||
fnMu.Lock()
|
||||
defer fnMu.Unlock()
|
||||
return fn(remote, item, isDir)
|
||||
}
|
||||
|
||||
// Schedule every shard; SetLimit bounds concurrency, so more shards than
|
||||
// workers gives free load balancing as workers pick up the next shard.
|
||||
var total atomic.Int64
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(f.opt.ListParallelism)
|
||||
for _, sh := range shards {
|
||||
if gCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
// Clone opts per shard so concurrent shards don't share StartFrom/EndBefore.
|
||||
shardOpts := *opts
|
||||
if sh.startFrom != "" {
|
||||
startFrom := sh.startFrom
|
||||
shardOpts.StartFrom = &startFrom
|
||||
} else {
|
||||
shardOpts.StartFrom = nil
|
||||
}
|
||||
if sh.endBefore != "" {
|
||||
endBefore := sh.endBefore
|
||||
shardOpts.EndBefore = &endBefore
|
||||
} else {
|
||||
shardOpts.EndBefore = nil
|
||||
}
|
||||
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
|
||||
total.Add(int64(n))
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := g.Wait()
|
||||
found := int(total.Load())
|
||||
if isEndBeforeUnsupported(err) {
|
||||
// The account can't honour the shards' endBefore bounds, so parallel
|
||||
// listing can't be used on it.
|
||||
f.arrowXMLFallback.Store(true)
|
||||
if found == 0 {
|
||||
// No shard delivered anything yet (the normal case - Arrow listing
|
||||
// enablement is per account so every shard fails on its first
|
||||
// page) so it is safe to relist sequentially.
|
||||
fs.Debugf(f, "Parallel Arrow listing not supported by this account (%v) - listing sequentially", err)
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
// Some entries were already delivered so a rerun would duplicate them.
|
||||
return found, fmt.Errorf("parallel listing aborted after partial results (retry, or unset list_parallelism): %w", err)
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
|
||||
// isEndBeforeUnsupported reports whether err means the service can't honour
|
||||
// the endBefore listing bound, so sharded parallel listing must not be used.
|
||||
//
|
||||
// The service rejects the endBefore parameter with 400
|
||||
// OperationNotSupportedWithFeatureMissing "The requested operation is not
|
||||
// allowed as EndBefore parameter support is missing on this account" -
|
||||
// endBefore is only supported on the Arrow listing path of accounts with
|
||||
// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
|
||||
// accept an XML page with endBefore set, but the service never sends one.)
|
||||
func isEndBeforeUnsupported(err error) bool {
|
||||
var respErr *azcore.ResponseError
|
||||
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
|
||||
}
|
||||
|
||||
// arrowShardTarget is the number of ladder boundaries to use for the given
|
||||
// parallelism (a few per worker for load balancing).
|
||||
func arrowShardTarget(parallelism int) int {
|
||||
return parallelism * 4
|
||||
}
|
||||
|
||||
// arrowLadderBoundaries returns up to target sorted ENCODED-space boundaries of
|
||||
// the form directory+c, used to split the keyspace into shards.
|
||||
//
|
||||
// The partition is always correct (no dup/gap) regardless of the alphabet: keys
|
||||
// whose next byte is outside the alphabet simply fall into an adjacent shard;
|
||||
// the alphabet only affects balance. A single-case (digit+lowercase) alphabet
|
||||
// is deliberate: the boundaries are then monotonic under BOTH byte order (the
|
||||
// listing order) AND the service's case-insensitive startFrom/endBefore
|
||||
// validation, so server-side endBefore is accepted. (A mixed-case ladder gives
|
||||
// pairs like [".../Z", ".../a") which are byte-ascending but rejected 400 as
|
||||
// "endbefore precedes startfrom".) directory is already encoded; the alphabet
|
||||
// is ASCII (encoder-invariant) and ascending, so the result is sorted.
|
||||
func arrowLadderBoundaries(directory string, target int) []string {
|
||||
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
if target < 1 {
|
||||
return nil
|
||||
}
|
||||
if target > len(alphabet) {
|
||||
target = len(alphabet)
|
||||
}
|
||||
points := make([]string, 0, target)
|
||||
last := -1
|
||||
for i := 0; i < target; i++ {
|
||||
idx := i * len(alphabet) / target
|
||||
if idx == last {
|
||||
continue
|
||||
}
|
||||
last = idx
|
||||
points = append(points, directory+string(alphabet[idx]))
|
||||
}
|
||||
return points
|
||||
}
|
||||
@@ -1,219 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testArrowList exercises the experimental Apache Arrow listing.
|
||||
//
|
||||
// It is skipped automatically if Arrow listing is not enabled on the test
|
||||
// account (the server returns XML, or 409 on HNS accounts).
|
||||
func (f *Fs) testArrowList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
containerName, _ := f.split("probe")
|
||||
if containerName == "" {
|
||||
t.Skip("Arrow list test needs a container in the remote root")
|
||||
}
|
||||
|
||||
// withArrow runs fn with UseArrowList temporarily set.
|
||||
withArrow := func(on bool, fn func()) {
|
||||
old := f.opt.UseArrowList
|
||||
f.opt.UseArrowList = on
|
||||
defer func() { f.opt.UseArrowList = old }()
|
||||
fn()
|
||||
}
|
||||
|
||||
// withParallel runs fn with ListParallelism temporarily set.
|
||||
withParallel := func(n int, fn func()) {
|
||||
old := f.opt.ListParallelism
|
||||
f.opt.ListParallelism = n
|
||||
defer func() { f.opt.ListParallelism = old }()
|
||||
fn()
|
||||
}
|
||||
|
||||
// Create a set of known objects under a unique directory: flat files plus
|
||||
// one in a subdirectory (so we exercise both the flat and delimited paths).
|
||||
dir := "arrow-list-" + random.String(8)
|
||||
// Includes upper-case and digit first-characters: these byte-sort between
|
||||
// the digit and lowercase ladder boundaries, so they verify the server's
|
||||
// endBefore filtering is byte-ordered (not case-insensitive, which would
|
||||
// drop them from a shard and make parallel != sequential).
|
||||
objects := []string{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt", "sub/x.txt", "Foo.txt", "Zoo.txt", "0num.txt", "9end.txt"}
|
||||
wantSizes := map[string]int64{}
|
||||
created := make([]string, 0, len(objects))
|
||||
for i, n := range objects {
|
||||
contents := random.String(10 + i)
|
||||
remote := dir + "/" + n
|
||||
item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
|
||||
_ = fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
created = append(created, remote)
|
||||
wantSizes[remote] = int64(len(contents))
|
||||
}
|
||||
defer func() {
|
||||
for _, remote := range created {
|
||||
if o, err := f.NewObject(ctx, remote); err == nil {
|
||||
_ = o.Remove(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// listing returns remote->size for dir (dirs reported as size -1).
|
||||
listing := func(recurse bool) (map[string]int64, error) {
|
||||
got := map[string]int64{}
|
||||
add := func(entries fs.DirEntries) error {
|
||||
for _, e := range entries {
|
||||
if o, ok := e.(fs.Object); ok {
|
||||
got[e.Remote()] = o.Size()
|
||||
} else {
|
||||
got[e.Remote()] = -1
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if recurse {
|
||||
return got, f.ListR(ctx, dir, add)
|
||||
}
|
||||
entries, err := f.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = add(entries)
|
||||
return got, nil
|
||||
}
|
||||
|
||||
// Probe: is Arrow available? If not, skip cleanly.
|
||||
var probeErr error
|
||||
withArrow(true, func() { _, probeErr = listing(true) })
|
||||
if probeErr != nil {
|
||||
if strings.Contains(probeErr.Error(), "OperationNotSupportedWithFeatureMissing") || strings.Contains(probeErr.Error(), "409") {
|
||||
t.Skipf("Arrow listing not available on this account: %v", probeErr)
|
||||
}
|
||||
require.NoError(t, probeErr)
|
||||
}
|
||||
|
||||
// Arrow listing must equal XML listing, recursively and per-directory
|
||||
// (including the "sub" subdirectory).
|
||||
t.Run("EqualsXML", func(t *testing.T) {
|
||||
for _, recurse := range []bool{true, false} {
|
||||
var xml, arrow map[string]int64
|
||||
var err error
|
||||
withArrow(false, func() { xml, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
withArrow(true, func() { arrow, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, xml, arrow, "recurse=%v: Arrow listing differs from XML", recurse)
|
||||
}
|
||||
|
||||
// Recursive Arrow listing contains every object with the right size.
|
||||
var arrow map[string]int64
|
||||
withArrow(true, func() { arrow, _ = listing(true) })
|
||||
for remote, size := range wantSizes {
|
||||
assert.Equal(t, size, arrow[remote], remote)
|
||||
}
|
||||
|
||||
// Single-directory Arrow listing reports "sub" as a directory.
|
||||
var entries fs.DirEntries
|
||||
withArrow(true, func() {
|
||||
var err error
|
||||
entries, err = f.List(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
foundSubDir := false
|
||||
for _, e := range entries {
|
||||
if e.Remote() == dir+"/sub" {
|
||||
_, isObj := e.(fs.Object)
|
||||
foundSubDir = !isObj
|
||||
}
|
||||
}
|
||||
assert.True(t, foundSubDir, "expected %q to be listed as a directory", dir+"/sub")
|
||||
})
|
||||
|
||||
// Parallel listing must equal sequential listing, recursively and
|
||||
// per-directory, including the "sub" subdirectory.
|
||||
t.Run("ParallelEqualsSequential", func(t *testing.T) {
|
||||
for _, recurse := range []bool{true, false} {
|
||||
var seq, par map[string]int64
|
||||
var err error
|
||||
withArrow(true, func() {
|
||||
withParallel(0, func() { seq, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
withParallel(8, func() { par, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
})
|
||||
assert.Equal(t, seq, par, "recurse=%v: parallel listing differs from sequential", recurse)
|
||||
}
|
||||
})
|
||||
|
||||
// A tiny list_chunk forces each shard to span multiple Arrow pages, so this
|
||||
// exercises pagination (startFrom+endBefore+marker resent together) inside a
|
||||
// shard. The "sub/" boundary puts a.txt..e.txt in one shard => >1 page at
|
||||
// chunk size 2.
|
||||
t.Run("ParallelMultiPage", func(t *testing.T) {
|
||||
oldChunk := f.opt.ListChunkSize
|
||||
f.opt.ListChunkSize = 2
|
||||
defer func() { f.opt.ListChunkSize = oldChunk }()
|
||||
var seq, par map[string]int64
|
||||
var err error
|
||||
withArrow(true, func() {
|
||||
withParallel(0, func() { seq, err = listing(true) })
|
||||
require.NoError(t, err)
|
||||
withParallel(8, func() { par, err = listing(true) })
|
||||
require.NoError(t, err)
|
||||
})
|
||||
assert.Equal(t, seq, par)
|
||||
for remote, size := range wantSizes {
|
||||
assert.Equal(t, size, par[remote], remote)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestArrowLadderBoundaries(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
directory string
|
||||
target int
|
||||
}{
|
||||
{"", 8},
|
||||
{"dir/", 16},
|
||||
{"dir/", 100}, // capped at alphabet size
|
||||
{"dir/", 1},
|
||||
} {
|
||||
points := arrowLadderBoundaries(tc.directory, tc.target)
|
||||
require.NotEmpty(t, points)
|
||||
assert.LessOrEqual(t, len(points), 62)
|
||||
for i, p := range points {
|
||||
assert.True(t, strings.HasPrefix(p, tc.directory), "boundary %q lacks prefix %q", p, tc.directory)
|
||||
if i > 0 {
|
||||
assert.Less(t, points[i-1], p, "boundaries not strictly ascending")
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Nil(t, arrowLadderBoundaries("dir/", 0))
|
||||
// alphabet is single-case (digit+lowercase) so boundaries are monotonic
|
||||
// under both byte order and case-insensitive order
|
||||
pts := arrowLadderBoundaries("dir/", 36)
|
||||
for _, p := range pts {
|
||||
assert.Equal(t, p, strings.ToLower(p), "boundary must be lowercase: %q", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEndBeforeUnsupported(t *testing.T) {
|
||||
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
|
||||
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
|
||||
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
|
||||
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
|
||||
assert.False(t, isEndBeforeUnsupported(nil))
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
@@ -284,9 +283,6 @@ type transporter struct {
|
||||
|
||||
// Make a new transporter
|
||||
func newTransporter(ctx context.Context) transporter {
|
||||
// Set the User-Agent to include the Microsoft Partner Network prefix
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.UserAgent = "APN/1.0 rclone/1.0 rclone/" + strings.TrimPrefix(fs.Version, "v")
|
||||
return transporter{
|
||||
RoundTripper: fshttp.NewTransport(ctx),
|
||||
}
|
||||
@@ -297,13 +293,6 @@ func (tr transporter) Do(req *http.Request) (*http.Response, error) {
|
||||
return tr.RoundTripper.RoundTrip(req)
|
||||
}
|
||||
|
||||
// Transporter returns the policy.Transporter rclone uses for Azure SDK
|
||||
// clients (an fshttp based transport with the APN user agent), for callers
|
||||
// which build their own azcore pipelines.
|
||||
func Transporter(ctx context.Context) policy.Transporter {
|
||||
return newTransporter(ctx)
|
||||
}
|
||||
|
||||
// NewClientOpts should be passed to configure NewClient
|
||||
type NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct {
|
||||
DefaultBaseURL string // Base URL, eg blob.core.windows.net
|
||||
|
||||
+81
-380
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
@@ -48,8 +47,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/transferaccounter"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -68,19 +65,8 @@ const (
|
||||
defaultChunkSize = 4 * fs.Mebi
|
||||
defaultAccessTier = blob.AccessTier("") // FIXME AccessTierNone
|
||||
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
|
||||
sasCopyStartSkew = 15 * time.Minute // how far back SAS should start to allow for clock skew
|
||||
)
|
||||
|
||||
// setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
|
||||
// on every request. This prevents the Go HTTP transport from automatically
|
||||
// decompressing gzip-encoded blobs on download.
|
||||
type setAcceptEncodingGzip struct{}
|
||||
|
||||
func (p setAcceptEncodingGzip) Do(req *policy.Request) (*http.Response, error) {
|
||||
req.Raw().Header.Set("Accept-Encoding", "gzip")
|
||||
return req.Next()
|
||||
}
|
||||
|
||||
var (
|
||||
errCantUpdateArchiveTierBlobs = fserrors.NoRetryError(errors.New("can't update archive tier blob without --azureblob-archive-tier-delete"))
|
||||
|
||||
@@ -238,40 +224,6 @@ minutes per megabyte on average, it will time out (
|
||||
avoid the time out.`,
|
||||
Default: maxListChunkSize,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_arrow_list",
|
||||
Help: `Use the Apache Arrow listing format.
|
||||
|
||||
If set, directory listings are fetched using the ListBlobs Apache
|
||||
Arrow response format instead of XML. Arrow responses are smaller and
|
||||
much cheaper to parse, making listings of large containers several
|
||||
times faster. Combine with "list_parallelism" for the biggest gains.
|
||||
|
||||
"Blob Listing with Apache Arrow" is in public preview at Microsoft and
|
||||
is only supported on flat namespace accounts. On accounts with a
|
||||
hierarchical namespace (ADLS Gen2), or where the feature is otherwise
|
||||
unavailable, the server returns XML and the listing transparently
|
||||
falls back to the normal XML path (logged at debug level).`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "list_parallelism",
|
||||
Help: `Number of parallel shards to list a directory with.
|
||||
|
||||
If set greater than 1, the blob name keyspace of each directory is
|
||||
split into ranges which are listed concurrently using the Arrow
|
||||
startFrom/endBefore range parameters. This can dramatically speed up
|
||||
listing containers with millions of objects, for both recursive
|
||||
(ListR) and single directory listings. Speed keeps improving up to a
|
||||
parallelism of around 30.
|
||||
|
||||
This has no effect unless "use_arrow_list" is also set, as Arrow is the
|
||||
only listing path that supports server-side name ranges. If the
|
||||
account does not support range listing (e.g. it has a hierarchical
|
||||
namespace) the listing falls back to sequential. The default of 0 (or
|
||||
1) lists sequentially.`,
|
||||
Default: 0,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "access_tier",
|
||||
Help: `Access tier of blob: hot, cool, cold or archive.
|
||||
@@ -365,22 +317,7 @@ creates an empty object ending with "/", to persist the folder.
|
||||
|
||||
This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
|
||||
the Microsoft standard.
|
||||
|
||||
This is not needed on storage accounts with a hierarchical namespace
|
||||
(see the hns option) as these always support empty directories.
|
||||
`,
|
||||
}, {
|
||||
Name: "hns",
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
Help: `Set if the storage account has a hierarchical namespace.
|
||||
|
||||
Storage accounts with hierarchical namespace enabled (Azure Data Lake
|
||||
Storage Gen2) have real directories which can be empty.
|
||||
|
||||
If this is unset then rclone will read it from the storage account
|
||||
and save it in the config file.
|
||||
`,
|
||||
}, {
|
||||
Name: "no_check_container",
|
||||
Help: `If set, don't attempt to check the container exists or create it.
|
||||
@@ -413,19 +350,6 @@ rclone does if you know the container exists already.
|
||||
Default: "",
|
||||
Exclusive: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "decompress",
|
||||
Help: `If set this will decompress gzip encoded objects.
|
||||
|
||||
It is possible to upload objects to Azure Blob Storage with "Content-Encoding: gzip"
|
||||
set. Normally rclone will download these files as compressed objects.
|
||||
|
||||
If this flag is set then rclone will decompress these files with
|
||||
"Content-Encoding: gzip" as they are received. This means that rclone
|
||||
can't check the size and hash but the file contents will be decompressed.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}}),
|
||||
})
|
||||
}
|
||||
@@ -440,19 +364,15 @@ type Options struct {
|
||||
UseCopyBlob bool `config:"use_copy_blob"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
ListChunkSize uint `config:"list_chunk"`
|
||||
UseArrowList bool `config:"use_arrow_list"`
|
||||
ListParallelism int `config:"list_parallelism"`
|
||||
AccessTier string `config:"access_tier"`
|
||||
ArchiveTierDelete bool `config:"archive_tier_delete"`
|
||||
DisableCheckSum bool `config:"disable_checksum"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
PublicAccess string `config:"public_access"`
|
||||
DirectoryMarkers bool `config:"directory_markers"`
|
||||
HNS fs.Tristate `config:"hns"`
|
||||
NoCheckContainer bool `config:"no_check_container"`
|
||||
NoHeadObject bool `config:"no_head_object"`
|
||||
DeleteSnapshots string `config:"delete_snapshots"`
|
||||
Decompress bool `config:"decompress"`
|
||||
}
|
||||
|
||||
// Fs represents a remote azure server
|
||||
@@ -464,7 +384,6 @@ type Fs struct {
|
||||
features *fs.Features // optional features
|
||||
cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache
|
||||
cntSVCcache map[string]*container.Client // reference to containerClient per container
|
||||
arrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retried
|
||||
svc *service.Client // client to access azblob
|
||||
cred azcore.TokenCredential // how to generate tokens (may be nil)
|
||||
usingSharedKeyCred bool // set if using shared key credentials
|
||||
@@ -472,15 +391,12 @@ type Fs struct {
|
||||
rootContainer string // container part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
isLimited bool // if limited to one container
|
||||
hns bool // if the storage account has a hierarchical namespace
|
||||
cache *bucket.Cache // cache for container creation status
|
||||
pacer *fs.Pacer // To pace and retry the API calls
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
copyToken *pacer.TokenDispenser // global multipart copy concurrency limiter
|
||||
publicAccess container.PublicAccessType // Container Public Access Level
|
||||
|
||||
warnCompressed sync.Once // warn once about compressed files
|
||||
|
||||
// user delegation cache
|
||||
userDelegationMu sync.Mutex
|
||||
userDelegation *service.UserDelegationCredential
|
||||
@@ -489,16 +405,15 @@ type Fs struct {
|
||||
|
||||
// Object describes an azure object
|
||||
type Object struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
accessTier blob.AccessTier // Blob Access Tier
|
||||
meta map[string]string // blob metadata - take metadataMu when accessing
|
||||
tags map[string]string // blob tags
|
||||
contentEncoding *string // Content-Encoding of the object
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
accessTier blob.AccessTier // Blob Access Tier
|
||||
meta map[string]string // blob metadata - take metadataMu when accessing
|
||||
tags map[string]string // blob tags
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -539,7 +454,7 @@ func parsePath(path string) (root string) {
|
||||
// relative to f.root
|
||||
func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
|
||||
containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
|
||||
if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
|
||||
if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
|
||||
containerPath = containerPath[:len(containerPath)-1]
|
||||
}
|
||||
return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
|
||||
@@ -587,7 +502,8 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
// General errors from:
|
||||
// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
|
||||
// Blob specific errors from:
|
||||
@@ -664,9 +580,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if opt.ListChunkSize > maxListChunkSize {
|
||||
return nil, fmt.Errorf("blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
|
||||
}
|
||||
if opt.ListParallelism > 1 && !opt.UseArrowList {
|
||||
fs.Logf(nil, "azureblob: list_parallelism has no effect without use_arrow_list - listing sequentially")
|
||||
}
|
||||
|
||||
if opt.AccessTier == "" {
|
||||
opt.AccessTier = string(defaultAccessTier)
|
||||
@@ -706,6 +619,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ServerSideAcrossConfigs: true,
|
||||
DoubleSlash: true,
|
||||
}).Fill(ctx, f)
|
||||
if opt.DirectoryMarkers {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
|
||||
DefaultBaseURL: storageDefaultBaseURL,
|
||||
@@ -717,9 +634,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential,
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
// Override the automatic decompression in the transport
|
||||
// to download compressed files as-is
|
||||
policyClientOptions.PerCallPolicies = append(policyClientOptions.PerCallPolicies, setAcceptEncodingGzip{})
|
||||
options.ClientOptions = policyClientOptions
|
||||
},
|
||||
}
|
||||
@@ -738,28 +652,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.isLimited = true
|
||||
}
|
||||
|
||||
if opt.HNS.Valid {
|
||||
f.hns = opt.HNS.Value
|
||||
} else if !f.anonymous {
|
||||
containerName := f.rootContainer
|
||||
if containerName == "" {
|
||||
containerName = res.Container
|
||||
}
|
||||
f.hns, err = f.readHNS(ctx, containerName)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
|
||||
} else {
|
||||
m.Set("hns", strconv.FormatBool(f.hns))
|
||||
}
|
||||
}
|
||||
if f.hns {
|
||||
fs.Debugf(f, "Storage account has a hierarchical namespace")
|
||||
}
|
||||
if f.dirMarkers() {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
fs.Debugf(f, "Using directory markers")
|
||||
}
|
||||
|
||||
if f.rootContainer != "" && f.rootDirectory != "" {
|
||||
// Check to see if the (container,directory) is actually an existing file
|
||||
oldRoot := f.root
|
||||
@@ -780,39 +672,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// readHNS reads whether the storage account has a hierarchical namespace
|
||||
//
|
||||
// If containerName is set then the container client is used which works
|
||||
// with credentials limited to that container.
|
||||
func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
|
||||
var isHNS *bool
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
if containerName != "" {
|
||||
var resp container.GetAccountInfoResponse
|
||||
resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
} else {
|
||||
var resp service.GetAccountInfoResponse
|
||||
resp, err = f.svc.GetAccountInfo(ctx, nil)
|
||||
isHNS = resp.IsHierarchicalNamespaceEnabled
|
||||
}
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if isHNS == nil {
|
||||
return false, errors.New("not returned by server")
|
||||
}
|
||||
return *isHNS, nil
|
||||
}
|
||||
|
||||
// dirMarkers returns true if directories are persisted with directory markers
|
||||
func (f *Fs) dirMarkers() bool {
|
||||
return f.opt.DirectoryMarkers || f.hns
|
||||
}
|
||||
|
||||
// return the container client for the container passed in
|
||||
func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
|
||||
f.cntSVCcacheMu.Lock()
|
||||
@@ -850,12 +709,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error fs.ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
o, err := f.newObjectWithInfo(ctx, remote, nil)
|
||||
if err == fs.ErrorNotAFile {
|
||||
// The blob is a directory
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, err
|
||||
return f.newObjectWithInfo(ctx, remote, nil)
|
||||
}
|
||||
|
||||
// getBlobSVC creates a blob client
|
||||
@@ -924,15 +778,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
|
||||
lowerKey := strings.ToLower(k)
|
||||
switch lowerKey {
|
||||
case "cache-control":
|
||||
headers.BlobCacheControl = new(v)
|
||||
headers.BlobCacheControl = pString(v)
|
||||
case "content-disposition":
|
||||
headers.BlobContentDisposition = new(v)
|
||||
headers.BlobContentDisposition = pString(v)
|
||||
case "content-encoding":
|
||||
headers.BlobContentEncoding = new(v)
|
||||
headers.BlobContentEncoding = pString(v)
|
||||
case "content-language":
|
||||
headers.BlobContentLanguage = new(v)
|
||||
headers.BlobContentLanguage = pString(v)
|
||||
case "content-type":
|
||||
headers.BlobContentType = new(v)
|
||||
headers.BlobContentType = pString(v)
|
||||
case "x-ms-tags":
|
||||
parsed, perr := parseXMsTags(v)
|
||||
if perr != nil {
|
||||
@@ -1169,33 +1023,18 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
|
||||
if endsWithSlash || remote == "" {
|
||||
return true
|
||||
}
|
||||
return hasDirMetadata(metadata)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether the metadata marks the blob as a directory
|
||||
func hasDirMetadata(metadata map[string]*string) bool {
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
// Note that metadata with hdi_isfolder = true seems to be a
|
||||
// defacto standard for marking blobs as directories.
|
||||
// Note also that the metadata hasn't been normalised to lower case yet
|
||||
for k, v := range metadata {
|
||||
if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Returns whether props are for a directory on a storage account with
|
||||
// a hierarchical namespace
|
||||
//
|
||||
// These ignore a trailing "/" on the name so a file can be returned
|
||||
// when looking for a directory.
|
||||
func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
|
||||
return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
|
||||
}
|
||||
|
||||
// listFn is called from list to handle an object
|
||||
type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
|
||||
|
||||
@@ -1221,7 +1060,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
delimiter = "/"
|
||||
}
|
||||
|
||||
opts := &container.ListBlobsHierarchyOptions{
|
||||
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &container.ListBlobsHierarchyOptions{
|
||||
// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
|
||||
Include: container.ListBlobsInclude{
|
||||
Copy: false,
|
||||
@@ -1232,53 +1071,8 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
|
||||
},
|
||||
Prefix: &directory,
|
||||
MaxResults: &maxResults,
|
||||
}
|
||||
// Request the Apache Arrow listing format. The SDK pager requests an
|
||||
// Arrow IPC stream and decodes it, falling back to XML if the account
|
||||
// doesn't have Arrow listing enabled. Skip the maxResults==1 probe
|
||||
// (isEmpty) which doesn't benefit.
|
||||
useArrow := f.opt.UseArrowList && maxResults != 1
|
||||
if useArrow {
|
||||
opts.ResponseFormat = container.StorageResponseFormatArrow
|
||||
}
|
||||
|
||||
var foundItems int
|
||||
var err error
|
||||
if useArrow && f.opt.ListParallelism > 1 && !f.arrowXMLFallback.Load() {
|
||||
// Arrow exposes server-side startFrom/endBefore name ranges, so the
|
||||
// keyspace can be sharded and listed concurrently.
|
||||
foundItems, err = f.listArrowParallel(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
} else {
|
||||
foundItems, err = f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if f.dirMarkers() && foundItems == 0 && directory != "" {
|
||||
// Determine whether the directory exists or not by whether it has a marker
|
||||
props, err := f.readMetaData(ctx, containerName, directory)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
if f.hns && !isHNSDirectory(props) {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// listBlobsPager runs the hierarchy listing described by opts, calling fn for
|
||||
// each blob and subdirectory, and returns the number of raw items seen.
|
||||
// delimiter selects flat (recurse) vs hierarchical listing. If opts requests
|
||||
// the Apache Arrow format and the service answers with XML (Arrow listing
|
||||
// not enabled) a debug message is logged.
|
||||
func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
|
||||
useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
|
||||
pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
|
||||
checkedArrow := false
|
||||
})
|
||||
foundItems := 0
|
||||
for pager.More() {
|
||||
var response container.ListBlobsHierarchyResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -1288,22 +1082,12 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
// If Arrow was requested but the service answered with XML, Arrow
|
||||
// listing is not enabled on this account; the listing is still correct
|
||||
// but not accelerated.
|
||||
if useArrow && !checkedArrow && err == nil {
|
||||
checkedArrow = true
|
||||
if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
|
||||
fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
|
||||
if storageErr, ok := err.(*azcore.ResponseError); ok && (storageErr.ErrorCode == string(bloberror.ContainerNotFound) || storageErr.StatusCode == http.StatusNotFound) {
|
||||
return foundItems, fs.ErrorDirNotFound
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return foundItems, err
|
||||
return err
|
||||
}
|
||||
// Advance marker to next
|
||||
// marker = response.NextMarker
|
||||
@@ -1318,24 +1102,12 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
fs.Debugf(f, "Nil name received")
|
||||
continue
|
||||
}
|
||||
// Storage accounts with a hierarchical namespace return
|
||||
// the file "dir" when listing "dir/"
|
||||
if !strings.HasPrefix(*file.Name, directory) {
|
||||
fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
|
||||
foundItems--
|
||||
continue
|
||||
}
|
||||
remote := f.opt.Enc.ToStandardPath(*file.Name)
|
||||
if !strings.HasPrefix(remote, prefix) {
|
||||
fs.Debugf(f, "Odd name received %q", remote)
|
||||
continue
|
||||
}
|
||||
// ContentLength is documented as nullable in the Arrow listing schema
|
||||
size := int64(-1)
|
||||
if file.Properties.ContentLength != nil {
|
||||
size = *file.Properties.ContentLength
|
||||
}
|
||||
isDirectory := isDirectoryMarker(size, file.Metadata, remote)
|
||||
isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
|
||||
if isDirectory {
|
||||
// Don't insert the root directory
|
||||
if remote == f.opt.Enc.ToStandardPath(directory) {
|
||||
@@ -1351,7 +1123,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
// Send object
|
||||
err = fn(remote, file, isDirectory)
|
||||
if err != nil {
|
||||
return foundItems, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Send the subdirectories
|
||||
@@ -1366,13 +1138,6 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
fs.Debugf(f, "Odd directory name received %q", remote)
|
||||
continue
|
||||
}
|
||||
// Don't insert the root directory. The Arrow listing returns a
|
||||
// directory marker blob with the same name as the listed directory
|
||||
// as a BlobPrefix, whereas XML returns it as a blob (handled
|
||||
// above).
|
||||
if remote == f.opt.Enc.ToStandardPath(directory) {
|
||||
continue
|
||||
}
|
||||
remote = remote[len(prefix):]
|
||||
// Trim one slash off the remote name
|
||||
remote, _ = strings.CutSuffix(remote, "/")
|
||||
@@ -1385,11 +1150,21 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
|
||||
// Send object
|
||||
err = fn(remote, nil, true)
|
||||
if err != nil {
|
||||
return foundItems, err
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return foundItems, nil
|
||||
if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
|
||||
// Determine whether the directory exists or not by whether it has a marker
|
||||
_, err := f.readMetaData(ctx, containerName, directory)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert a list item into a DirEntry
|
||||
@@ -1586,7 +1361,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
|
||||
func (f *Fs) listContainersToFn(fn listContainerFn) error {
|
||||
max := int32(f.opt.ListChunkSize)
|
||||
pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
|
||||
Include: service.ListContainersInclude{Metadata: true},
|
||||
Include: service.ListContainersInclude{Metadata: true, Deleted: true},
|
||||
MaxResults: &max,
|
||||
})
|
||||
ctx := context.Background()
|
||||
@@ -1636,7 +1411,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
|
||||
// Create directory marker file and parents
|
||||
func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
|
||||
if !f.dirMarkers() || container == "" {
|
||||
if !f.opt.DirectoryMarkers || container == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1693,11 +1468,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
|
||||
// mkdirParent creates the parent bucket/directory if it doesn't exist
|
||||
func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
|
||||
if f.hns {
|
||||
// Parent directories are created by the upload
|
||||
container, _ := f.split(remote)
|
||||
return f.makeContainer(ctx, container)
|
||||
}
|
||||
remote, _ = strings.CutSuffix(remote, "/")
|
||||
dir := path.Dir(remote)
|
||||
if dir == "/" || dir == "." {
|
||||
@@ -1803,9 +1573,6 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
|
||||
// Returns an error if it isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
container, directory := f.split(dir)
|
||||
if f.hns && container != "" && directory != "" {
|
||||
return f.rmdirHNS(ctx, container, directory)
|
||||
}
|
||||
// Remove directory marker file
|
||||
if f.opt.DirectoryMarkers && container != "" && directory != "" {
|
||||
o := &Object{
|
||||
@@ -1828,39 +1595,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return f.deleteContainer(ctx, container)
|
||||
}
|
||||
|
||||
// rmdirHNS removes the directory on a storage account with a
|
||||
// hierarchical namespace
|
||||
//
|
||||
// The directory must be deleted without a trailing "/" and the server
|
||||
// refuses to delete it if it isn't empty.
|
||||
func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
|
||||
directory, _ = strings.CutSuffix(directory, "/")
|
||||
// Check it is a directory as deleting a file would succeed
|
||||
props, err := f.readMetaData(ctx, container, directory)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isHNSDirectory(props) {
|
||||
return fs.ErrorIsFile
|
||||
}
|
||||
blb := f.getBlobSVC(container, directory)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err := blb.Delete(ctx, nil)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if storageErr.StatusCode == http.StatusNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Precision of the remote
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Nanosecond
|
||||
@@ -1896,7 +1630,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
|
||||
}
|
||||
|
||||
// Validity window
|
||||
start := time.Now().UTC().Add(-sasCopyStartSkew)
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(2 * sasCopyValidity)
|
||||
startStr := start.Format(time.RFC3339)
|
||||
expiryStr := expiry.Format(time.RFC3339)
|
||||
@@ -1945,7 +1679,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
|
||||
// Build the SAS values
|
||||
perms := sas.BlobPermissions{Read: true}
|
||||
container, containerPath := o.split()
|
||||
start := time.Now().UTC().Add(-sasCopyStartSkew)
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(sasCopyValidity)
|
||||
vals := sas.BlobSignatureValues{
|
||||
StartTime: start,
|
||||
@@ -2267,10 +2001,6 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.MD5 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// If decompressing, erase the hash
|
||||
if o.size < 0 {
|
||||
return "", nil
|
||||
}
|
||||
// Convert base64 encoded md5 into lower case hex
|
||||
if o.md5 == "" {
|
||||
return "", nil
|
||||
@@ -2396,13 +2126,6 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *blob.GetPropertiesRe
|
||||
o.accessTier = blob.AccessTier(*info.AccessTier)
|
||||
}
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2415,31 +2138,12 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
|
||||
} else {
|
||||
size = *info.ContentLength
|
||||
}
|
||||
// On a range request Content-Length is the length of the range, not the object, so take the
|
||||
// object's size from the total in Content-Range instead.
|
||||
if info.ContentRange != nil {
|
||||
contentRange, err := rest.ParseContentRange(*info.ContentRange)
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
|
||||
} else if contentRange.Size >= 0 {
|
||||
size = contentRange.Size
|
||||
}
|
||||
}
|
||||
if isDirectoryMarker(size, metadata, o.remote) {
|
||||
return fs.ErrorNotAFile
|
||||
}
|
||||
// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
|
||||
// this as base64 encoded string.
|
||||
//
|
||||
// On a range request the service returns the whole-blob MD5 in the
|
||||
// x-ms-blob-content-md5 header (BlobContentMD5) and leaves Content-MD5
|
||||
// (ContentMD5) empty, so prefer BlobContentMD5. Never overwrite a known
|
||||
// hash with an empty one, or a ranged read would drop the object's MD5.
|
||||
if md5 := info.BlobContentMD5; len(md5) > 0 {
|
||||
o.md5 = base64.StdEncoding.EncodeToString(md5)
|
||||
} else if len(info.ContentMD5) > 0 {
|
||||
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
|
||||
}
|
||||
o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
|
||||
if info.ContentType == nil {
|
||||
o.mimeType = ""
|
||||
} else {
|
||||
@@ -2458,12 +2162,21 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
|
||||
// o.accessTier = blob.AccessTier(*info.AccessTier)
|
||||
// }
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
// If it was a Range request, the size is wrong, so correct it
|
||||
if info.ContentRange != nil {
|
||||
contentRange := *info.ContentRange
|
||||
slash := strings.IndexRune(contentRange, '/')
|
||||
if slash >= 0 {
|
||||
i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
|
||||
if err == nil {
|
||||
o.size = i
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(o, "Failed to find length in %q", contentRange)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -2503,13 +2216,6 @@ func (o *Object) decodeMetaDataFromBlob(info *container.BlobItem) (err error) {
|
||||
o.accessTier = *info.Properties.AccessTier
|
||||
}
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.Properties.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2685,20 +2391,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode metadata for download: %w", err)
|
||||
}
|
||||
|
||||
// Decompress body if necessary
|
||||
if downloadResponse.ContentEncoding != nil && *downloadResponse.ContentEncoding == "gzip" {
|
||||
if o.fs.opt.Decompress {
|
||||
return readers.NewGzipReader(downloadResponse.Body)
|
||||
}
|
||||
o.fs.warnCompressed.Do(func() {
|
||||
fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --azureblob-decompress to override")
|
||||
})
|
||||
}
|
||||
|
||||
return downloadResponse.Body, nil
|
||||
}
|
||||
|
||||
// Converts a string into a pointer to a string
|
||||
func pString(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
|
||||
type readSeekCloser struct {
|
||||
io.Reader
|
||||
@@ -2853,7 +2553,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
// isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
|
||||
// returning true if it is found
|
||||
func isInvalidBlockOrBlob(err error) bool {
|
||||
if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
|
||||
var storageErr *azcore.ResponseError
|
||||
if errors.As(err, &storageErr) {
|
||||
return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
|
||||
}
|
||||
return false
|
||||
@@ -3225,7 +2926,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
|
||||
// Start with default content-type based on source
|
||||
ui.httpHeaders = blob.HTTPHeaders{
|
||||
BlobContentType: new(fs.MimeType(ctx, src)),
|
||||
BlobContentType: pString(fs.MimeType(ctx, src)),
|
||||
}
|
||||
|
||||
// Apply mapped metadata/headers/tags if requested
|
||||
@@ -3271,15 +2972,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
|
||||
o.tags[parts[0]] = parts[1]
|
||||
}
|
||||
case "cache-control":
|
||||
ui.httpHeaders.BlobCacheControl = new(value)
|
||||
ui.httpHeaders.BlobCacheControl = pString(value)
|
||||
case "content-disposition":
|
||||
ui.httpHeaders.BlobContentDisposition = new(value)
|
||||
ui.httpHeaders.BlobContentDisposition = pString(value)
|
||||
case "content-encoding":
|
||||
ui.httpHeaders.BlobContentEncoding = new(value)
|
||||
ui.httpHeaders.BlobContentEncoding = pString(value)
|
||||
case "content-language":
|
||||
ui.httpHeaders.BlobContentLanguage = new(value)
|
||||
ui.httpHeaders.BlobContentLanguage = pString(value)
|
||||
case "content-type":
|
||||
ui.httpHeaders.BlobContentType = new(value)
|
||||
ui.httpHeaders.BlobContentType = pString(value)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -17,7 +14,6 @@ import (
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
@@ -54,58 +50,6 @@ func TestBlockIDCreator(t *testing.T) {
|
||||
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
|
||||
}
|
||||
|
||||
func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
|
||||
pInt64 := func(i int64) *int64 { return &i }
|
||||
pString := func(s string) *string { return &s }
|
||||
newTestObject := func() *Object {
|
||||
return &Object{
|
||||
fs: &Fs{},
|
||||
remote: "test.bin",
|
||||
size: -2, // sentinel to check it gets set
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("WholeBlob", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(12345)
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(12345), o.size)
|
||||
})
|
||||
|
||||
t.Run("RangeRequest", func(t *testing.T) {
|
||||
// On a ranged download Content-Length is the chunk length so the
|
||||
// size must come from the total in Content-Range
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("bytes 67108864-134217727/169721004032")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(169721004032), o.size)
|
||||
})
|
||||
|
||||
t.Run("BadContentRange", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("potato")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(67108864), o.size)
|
||||
})
|
||||
|
||||
t.Run("UnknownLength", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(-1), o.size)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCopySASTimingConstants(t *testing.T) {
|
||||
require.Greater(t, sasCopyStartSkew, time.Duration(0))
|
||||
require.Greater(t, sasCopyValidity, sasCopyStartSkew)
|
||||
}
|
||||
|
||||
func (f *Fs) testFeatures(t *testing.T) {
|
||||
// Check first feature flags are set on this remote
|
||||
enabled := f.Features().SetTier
|
||||
@@ -206,66 +150,10 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
|
||||
require.NoError(t, dst.Remove(ctx))
|
||||
}
|
||||
|
||||
func gz(t *testing.T, s string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
_, err := zw.Write([]byte(s))
|
||||
require.NoError(t, err)
|
||||
err = zw.Close()
|
||||
require.NoError(t, err)
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func md5sum(t *testing.T, s string) string {
|
||||
hash := md5.Sum([]byte(s))
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
func (f *Fs) testGzipEncoding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
original := random.String(1000)
|
||||
contents := gz(t, original)
|
||||
|
||||
item := fstest.NewItem("test-gzip", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
metadata := fs.Metadata{
|
||||
"content-encoding": "gzip",
|
||||
"content-type": "text/plain",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", metadata)
|
||||
defer func() {
|
||||
assert.NoError(t, obj.Remove(ctx))
|
||||
}()
|
||||
o := obj.(*Object)
|
||||
|
||||
// Test that the gzipped file we uploaded can be
|
||||
// downloaded with and without decompression
|
||||
checkDownload := func(wantContents string, wantSize int64, wantHash string) {
|
||||
gotContents := fstests.ReadObject(ctx, t, o, -1)
|
||||
assert.Equal(t, wantContents, gotContents)
|
||||
assert.Equal(t, wantSize, o.Size())
|
||||
gotHash, err := o.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantHash, gotHash)
|
||||
}
|
||||
|
||||
t.Run("NoDecompress", func(t *testing.T) {
|
||||
checkDownload(contents, int64(len(contents)), md5sum(t, contents))
|
||||
})
|
||||
t.Run("Decompress", func(t *testing.T) {
|
||||
f.opt.Decompress = true
|
||||
defer func() {
|
||||
f.opt.Decompress = false
|
||||
}()
|
||||
checkDownload(original, -1, "")
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Features", f.testFeatures)
|
||||
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
||||
t.Run("Metadata", f.testMetadataPaths)
|
||||
t.Run("GzipEncoding", f.testGzipEncoding)
|
||||
t.Run("ArrowList", f.testArrowList)
|
||||
}
|
||||
|
||||
// helper to read blob properties for an object
|
||||
@@ -325,14 +213,6 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
|
||||
}
|
||||
|
||||
// helper to read blob tags for an object
|
||||
// skipIfHNS skips the test if the storage account has a hierarchical
|
||||
// namespace as these don't support blob tags
|
||||
func skipIfHNS(t *testing.T, f *Fs) {
|
||||
if f.hns {
|
||||
t.Skip("blob tags not supported with hierarchical namespace")
|
||||
}
|
||||
}
|
||||
|
||||
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
||||
ao := o.(*Object)
|
||||
blb := ao.getBlobSVC()
|
||||
@@ -418,7 +298,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart upload
|
||||
t.Run("PutSinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -434,7 +313,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart upload
|
||||
t.Run("PutMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize + 2048))
|
||||
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
@@ -622,7 +500,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Singlepart copy
|
||||
t.Run("CopySinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
@@ -648,7 +525,6 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
|
||||
// Tags: Multipart copy
|
||||
t.Run("CopyMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
||||
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || solaris || js
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
@@ -58,9 +58,6 @@ const (
|
||||
maxFileSize = 4 * fs.Tebi
|
||||
defaultChunkSize = 4 * fs.Mebi
|
||||
storageDefaultBaseURL = "file.core.windows.net"
|
||||
// smbTimePrecision is the precision the server stores
|
||||
// modtimes with (100 ns FILETIME ticks)
|
||||
smbTimePrecision = 100 * time.Nanosecond
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -176,10 +173,6 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
options.ClientOptions = policyClientOptions
|
||||
// FileRequestIntent is required for TokenCredential (OAuth)
|
||||
// auth; the SDK omits it for shared-key / SAS.
|
||||
backup := service.ShareTokenIntentBackup
|
||||
options.FileRequestIntent = &backup
|
||||
},
|
||||
}
|
||||
res, err := auth.NewClient(ctx, conf, &opt.Options)
|
||||
@@ -253,10 +246,10 @@ func (f *Fs) Features() *fs.Features {
|
||||
|
||||
// Precision return the precision of this Fs
|
||||
//
|
||||
// The SMB last write time properties are ISO 8601 with 100 ns (FILETIME) precision
|
||||
// One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds
|
||||
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return smbTimePrecision
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
@@ -601,10 +594,7 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to set modTime: %w", err)
|
||||
}
|
||||
// Truncate to the precision the server stores the modtime with
|
||||
// to keep the in-memory modtime identical to the one a fresh
|
||||
// listing returns.
|
||||
o.modTime = t.Truncate(smbTimePrecision)
|
||||
o.modTime = t
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -650,6 +640,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
|
||||
@@ -691,9 +686,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
// Count the bytes read so short reads from the source can be detected
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
// Measure the size if it is unknown
|
||||
if sizeUnknown {
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
}
|
||||
|
||||
// Check we have a source MD5 hash...
|
||||
if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
|
||||
@@ -728,21 +725,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if sizeUnknown {
|
||||
// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
|
||||
size = int64(counter.BytesRead())
|
||||
} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
|
||||
// The file was created at the declared size so a short source
|
||||
// leaves the remainder padded with zeroes
|
||||
if isNewlyCreated {
|
||||
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
|
||||
fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
|
||||
}
|
||||
} else if bytesRead < size {
|
||||
// Truncate the file to the bytes actually received so it
|
||||
// is not left padded to the declared size with zeroes
|
||||
if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
|
||||
fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if hashUnknown {
|
||||
md5Hash = hasher.Sum(nil)
|
||||
@@ -783,12 +765,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return fmt.Errorf("update: failed to set properties: %w", err)
|
||||
}
|
||||
|
||||
// Make sure Object is in sync, truncating the modtime to the
|
||||
// precision the server stores it with so it is identical to the
|
||||
// one a fresh listing returns
|
||||
// Make sure Object is in sync
|
||||
o.size = size
|
||||
o.md5 = md5Hash
|
||||
o.modTime = modTime.Truncate(smbTimePrecision)
|
||||
o.modTime = modTime
|
||||
o.contentType = contentType
|
||||
return nil
|
||||
}
|
||||
@@ -813,8 +793,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
|
||||
}
|
||||
opt := file.RenameOptions{
|
||||
IgnoreReadOnly: new(true),
|
||||
ReplaceIfExists: new(true),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
ReplaceIfExists: ptr(true),
|
||||
}
|
||||
dstAbsPath := f.absPath(remote)
|
||||
fc := srcObj.fileClient()
|
||||
@@ -859,8 +839,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
opt := directory.RenameOptions{
|
||||
IgnoreReadOnly: new(false),
|
||||
ReplaceIfExists: new(false),
|
||||
IgnoreReadOnly: ptr(false),
|
||||
ReplaceIfExists: ptr(false),
|
||||
}
|
||||
dstAbsPath := dstFs.absPath(dstRemote)
|
||||
dirClient := srcFs.dirClient(srcRemote)
|
||||
@@ -899,8 +879,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
ChangeTime: file.SourceCopyFileChangeTime{},
|
||||
CreationTime: file.SourceCopyFileCreationTime{},
|
||||
LastWriteTime: file.SourceCopyFileLastWriteTime{},
|
||||
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: new(true),
|
||||
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
},
|
||||
}
|
||||
srcURL := srcObj.fileClient().URL()
|
||||
|
||||
+1
-25
@@ -191,11 +191,6 @@ If the custom endpoint rewrites the requests for authentication,
|
||||
e.g., in Cloudflare Workers, this header needs to be handled properly.
|
||||
Leave blank if you want to use the endpoint provided by Backblaze.
|
||||
|
||||
Rclone will not follow a redirect from this endpoint which downgrades
|
||||
HTTPS to HTTP, and will not send the "Authorization" header or the
|
||||
SSE-C key headers once a redirect has left the host and port of this
|
||||
endpoint.
|
||||
|
||||
The URL provided here SHOULD have the protocol and SHOULD NOT have
|
||||
a trailing slash or specify the /file/bucket subpath as rclone will
|
||||
request files with "{download_url}/file/{bucket_name}/{path}".
|
||||
@@ -534,25 +529,6 @@ func (f *Fs) setRoot(root string) {
|
||||
f.rootBucket, f.rootDirectory = bucket.Split(f.root)
|
||||
}
|
||||
|
||||
// redirectSecretHeaders are the request headers carrying secrets that
|
||||
// must not be forwarded when a redirect leaves the original host.
|
||||
var redirectSecretHeaders = []string{
|
||||
"Authorization",
|
||||
sseAlgorithmHeader,
|
||||
sseKeyHeader,
|
||||
sseMd5Header,
|
||||
}
|
||||
|
||||
// newClient makes the http client used for all B2 requests.
|
||||
//
|
||||
// The download_url option sends the requests to a host the user does
|
||||
// not control so it strips headers on cross host redirects.
|
||||
func newClient(ctx context.Context) *http.Client {
|
||||
client := fshttp.NewClient(ctx)
|
||||
client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
|
||||
return client
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, bucket:path
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
@@ -605,7 +581,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
name: name,
|
||||
opt: *opt,
|
||||
ci: ci,
|
||||
srv: rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
|
||||
srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
|
||||
cache: bucket.NewCache(),
|
||||
_bucketID: make(map[string]string, 1),
|
||||
_bucketType: make(map[string]string, 1),
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -619,60 +617,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// redirectTestHeaders is a deliberately literal copy of
|
||||
// redirectSecretHeaders so that a header dropped from the production
|
||||
// list fails TestRedirectSecretHeaders rather than silently losing
|
||||
// test coverage.
|
||||
var redirectTestHeaders = map[string]string{
|
||||
"Authorization": "secret-token",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Key": "secret-key",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Key-Md5": "secret-md5",
|
||||
}
|
||||
|
||||
func TestRedirectSecretHeaders(t *testing.T) {
|
||||
want := make([]string, 0, len(redirectTestHeaders))
|
||||
for header := range redirectTestHeaders {
|
||||
want = append(want, header)
|
||||
}
|
||||
sort.Strings(want)
|
||||
got := append([]string(nil), redirectSecretHeaders...)
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
// redirectTestRequest makes a GET request to rawURL carrying the
|
||||
// secret headers plus a harmless marker header.
|
||||
func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
|
||||
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
|
||||
require.NoError(t, err)
|
||||
for header, value := range redirectTestHeaders {
|
||||
req.Header.Set(header, value)
|
||||
}
|
||||
req.Header.Set("X-Bz-Test-Mode", "marker")
|
||||
return req
|
||||
}
|
||||
|
||||
func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
|
||||
client := newClient(context.Background())
|
||||
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for header := range redirectTestHeaders {
|
||||
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
|
||||
}
|
||||
assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer initialServer.Close()
|
||||
|
||||
resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
}
|
||||
+4
-11
@@ -45,7 +45,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/youmark/pkcs8"
|
||||
)
|
||||
@@ -1461,7 +1460,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
// Item in the cache has the same or newer SequenceID than
|
||||
// this event. Ignore this event, it must be old.
|
||||
f.itemMetaCacheMu.Unlock()
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1692,7 +1691,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// upload does a single non-multipart upload
|
||||
//
|
||||
// This is recommended for less than 50 MiB of content
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
upload := api.UploadFile{
|
||||
Name: o.fs.opt.Enc.FromStandardName(leaf),
|
||||
ContentModifiedAt: api.Time(modTime),
|
||||
@@ -1704,10 +1703,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
|
||||
var resp *http.Response
|
||||
var result api.FolderItems
|
||||
counter := readers.NewCountingReader(in)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Body: counter,
|
||||
Body: in,
|
||||
MultipartMetadataName: "attributes",
|
||||
MultipartContentName: "contents",
|
||||
MultipartFileName: upload.Name,
|
||||
@@ -1727,11 +1725,6 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload.
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
if result.TotalCount != 1 || len(result.Entries) != 1 {
|
||||
return fmt.Errorf("failed to upload %v - not sure why", o)
|
||||
}
|
||||
@@ -1761,7 +1754,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Upload with simple or multipart
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
|
||||
} else {
|
||||
err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
}
|
||||
|
||||
+15
-23
@@ -3,6 +3,7 @@
|
||||
package box
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
@@ -19,7 +20,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
|
||||
}
|
||||
|
||||
// uploadPart uploads a part in an upload session
|
||||
//
|
||||
// chunk must contain exactly the part's data and is read from the start
|
||||
// on each attempt.
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
hasher := sha1.New()
|
||||
chunkSize, err := io.Copy(hasher, chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
chunkSize := int64(len(chunk))
|
||||
sha1sum := sha1.Sum(chunk)
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/files/upload_sessions/" + SessionID,
|
||||
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
|
||||
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Digest": sha1Digest(hasher.Sum(nil)),
|
||||
"Digest": sha1Digest(sha1sum[:]),
|
||||
},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = chunk.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = wrap(chunk)
|
||||
opts.Body = wrap(bytes.NewReader(chunk))
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -223,25 +213,27 @@ outer:
|
||||
|
||||
reqSize := min(remaining, chunkSize)
|
||||
|
||||
// Read the chunk into memory from the global pool, hashing
|
||||
// it for the whole file hash (must be done sequentially)
|
||||
rw := multipart.NewRW().Reserve(reqSize)
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
|
||||
// Make a block of memory
|
||||
buf := make([]byte, reqSize)
|
||||
|
||||
// Read the chunk
|
||||
_, err = io.ReadFull(in, buf)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
err = fmt.Errorf("multipart upload failed to read source: %w", err)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Make the global hash (must be done sequentially)
|
||||
_, _ = hash.Write(buf)
|
||||
|
||||
// Transfer the chunk
|
||||
wg.Add(1)
|
||||
o.fs.uploadToken.Get()
|
||||
go func(part int, position int64) {
|
||||
defer wg.Done()
|
||||
defer o.fs.uploadToken.Put()
|
||||
defer func() { _ = rw.Close() }()
|
||||
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
|
||||
select {
|
||||
|
||||
Vendored
+5
-10
@@ -503,19 +503,15 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
|
||||
}
|
||||
|
||||
go func() {
|
||||
// Wait on the timer and the stop channel together so that a stop
|
||||
// signalled by StopBackgroundRunners is honoured immediately.
|
||||
timer := time.NewTimer(time.Duration(f.opt.ChunkCleanInterval))
|
||||
defer timer.Stop()
|
||||
for {
|
||||
time.Sleep(time.Duration(f.opt.ChunkCleanInterval))
|
||||
select {
|
||||
case <-f.cleanupChan:
|
||||
fs.Infof(f, "stopping cleanup")
|
||||
return
|
||||
case <-timer.C:
|
||||
default:
|
||||
fs.Debugf(f, "starting cleanup")
|
||||
f.CleanUpCache(false)
|
||||
timer.Reset(time.Duration(f.opt.ChunkCleanInterval))
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -567,10 +563,9 @@ Eg
|
||||
})
|
||||
|
||||
rc.Add(rc.Call{
|
||||
Path: "cache/stats",
|
||||
NoAuth: true,
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Path: "cache/stats",
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Help: `
|
||||
Show statistics for the cache remote.
|
||||
`,
|
||||
|
||||
@@ -801,12 +801,9 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
|
||||
fs.Debugf(f, "skip orphan data chunk %q", remote)
|
||||
break
|
||||
}
|
||||
if mainObject == nil && !f.useMeta && !isSpecial {
|
||||
if mainObject == nil && !f.useMeta {
|
||||
// this is the "nometa" case
|
||||
// create dummy chunked object without metadata
|
||||
//
|
||||
// not for temporary or control chunks as they may be all
|
||||
// that is left of an interrupted upload
|
||||
mainObject = f.newObject(mainRemote, nil, nil)
|
||||
byRemote[mainRemote] = mainObject
|
||||
if !badEntry[mainRemote] {
|
||||
|
||||
@@ -43,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
@@ -627,22 +627,6 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
|
||||
_ = file.Remove(ctx)
|
||||
}
|
||||
|
||||
// testOrphanTempChunks checks that temporary chunks left behind by an
|
||||
// interrupted upload are not listed and don't break the listing
|
||||
func testOrphanTempChunks(t *testing.T, f *Fs) {
|
||||
const dir = "orphan"
|
||||
ctx := context.Background()
|
||||
defer func() {
|
||||
_ = operations.Purge(ctx, f.base, dir)
|
||||
}()
|
||||
|
||||
modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
|
||||
item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
|
||||
fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
|
||||
|
||||
fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
|
||||
}
|
||||
|
||||
func testMetadataInput(t *testing.T, f *Fs) {
|
||||
const minChunkForTest = 50
|
||||
if f.opt.ChunkSize < minChunkForTest {
|
||||
@@ -944,9 +928,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("BackwardsCompatibility", func(t *testing.T) {
|
||||
testBackwardsCompatibility(t, f)
|
||||
})
|
||||
t.Run("OrphanTempChunks", func(t *testing.T) {
|
||||
testOrphanTempChunks(t, f)
|
||||
})
|
||||
t.Run("ChunkerServerSideMove", func(t *testing.T) {
|
||||
testChunkerServerSideMove(t, f)
|
||||
})
|
||||
|
||||
@@ -34,13 +34,6 @@ import (
|
||||
"github.com/zeebo/blake3"
|
||||
)
|
||||
|
||||
// Pacer settings - tuned to match other rclone backends
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Cloudinary shouldn't have a trailing dot if there is no path
|
||||
func cldPathDir(somePath string) string {
|
||||
if somePath == "" || somePath == "." {
|
||||
@@ -196,7 +189,7 @@ func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (f
|
||||
root: root,
|
||||
opt: *opt,
|
||||
cld: cld,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
|
||||
srv: rest.NewClient(client),
|
||||
}
|
||||
|
||||
@@ -472,8 +465,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
if updateOptions, ok := option.(*api.UpdateOptions); ok {
|
||||
if updateOptions.PublicID != "" {
|
||||
updateObject = true
|
||||
params.Overwrite = new(true)
|
||||
params.Invalidate = new(true)
|
||||
params.Overwrite = SDKApi.Bool(true)
|
||||
params.Invalidate = SDKApi.Bool(true)
|
||||
params.PublicID = updateOptions.PublicID
|
||||
params.ResourceType = updateOptions.ResourceType
|
||||
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
|
||||
@@ -593,18 +586,21 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
// Cloudinary returns rate-limit windows as "Try again on YYYY-MM-DD HH:MM:SS UTC"
|
||||
// embedded in the error message - honour it as a retry-after.
|
||||
if err != nil {
|
||||
tryAgain := "Try again on "
|
||||
if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
|
||||
layout := "2006-01-02 15:04:05 UTC"
|
||||
dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
|
||||
if timestamp, err2 := time.Parse(layout, dateStr); err2 == nil {
|
||||
timestamp, err2 := time.Parse(layout, dateStr)
|
||||
if err2 == nil {
|
||||
return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
|
||||
}
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "Retrying API error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
|
||||
@@ -1104,7 +1104,7 @@ func (o *Object) Remote() string {
|
||||
newPath, err := o.u.pathAdjustment.do(o.Object.String())
|
||||
if err != nil {
|
||||
fs.Errorf(o.Object, "Bad object: %v", err)
|
||||
return ""
|
||||
return err.Error()
|
||||
}
|
||||
return newPath
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -564,19 +563,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
|
||||
|
||||
// cache small files in memory and do normal upload
|
||||
rw := multipart.NewRW()
|
||||
defer fs.CheckClose(rw, &err)
|
||||
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
|
||||
if err == io.EOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, rw, src, options...)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
buf := make([]byte, f.opt.RAMCacheLimit)
|
||||
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
|
||||
}
|
||||
|
||||
// Need to include what we already read
|
||||
in = &ReadCloserWrapper{
|
||||
Reader: io.MultiReader(rw, in),
|
||||
Reader: io.MultiReader(bytes.NewReader(buf), in),
|
||||
Closer: in,
|
||||
}
|
||||
|
||||
@@ -703,16 +698,6 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared -
|
||||
// compressed data is stored under a name containing the
|
||||
// original size so a mismatch makes the object unreadable.
|
||||
if size := src.Size(); size >= 0 && meta.Size != size {
|
||||
if removeErr := dataObject.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
|
||||
|
||||
// meta data upload may fail. in this case we try to remove the original object
|
||||
@@ -1085,35 +1070,6 @@ type ObjectMetadata struct {
|
||||
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
|
||||
}
|
||||
|
||||
// validate checks metadata read from the wrapped remote before it is used to
|
||||
// construct a compressed reader.
|
||||
func (meta *ObjectMetadata) validate() error {
|
||||
if meta.Mode != Gzip {
|
||||
return nil
|
||||
}
|
||||
if meta.CompressionMetadataGzip == nil {
|
||||
return errors.New("missing gzip metadata")
|
||||
}
|
||||
if meta.CompressionMetadataGzip.BlockSize <= 0 {
|
||||
return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
|
||||
}
|
||||
if meta.Size != meta.CompressionMetadataGzip.Size {
|
||||
return errors.New("gzip metadata size does not match object size")
|
||||
}
|
||||
if meta.Size < 0 {
|
||||
return errors.New("invalid gzip object size")
|
||||
}
|
||||
blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
|
||||
blocks := meta.Size / blockSize
|
||||
if meta.Size%blockSize != 0 {
|
||||
blocks++
|
||||
}
|
||||
if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
|
||||
return errors.New("gzip block data is incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Object with external metadata
|
||||
type Object struct {
|
||||
fs.Object // Wraps around data object for this object
|
||||
@@ -1137,9 +1093,6 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
|
||||
if err = jr.Decode(meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = meta.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
@@ -1157,15 +1110,6 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return objErr
|
||||
}
|
||||
|
||||
// countingDiscard is an io.Writer which discards its input and counts
|
||||
// the bytes written
|
||||
type countingDiscard int64
|
||||
|
||||
func (c *countingDiscard) Write(p []byte) (int, error) {
|
||||
*c += countingDiscard(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
|
||||
type ReadCloserWrapper struct {
|
||||
io.Reader
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/buengese/sgzip"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
@@ -16,93 +14,6 @@ import (
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
func TestObjectMetadataValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
meta ObjectMetadata
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "nil gzip metadata",
|
||||
meta: ObjectMetadata{Mode: Gzip},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "zero block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 0,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "negative block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: -1,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "mismatched sizes",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 2048,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "short block data",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1, 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := test.meta.validate()
|
||||
if (err != nil) != test.want {
|
||||
t.Fatalf("validate() error = %v, want error: %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var defaultOpt = fstests.Opt{
|
||||
RemoteName: "TestCompress:",
|
||||
NilObject: (*Object)(nil),
|
||||
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type gzipModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
|
||||
// PutStream. It reads each body twice when it is seekable to model a
|
||||
// backend retrying an upload.
|
||||
type putRecorder struct {
|
||||
fs.Fs
|
||||
t *testing.T
|
||||
stream bool // expose PutStream from the wrapped Fs
|
||||
size int64
|
||||
seekable bool
|
||||
body []byte
|
||||
streamed bool
|
||||
}
|
||||
|
||||
func (r *putRecorder) Features() *fs.Features {
|
||||
features := *r.Fs.Features()
|
||||
features.PutStream = nil
|
||||
if r.stream {
|
||||
features.PutStream = r.PutStream
|
||||
}
|
||||
return &features
|
||||
}
|
||||
|
||||
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
|
||||
r.size = src.Size()
|
||||
body, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
r.seekable = true
|
||||
_, err = seeker.Seek(0, io.SeekStart)
|
||||
require.NoError(r.t, err)
|
||||
again, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
assert.Equal(r.t, body, again, "retried body should be identical")
|
||||
}
|
||||
r.body = body
|
||||
return body
|
||||
}
|
||||
|
||||
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
body := r.record(in, src)
|
||||
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
|
||||
}
|
||||
|
||||
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
r.streamed = true
|
||||
return r.Put(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// TestRcat checks the three upload paths rcat takes: small files buffered
|
||||
// in memory and uploaded with a known size and a seekable body, large
|
||||
// files streamed when the wrapped remote supports it, and large files
|
||||
// spooled to a temporary file when it doesn't. Pool pages taken for the
|
||||
// buffer must be returned on every path.
|
||||
func TestRcat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const limit = 1024
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
stream bool
|
||||
wantSize int64
|
||||
}{
|
||||
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
|
||||
{name: "Empty", size: 0, stream: true, wantSize: 0},
|
||||
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
|
||||
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
|
||||
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
|
||||
require.NoError(t, err)
|
||||
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
|
||||
f := &Fs{Fs: rec}
|
||||
f.opt.RAMCacheLimit = limit
|
||||
content := []byte(random.String(test.size))
|
||||
inUse := pool.Global().InUse()
|
||||
|
||||
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
|
||||
|
||||
assert.Equal(t, test.wantSize, rec.size)
|
||||
assert.Equal(t, content, rec.body)
|
||||
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
|
||||
if test.size < limit {
|
||||
assert.True(t, rec.seekable, "small file body should be seekable")
|
||||
}
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, content, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsCompressible checks the compressibility heuristic on data at
|
||||
// both ends of the scale for every compression mode which has one.
|
||||
func TestIsCompressible(t *testing.T) {
|
||||
compressible := bytes.Repeat([]byte("compress me "), 4096)
|
||||
incompressible := make([]byte, len(compressible))
|
||||
_, err := io.ReadFull(rand.Reader, incompressible)
|
||||
require.NoError(t, err)
|
||||
for name, handler := range map[string]compressionModeHandler{
|
||||
"gzip": &gzipModeHandler{},
|
||||
"zstd": &zstdModeHandler{},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
@@ -22,7 +23,7 @@ type zstdModeHandler struct{}
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
@@ -36,7 +37,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
|
||||
+13
-40
@@ -11,7 +11,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -118,8 +117,8 @@ func (mode NameEncryptionMode) String() (out string) {
|
||||
return out
|
||||
}
|
||||
|
||||
// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
|
||||
type FileNameEncoding interface {
|
||||
// fileNameEncoding are the encoding methods dealing with encrypted file names
|
||||
type fileNameEncoding interface {
|
||||
EncodeToString(src []byte) string
|
||||
DecodeString(s string) ([]byte, error)
|
||||
}
|
||||
@@ -154,7 +153,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
|
||||
}
|
||||
|
||||
// NewNameEncoding creates a NameEncoding from a string
|
||||
func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
|
||||
func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
|
||||
s = strings.ToLower(s)
|
||||
switch s {
|
||||
case "base32":
|
||||
@@ -176,7 +175,7 @@ type Cipher struct {
|
||||
nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto
|
||||
block gocipher.Block
|
||||
mode NameEncryptionMode
|
||||
fileNameEnc FileNameEncoding
|
||||
fileNameEnc fileNameEncoding
|
||||
buffers sync.Pool // encrypt/decrypt buffers
|
||||
cryptoRand io.Reader // read crypto random numbers from here
|
||||
dirNameEncrypt bool
|
||||
@@ -185,7 +184,7 @@ type Cipher struct {
|
||||
}
|
||||
|
||||
// newCipher initialises the cipher. If salt is "" then it uses a built in salt val
|
||||
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
|
||||
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
|
||||
c := &Cipher{
|
||||
mode: mode,
|
||||
fileNameEnc: enc,
|
||||
@@ -487,12 +486,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
}
|
||||
|
||||
// encryptFileName encrypts a file path
|
||||
//
|
||||
// If stripVersion is set, a version string (as used by --b2-versions)
|
||||
// on the last segment is kept in plain text. Set it for file names
|
||||
// only: directory names must encrypt identically whether passed alone
|
||||
// or as the parent of a file name.
|
||||
func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
|
||||
func (c *Cipher) encryptFileName(in string) string {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
// Skip directory name encryption if the user chose to
|
||||
@@ -505,7 +499,7 @@ func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
|
||||
// of the file name gets encrypted/obfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -535,24 +529,19 @@ func (c *Cipher) EncryptFileName(in string) string {
|
||||
if c.mode == NameEncryptionOff {
|
||||
return in + c.encryptedSuffix
|
||||
}
|
||||
return c.encryptFileName(in, true)
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// EncryptDirName encrypts a directory path
|
||||
//
|
||||
// Unlike EncryptFileName, a version string on the last segment is
|
||||
// encrypted along with the rest of the name.
|
||||
func (c *Cipher) EncryptDirName(in string) string {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in
|
||||
}
|
||||
return c.encryptFileName(in, false)
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// decryptFileName decrypts a file path
|
||||
//
|
||||
// stripVersion is as for encryptFileName.
|
||||
func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
|
||||
func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
var err error
|
||||
@@ -566,7 +555,7 @@ func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
|
||||
// of the file name gets decrypted/deobfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -612,31 +601,15 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
|
||||
// Leave the version string on, if it was there
|
||||
return decrypted, nil
|
||||
}
|
||||
return c.decryptFileName(in, true)
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// DecryptDirName decrypts a directory path
|
||||
//
|
||||
// Unlike DecryptFileName, a version string on the last segment is
|
||||
// expected to be part of the encrypted name. Directory names created
|
||||
// by rclone before v1.76 had a version-like suffix left in plain
|
||||
// text; these are still decrypted so that they appear in listings,
|
||||
// but they can't be opened or removed until renamed on the
|
||||
// underlying remote.
|
||||
func (c *Cipher) DecryptDirName(in string) (string, error) {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in, nil
|
||||
}
|
||||
out, err := c.decryptFileName(in, false)
|
||||
if err != nil && version.Match(path.Base(in)) {
|
||||
var legacyErr error
|
||||
out, legacyErr = c.decryptFileName(in, true)
|
||||
if legacyErr == nil {
|
||||
fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// NameEncryptionMode returns the encryption mode in use for names
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -683,43 +682,6 @@ func TestNonStandardDecryptDirName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test directories whose name looks like it has a version string -
|
||||
// these encrypt verbatim, so that EncryptDirName agrees with the same
|
||||
// directory encrypted as the parent of a file name, whereas file names
|
||||
// keep the version string in plain text
|
||||
func TestVersionedDirName(t *testing.T) {
|
||||
const dir = "dir-v2001-02-03-040506-123"
|
||||
enc, err := NewNameEncoding("base32")
|
||||
require.NoError(t, err)
|
||||
for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
|
||||
c, err := newCipher(mode, "", "", true, enc)
|
||||
require.NoError(t, err)
|
||||
what := fmt.Sprintf("Testing mode=%v", mode)
|
||||
// Check EncryptDirName matches the parent of an encrypted file name
|
||||
encryptedDir := c.EncryptDirName(dir)
|
||||
encryptedFile := c.EncryptFileName(dir + "/file.txt")
|
||||
assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
|
||||
// Check the encrypted directory name round trips OK
|
||||
decryptedDir, err := c.DecryptDirName(encryptedDir)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// Check a file with the same name keeps its version string in plain text
|
||||
encryptedFile = c.EncryptFileName(dir)
|
||||
assert.NotEqual(t, encryptedDir, encryptedFile, what)
|
||||
assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
|
||||
decryptedFile, err := c.DecryptFileName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedFile, what)
|
||||
// Check a directory with a legacy encrypted name (the file form) still decrypts
|
||||
decryptedDir, err = c.DecryptDirName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// but a name which is bad in both forms is still an error
|
||||
_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
|
||||
assert.Error(t, err, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSize(t *testing.T) {
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
|
||||
for _, test := range []struct {
|
||||
|
||||
@@ -1179,11 +1179,6 @@ func (o *ObjectInfo) Size() int64 {
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
|
||||
// If the data is unchanged then the hash of the source is the
|
||||
// hash of the object which will be uploaded
|
||||
if o.f.opt.NoDataEncryption {
|
||||
return o.ObjectInfo.Hash(ctx, hash)
|
||||
}
|
||||
var srcObj fs.Object
|
||||
var ok bool
|
||||
// Get the underlying object if there is one
|
||||
|
||||
@@ -84,10 +84,6 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
|
||||
|
||||
// Test ObjectInfo.Hash
|
||||
wantHash := md5.Sum(outBuf.Bytes())
|
||||
if f.opt.NoDataEncryption {
|
||||
// If the data isn't encrypted, the hash should be that of the plaintext
|
||||
wantHash = md5.Sum([]byte(contents))
|
||||
}
|
||||
gotHash, err := src.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
// Package api has type definitions for dosya.dev
|
||||
package api
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Response is the base response wrapper
|
||||
type Response struct {
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Error is an error response from the API.
|
||||
//
|
||||
// StatusCode and Status are filled in from the HTTP response. The API
|
||||
// sets ErrorCode on the errors a client is expected to act on rather
|
||||
// than just report, e.g. "concurrent_upload_limit".
|
||||
type Error struct {
|
||||
StatusCode int `json:"-"`
|
||||
Status string `json:"-"`
|
||||
OK bool `json:"ok"`
|
||||
Message string `json:"error"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
|
||||
}
|
||||
|
||||
// Error satisfies the error interface
|
||||
func (e *Error) Error() string {
|
||||
out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
|
||||
if e.Message != "" {
|
||||
out += ": " + e.Message
|
||||
}
|
||||
if e.ErrorCode != "" {
|
||||
out += " (" + e.ErrorCode + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileItem represents a file from the dosya API
|
||||
type FileItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
UpdatedAt float64 `json:"updated_at"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
|
||||
// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
|
||||
// are never re-read to hash the whole object - the same limitation S3 has).
|
||||
ContentHash string `json:"content_hash"`
|
||||
}
|
||||
|
||||
// FolderItem represents a folder from the dosya API
|
||||
type FolderItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
FileCount int `json:"file_count"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
}
|
||||
|
||||
// ListResponse is the response from GET /api/files
|
||||
type ListResponse struct {
|
||||
Response
|
||||
Files []FileItem `json:"files"`
|
||||
Folders []FolderItem `json:"folders"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
// Pagination describes which page of files a ListResponse holds
|
||||
type Pagination struct {
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest is the request body for POST /api/folders
|
||||
type CreateFolderRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse is the response from POST /api/folders
|
||||
type CreateFolderResponse struct {
|
||||
Response
|
||||
Folder struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Workspace string `json:"workspace_id"`
|
||||
} `json:"folder"`
|
||||
}
|
||||
|
||||
// DeleteFolderResponse is the response from DELETE /api/folders/:id
|
||||
type DeleteFolderResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
Complete bool `json:"complete"`
|
||||
Remaining int `json:"remaining"`
|
||||
FilesAffected int `json:"files_affected"`
|
||||
FoldersRemoved int `json:"folders_removed"`
|
||||
}
|
||||
|
||||
// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
|
||||
type RenameFolderRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFolderRequest is the request body for PUT /api/folders/:id/move
|
||||
type MoveFolderRequest struct {
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// MoveFolderResponse is the response from PUT /api/folders/:id/move
|
||||
type MoveFolderResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// UploadInitRequest is the request body for POST /api/upload/init
|
||||
type UploadInitRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Region string `json:"region,omitempty"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
FileID *string `json:"file_id,omitempty"`
|
||||
}
|
||||
|
||||
// UploadInitResponse is the response from POST /api/upload/init
|
||||
type UploadInitResponse struct {
|
||||
Response
|
||||
SessionID string `json:"session_id"`
|
||||
UploadURL string `json:"upload_url"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Resumable *struct {
|
||||
PartSize int64 `json:"part_size"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
PartUploadURL string `json:"part_upload_url"`
|
||||
CompleteURL string `json:"complete_url"`
|
||||
StatusURL string `json:"status_url"`
|
||||
} `json:"resumable"`
|
||||
}
|
||||
|
||||
// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
|
||||
type UploadCompleteResponse struct {
|
||||
Response
|
||||
File struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Version int `json:"version"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
|
||||
// "" for a multipart upload, which the server does not hash.
|
||||
ContentHash string `json:"content_hash"`
|
||||
} `json:"file"`
|
||||
}
|
||||
|
||||
// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
|
||||
type UploadPartResponse struct {
|
||||
Response
|
||||
PartNumber int `json:"part_number"`
|
||||
ETag string `json:"etag"`
|
||||
BytesUploaded int64 `json:"bytes_uploaded"`
|
||||
PartsCompleted int `json:"parts_completed"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
}
|
||||
|
||||
// DeleteFileResponse is the response from DELETE /api/files/:id
|
||||
type DeleteFileResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
}
|
||||
|
||||
// RenameFileRequest is the request body for PUT /api/files/:id/rename
|
||||
type RenameFileRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// RenameFileResponse is the response from PUT /api/files/:id/rename
|
||||
type RenameFileResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFileRequest is the request body for PUT /api/files/:id/move
|
||||
type MoveFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest is the request body for POST /api/files/:id/copy
|
||||
type CopyFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileResponse is the response from POST /api/files/:id/copy
|
||||
type CopyFileResponse struct {
|
||||
Response
|
||||
FileID string `json:"file_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// ShareLinkRequest is the request body for POST /api/files/:id/share
|
||||
type ShareLinkRequest struct {
|
||||
ExpiresInDays *int `json:"expires_in_days,omitempty"`
|
||||
}
|
||||
|
||||
// ShareLinkResponse is the response from POST /api/files/:id/share
|
||||
type ShareLinkResponse struct {
|
||||
Response
|
||||
Link struct {
|
||||
ID string `json:"id"`
|
||||
Token string `json:"token"`
|
||||
URL string `json:"url"`
|
||||
ExpiresAt *int64 `json:"expires_at"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
} `json:"link"`
|
||||
}
|
||||
|
||||
// Workspace is one entry of the workspace list
|
||||
type Workspace struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// WorkspaceListResponse is the response from GET /api/workspaces
|
||||
type WorkspaceListResponse struct {
|
||||
Response
|
||||
Workspaces []Workspace `json:"workspaces"`
|
||||
}
|
||||
|
||||
// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
|
||||
type WorkspaceInfoResponse struct {
|
||||
Response
|
||||
Storage struct {
|
||||
Used int64 `json:"used"`
|
||||
Total int64 `json:"total"`
|
||||
Free int64 `json:"free"`
|
||||
} `json:"storage"`
|
||||
}
|
||||
@@ -1,772 +0,0 @@
|
||||
// Package dosya provides an interface to the dosya.dev cloud storage system.
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
rootID = "root"
|
||||
defaultBaseURL = "https://api.dosya.dev"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2
|
||||
attackConstant = 0
|
||||
)
|
||||
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "dosya",
|
||||
Description: "dosya.dev",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
|
||||
Name: "api_key",
|
||||
Sensitive: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "workspace_id",
|
||||
Help: `Your workspace ID.
|
||||
|
||||
Leave this blank to use the one workspace the API key can reach, which
|
||||
is what a key restricted to a single workspace does. If the key can
|
||||
reach several, rclone will refuse and list their IDs for you to choose
|
||||
from.`,
|
||||
}, {
|
||||
Help: "API base URL.",
|
||||
Name: "api_url",
|
||||
Default: defaultBaseURL,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
Help: `Concurrency for multipart uploads.
|
||||
|
||||
This is the number of parts of the same file that are uploaded
|
||||
concurrently. The first part is always uploaded on its own, because
|
||||
the server creates the upload when it receives it.
|
||||
|
||||
Each part in flight is held in memory, so memory use grows with
|
||||
this value times the part size the server chooses.
|
||||
|
||||
If you are uploading large files over high-speed links and these
|
||||
uploads do not fully utilize your bandwidth, then increasing this
|
||||
may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeSingleQuote |
|
||||
encoder.EncodeDoubleQuote |
|
||||
encoder.EncodeLtGt |
|
||||
encoder.EncodeCtl |
|
||||
encoder.EncodeLeftSpace |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
APIKey string `config:"api_key"`
|
||||
WorkspaceID string `config:"workspace_id"`
|
||||
APIURL string `config:"api_url"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dosya.dev filesystem
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
features *fs.Features
|
||||
rest *rest.Client
|
||||
pacer *fs.Pacer
|
||||
dirCache *dircache.DirCache
|
||||
}
|
||||
|
||||
// Object represents a dosya.dev file
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
file api.FileItem
|
||||
}
|
||||
|
||||
// findWorkspace fills in the workspace ID when the config leaves it blank.
|
||||
//
|
||||
// The ID is not shown anywhere in the dosya.dev web interface, so requiring
|
||||
// it meant asking people to find it in a developer console. A key restricted
|
||||
// to one workspace can only mean that one, and an account with a single
|
||||
// workspace has only one possible answer; anything else genuinely has to be
|
||||
// chosen, so list the choices in the error.
|
||||
func (f *Fs) findWorkspace(ctx context.Context) error {
|
||||
workspaces, err := f.listWorkspaces(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch len(workspaces) {
|
||||
case 0:
|
||||
return errors.New("no workspaces found for this API key")
|
||||
case 1:
|
||||
f.opt.WorkspaceID = workspaces[0].ID
|
||||
fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
|
||||
return nil
|
||||
}
|
||||
var choices strings.Builder
|
||||
for _, w := range workspaces {
|
||||
_, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
|
||||
}
|
||||
return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
|
||||
len(workspaces), choices.String())
|
||||
}
|
||||
|
||||
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
||||
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
|
||||
listing, err := f.listFilesAndFolders(ctx, pathID)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, folder := range listing.Folders {
|
||||
if f.opt.Enc.ToStandardName(folder.Name) == leaf {
|
||||
return folder.ID, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// CreateDir makes a directory with pathID as parent and name leaf
|
||||
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
|
||||
name := f.opt.Enc.FromStandardName(leaf)
|
||||
resp, err := f.createFolder(ctx, name, pathID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.Folder.ID, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("dosya.dev root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
// Modification times are stored and returned to the second (the API
|
||||
// keeps them as Unix seconds, sent on upload via the source-mtime header).
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash types of the filesystem
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
// The server computes and stores the SHA-256 of a single-shot upload's
|
||||
// bytes; multipart uploads have none (see Object.Hash).
|
||||
return hash.NewHashSet(hash.SHA256)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(config, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
|
||||
}
|
||||
|
||||
// No DuplicateFiles: an upload that names no existing file but whose
|
||||
// (workspace, folder, name) is already taken becomes a new VERSION of
|
||||
// that file rather than a second row, so a name is unique within a
|
||||
// folder and the same upload twice returns the same file id.
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
ReadMimeType: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
|
||||
f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
|
||||
|
||||
if f.opt.WorkspaceID == "" {
|
||||
err = f.findWorkspace(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, rootID, f)
|
||||
|
||||
// Find the current root
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// Assume it is a file
|
||||
newRoot, remote := dircache.SplitPath(root)
|
||||
tempF := *f
|
||||
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
|
||||
tempF.root = newRoot
|
||||
// Make new Fs which is the parent
|
||||
err = tempF.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// No root so return old f
|
||||
return f, nil
|
||||
}
|
||||
_, err := tempF.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// File doesn't exist so return old f
|
||||
return f, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
f.features.Fill(ctx, &tempF)
|
||||
f.dirCache = tempF.dirCache
|
||||
f.root = tempF.root
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, folder := range listing.Folders {
|
||||
folderName := f.opt.Enc.ToStandardName(folder.Name)
|
||||
fullPath := getRemote(dir, folderName)
|
||||
modTime := time.Unix(int64(folder.CreatedAt), 0)
|
||||
d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
|
||||
entries = append(entries, d)
|
||||
f.dirCache.Put(fullPath, folder.ID)
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
fileName := f.opt.Enc.ToStandardName(file.Name)
|
||||
remote := getRemote(dir, fileName)
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}
|
||||
entries = append(entries, o)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
if f.opt.Enc.ToStandardName(file.Name) == leaf {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
// Put uploads a new file
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
existingObj, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return existingObj, existingObj.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// put uploads src as a new file
|
||||
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
remote := src.Remote()
|
||||
|
||||
_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: fileFromUpload(resp, modTime),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fileFromUpload builds the FileItem for the object an upload just created.
|
||||
// The single-shot upload response omits updated_at, so the modification time
|
||||
// is taken from what was sent (modTime) rather than defaulting to the create
|
||||
// time - it is the time the server stored via the source-mtime header. The
|
||||
// SHA-256 comes straight from the response (empty for a multipart upload).
|
||||
func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
|
||||
updatedAt := resp.File.CreatedAt
|
||||
if !modTime.IsZero() {
|
||||
updatedAt = float64(modTime.Unix())
|
||||
}
|
||||
return api.FileItem{
|
||||
ID: resp.File.ID,
|
||||
Name: resp.File.Name,
|
||||
SizeBytes: resp.File.SizeBytes,
|
||||
MimeType: resp.File.MimeType,
|
||||
Extension: resp.File.Extension,
|
||||
Region: resp.File.Region,
|
||||
CreatedAt: resp.File.CreatedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
ContentHash: resp.File.ContentHash,
|
||||
}
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
return err
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The server checks emptiness itself (?if_empty=1) and answers
|
||||
// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
|
||||
err = f.removeFolder(ctx, directoryID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move src to this remote using server-side move operations
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't move - not same remote type")
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
|
||||
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find destination directory
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
needsMove := srcDirID != dstDirID
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
|
||||
// Moving and renaming in the same request (rather than a moveFile call
|
||||
// followed by a separate renameFile call) matters when both are needed:
|
||||
// a two-step move-then-rename can transiently place the file's CURRENT
|
||||
// name into a destination folder that already holds an unrelated file
|
||||
// with that name, even though the FINAL name is free - the server only
|
||||
// allows one active name per folder, so that intermediate state 409s.
|
||||
if needsMove || needsRename {
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Return new object. file.Name must reflect the destination leaf, not
|
||||
// the source's - a caller that reuses this object as the src of a
|
||||
// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
|
||||
// recomputes srcLeaf from file.Name; a stale name there makes that
|
||||
// second move see srcLeaf == dstLeaf and skip the rename entirely,
|
||||
// silently leaving the object under the wrong name.
|
||||
newFile := srcObj.file
|
||||
newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: newFile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
|
||||
srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srcLeaf := srcRemote
|
||||
if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
|
||||
srcLeaf = srcRemote[idx+1:]
|
||||
}
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
|
||||
// Move folder to new parent only if parent changed. The new name goes
|
||||
// in the same request, as for files in Move: a move followed by a
|
||||
// separate rename can collide with an unrelated folder sharing the
|
||||
// CURRENT name in the destination even though the final name is free.
|
||||
if srcDirectoryID != dstDirectoryID {
|
||||
renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move directory: %w", err)
|
||||
}
|
||||
if renamed {
|
||||
needsRename = false
|
||||
}
|
||||
}
|
||||
|
||||
if needsRename {
|
||||
err = f.renameFolder(ctx, srcID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side copy operations
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The copy API creates a new row named after the source file when
|
||||
// copying across folders, which collides with one already at the
|
||||
// destination. Copy under a temporary name, then delete the old
|
||||
// destination, then move the copy onto the name - so a copy that fails
|
||||
// leaves the destination as it was. Deleting it up front lost it
|
||||
// (prior art: backend/box).
|
||||
existingObj, err := f.NewObject(ctx, remote)
|
||||
switch err {
|
||||
case nil:
|
||||
tempRemote := remote + "-rclone-copy-" + random.String(8)
|
||||
fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
|
||||
tempObj, err := f.Copy(ctx, src, tempRemote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs.Debugf(tempRemote, "moving to real name %v", remote)
|
||||
if err := existingObj.Remove(ctx); err != nil {
|
||||
return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
|
||||
}
|
||||
return f.Move(ctx, tempObj, remote)
|
||||
case fs.ErrorObjectNotFound:
|
||||
// nothing at the destination, so copy straight onto the name
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dstName := f.opt.Enc.FromStandardName(dstLeaf)
|
||||
|
||||
resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
|
||||
// Servers that predate the name field ignore it and name the copy after
|
||||
// the source, or "Copy of <original>" in the same folder. Rename to the
|
||||
// requested destination name if it differs.
|
||||
if resp.Name != dstName {
|
||||
err = f.renameFile(ctx, resp.FileID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't rename copied file: %w", err)
|
||||
}
|
||||
resp.Name = dstName
|
||||
}
|
||||
|
||||
// A server-side copy is byte-identical to the source and the API
|
||||
// preserves the source's modification time and content hash onto the
|
||||
// copy, so the returned object must report the SOURCE's modtime and hash,
|
||||
// not the copy's wall-clock time. Reading "now" here made the object
|
||||
// rclone holds in memory disagree with the one read back from the remote
|
||||
// (its Fingerprint check compares the two exactly).
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: api.FileItem{
|
||||
ID: resp.FileID,
|
||||
Name: resp.Name,
|
||||
SizeBytes: srcObj.file.SizeBytes,
|
||||
MimeType: srcObj.file.MimeType,
|
||||
Extension: srcObj.file.Extension,
|
||||
Region: srcObj.file.Region,
|
||||
CreatedAt: srcObj.file.CreatedAt,
|
||||
UpdatedAt: srcObj.file.UpdatedAt,
|
||||
ContentHash: srcObj.file.ContentHash,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Purge deletes all the files and the container
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
|
||||
err = f.removeFolder(ctx, directoryID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
obj := o.(*Object)
|
||||
|
||||
resp, err := f.createShareLink(ctx, obj.file.ID, expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return resp.Link.URL, nil
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing as an
|
||||
// optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
info, err := f.getWorkspaceInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage := &fs.Usage{
|
||||
Used: fs.NewUsageValue(info.Storage.Used),
|
||||
Total: fs.NewUsageValue(info.Storage.Total),
|
||||
Free: fs.NewUsageValue(info.Storage.Free),
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Object methods
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// String returns a description of the Object
|
||||
func (o *Object) String() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// ModTime returns the modification date of the file
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
if o.file.UpdatedAt > 0 {
|
||||
return time.Unix(int64(o.file.UpdatedAt), 0)
|
||||
}
|
||||
return time.Unix(int64(o.file.CreatedAt), 0)
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return o.file.SizeBytes
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.SHA256 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// May be "" - a multipart-uploaded object carries no server-side hash,
|
||||
// which rclone treats as "hash unavailable for this object" (as it does
|
||||
// for S3 multipart objects).
|
||||
return o.file.ContentHash, nil
|
||||
}
|
||||
|
||||
// Storable says whether this object can be stored
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
return fs.ErrorCantSetModTime
|
||||
}
|
||||
|
||||
// Open opens the file for read
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
fs.FixRangeOption(options, o.file.SizeBytes)
|
||||
return o.fs.downloadFile(ctx, o.file.ID, options)
|
||||
}
|
||||
|
||||
// Update replaces the file content
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
// Find the directory ID for this file
|
||||
_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload as new version of existing file
|
||||
fileID := o.file.ID
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update object metadata
|
||||
o.file = fileFromUpload(resp, modTime)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove removes this object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.deleteFile(ctx, o.file.ID)
|
||||
}
|
||||
|
||||
// MimeType of an Object if known, "" otherwise
|
||||
func (o *Object) MimeType(ctx context.Context) string {
|
||||
return o.file.MimeType
|
||||
}
|
||||
|
||||
// ID returns the ID of the Object if known, or "" if not
|
||||
func (o *Object) ID() string {
|
||||
return o.file.ID
|
||||
}
|
||||
|
||||
func getRemote(dir, fileName string) string {
|
||||
if dir == "" {
|
||||
return fileName
|
||||
}
|
||||
return dir + "/" + fileName
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ dircache.DirCacher = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
)
|
||||
@@ -1,880 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// sourceMTimeHeader is the request header the upload endpoints read to set a
|
||||
// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
|
||||
const sourceMTimeHeader = "x-dosya-source-mtime"
|
||||
|
||||
// modTimeHeaders returns the ExtraHeaders that carry a source modification time
|
||||
// on an upload, or nil when there is nothing plausible to send. The server
|
||||
// clamps implausible values itself, but a zero time is never worth a header.
|
||||
func modTimeHeaders(modTime time.Time) map[string]string {
|
||||
if modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
|
||||
}
|
||||
|
||||
// workspaceHintHeader names the workspace a by-id request is about. The API
|
||||
// maps an id to its workspace through an index the workspace fills about a
|
||||
// second after a write, so an id this client was handed moments ago can still
|
||||
// miss; the hint lets the server look for it in the named workspace instead.
|
||||
// It only locates: every route still checks membership against the workspace
|
||||
// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
|
||||
const workspaceHintHeader = "X-Dosya-Workspace"
|
||||
|
||||
// byIDHeaders returns the ExtraHeaders every /api/files/:id and
|
||||
// /api/folders/:id call carries, or nil when no workspace is configured.
|
||||
func (f *Fs) byIDHeaders() map[string]string {
|
||||
if f.opt.WorkspaceID == "" {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
|
||||
}
|
||||
|
||||
// retryErrorCodes is a slice of error codes that we will retry
|
||||
var retryErrorCodes = []int{
|
||||
429, // Too Many Requests
|
||||
500, // Internal Server Error
|
||||
502, // Bad Gateway
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
|
||||
if retry {
|
||||
if d, ok := retryAfter(resp); ok {
|
||||
// The API answers 503 + Retry-After while a workspace is being
|
||||
// moved between storage homes, and Cloudflare sends it with 429.
|
||||
// The pacer's own backoff tops out at maxSleep, far short of that.
|
||||
fs.Debugf(nil, "Server asked to retry after %v", d)
|
||||
err = pacer.RetryAfterError(err, d)
|
||||
}
|
||||
}
|
||||
return retry, err
|
||||
}
|
||||
|
||||
// maxRetryAfter caps how long a single Retry-After is honoured for
|
||||
const maxRetryAfter = 5 * time.Minute
|
||||
|
||||
// retryAfter reads a 429 or 503 response's Retry-After header, which the API
|
||||
// sends as whole seconds
|
||||
func retryAfter(resp *http.Response) (time.Duration, bool) {
|
||||
if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
|
||||
return 0, false
|
||||
}
|
||||
seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
|
||||
if err != nil || seconds <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
|
||||
}
|
||||
|
||||
// errorHandler decodes a non-2xx response into an *api.Error so callers can
|
||||
// act on the machine-readable error_code rather than the message text.
|
||||
func errorHandler(resp *http.Response) error {
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading error out of body: %w", err)
|
||||
}
|
||||
apiErr := &api.Error{
|
||||
StatusCode: resp.StatusCode,
|
||||
Status: resp.Status,
|
||||
}
|
||||
if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
|
||||
// Not the API's JSON shape (e.g. an edge response that never
|
||||
// reached the API) - keep the body text so it isn't lost.
|
||||
apiErr.ErrorCode = ""
|
||||
apiErr.Message = strings.TrimSpace(string(body))
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
||||
// The workspace caps the number of in-flight uploads per user. The API
|
||||
// refuses an upload/init past that cap with a 400 carrying
|
||||
// error_code "concurrent_upload_limit", and documents it as an error to
|
||||
// wait on rather than give up on: slots free as uploads finish, and the
|
||||
// server marks sessions idle for 10 minutes as failed, so waiting up to
|
||||
// that long guarantees progress unless the workspace really is saturated.
|
||||
// A 400 is deliberately outside the pacer's retry codes, so the wait
|
||||
// lives in initUpload instead of shouldRetry.
|
||||
var (
|
||||
concurrentUploadRetryDelay = 5 * time.Second
|
||||
concurrentUploadWaitMax = 10 * time.Minute
|
||||
)
|
||||
|
||||
// abortUploadTimeout bounds the abort of a session the client is giving up on,
|
||||
// which runs outside the transfer's context so a cancelled transfer still
|
||||
// sends it
|
||||
const abortUploadTimeout = 10 * time.Second
|
||||
|
||||
// isConcurrentUploadLimit reports whether err is the API refusing an
|
||||
// upload because the workspace's concurrent upload limit is reached
|
||||
func isConcurrentUploadLimit(err error) bool {
|
||||
var apiErr *api.Error
|
||||
return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
|
||||
}
|
||||
|
||||
// listPageSize is the largest page GET /api/files will serve
|
||||
const listPageSize = 500
|
||||
|
||||
// listFilesAndFolders lists files and folders in a directory
|
||||
//
|
||||
// The API pages files (at most listPageSize per page) but returns every
|
||||
// folder on each page, so the files from all pages are gathered and the
|
||||
// folders are taken from the first page only. Pages are ordered by name,
|
||||
// which the API tie-breaks on id, so offsets are stable between requests.
|
||||
func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
|
||||
var listing *api.ListResponse
|
||||
for page := 1; ; page++ {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files",
|
||||
Parameters: map[string][]string{
|
||||
"workspace_id": {f.opt.WorkspaceID},
|
||||
"per_page": {strconv.Itoa(listPageSize)},
|
||||
"page": {strconv.Itoa(page)},
|
||||
"sort": {"name"},
|
||||
"dir": {"asc"},
|
||||
},
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
opts.Parameters["folder_id"] = []string{folderID}
|
||||
}
|
||||
|
||||
var result api.ListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list files: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if listing == nil {
|
||||
listing = &result
|
||||
} else {
|
||||
listing.Files = append(listing.Files, result.Files...)
|
||||
}
|
||||
if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
|
||||
return listing, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// createFolder creates a folder
|
||||
func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
|
||||
request := api.CreateFolderRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
Name: name,
|
||||
}
|
||||
if parentID != "" && parentID != rootID {
|
||||
request.ParentID = &parentID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/folders",
|
||||
}
|
||||
|
||||
var result api.CreateFolderResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// removeFolder deletes a folder
|
||||
//
|
||||
// ?permanent=1 trashes and purges a live folder in one request on today's
|
||||
// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
|
||||
// HTTP 409 "Folder is not empty" instead of purging a folder that still
|
||||
// holds a live file or a live child folder; that 409 is turned into
|
||||
// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
|
||||
// error, and it is never retried (409 is not one of retryErrorCodes). A big
|
||||
// subtree is purged in bounded passes (202, complete=false), so keep
|
||||
// calling until the purge is complete. An older server that ignores
|
||||
// permanent=1 only trashes on the first call (answering permanent=false),
|
||||
// so the loop's second pass sends the plain DELETE that purges it.
|
||||
func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
|
||||
parameters := map[string][]string{"permanent": {"1"}}
|
||||
if ifEmpty {
|
||||
parameters["if_empty"] = []string{"1"}
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/folders/" + folderID,
|
||||
Parameters: parameters,
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
for pass := 0; ; pass++ {
|
||||
var result api.DeleteFolderResponse
|
||||
var statusCode int
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
if resp != nil {
|
||||
statusCode = resp.StatusCode
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if ifEmpty && statusCode == http.StatusConflict {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if pass == 0 {
|
||||
return fmt.Errorf("couldn't remove folder: %w", err)
|
||||
}
|
||||
return fmt.Errorf("couldn't permanently remove folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if result.Permanent && result.Complete {
|
||||
return nil
|
||||
}
|
||||
if pass >= maxFolderPurgePasses {
|
||||
return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
|
||||
const maxFolderPurgePasses = 1000
|
||||
|
||||
// renameFolder renames a folder
|
||||
func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
|
||||
request := api.RenameFolderRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFolder moves a folder to a new parent
|
||||
func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
|
||||
request := api.MoveFolderRequest{Name: newName}
|
||||
if newParentID != "" && newParentID != rootID {
|
||||
request.ParentID = &newParentID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.MoveFolderResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("couldn't move folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return false, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
// Servers that predate the name field ignore it and answer without a
|
||||
// name, leaving the caller to rename separately
|
||||
return newName != "" && result.Name == newName, nil
|
||||
}
|
||||
|
||||
// deleteFile deletes a file
|
||||
//
|
||||
// ?permanent=1 trashes and purges it in one request on today's API. An
|
||||
// older server that ignores the flag only soft-deletes it, answering
|
||||
// permanent=false, so a second DELETE on the now soft-deleted file purges
|
||||
// it from the database and R2, as it always has.
|
||||
func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/files/" + fileID,
|
||||
Parameters: map[string][]string{"permanent": {"1"}},
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.DeleteFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't delete file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
|
||||
// Answered into its own value so a field the second response omits
|
||||
// can't be read off the first.
|
||||
if !result.Permanent {
|
||||
var permanent api.DeleteFileResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't permanently delete file: %w", err)
|
||||
}
|
||||
if !permanent.OK {
|
||||
return fmt.Errorf("API error: %s", permanent.Error)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renameFile renames a file
|
||||
func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
|
||||
request := api.RenameFileRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.RenameFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFile moves a file to a new folder and, if newName is non-empty,
|
||||
// renames it in the same request
|
||||
func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
|
||||
request := api.MoveFileRequest{}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
if newName != "" {
|
||||
request.Name = &newName
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies a file to a folder
|
||||
func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
|
||||
request := api.CopyFileRequest{Name: name}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/copy",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.CopyFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// downloadFile downloads a file by its ID
|
||||
//
|
||||
// The download endpoint answers 302 with a presigned R2 URL in Location, so
|
||||
// this is two calls. Both go through the rest client, which is built on
|
||||
// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
|
||||
// --user-agent, --bind, --dump and the proxy settings apply to downloads as
|
||||
// they do to every other call, and errorHandler and shouldRetry decide the
|
||||
// errors and the retries uniformly.
|
||||
func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
|
||||
// NoRedirect stops the client following the 302 with the API's
|
||||
// Authorization header attached. A 302 is not a 2xx, so Call reports it
|
||||
// as an error (having read and closed the body); the Location header
|
||||
// is what actually matters, and finding it clears the error, as in
|
||||
// backend/linkbox.
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files/" + fileID + "/download",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
NoRedirect: true,
|
||||
}
|
||||
var downloadURL string
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
if resp != nil {
|
||||
if location := resp.Header.Get("Location"); location != "" {
|
||||
downloadURL, err = location, nil
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get download URL: %w", err)
|
||||
}
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("no download URL in redirect response")
|
||||
}
|
||||
|
||||
// Fetch the object from the presigned URL. It carries its own
|
||||
// credentials and R2 refuses a request that also has an Authorization
|
||||
// header, so the client-level bearer token is removed by setting the
|
||||
// header empty: the rest client skips a header with an empty value.
|
||||
opts = rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: downloadURL,
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{"Authorization": ""},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't download file: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// uploadSmallFile uploads a file smaller than 10MB in a single request
|
||||
func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
Body: in,
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
// The body is a stream which can't be rewound, so a low level retry
|
||||
// would resend whatever is left of it - nothing - and the server
|
||||
// would store that as an empty file. Don't retry here; rclone's high
|
||||
// level retries redo the whole transfer instead.
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadPart uploads a single part of a multipart upload
|
||||
func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
|
||||
size := int64(len(part))
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
var result api.UploadPartResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
// A fresh reader per attempt so a retry resends the whole part
|
||||
opts.Body = bytes.NewReader(part)
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// completeUpload finalizes a multipart upload
|
||||
func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/" + sessionID + "/complete",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't complete upload: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// initUpload initializes a new upload session
|
||||
func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
|
||||
request := api.UploadInitRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
FileName: fileName,
|
||||
FileSize: fileSize,
|
||||
MimeType: mimeType,
|
||||
FileID: fileID,
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/init",
|
||||
}
|
||||
|
||||
var result api.UploadInitResponse
|
||||
deadline := time.Now().Add(concurrentUploadWaitMax)
|
||||
for {
|
||||
result = api.UploadInitResponse{}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", err)
|
||||
}
|
||||
fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
|
||||
case <-time.After(concurrentUploadRetryDelay):
|
||||
}
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadParts reads the source part by part and uploads the parts.
|
||||
//
|
||||
// Part 1 goes on its own: the server creates the R2 multipart upload (and a
|
||||
// new file's id and key) when it receives the first part, so parts sent
|
||||
// before that has finished would each start an upload of their own. The
|
||||
// rest go out opt.UploadConcurrency at a time. A part is only read from the
|
||||
// source once a buffer is free, so at most that many parts are held in
|
||||
// memory, and a short source is still found by the reader before the part
|
||||
// it could not fill is sent.
|
||||
func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
|
||||
readPart := func(buf []byte, partNum int) ([]byte, error) {
|
||||
currentPartSize := partSize
|
||||
if partNum == totalParts {
|
||||
currentPartSize = size - partSize*int64(totalParts-1)
|
||||
}
|
||||
part := buf[:currentPartSize]
|
||||
if _, err := io.ReadFull(in, part); err != nil {
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
|
||||
}
|
||||
return part, nil
|
||||
}
|
||||
|
||||
concurrency := max(f.opt.UploadConcurrency, 1)
|
||||
buffers := make(chan []byte, concurrency)
|
||||
buffers <- make([]byte, partSize)
|
||||
allocated := 1
|
||||
|
||||
first := <-buffers
|
||||
part, err := readPart(first, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
buffers <- first
|
||||
|
||||
// A failed read has to take the parts already in flight down with it,
|
||||
// or they upload to completion while the error waits on Wait.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var readErr error
|
||||
for partNum := 2; partNum <= totalParts; partNum++ {
|
||||
// Take a free buffer, allocating one while under the limit.
|
||||
var buf []byte
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
default:
|
||||
if allocated < concurrency {
|
||||
buf = make([]byte, partSize)
|
||||
allocated++
|
||||
} else {
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
case <-gCtx.Done():
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf == nil {
|
||||
break // a part failed; its error comes back from Wait
|
||||
}
|
||||
part, err := readPart(buf, partNum)
|
||||
if err != nil {
|
||||
readErr = err
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
defer func() { buffers <- buf }()
|
||||
_, err := f.uploadPart(gCtx, part, sessionID, partNum)
|
||||
return err
|
||||
})
|
||||
}
|
||||
err = g.Wait()
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// abortUpload retires an upload session the client is giving up on.
|
||||
//
|
||||
// Best effort: the server reclaims an abandoned session itself, so a failure
|
||||
// here - including an older server with no such route - costs the slot the
|
||||
// wait it would have cost anyway and is not worth failing an upload over.
|
||||
// Sent on a context of its own so a cancelled transfer still gives its slot
|
||||
// back rather than leaving it to the sweep.
|
||||
func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
|
||||
defer cancel()
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
NoResponse: true,
|
||||
}
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadFile handles the full upload flow (small or multipart)
|
||||
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
|
||||
// Both refusals live here so a new file and a new version of an existing
|
||||
// one answer the same way: upload/init needs a byte count up front, and
|
||||
// the server stores no zero-length object.
|
||||
if size < 0 {
|
||||
return nil, errors.New("can't upload files with unknown size")
|
||||
}
|
||||
if size == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
leaf := f.opt.Enc.FromStandardName(path.Base(remote))
|
||||
|
||||
initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A session abandoned here holds one of the workspace's upload slots
|
||||
// until the server's sweep retires it ten minutes later, and rclone's
|
||||
// high-level retry opens a fresh session rather than resuming this one -
|
||||
// so without this, a few failed attempts use up every slot.
|
||||
defer func() {
|
||||
if err != nil {
|
||||
f.abortUpload(ctx, initResp.SessionID)
|
||||
}
|
||||
}()
|
||||
|
||||
// Count what the source actually supplies, so a source which ends
|
||||
// early is reported as an error rather than stored as a truncated
|
||||
// file.
|
||||
counter := readers.NewCountingReader(in)
|
||||
|
||||
// Small file upload (single PUT, no resumable)
|
||||
if initResp.Resumable == nil {
|
||||
resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := int64(counter.BytesRead()); got != size {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Multipart upload. Each part is read fully into memory before it is
|
||||
// sent, so a short source is caught before the part goes out and a
|
||||
// part which has to be retried is resent with all of its bytes.
|
||||
partSize := initResp.Resumable.PartSize
|
||||
totalParts := initResp.Resumable.TotalParts
|
||||
// Both bounds matter. partSize*(totalParts-1) >= size means a part would
|
||||
// have nothing to carry; size > partSize*totalParts means the last part
|
||||
// is bigger than partSize, and readPart would slice past its buffer and
|
||||
// panic.
|
||||
if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
|
||||
return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
|
||||
}
|
||||
if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.completeUpload(ctx, initResp.SessionID, modTime)
|
||||
}
|
||||
|
||||
// createShareLink creates a public share link for a file
|
||||
func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
|
||||
request := api.ShareLinkRequest{}
|
||||
if expire < fs.DurationOff {
|
||||
days := int(time.Duration(expire).Hours() / 24)
|
||||
if days < 1 {
|
||||
days = 1
|
||||
}
|
||||
request.ExpiresInDays = &days
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/share",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.ShareLinkResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create share link: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// getWorkspaceInfo fetches workspace storage info
|
||||
// listWorkspaces lists the workspaces this credential can reach. An API key
|
||||
// pinned to a workspace sees exactly one: its own.
|
||||
func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces",
|
||||
}
|
||||
|
||||
var result api.WorkspaceListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list workspaces: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return result.Workspaces, nil
|
||||
}
|
||||
|
||||
func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces/" + f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
var result api.WorkspaceInfoResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get workspace info: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
header string
|
||||
want time.Duration
|
||||
ok bool
|
||||
}{
|
||||
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
|
||||
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
|
||||
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
|
||||
{"missing header", http.StatusServiceUnavailable, "", 0, false},
|
||||
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
|
||||
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
|
||||
if tc.header != "" {
|
||||
resp.Header.Set("Retry-After", tc.header)
|
||||
}
|
||||
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
|
||||
assert.True(t, retry)
|
||||
d, ok := pacer.IsRetryAfter(err)
|
||||
assert.Equal(t, tc.ok, ok)
|
||||
if tc.ok {
|
||||
assert.Equal(t, tc.want, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
|
||||
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
|
||||
resp.Header.Set("Retry-After", "30")
|
||||
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
|
||||
assert.False(t, retry)
|
||||
_, ok := pacer.IsRetryAfter(err)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestRemoveFolderPurgesPermanently(t *testing.T) {
|
||||
// a big subtree purges in bounded passes: 202 complete=false repeats
|
||||
// the same request until complete=true
|
||||
answers := []struct {
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
|
||||
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
|
||||
}
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "DELETE", r.Method)
|
||||
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
|
||||
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
|
||||
a := answers[calls]
|
||||
calls++
|
||||
return jsonResp(a.status, a.body), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
|
||||
assert.Equal(t, len(answers), calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "locked file")
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", true)
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRmdirMakesOneRequest(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
require.NoError(t, f.Rmdir(context.Background(), "sub"))
|
||||
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
|
||||
}
|
||||
|
||||
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
err := f.Rmdir(context.Background(), "sub")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
}
|
||||
|
||||
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
permanent bool
|
||||
wantCalls int
|
||||
}{
|
||||
{"today's API purges in one call", true, 1},
|
||||
{"an older server ignores the flag and still gets a second DELETE", false, 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
|
||||
assert.Equal(t, tc.wantCalls, calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// decodeBody reads a JSON request body into a map
|
||||
func decodeBody(t *testing.T, r *http.Request) map[string]any {
|
||||
b, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
var m map[string]any
|
||||
require.NoError(t, json.Unmarshal(b, &m))
|
||||
return m
|
||||
}
|
||||
|
||||
func TestMoveFolderSendsName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
newName string
|
||||
respBody string
|
||||
wantRenamed bool
|
||||
}{
|
||||
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
|
||||
{"older server ignores name", "dst", `{"ok":true}`, false},
|
||||
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_parent", body["parent_id"])
|
||||
if tc.newName == "" {
|
||||
assert.NotContains(t, body, "name")
|
||||
} else {
|
||||
assert.Equal(t, tc.newName, body["name"])
|
||||
}
|
||||
return jsonResp(http.StatusOK, tc.respBody), nil
|
||||
}))
|
||||
|
||||
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantRenamed, renamed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyFileSendsName(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_dst", body["folder_id"])
|
||||
assert.Equal(t, "final.txt", body["name"])
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
|
||||
}))
|
||||
|
||||
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_2", resp.FileID)
|
||||
assert.Equal(t, "final.txt", resp.Name)
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
|
||||
f := &Fs{}
|
||||
assert.Equal(t, time.Second, f.Precision())
|
||||
assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
|
||||
// Exactly SHA-256, nothing else.
|
||||
assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
|
||||
}
|
||||
|
||||
func TestObjectHashReturnsContentHash(t *testing.T) {
|
||||
o := &Object{file: api.FileItem{ContentHash: "abc123"}}
|
||||
|
||||
got, err := o.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "abc123", got)
|
||||
|
||||
// A multipart object with no server-side hash yields "" and no error,
|
||||
// exactly as an S3 multipart object does.
|
||||
empty := &Object{file: api.FileItem{ContentHash: ""}}
|
||||
got, err = empty.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", got)
|
||||
|
||||
// Any other algorithm is unsupported.
|
||||
_, err = o.Hash(context.Background(), hash.MD5)
|
||||
assert.ErrorIs(t, err, hash.ErrUnsupported)
|
||||
}
|
||||
|
||||
func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
|
||||
assert.True(t, mt.Equal(o.ModTime(context.Background())))
|
||||
}
|
||||
|
||||
func TestModTimeHeaders(t *testing.T) {
|
||||
assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
|
||||
h := modTimeHeaders(mt)
|
||||
require.NotNil(t, h)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
|
||||
}
|
||||
|
||||
func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
|
||||
resp := &api.UploadCompleteResponse{}
|
||||
resp.File.ID = "fil_1"
|
||||
resp.File.Name = "x.txt"
|
||||
resp.File.SizeBytes = 10
|
||||
resp.File.CreatedAt = 1_000
|
||||
resp.File.ContentHash = "deadbeef"
|
||||
|
||||
mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
|
||||
item := fileFromUpload(resp, mt)
|
||||
assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
|
||||
assert.Equal(t, "deadbeef", item.ContentHash)
|
||||
assert.Equal(t, float64(1_000), item.CreatedAt)
|
||||
|
||||
// A zero modtime falls back to the create time.
|
||||
item = fileFromUpload(resp, time.Time{})
|
||||
assert.Equal(t, float64(1_000), item.UpdatedAt)
|
||||
}
|
||||
|
||||
// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
|
||||
// fake transport and asserts the source-mtime header actually rides on the PUT.
|
||||
func TestUploadSendsSourceMTimeHeader(t *testing.T) {
|
||||
var putMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
|
||||
}
|
||||
switch {
|
||||
case r.Method == "POST" && r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
|
||||
putMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "h", resp.File.ContentHash)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
|
||||
// the completion request for a multipart upload (the server stores the modtime
|
||||
// there, not on the individual parts).
|
||||
func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
|
||||
var completeMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body)
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
|
||||
case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
case r.URL.Path == "/api/upload/upl_1/complete":
|
||||
completeMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
|
||||
// server-side Copy returns: it must report the SOURCE's modtime and hash (the
|
||||
// copy is byte-identical and the API preserves both), not the copy's wall
|
||||
// clock. Reading "now" here made rclone's in-memory object disagree with the
|
||||
// one read back from the remote and failed FsCopy's fingerprint check.
|
||||
func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case r.Method == "GET" && r.URL.Path == "/api/files":
|
||||
// NewObject's existence probe on the destination: nothing there.
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
|
||||
case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
|
||||
mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
|
||||
ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
|
||||
ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
|
||||
CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
|
||||
}}
|
||||
|
||||
dst, err := f.Copy(context.Background(), src, "f-copy.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
|
||||
assert.Equal(t, int64(100), dst.Size())
|
||||
got, err := dst.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "srchash", got, "copy keeps the source hash")
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
|
||||
|
||||
func TestErrorHandlerDecodesAPIError(t *testing.T) {
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Status: "400 Bad Request",
|
||||
Body: io.NopCloser(strings.NewReader(concurrentLimitBody)),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
|
||||
assert.Contains(t, err.Error(), "uploads in progress")
|
||||
assert.Contains(t, err.Error(), "concurrent_upload_limit")
|
||||
}
|
||||
|
||||
func TestErrorHandlerNonJSONBody(t *testing.T) {
|
||||
// e.g. an edge (Cloudflare) response that never reached the API
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Status: "429 Too Many Requests",
|
||||
Body: io.NopCloser(strings.NewReader("error code: 1015\n")),
|
||||
}
|
||||
err := errorHandler(resp)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
|
||||
assert.Equal(t, "", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "429")
|
||||
assert.Contains(t, err.Error(), "error code: 1015")
|
||||
}
|
||||
|
||||
func TestShouldRetryAPIErrors(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// The concurrent upload limit is handled by initUpload's slot wait,
|
||||
// not by the pacer's low level retries.
|
||||
limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
|
||||
retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
|
||||
assert.False(t, retry)
|
||||
assert.Equal(t, limitErr, err)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
|
||||
assert.True(t, retry)
|
||||
|
||||
retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
|
||||
assert.False(t, retry)
|
||||
}
|
||||
|
||||
// swapWaitTimings shortens the slot wait timings for a test and returns a
|
||||
// function which restores them
|
||||
func swapWaitTimings(delay, max time.Duration) func() {
|
||||
oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
|
||||
concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
|
||||
return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
|
||||
}
|
||||
|
||||
// newTestFs makes an Fs which talks to srv
|
||||
func newTestFs(srv *httptest.Server) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// limitServer answers the first n init calls with the concurrent upload
|
||||
// limit error and the rest with success
|
||||
func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/upload/init", r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if calls.Add(1) <= n {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(concurrentLimitBody))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||||
}))
|
||||
}
|
||||
|
||||
func TestInitUploadWaitsForFreeSlot(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 2, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, resp.OK)
|
||||
assert.Equal(t, int32(3), calls.Load())
|
||||
}
|
||||
|
||||
func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
|
||||
defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
var apiErr *api.Error
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
|
||||
assert.Contains(t, err.Error(), "couldn't init upload")
|
||||
assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
|
||||
}
|
||||
|
||||
func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
|
||||
defer swapWaitTimings(time.Minute, time.Hour)()
|
||||
var calls atomic.Int32
|
||||
srv := limitServer(t, 1<<30, &calls)
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeListServer serves GET /api/files the way the API does: files paged at
|
||||
// per_page, every folder repeated on each page
|
||||
func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/files", r.URL.Path)
|
||||
q := r.URL.Query()
|
||||
assert.Equal(t, "name", q.Get("sort"))
|
||||
assert.Equal(t, "asc", q.Get("dir"))
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
perPage, _ := strconv.Atoi(q.Get("per_page"))
|
||||
require.Positive(t, page)
|
||||
require.Positive(t, perPage)
|
||||
*pages = append(*pages, page)
|
||||
|
||||
var files []string
|
||||
for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
|
||||
files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
|
||||
}
|
||||
totalPages := max(1, (totalFiles+perPage-1)/perPage)
|
||||
body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
|
||||
`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
|
||||
strings.Join(files, ","), page, perPage, totalFiles, totalPages)
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersFollowsPages(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1, 2, 3}, pages)
|
||||
require.Len(t, listing.Files, 2*listPageSize+7)
|
||||
assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
|
||||
// folders repeat on every page and must not be duplicated
|
||||
assert.Len(t, listing.Folders, 1)
|
||||
}
|
||||
|
||||
func TestListFilesAndFoldersSinglePage(t *testing.T) {
|
||||
for _, total := range []int{0, 1, listPageSize} {
|
||||
t.Run(strconv.Itoa(total), func(t *testing.T) {
|
||||
var pages []int
|
||||
f := newFakeFs(fakeListServer(t, total, &pages))
|
||||
|
||||
listing, err := f.listFilesAndFolders(context.Background(), rootID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, []int{1}, pages)
|
||||
assert.Len(t, listing.Files, total)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// parallelUploadServer answers the multipart endpoints without holding a
|
||||
// lock across a request, so parts really do overlap, and records what the
|
||||
// ordering rules below are about: how many parts were in flight at once, and
|
||||
// whether any part started before part 1 had finished.
|
||||
//
|
||||
// Part 1 matters because the API creates the R2 multipart upload (and, for a
|
||||
// new file, its id and key) on the first part it sees. Parts that race ahead
|
||||
// of part 1 would each create their own upload, and only one of those would
|
||||
// be completed.
|
||||
type parallelUploadServer struct {
|
||||
t *testing.T
|
||||
resumable string
|
||||
partDelay time.Duration
|
||||
failPart int // answer this part with a 400, 0 for none
|
||||
mu sync.Mutex
|
||||
inFlight int
|
||||
maxFlight int
|
||||
part1Done bool
|
||||
earlyParts []int // parts that started before part 1 finished
|
||||
parts map[int]string
|
||||
completes int
|
||||
aborts int
|
||||
}
|
||||
|
||||
func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
|
||||
s.mu.Lock()
|
||||
if n != 1 && !s.part1Done {
|
||||
s.earlyParts = append(s.earlyParts, n)
|
||||
}
|
||||
s.inFlight++
|
||||
s.maxFlight = max(s.maxFlight, s.inFlight)
|
||||
s.mu.Unlock()
|
||||
|
||||
time.Sleep(s.partDelay)
|
||||
|
||||
s.mu.Lock()
|
||||
s.inFlight--
|
||||
if n == 1 {
|
||||
s.part1Done = true
|
||||
}
|
||||
s.parts[n] = string(body)
|
||||
fail := n == s.failPart
|
||||
s.mu.Unlock()
|
||||
|
||||
if fail {
|
||||
return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.mu.Lock()
|
||||
s.completes++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.mu.Lock()
|
||||
s.aborts++
|
||||
s.mu.Unlock()
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
|
||||
return ¶llelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
|
||||
}
|
||||
|
||||
const tenParts = `{"part_size":2,"total_parts":10}`
|
||||
|
||||
func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
content := "aabbccddeeffgghhiijj"
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
|
||||
assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
for i := 1; i <= 10; i++ {
|
||||
assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 1
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, srv.maxFlight)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
|
||||
srv := newParallelServer(t, tenParts)
|
||||
srv.failPart = 6
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "part 6")
|
||||
assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
|
||||
// 20 bytes declared, 15 supplied: the short read is found by the
|
||||
// reader, whichever parts are already in flight, and nothing completes.
|
||||
srv := newParallelServer(t, tenParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
f.opt.UploadConcurrency = 4
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Equal(t, 0, srv.completes)
|
||||
assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Test Dosya filesystem interface
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestDosya:",
|
||||
})
|
||||
}
|
||||
@@ -1,247 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// rtFunc adapts a function into an http.RoundTripper so the upload path
|
||||
// can be driven without a network. Unlike a real server it always reads
|
||||
// the whole request body and answers, which is exactly the misbehaviour
|
||||
// (accepting a short body as a complete upload) these tests guard against.
|
||||
type rtFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func jsonResp(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{"Content-Type": {"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
ContentLength: int64(len(body)),
|
||||
}
|
||||
}
|
||||
|
||||
func newFakeFs(rt http.RoundTripper) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
name: "test",
|
||||
opt: Options{WorkspaceID: "ws_test"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(
|
||||
pacer.MinSleep(time.Millisecond),
|
||||
pacer.MaxSleep(time.Millisecond),
|
||||
pacer.DecayConstant(decayConstant),
|
||||
pacer.AttackConstant(attackConstant),
|
||||
)),
|
||||
}
|
||||
f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
|
||||
return f
|
||||
}
|
||||
|
||||
// fakeUploadServer records what an upload sends and lets a test inject
|
||||
// failures per request
|
||||
type fakeUploadServer struct {
|
||||
t *testing.T
|
||||
mu sync.Mutex
|
||||
resumable string // JSON for the init "resumable" field, "" for a single PUT
|
||||
puts [][]byte // bodies of PUT /api/upload/<id>
|
||||
parts map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
|
||||
completes int
|
||||
aborts int // DELETE /api/upload/<id>
|
||||
failFirst map[string]bool // path -> answer the first request with a 502
|
||||
seenBefore map[string]bool
|
||||
}
|
||||
|
||||
func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
|
||||
return &fakeUploadServer{
|
||||
t: t,
|
||||
resumable: resumable,
|
||||
parts: map[int][][]byte{},
|
||||
failFirst: map[string]bool{},
|
||||
seenBefore: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var body []byte
|
||||
if r.Body != nil {
|
||||
var err error
|
||||
body, err = io.ReadAll(r.Body)
|
||||
require.NoError(s.t, err)
|
||||
}
|
||||
path := r.URL.Path
|
||||
// Bodies are recorded for every attempt, including the ones answered
|
||||
// with a 502, so tests can see exactly what each attempt carried.
|
||||
fail := s.failFirst[path] && !s.seenBefore[path]
|
||||
s.seenBefore[path] = true
|
||||
bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
|
||||
switch {
|
||||
case r.Method == "POST" && path == "/api/upload/init":
|
||||
if s.resumable != "" {
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && path == "/api/upload/upl_1":
|
||||
s.puts = append(s.puts, body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
|
||||
case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
|
||||
var n int
|
||||
_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
|
||||
require.NoError(s.t, err)
|
||||
s.parts[n] = append(s.parts[n], body)
|
||||
if fail {
|
||||
return bad, nil
|
||||
}
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
|
||||
case r.Method == "POST" && path == "/api/upload/upl_1/complete":
|
||||
s.completes++
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
case r.Method == "DELETE" && path == "/api/upload/upl_1":
|
||||
s.aborts++
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
s.t.Fatalf("unexpected request %s %s", r.Method, path)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestUploadSmallSendsWholeBody(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := strings.Repeat("a", 100)
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
require.Len(t, srv.puts, 1)
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
func TestUploadSmallRejectsShortSource(t *testing.T) {
|
||||
// The source declares 100 bytes but only supplies 60. A server which
|
||||
// accepts that as a complete upload must not turn into a success.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
|
||||
}
|
||||
|
||||
func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
|
||||
// A retried PUT would resend an already-consumed reader, i.e. an
|
||||
// empty body, which the server would store as an empty file. The
|
||||
// single PUT therefore must not be retried at the low level.
|
||||
srv := newFakeUploadServer(t, "")
|
||||
srv.failFirst["/api/upload/upl_1"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
content := strings.Repeat("a", 100)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
|
||||
assert.Equal(t, content, string(srv.puts[0]))
|
||||
}
|
||||
|
||||
const threeParts = `{"part_size":4,"total_parts":3}`
|
||||
|
||||
func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
content := "0123456789"
|
||||
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_1", resp.File.ID)
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
require.Len(t, srv.parts, 3)
|
||||
assert.Equal(t, "0123", string(srv.parts[1][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
}
|
||||
|
||||
func TestUploadMultipartRejectsShortSource(t *testing.T) {
|
||||
// 10 bytes declared, 8 supplied: the last part must never be sent
|
||||
// and the upload must not be completed.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
|
||||
assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
|
||||
assert.Equal(t, 0, srv.completes, "upload must not be completed")
|
||||
}
|
||||
|
||||
func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
|
||||
// A part which fails transiently must be resent with all its bytes,
|
||||
// not with whatever is left of a consumed reader.
|
||||
srv := newFakeUploadServer(t, threeParts)
|
||||
srv.failFirst["/api/upload/upl_1/part/2"] = true
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
|
||||
assert.Equal(t, "4567", string(srv.parts[2][0]))
|
||||
assert.Equal(t, "4567", string(srv.parts[2][1]))
|
||||
assert.Equal(t, "89", string(srv.parts[3][0]))
|
||||
assert.Equal(t, 1, srv.completes)
|
||||
}
|
||||
|
||||
// An upload that fails after init has to hand its session back: the session
|
||||
// holds one of the workspace's upload slots for ten minutes otherwise, and
|
||||
// rclone's high-level retry opens a new one rather than resuming it.
|
||||
func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
resumable string
|
||||
content string
|
||||
size int64
|
||||
}{
|
||||
{"small file short source", "", "0123", 10},
|
||||
{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, tc.resumable)
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
|
||||
"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, srv.aborts)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A successful upload must not abort anything
|
||||
func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
|
||||
srv := newFakeUploadServer(t, "")
|
||||
f := newFakeFs(rtFunc(srv.roundTrip))
|
||||
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
|
||||
"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, srv.aborts)
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Every by-id call names the workspace, so the API can find an id it minted
|
||||
// moments ago before its file_home index row lands.
|
||||
func TestByIDCallsSendWorkspaceHint(t *testing.T) {
|
||||
seen := map[string]string{}
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
|
||||
switch r.URL.Path {
|
||||
case "/api/folders/fld_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
case "/api/files/fil_1":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
|
||||
case "/api/folders/fld_1/move":
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
|
||||
default:
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
}
|
||||
}))
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, f.removeFolder(ctx, "fld_1", false))
|
||||
require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
|
||||
_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.deleteFile(ctx, "fil_1"))
|
||||
require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
|
||||
require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
|
||||
_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
|
||||
require.NoError(t, err)
|
||||
_, err = f.createShareLink(ctx, "fil_1", 0)
|
||||
require.NoError(t, err)
|
||||
|
||||
for call, header := range seen {
|
||||
assert.Equal(t, "ws_test", header, call)
|
||||
}
|
||||
assert.Len(t, seen, 8)
|
||||
}
|
||||
|
||||
// The download's API request carries the hint; the presigned R2 fetch must not.
|
||||
func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
|
||||
var apiHint, blobHint string
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/files/fil_1/download" {
|
||||
apiHint = r.Header.Get("X-Dosya-Workspace")
|
||||
http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
|
||||
return
|
||||
}
|
||||
blobHint = r.Header.Get("X-Dosya-Workspace")
|
||||
_, _ = w.Write([]byte("bytes"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
f := newTestFs(srv)
|
||||
f.opt.APIURL = srv.URL
|
||||
|
||||
rc, err := f.downloadFile(context.Background(), "fil_1", nil)
|
||||
require.NoError(t, err)
|
||||
_ = rc.Close()
|
||||
|
||||
assert.Equal(t, "ws_test", apiHint)
|
||||
assert.Equal(t, "", blobHint)
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeWorkspaceServer answers GET /api/workspaces with the given body
|
||||
func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
|
||||
return func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "/api/workspaces", r.URL.Path)
|
||||
*calls++
|
||||
return jsonResp(http.StatusOK, body), nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.NoError(t, f.findWorkspace(context.Background()))
|
||||
assert.Equal(t, "ws_only", f.opt.WorkspaceID)
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceListsTheChoices(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
|
||||
`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
err := f.findWorkspace(context.Background())
|
||||
require.Error(t, err)
|
||||
// the ids have to be in the message: they are the only place the user
|
||||
// can read them, which is the whole point of listing them
|
||||
assert.Contains(t, err.Error(), "ws_one (One)")
|
||||
assert.Contains(t, err.Error(), "ws_two (Two)")
|
||||
assert.Empty(t, f.opt.WorkspaceID)
|
||||
}
|
||||
|
||||
func TestFindWorkspaceWithNone(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
|
||||
f.opt.WorkspaceID = ""
|
||||
|
||||
require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
|
||||
}
|
||||
@@ -21,7 +21,7 @@ type User struct {
|
||||
Avatar string `json:"avatar"`
|
||||
ModelType string `json:"model_type"`
|
||||
OwnsEntry bool `json:"owns_entry"`
|
||||
EntryPermissions any `json:"entry_permissions"`
|
||||
EntryPermissions []any `json:"entry_permissions"`
|
||||
DisplayName string `json:"display_name"`
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ type Item struct {
|
||||
FileHash any `json:"file_hash"`
|
||||
Public bool `json:"public"`
|
||||
Thumbnail bool `json:"thumbnail"`
|
||||
MuxStatus any `json:"mux_status"`
|
||||
ThumbnailURL any `json:"thumbnail_url"`
|
||||
WorkspaceID int `json:"workspace_id"`
|
||||
IsEncrypted int `json:"is_encrypted"`
|
||||
@@ -76,9 +77,9 @@ type Listing struct {
|
||||
Data []Item `json:"data"`
|
||||
From int `json:"from"`
|
||||
LastPage int `json:"last_page"`
|
||||
NextPageURL string `json:"next_page_url"`
|
||||
NextPage int `json:"next_page"`
|
||||
PerPage int `json:"per_page"`
|
||||
PrevPageURL string `json:"prev_page_url"`
|
||||
PrevPage int `json:"prev_page"`
|
||||
To int `json:"to"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
@@ -170,7 +171,7 @@ type MultiPartCreateRequest struct {
|
||||
Mime string `json:"mime"`
|
||||
Size int64 `json:"size"`
|
||||
Extension string `json:"extension"`
|
||||
ParentID json.Number `json:"parentId"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
@@ -221,7 +222,7 @@ type MultiPartEntriesRequest struct {
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
ClientExtension string `json:"clientExtension"`
|
||||
ParentID json.Number `json:"parentId"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
|
||||
+36
-123
@@ -54,7 +54,7 @@ const (
|
||||
rootURL = baseURL + "api/v1"
|
||||
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
|
||||
minChunkSize = fs.SizeSuffix(1024 * 1024 * 5)
|
||||
defaultUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024) // as per https://docs.drime.cloud/uploads-guide
|
||||
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -146,6 +146,14 @@ and these uploads do not fully utilize your bandwidth, then increasing
|
||||
this may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: `Cutoff for switching to chunked upload.
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -270,11 +278,6 @@ var retryErrorCodes = []int{
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
509, // Bandwidth Limit Exceeded
|
||||
520, // Cloudflare: Web server returns an unknown error
|
||||
521, // Cloudflare: Web server is down
|
||||
522, // Cloudflare: Connection timed out
|
||||
523, // Cloudflare: Origin is unreachable
|
||||
524, // Cloudflare: A timeout occurred
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
@@ -524,7 +527,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
if dirID != "" {
|
||||
opts.Parameters.Add("folderId", dirID)
|
||||
opts.Parameters.Add("parentIds", dirID)
|
||||
}
|
||||
if directoriesOnly {
|
||||
opts.Parameters.Add("type", api.ItemTypeFolder)
|
||||
@@ -562,10 +565,10 @@ OUTER:
|
||||
break OUTER
|
||||
}
|
||||
}
|
||||
if result.CurrentPage == result.LastPage {
|
||||
if result.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
page = result.CurrentPage + 1
|
||||
page = result.NextPage
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
@@ -692,27 +695,13 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
|
||||
// deleteObject removes an object by ID
|
||||
func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
if f.opt.HardDelete {
|
||||
// Deleting forever doesn't invalidate the server's cache of
|
||||
// the parent folder listing, so the entry keeps being listed
|
||||
// for about a second. Moving to the trash first does.
|
||||
err := f.deleteEntry(ctx, id, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return f.deleteEntry(ctx, id, f.opt.HardDelete)
|
||||
}
|
||||
|
||||
// deleteEntry moves the entry with ID to the trash or deletes it forever
|
||||
func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) error {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/file-entries/delete",
|
||||
}
|
||||
request := api.DeleteRequest{
|
||||
EntryIDs: []string{id},
|
||||
DeleteForever: deleteForever,
|
||||
DeleteForever: f.opt.HardDelete,
|
||||
}
|
||||
var result api.DeleteResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -720,11 +709,6 @@ func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) err
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
// If Drime returns 422 with invalid entry ID, the object is already gone.
|
||||
// Map this to a standard not-found error
|
||||
if strings.Contains(err.Error(), "entry ids is invalid") {
|
||||
return fs.ErrorObjectNotFound
|
||||
}
|
||||
return fmt.Errorf("failed to delete item: %w", err)
|
||||
}
|
||||
// Check the individual result codes also
|
||||
@@ -798,16 +782,10 @@ func (f *Fs) patch(ctx context.Context, id, attribute string, value string) (ite
|
||||
Name: value,
|
||||
}
|
||||
var result api.UpdateItemResponse
|
||||
// The drime origin returns a malformed response (seen by Cloudflare as
|
||||
// a 520) for a literal PUT to this endpoint, so use a POST with Laravel
|
||||
// method spoofing instead - the API routes it to the same handler.
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Method: "PUT",
|
||||
Path: "/file-entries/" + id,
|
||||
Parameters: url.Values{},
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-HTTP-Method-Override": "PUT",
|
||||
},
|
||||
}
|
||||
if f.opt.WorkspaceID != "" {
|
||||
opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
|
||||
@@ -941,48 +919,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return dstObj, nil
|
||||
}
|
||||
|
||||
// countDirChildren returns the number of direct children of dirID.
|
||||
func (f *Fs) countDirChildren(ctx context.Context, dirID string) (int, error) {
|
||||
n := 0
|
||||
_, err := f.listAll(ctx, dirID, false, false, "", func(*api.Item) bool {
|
||||
n++
|
||||
return false
|
||||
})
|
||||
return n, err
|
||||
}
|
||||
|
||||
// waitForDirChildren polls dirID until it lists at least want direct
|
||||
// children, or the timeout elapses. Drime's folder rename has visible
|
||||
// eventual-consistency on the children listing - the rename returns
|
||||
// success immediately but the renamed folder lists as empty for a
|
||||
// short window - so callers that rename a folder use this to wait for
|
||||
// the listing to settle before returning (see issue #9450).
|
||||
func (f *Fs) waitForDirChildren(ctx context.Context, dirID string, want int) error {
|
||||
const timeout = 30 * time.Second
|
||||
deadline := time.Now().Add(timeout)
|
||||
backoff := 100 * time.Millisecond
|
||||
for {
|
||||
got, err := f.countDirChildren(ctx, dirID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if got >= want {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("timed out waiting for directory listing to settle: got %d, want %d", got, want)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 2*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
// using server-side move operations.
|
||||
//
|
||||
@@ -1003,25 +939,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
return err
|
||||
}
|
||||
|
||||
// Count children before the move so we can wait for the server's
|
||||
// children listing to reflect the rename - see waitForDirChildren.
|
||||
preCount, err := f.countDirChildren(ctx, srcID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DirMove: count source children: %w", err)
|
||||
}
|
||||
|
||||
// Do the move
|
||||
info, err := f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
_, err = f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if preCount > 0 {
|
||||
if err := f.waitForDirChildren(ctx, info.ID.String(), preCount); err != nil {
|
||||
fs.Logf(f, "DirMove: %v (continuing)", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
@@ -1054,28 +976,17 @@ func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err
|
||||
}
|
||||
|
||||
// copy and rename a file or folder to directoryID with leaf
|
||||
//
|
||||
// If existing is set it is removed once the copy has succeeded.
|
||||
func (f *Fs) copyTo(ctx context.Context, srcID, dstLeaf, dstDirectoryID string, existing fs.Object) (info *api.Item, err error) {
|
||||
func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
|
||||
// Can have duplicates so don't have to be careful here
|
||||
|
||||
// Copy to dstDirectoryID first
|
||||
info, err = f.copy(ctx, srcID, dstDirectoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The server names the copy "name (1)" if the destination
|
||||
// directory already has an entry called "name" and refuses to
|
||||
// rename it to "name" until that entry is removed.
|
||||
if existing != nil {
|
||||
fs.Debugf(existing, "Server side copy: removing existing object after successful copy")
|
||||
err = existing.Remove(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Rename if required
|
||||
if info.Name != f.opt.Enc.FromStandardName(dstLeaf) {
|
||||
if srcLeaf != dstLeaf {
|
||||
info, err = f.rename(ctx, info.ID.String(), dstLeaf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1099,6 +1010,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
srcLeaf := path.Base(srcObj.remote)
|
||||
|
||||
srcPath := srcObj.fs.rootSlash() + srcObj.remote
|
||||
dstPath := f.rootSlash() + remote
|
||||
if srcPath == dstPath {
|
||||
@@ -1107,8 +1020,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
|
||||
// Find existing object
|
||||
existingObj, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
existingObj = nil
|
||||
if err == nil {
|
||||
defer func() {
|
||||
// Don't remove existing object if returning an error
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
|
||||
err = existingObj.Remove(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Create temporary object
|
||||
@@ -1118,7 +1038,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
}
|
||||
|
||||
// Copy the object
|
||||
info, err := f.copyTo(ctx, srcObj.id, dstLeaf, dstDirectoryID, existingObj)
|
||||
info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1176,11 +1096,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
// Send just the leaf as relativePath, matching the single-part /uploads
|
||||
// convention. The file is placed by parentId; sending an absolute path
|
||||
// here makes the server build folders from it and drop "0" path segments.
|
||||
relPath := f.opt.Enc.FromStandardName(leaf)
|
||||
|
||||
// Temporary Object under construction
|
||||
o := &Object{
|
||||
fs: f,
|
||||
@@ -1214,7 +1129,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
Size: createSize,
|
||||
Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
|
||||
ParentID: json.Number(directoryID),
|
||||
RelativePath: relPath,
|
||||
RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
@@ -1241,6 +1156,8 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
if ext == "" {
|
||||
ext = "bin"
|
||||
}
|
||||
rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
|
||||
|
||||
chunkWriter := &drimeChunkWriter{
|
||||
uploadID: resp.UploadID,
|
||||
key: resp.Key,
|
||||
@@ -1253,7 +1170,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
mime: mime,
|
||||
extension: ext,
|
||||
parentID: json.Number(directoryID),
|
||||
relativePath: relPath,
|
||||
relativePath: rel,
|
||||
}
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(chunkSize),
|
||||
@@ -1613,11 +1530,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Debugf(o, "Removing old object on successful upload")
|
||||
deleteErr := o.fs.deleteObject(ctx, id)
|
||||
if deleteErr != nil {
|
||||
if errors.Is(deleteErr, fs.ErrorObjectNotFound) {
|
||||
fs.Debugf(o, "Old object already deleted, safely ignoring")
|
||||
} else {
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
}
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
+15
-82
@@ -144,7 +144,6 @@ var (
|
||||
_importFormats map[string][]string // allowed import MIME type conversions
|
||||
templatesOnce sync.Once // parse link templates only once
|
||||
_linkTemplates map[string]*template.Template // available link types
|
||||
gdocsWarnOnce sync.Once // warn once about skipped non-exportable Google documents
|
||||
)
|
||||
|
||||
// rwChoices type for fs.Bits
|
||||
@@ -208,10 +207,7 @@ func driveOAuthOptions() []fs.Option {
|
||||
opts := []fs.Option{}
|
||||
for _, opt := range oauthutil.SharedOptions {
|
||||
if opt.Name == config.ConfigClientID {
|
||||
opt.Help = "Google Application Client Id\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nThe shared client_id is being retired and will stop working during 2026, so creating your own is now strongly recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own."
|
||||
}
|
||||
if opt.Name == config.ConfigClientSecret {
|
||||
opt.Help = "Google Application Client Secret\nLeave blank to use rclone's shared client_id, or if you are using a service account.\nIf you created your own client_id then enter its client secret here."
|
||||
opt.Help = "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance."
|
||||
}
|
||||
opts = append(opts, opt)
|
||||
}
|
||||
@@ -225,7 +221,7 @@ func init() {
|
||||
Description: "Google Drive",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
@@ -233,7 +229,7 @@ func init() {
|
||||
return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
|
||||
}
|
||||
|
||||
switch configIn.State {
|
||||
switch config.State {
|
||||
case "":
|
||||
// Fill in the scopes
|
||||
driveConfig.Scopes = driveScopes(opt.Scope)
|
||||
@@ -244,43 +240,24 @@ func init() {
|
||||
}
|
||||
|
||||
if opt.ServiceAccountFile == "" && opt.ServiceAccountCredentials == "" && !opt.EnvAuth {
|
||||
return fs.ConfigGoto("client_id")
|
||||
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
|
||||
OAuth2Config: driveConfig,
|
||||
})
|
||||
}
|
||||
return fs.ConfigGoto("teamdrive")
|
||||
case "client_id":
|
||||
if clientID, _ := m.Get(config.ConfigClientID); clientID != "" {
|
||||
return fs.ConfigGoto("oauth")
|
||||
}
|
||||
return oauthutil.SharedClientIDConfigConfirm("client_id_warning", "Google Drive", "https://rclone.org/drive/#making-your-own-client-id")
|
||||
case "client_id_warning":
|
||||
if configIn.Result == "true" {
|
||||
// Continue using the shared client_id
|
||||
return fs.ConfigGoto("oauth")
|
||||
}
|
||||
return fs.ConfigInput("client_id_set", config.ConfigClientID, "Google Application Client Id")
|
||||
case "client_id_set":
|
||||
m.Set(config.ConfigClientID, configIn.Result)
|
||||
return fs.ConfigInput("client_secret_set", config.ConfigClientSecret, "Google Application Client Secret")
|
||||
case "client_secret_set":
|
||||
m.Set(config.ConfigClientSecret, configIn.Result)
|
||||
return fs.ConfigGoto("oauth")
|
||||
case "oauth":
|
||||
return oauthutil.ConfigOut("teamdrive", &oauthutil.Options{
|
||||
OAuth2Config: driveConfig,
|
||||
})
|
||||
case "teamdrive":
|
||||
if opt.TeamDriveID == "" {
|
||||
return fs.ConfigConfirm("teamdrive_ok", false, "config_change_team_drive", "Configure this as a Shared Drive (Team Drive)?\n")
|
||||
}
|
||||
return fs.ConfigConfirm("teamdrive_change", false, "config_change_team_drive", fmt.Sprintf("Change current Shared Drive (Team Drive) ID %q?\n", opt.TeamDriveID))
|
||||
case "teamdrive_ok":
|
||||
if configIn.Result == "false" {
|
||||
if config.Result == "false" {
|
||||
m.Set("team_drive", "")
|
||||
return nil, nil
|
||||
}
|
||||
return fs.ConfigGoto("teamdrive_config")
|
||||
case "teamdrive_change":
|
||||
if configIn.Result == "false" {
|
||||
if config.Result == "false" {
|
||||
return nil, nil
|
||||
}
|
||||
return fs.ConfigGoto("teamdrive_config")
|
||||
@@ -301,14 +278,14 @@ func init() {
|
||||
return teamDrive.Id, teamDrive.Name
|
||||
})
|
||||
case "teamdrive_final":
|
||||
driveID := configIn.Result
|
||||
driveID := config.Result
|
||||
m.Set("team_drive", driveID)
|
||||
m.Set("root_folder_id", "")
|
||||
opt.TeamDriveID = driveID
|
||||
opt.RootFolderID = ""
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown state %q", configIn.State)
|
||||
return nil, fmt.Errorf("unknown state %q", config.State)
|
||||
},
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
System: systemMetadataInfo,
|
||||
@@ -1325,7 +1302,6 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
|
||||
return nil, fmt.Errorf("failed to create client from environment: %w", err)
|
||||
}
|
||||
} else {
|
||||
oauthutil.SharedClientIDWarning(name, "Google Drive", "https://rclone.org/drive/#making-your-own-client-id", m)
|
||||
oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create oauth client: %w", err)
|
||||
@@ -1710,9 +1686,6 @@ func (f *Fs) newObjectWithExportInfo(
|
||||
// If item MimeType is in the ExportFormats then it is a google doc
|
||||
if !isDocument {
|
||||
fs.Debugf(remote, "Ignoring unknown document type %q", info.MimeType)
|
||||
gdocsWarnOnce.Do(func() {
|
||||
fs.Logf(remote, "Skipping unexportable google document %q. Use --drive-show-all-gdocs to include them in server side copy and move", info.MimeType)
|
||||
})
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
if extension == "" {
|
||||
@@ -2439,40 +2412,12 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
|
||||
return newItem, nil
|
||||
}
|
||||
|
||||
// shortcutLoop reports whether a folder shortcut at remote pointing to
|
||||
// the directory targetID would create a recursive loop.
|
||||
//
|
||||
// This is the case when targetID is the ID of remote's parent or any
|
||||
// other ancestor directory, as descending into it would list an
|
||||
// ancestor again forever. The ancestor IDs are looked up in the
|
||||
// directory cache which is always populated for the parents of remote
|
||||
// as they are listed before their children.
|
||||
func (f *Fs) shortcutLoop(remote, targetID string) bool {
|
||||
for dir := path.Dir(remote); ; dir = path.Dir(dir) {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
if id, ok := f.dirCache.Get(dir); ok && actualID(id) == targetID {
|
||||
return true
|
||||
}
|
||||
if dir == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// itemToDirEntry converts a drive.File to an fs.DirEntry.
|
||||
// When the drive.File cannot be represented as an fs.DirEntry
|
||||
// (nil, nil) is returned.
|
||||
func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File) (entry fs.DirEntry, err error) {
|
||||
switch {
|
||||
case item.MimeType == driveFolderType:
|
||||
// A folder shortcut pointing at one of its own ancestors
|
||||
// would make rclone recurse forever, so drop it.
|
||||
if isShortcutID(item.Id) && f.shortcutLoop(remote, actualID(item.Id)) {
|
||||
fs.Errorf(remote, "Ignoring folder shortcut as it points to an ancestor directory creating a loop")
|
||||
return nil, nil
|
||||
}
|
||||
// cache the directory ID for later lookups
|
||||
f.dirCache.Put(remote, item.Id)
|
||||
// cache the resource key for later lookups
|
||||
@@ -2766,12 +2711,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
}
|
||||
trashedFiles := false
|
||||
if check {
|
||||
// trashedFiles is only consulted to decide between trashing and hard deleting
|
||||
// below, so when the directory is going to be trashed anyway there is no reason
|
||||
// to enumerate trashed children: let the server filter them out instead of
|
||||
// paging through them. Hard deletes still need to see them (#1040).
|
||||
includeAll := !f.opt.UseTrash || f.opt.TrashedOnly
|
||||
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, includeAll, func(item *drive.File) bool {
|
||||
found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
|
||||
if !item.Trashed {
|
||||
fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
|
||||
return true
|
||||
@@ -3094,18 +3034,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
remote = remote[:len(remote)-len(ext)]
|
||||
}
|
||||
|
||||
// Find the ID of the parent to remove the file from.
|
||||
var srcParentID string
|
||||
if len(srcObj.parents) == 1 {
|
||||
srcParentID = srcObj.parents[0]
|
||||
} else {
|
||||
var err error
|
||||
_, srcParentID, err = srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcParentID = actualID(srcParentID)
|
||||
_, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcParentID = actualID(srcParentID)
|
||||
|
||||
// Temporary Object under construction
|
||||
dstInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
|
||||
|
||||
@@ -8,14 +8,10 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
stdsync "sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -28,114 +24,13 @@ import (
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
service, err := drive.NewService(
|
||||
context.Background(),
|
||||
option.WithHTTPClient(server.Client()),
|
||||
option.WithEndpoint(server.URL+"/"),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return &Fs{
|
||||
svc: service,
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
permissionsMu: new(stdsync.Mutex),
|
||||
permissions: make(map[string]*drive.Permission),
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionConcurrent(t *testing.T) {
|
||||
const requestCount = 2
|
||||
arrived := make(chan string, requestCount)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce stdsync.Once
|
||||
releaseRequests := func() {
|
||||
releaseOnce.Do(func() { close(release) })
|
||||
}
|
||||
defer releaseRequests()
|
||||
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
arrived <- permissionID
|
||||
<-release
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
type permissionResult struct {
|
||||
permission *drive.Permission
|
||||
inherited bool
|
||||
err error
|
||||
}
|
||||
results := make(chan permissionResult, requestCount)
|
||||
permissionIDs := []string{"permission-1", "permission-2"}
|
||||
for _, permissionID := range permissionIDs {
|
||||
go func() {
|
||||
permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
|
||||
results <- permissionResult{permission: permission, inherited: inherited, err: err}
|
||||
}()
|
||||
}
|
||||
|
||||
arrivedIDs := make(map[string]bool, requestCount)
|
||||
timer := time.NewTimer(time.Second)
|
||||
defer timer.Stop()
|
||||
for len(arrivedIDs) < requestCount {
|
||||
select {
|
||||
case permissionID := <-arrived:
|
||||
arrivedIDs[permissionID] = true
|
||||
case <-timer.C:
|
||||
releaseRequests()
|
||||
t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
|
||||
}
|
||||
}
|
||||
releaseRequests()
|
||||
|
||||
assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
|
||||
returnedIDs := make(map[string]bool, requestCount)
|
||||
for range requestCount {
|
||||
result := <-results
|
||||
require.NoError(t, result.err)
|
||||
require.NotNil(t, result.permission)
|
||||
assert.False(t, result.inherited)
|
||||
returnedIDs[result.permission.Id] = true
|
||||
}
|
||||
assert.Equal(t, arrivedIDs, returnedIDs)
|
||||
}
|
||||
|
||||
func TestInternalGetPermissionCacheHit(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests.Add(1)
|
||||
permissionID := path.Base(r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
|
||||
assert.NoError(t, err)
|
||||
}))
|
||||
|
||||
first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, inherited)
|
||||
|
||||
assert.Same(t, first, second)
|
||||
assert.Equal(t, int32(1), requests.Load())
|
||||
}
|
||||
|
||||
func TestDriveScopes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
@@ -250,34 +145,6 @@ func TestInternalFindExportFormat(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestShortcutLoop(t *testing.T) {
|
||||
f := &Fs{}
|
||||
f.dirCache = dircache.New("", "root", f)
|
||||
// root is FolderX with ID "X"
|
||||
f.dirCache.Put("", "X")
|
||||
// a real subfolder
|
||||
f.dirCache.Put("sub", "sub-id")
|
||||
// a nested subfolder reached via a shortcut, composite ID
|
||||
f.dirCache.Put("sub/nested", joinID("nested-id", "shortcut-id"))
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
remote string
|
||||
targetID string
|
||||
want bool
|
||||
}{
|
||||
{"shortcut to the root itself", "loop", "X", true},
|
||||
{"shortcut to the parent", "sub/loop", "sub-id", true},
|
||||
{"shortcut to a grandparent", "sub/nested/loop", "X", true},
|
||||
{"shortcut to a composite ancestor", "sub/nested/loop", "nested-id", true},
|
||||
{"shortcut to an unrelated folder", "sub/loop", "other-id", false},
|
||||
{"shortcut to a sibling", "sub/loop", "sub-id2", false},
|
||||
} {
|
||||
got := f.shortcutLoop(test.remote, test.targetID)
|
||||
assert.Equal(t, test.want, got, test.name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMimeTypesToExtension(t *testing.T) {
|
||||
for mimeType, extension := range _mimeTypeToExtension {
|
||||
extensions, err := mime.ExtensionsByType(mimeType)
|
||||
@@ -814,83 +681,6 @@ func (f *Fs) InternalTestSingleQuoteFolder(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestIntegration/FsMkdir/FsPutFiles/Internal/MoveDuplicateParent
|
||||
//
|
||||
// Check that moving a file removes it from its real parent even when there
|
||||
// are two directories with the same name and the directory cache resolves the
|
||||
// path to the wrong duplicate. Otherwise the file ends up with two parents
|
||||
// which fails on shared drives with a teamDrivesParentLimit error (#9472).
|
||||
func (f *Fs) InternalTestMoveDuplicateParent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if f.Features().Move == nil {
|
||||
t.Skip("Move not supported")
|
||||
}
|
||||
|
||||
rootID, err := f.dirCache.RootID(ctx, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
const dupName = "dup9472"
|
||||
const dstName = "dst9472"
|
||||
const fileName = "michael.txt"
|
||||
|
||||
// Create the first "dup9472" directory - this is the one the directory
|
||||
// cache will resolve the path to.
|
||||
dirA, err := f.createDir(ctx, rootID, dupName, nil)
|
||||
require.NoError(t, err)
|
||||
f.dirCache.Put(dupName, dirA.Id)
|
||||
|
||||
// Create a second directory with the SAME name and parent: a duplicate
|
||||
// the directory cache does not know about.
|
||||
dirB, err := f.createDir(ctx, rootID, dupName, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Destination directory
|
||||
dirDst, err := f.createDir(ctx, rootID, dstName, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_ = f.delete(ctx, dirA.Id, false)
|
||||
_ = f.delete(ctx, dirB.Id, false)
|
||||
_ = f.delete(ctx, dirDst.Id, false)
|
||||
f.dirCache.Flush()
|
||||
}()
|
||||
|
||||
// Upload a file directly into dirB (the duplicate the cache doesn't know).
|
||||
createInfo := &drive.File{
|
||||
Name: fileName,
|
||||
Parents: []string{dirB.Id},
|
||||
}
|
||||
var fileInfo *drive.File
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
fileInfo, err = f.svc.Files.Create(createInfo).
|
||||
Media(strings.NewReader("9472 duplicate parent test")).
|
||||
Fields(f.getFileFields(ctx)).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{dirB.Id}, fileInfo.Parents)
|
||||
|
||||
// Sanity check: the directory cache resolves the path to dirA, not the
|
||||
// dirB the file actually lives in.
|
||||
gotDirID, err := f.dirCache.FindDir(ctx, dupName, false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, dirA.Id, gotDirID, "directory cache should resolve to the wrong duplicate")
|
||||
|
||||
// Build the source object pointing at the file in dirB and move it.
|
||||
srcObj, err := f.newObjectWithInfo(ctx, dupName+"/"+fileName, fileInfo)
|
||||
require.NoError(t, err)
|
||||
|
||||
dstObj, err := f.Move(ctx, srcObj, dstName+"/"+fileName)
|
||||
require.NoError(t, err)
|
||||
|
||||
// The moved file must have exactly one parent: the destination.
|
||||
movedInfo, err := f.getFile(ctx, dstObj.(fs.IDer).ID(), "id,parents")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{dirDst.Id}, movedInfo.Parents)
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
// These tests all depend on each other so run them as nested tests
|
||||
t.Run("DocumentImport", func(t *testing.T) {
|
||||
@@ -911,7 +701,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Query", f.InternalTestQuery)
|
||||
t.Run("AgeQuery", f.InternalTestAgeQuery)
|
||||
t.Run("SingleQuoteFolder", f.InternalTestSingleQuoteFolder)
|
||||
t.Run("MoveDuplicateParent", f.InternalTestMoveDuplicateParent)
|
||||
t.Run("ShouldRetry", f.InternalTestShouldRetry)
|
||||
}
|
||||
|
||||
|
||||
@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
|
||||
|
||||
// getPermission returns permissions for the fileID and permissionID passed in
|
||||
func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
|
||||
f.permissionsMu.Lock()
|
||||
defer f.permissionsMu.Unlock()
|
||||
if useCache {
|
||||
f.permissionsMu.Lock()
|
||||
perm = f.permissions[permissionID]
|
||||
f.permissionsMu.Unlock()
|
||||
if perm != nil {
|
||||
return perm, false, nil
|
||||
}
|
||||
@@ -131,9 +131,7 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
|
||||
cleanPermission(perm)
|
||||
|
||||
// cache the permission
|
||||
f.permissionsMu.Lock()
|
||||
f.permissions[permissionID] = perm
|
||||
f.permissionsMu.Unlock()
|
||||
|
||||
return perm, inherited, err
|
||||
}
|
||||
|
||||
+18
-30
@@ -11,6 +11,7 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -21,7 +22,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/googleapi"
|
||||
@@ -112,13 +112,8 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
|
||||
}
|
||||
|
||||
// Make an http.Request for the range passed in
|
||||
//
|
||||
// The body is wrapped in readers.NoCloser so the transport can't upgrade it
|
||||
// to an io.Closer and close it after each attempt — the chunk buffer is
|
||||
// pool-backed, so an early close would free its pages and a retry would then
|
||||
// read a dead buffer. The upload loop owns the buffer's lifetime.
|
||||
func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
|
||||
req.ContentLength = reqSize
|
||||
totalSize := "*"
|
||||
if rx.ContentLength >= 0 {
|
||||
@@ -173,40 +168,37 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
start := int64(0)
|
||||
var StatusCode int
|
||||
var err error
|
||||
chunkSize := int64(rx.f.opt.ChunkSize)
|
||||
buf := make([]byte, int(rx.f.opt.ChunkSize))
|
||||
for finished := false; !finished; {
|
||||
reqSize := chunkSize
|
||||
var reqSize int64
|
||||
var chunk io.ReadSeeker
|
||||
if rx.ContentLength >= 0 {
|
||||
// If size known use repeatable reader for smoother bwlimit
|
||||
if start >= rx.ContentLength {
|
||||
break
|
||||
}
|
||||
reqSize = min(rx.ContentLength-start, chunkSize)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool so reads are
|
||||
// repeatable for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, rx.Media, reqSize)
|
||||
if rx.ContentLength < 0 {
|
||||
reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
|
||||
chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
|
||||
} else {
|
||||
// If size unknown read into buffer
|
||||
var n int
|
||||
n, err = readers.ReadFill(rx.Media, buf)
|
||||
if err == io.EOF {
|
||||
// Send the last chunk with the correct ContentLength
|
||||
// otherwise Google doesn't know we've finished
|
||||
rx.ContentLength = start + n
|
||||
rx.ContentLength = start + int64(n)
|
||||
finished = true
|
||||
err = nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqSize = n
|
||||
}
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
reqSize = int64(n)
|
||||
chunk = bytes.NewReader(buf[:reqSize])
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
err = rx.f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
|
||||
StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
|
||||
again, err := rx.f.shouldRetry(ctx, err)
|
||||
if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
|
||||
again = false
|
||||
@@ -214,13 +206,9 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
|
||||
}
|
||||
return again, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
|
||||
start += reqSize
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
package drive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
|
||||
// retried successfully. The chunk buffer is pool-backed and implements
|
||||
// io.Closer, so if it reaches the http transport unwrapped the transport
|
||||
// closes it after the failed attempt, returning its pages to the pool, and
|
||||
// the retry then reads a freed buffer.
|
||||
func TestResumableUploadRetry(t *testing.T) {
|
||||
content := bytes.Repeat([]byte("resumable"), 512)
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
if attempts.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
|
||||
client: server.Client(),
|
||||
}
|
||||
f.opt.ChunkSize = fs.SizeSuffix(len(content))
|
||||
rx := &resumableUpload{
|
||||
f: f,
|
||||
remote: "remote",
|
||||
URI: server.URL,
|
||||
Media: bytes.NewReader(content),
|
||||
MediaType: "application/octet-stream",
|
||||
ContentLength: int64(len(content)),
|
||||
}
|
||||
info, err := rx.Upload(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, info)
|
||||
assert.Equal(t, "fake-id", info.Id)
|
||||
assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
|
||||
}
|
||||
@@ -19,7 +19,7 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
|
||||
Entries: items,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
complete, err = f.srv.UploadSessionFinishBatchV2Context(ctx, arg)
|
||||
complete, err = f.srv.UploadSessionFinishBatchV2(arg)
|
||||
if retry, err := shouldRetryExclude(ctx, err); !retry {
|
||||
return retry, err
|
||||
}
|
||||
|
||||
+115
-275
@@ -51,7 +51,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -151,20 +150,14 @@ var (
|
||||
|
||||
// Gets an oauth config with the right scopes
|
||||
func getOauthConfig(m configmap.Mapper) *oauthutil.Config {
|
||||
impersonate, _ := m.Get("impersonate")
|
||||
impersonateAdmin, _ := m.Get("impersonate_admin")
|
||||
|
||||
// If not impersonating, use standard scopes
|
||||
if impersonate == "" && impersonateAdmin == "" {
|
||||
if impersonate, _ := m.Get("impersonate"); impersonate == "" {
|
||||
return dropboxConfig
|
||||
}
|
||||
// Make a copy of the config
|
||||
config := *dropboxConfig
|
||||
// Make a copy of the scopes with extra scopes requires appended
|
||||
config.Scopes = append(config.Scopes, "team_data.member")
|
||||
if impersonate != "" {
|
||||
config.Scopes = append(config.Scopes, "members.read")
|
||||
}
|
||||
config.Scopes = append(config.Scopes, "members.read", "team_data.member")
|
||||
return &config
|
||||
}
|
||||
|
||||
@@ -213,29 +206,6 @@ You will have to use your own App (setting your own client_id and
|
||||
client_secret) to use this option as currently rclone's default set of
|
||||
permissions doesn't include "members.read". This can be added once
|
||||
v1.55 or later is in use everywhere.
|
||||
`,
|
||||
Default: "",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "impersonate_admin",
|
||||
Help: `Team admin ID to use when performing actions as a team administrator.
|
||||
|
||||
This sets the Dropbox-API-Select-Admin header with the given team
|
||||
member ID (for example "dbmid:...").
|
||||
|
||||
This takes a team member ID directly rather than an email address.
|
||||
|
||||
Note that if you want to use impersonate_admin, you should make sure this
|
||||
flag is set when running "rclone config" as this will cause rclone to
|
||||
request the "team_data.member" scope which it won't normally.
|
||||
|
||||
Using the "team_data.member" scope will require a Dropbox Team Admin
|
||||
to approve during the OAuth flow.
|
||||
|
||||
You will have to use your own App (setting your own client_id and
|
||||
client_secret) to use this option as currently rclone's default set of
|
||||
permissions doesn't include "team_data.member".
|
||||
`,
|
||||
Default: "",
|
||||
Advanced: true,
|
||||
@@ -268,36 +238,6 @@ See also --dropbox-root-namespace for an alternative way to work with shared
|
||||
folders.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_shared_folders",
|
||||
Help: `Instructs rclone to skip all shared folders.
|
||||
|
||||
When set, any folder that is a shared folder mount point will be
|
||||
excluded from directory listings, regardless of ownership.
|
||||
This is useful if you prefer to back up shared folders separately
|
||||
using a separate remote configured with the shared folder namespace.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_unowned_folders",
|
||||
Help: `Instructs rclone to skip shared folders not owned by the current user.
|
||||
|
||||
When set, any folder that is a shared folder mount point and not
|
||||
owned by the current user will be excluded from directory listings.
|
||||
This is useful when backing up multiple Dropbox accounts that share
|
||||
common folders, to avoid duplicating the shared data across accounts.
|
||||
|
||||
Note: In Dropbox Business, 'Team Folders' are owned by the Team.
|
||||
For standard team members, these folders evaluate as 'unowned'
|
||||
(editor/viewer access) and will be excluded by this flag. To back up
|
||||
Team Folders, do not use this flag or run the backup using a Team Admin
|
||||
account.
|
||||
|
||||
If --dropbox-skip-shared-folders is also enabled, this flag has no effect.
|
||||
|
||||
This makes an extra API call per shared folder mount point.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "pacer_min_sleep",
|
||||
Default: defaultMinSleep,
|
||||
@@ -362,41 +302,38 @@ will fail to download them.
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
Impersonate string `config:"impersonate"`
|
||||
ImpersonateAdmin string `config:"impersonate_admin"`
|
||||
SharedFiles bool `config:"shared_files"`
|
||||
SharedFolders bool `config:"shared_folders"`
|
||||
SkipSharedFolders bool `config:"skip_shared_folders"`
|
||||
SkipUnownedFolders bool `config:"skip_unowned_folders"`
|
||||
BatchMode string `config:"batch_mode"`
|
||||
BatchSize int `config:"batch_size"`
|
||||
BatchTimeout fs.Duration `config:"batch_timeout"`
|
||||
AsyncBatch bool `config:"async_batch"`
|
||||
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
RootNsid string `config:"root_namespace"`
|
||||
ExportFormats fs.CommaSepList `config:"export_formats"`
|
||||
SkipExports bool `config:"skip_exports"`
|
||||
ShowAllExports bool `config:"show_all_exports"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
Impersonate string `config:"impersonate"`
|
||||
SharedFiles bool `config:"shared_files"`
|
||||
SharedFolders bool `config:"shared_folders"`
|
||||
BatchMode string `config:"batch_mode"`
|
||||
BatchSize int `config:"batch_size"`
|
||||
BatchTimeout fs.Duration `config:"batch_timeout"`
|
||||
AsyncBatch bool `config:"async_batch"`
|
||||
PacerMinSleep fs.Duration `config:"pacer_min_sleep"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
RootNsid string `config:"root_namespace"`
|
||||
ExportFormats fs.CommaSepList `config:"export_formats"`
|
||||
SkipExports bool `config:"skip_exports"`
|
||||
ShowAllExports bool `config:"show_all_exports"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dropbox server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
srv files.ContextClient // the connection to the dropbox server
|
||||
svc files.ContextClient // the connection to the dropbox server (unauthorized)
|
||||
sharing sharing.ContextClient // as above, but for generating sharing links
|
||||
users users.ContextClient // as above, but for accessing user information
|
||||
team team.ContextClient // for the Teams API
|
||||
slashRoot string // root with "/" prefix
|
||||
slashRootSlash string // root with "/" prefix and postfix
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
ns string // The namespace we are using or "" for none
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
ci *fs.ConfigInfo // global config
|
||||
features *fs.Features // optional features
|
||||
srv files.Client // the connection to the dropbox server
|
||||
svc files.Client // the connection to the dropbox server (unauthorized)
|
||||
sharing sharing.Client // as above, but for generating sharing links
|
||||
users users.Client // as above, but for accessing user information
|
||||
team team.Client // for the Teams API
|
||||
slashRoot string // root with "/" prefix, lowercase
|
||||
slashRootSlash string // root with "/" prefix and postfix, lowercase
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
ns string // The namespace we are using or "" for none
|
||||
batcher *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
|
||||
exportExts []exportExtension
|
||||
}
|
||||
@@ -578,7 +515,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
|
||||
// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
|
||||
f.team = team.NewContext(cfg)
|
||||
f.team = team.New(cfg)
|
||||
|
||||
if opt.Impersonate != "" {
|
||||
user := team.UserSelectorArg{
|
||||
@@ -589,7 +526,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
members := []*team.UserSelectorArg{&user}
|
||||
args := team.NewMembersGetInfoArgs(members)
|
||||
|
||||
memberIDs, err := f.team.MembersGetInfoContext(ctx, args)
|
||||
memberIDs, err := f.team.MembersGetInfo(args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
|
||||
}
|
||||
@@ -600,14 +537,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId
|
||||
}
|
||||
|
||||
if opt.ImpersonateAdmin != "" {
|
||||
cfg.AsAdminID = opt.ImpersonateAdmin
|
||||
}
|
||||
|
||||
f.srv = files.NewContext(cfg)
|
||||
f.svc = files.NewContext(ucfg)
|
||||
f.sharing = sharing.NewContext(cfg)
|
||||
f.users = users.NewContext(cfg)
|
||||
f.srv = files.New(cfg)
|
||||
f.svc = files.New(ucfg)
|
||||
f.sharing = sharing.New(cfg)
|
||||
f.users = users.New(cfg)
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: true,
|
||||
ReadMimeType: false,
|
||||
@@ -634,8 +567,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil // our root it empty so we probably want to list shared folders
|
||||
}
|
||||
|
||||
dir := path.Dir(f.root)
|
||||
if dir == "." {
|
||||
dir = f.root
|
||||
}
|
||||
|
||||
// root is not empty so we have find the right shared folder if it exists
|
||||
id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
|
||||
id, err := f.findSharedFolder(ctx, dir)
|
||||
if err != nil {
|
||||
// if we didn't find the specified shared folder we have to bail out here
|
||||
return nil, err
|
||||
@@ -669,7 +607,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// If root starts with / then use the actual root
|
||||
var acc *users.FullAccount
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
acc, err = f.users.GetCurrentAccountContext(ctx)
|
||||
acc, err = f.users.GetCurrentAccount()
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -689,7 +627,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
// See if the root is actually an object
|
||||
if f.root != "" {
|
||||
_, _, err = f.getFileMetadata(ctx, f.slashRoot)
|
||||
_, err = f.getFileMetadata(ctx, f.slashRoot)
|
||||
if err == nil {
|
||||
newRoot := path.Dir(f.root)
|
||||
if newRoot == "." {
|
||||
@@ -724,16 +662,15 @@ func (f *Fs) setRoot(root string) {
|
||||
}
|
||||
|
||||
type getMetadataResult struct {
|
||||
entry files.IsMetadata
|
||||
remoteIsExportPath bool
|
||||
notFound bool
|
||||
err error
|
||||
entry files.IsMetadata
|
||||
notFound bool
|
||||
err error
|
||||
}
|
||||
|
||||
// getMetadata gets the metadata for a file or directory
|
||||
func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
|
||||
res.err = f.pacer.Call(func() (bool, error) {
|
||||
res.entry, res.err = f.srv.GetMetadataContext(ctx, &files.GetMetadataArg{
|
||||
res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
|
||||
Path: f.opt.Enc.FromStandardPath(objPath),
|
||||
})
|
||||
return shouldRetry(ctx, res.err)
|
||||
@@ -776,7 +713,6 @@ func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportE
|
||||
ch <- getMetadataResult{notFound: true}
|
||||
return
|
||||
}
|
||||
res.remoteIsExportPath = true
|
||||
}
|
||||
|
||||
// Return our real result or error
|
||||
@@ -822,7 +758,7 @@ func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-ch
|
||||
}
|
||||
|
||||
// getFileMetadata gets the metadata for a file
|
||||
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, bool, error) {
|
||||
func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
|
||||
var res getMetadataResult
|
||||
|
||||
// Try all possible metadatas
|
||||
@@ -831,7 +767,7 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
|
||||
res = <-ch
|
||||
|
||||
if res.err != nil {
|
||||
return nil, false, res.err
|
||||
return nil, res.err
|
||||
}
|
||||
if !res.notFound {
|
||||
break
|
||||
@@ -839,17 +775,17 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
|
||||
}
|
||||
|
||||
if res.notFound {
|
||||
return nil, false, fs.ErrorObjectNotFound
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
fileInfo, ok := res.entry.(*files.FileMetadata)
|
||||
if !ok {
|
||||
if _, ok = res.entry.(*files.FolderMetadata); ok {
|
||||
return nil, false, fs.ErrorIsDir
|
||||
return nil, fs.ErrorIsDir
|
||||
}
|
||||
return nil, false, fs.ErrorNotAFile
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
return fileInfo, res.remoteIsExportPath, nil
|
||||
return fileInfo, nil
|
||||
}
|
||||
|
||||
// getDirMetadata gets the metadata for a directory
|
||||
@@ -908,7 +844,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Limit: 100,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFoldersContext(ctx, &arg)
|
||||
res, err = f.sharing.ListFolders(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -920,7 +856,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListFoldersContinueContext(ctx, &arg)
|
||||
res, err = f.sharing.ListFoldersContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -943,21 +879,13 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// sharedFolderName returns the shared folder name in root, which is its first
|
||||
// path component. root must be the trimmed root as produced by setRoot (no
|
||||
// leading slash).
|
||||
func sharedFolderName(root string) string {
|
||||
name, _, _ := strings.Cut(root, "/")
|
||||
return name
|
||||
}
|
||||
|
||||
// findSharedFolder find the id for a given shared folder name
|
||||
// somewhat annoyingly there is no endpoint to query a shared folder by it's name
|
||||
// so our only option is to iterate over all shared folders
|
||||
func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
|
||||
errFoundFile := errors.New("found file")
|
||||
err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
|
||||
if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
|
||||
if entry.(*fs.Dir).Remote() == name {
|
||||
id = entry.(*fs.Dir).ID()
|
||||
return errFoundFile
|
||||
}
|
||||
@@ -977,7 +905,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
|
||||
SharedFolderId: id,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.sharing.MountFolderContext(ctx, &arg)
|
||||
_, err := f.sharing.MountFolder(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -994,7 +922,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Limit: 100,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFilesContext(ctx, &arg)
|
||||
res, err = f.sharing.ListReceivedFiles(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1006,7 +934,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.sharing.ListReceivedFilesContinueContext(ctx, &arg)
|
||||
res, err = f.sharing.ListReceivedFilesContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1014,12 +942,13 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
}
|
||||
}
|
||||
for _, entry := range res.Entries {
|
||||
entryPath := f.opt.Enc.ToStandardName(entry.Name)
|
||||
fmt.Printf("%+v\n", entry)
|
||||
entryPath := entry.Name
|
||||
o := &Object{
|
||||
fs: f,
|
||||
url: entry.PreviewUrl,
|
||||
remote: entryPath,
|
||||
modTime: time.Time(*entry.TimeInvited),
|
||||
modTime: *entry.TimeInvited,
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1039,7 +968,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
|
||||
func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
|
||||
errFoundFile := errors.New("found file")
|
||||
err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
|
||||
if strings.EqualFold(entry.(*Object).remote, name) {
|
||||
if entry.(*Object).remote == name {
|
||||
o = entry.(*Object)
|
||||
return errFoundFile
|
||||
}
|
||||
@@ -1113,7 +1042,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
arg.Path = "" // Specify root folder as empty string
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContext(ctx, arg)
|
||||
res, err = f.srv.ListFolder(arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1131,7 +1060,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
Cursor: res.Cursor,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
res, err = f.srv.ListFolderContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1159,26 +1088,6 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
|
||||
remote := path.Join(dir, leaf)
|
||||
if folderInfo != nil {
|
||||
if folderInfo.SharingInfo != nil && folderInfo.SharingInfo.SharedFolderId != "" {
|
||||
if f.opt.SkipSharedFolders {
|
||||
fs.Debugf(remote, "Skipping shared folder")
|
||||
continue
|
||||
}
|
||||
if f.opt.SkipUnownedFolders {
|
||||
var sfMeta *sharing.SharedFolderMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var apiErr error
|
||||
sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
|
||||
return shouldRetry(ctx, apiErr)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Errorf(remote, "Failed to get shared folder metadata (defaulting to include): %v", err)
|
||||
} else if sfMeta != nil && sfMeta.AccessType != nil && sfMeta.AccessType.Tag != sharing.AccessLevelOwner {
|
||||
fs.Debugf(remote, "Skipping unowned shared folder")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id)
|
||||
err = list.Add(d)
|
||||
if err != nil {
|
||||
@@ -1255,7 +1164,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return cErr
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.CreateFolderV2Context(ctx, &arg2)
|
||||
_, err = f.srv.CreateFolderV2(&arg2)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -1273,41 +1182,34 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
|
||||
encRoot := f.opt.Enc.FromStandardPath(root)
|
||||
|
||||
if check {
|
||||
// ListFolder reports a missing path and a path that is a file, so it
|
||||
// checks that the directory exists and is empty in a single request.
|
||||
// check directory exists
|
||||
_, err = f.getDirMetadata(ctx, root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Rmdir: %w", err)
|
||||
}
|
||||
|
||||
// check directory empty
|
||||
arg := files.NewListFolderArg(encRoot)
|
||||
arg.Recursive = false
|
||||
arg.Limit = 1
|
||||
if root == "/" {
|
||||
arg.Path = "" // Specify root folder as empty string
|
||||
}
|
||||
var res *files.ListFolderResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
res, err = f.srv.ListFolderContext(ctx, arg)
|
||||
res, err = f.srv.ListFolder(arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
switch e := err.(type) {
|
||||
case files.ListFolderAPIError:
|
||||
if e.EndpointError != nil && e.EndpointError.Path != nil {
|
||||
switch e.EndpointError.Path.Tag {
|
||||
case files.LookupErrorNotFound:
|
||||
err = fs.ErrorDirNotFound
|
||||
case files.LookupErrorNotFolder:
|
||||
err = fs.ErrorIsFile
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Rmdir: %w", err)
|
||||
}
|
||||
if len(res.Entries) != 0 || res.HasMore {
|
||||
if len(res.Entries) != 0 {
|
||||
return errors.New("directory not empty")
|
||||
}
|
||||
}
|
||||
|
||||
// remove it
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.DeleteV2Context(ctx, &files.DeleteArg{Path: encRoot})
|
||||
_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
return err
|
||||
@@ -1366,7 +1268,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
|
||||
}
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.CopyV2Context(ctx, &arg)
|
||||
result, err = f.srv.CopyV2(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1427,7 +1329,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var err error
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.MoveV2Context(ctx, &arg)
|
||||
result, err = f.srv.MoveV2(&arg)
|
||||
switch e := err.(type) {
|
||||
case files.MoveV2APIError:
|
||||
// There seems to be a bit of eventual consistency here which causes this to
|
||||
@@ -1475,13 +1377,13 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
},
|
||||
}
|
||||
if expire < fs.DurationOff {
|
||||
expiryTime := dropbox.DBXTime(time.Now().Add(time.Duration(expire)).UTC().Round(time.Second))
|
||||
expiryTime := time.Now().Add(time.Duration(expire)).UTC().Round(time.Second)
|
||||
createArg.Settings.Expires = &expiryTime
|
||||
}
|
||||
|
||||
var linkRes sharing.IsSharedLinkMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -1490,7 +1392,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
fs.Debugf(absPath, "can't create link with expiry, trying without")
|
||||
createArg.Settings.Expires = nil
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
|
||||
linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
@@ -1504,7 +1406,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
}
|
||||
var listRes *sharing.ListSharedLinksResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
listRes, err = f.sharing.ListSharedLinksContext(ctx, &listArg)
|
||||
listRes, err = f.sharing.ListSharedLinks(&listArg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1565,7 +1467,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
},
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.srv.MoveV2Context(ctx, &arg)
|
||||
_, err = f.srv.MoveV2(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1579,7 +1481,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
var q *users.SpaceUsage
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
q, err = f.users.GetSpaceUsageContext(ctx)
|
||||
q, err = f.users.GetSpaceUsage()
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1666,7 +1568,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
|
||||
arg.Path = ""
|
||||
}
|
||||
|
||||
startCursor, err = f.srv.ListFolderGetLatestCursorContext(ctx, arg)
|
||||
startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
|
||||
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -1676,34 +1578,6 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
|
||||
return startCursor.Cursor, nil
|
||||
}
|
||||
|
||||
// trimPrefixFold returns s with the leading prefix removed, matching
|
||||
// case insensitively rune by rune so folded runes of differing UTF-8
|
||||
// length still match.
|
||||
//
|
||||
// If s is exactly the root that prefix names (prefix without its
|
||||
// trailing "/") it returns "". If prefix doesn't match it returns s
|
||||
// unchanged.
|
||||
func trimPrefixFold(s, prefix string) string {
|
||||
if strings.HasSuffix(prefix, "/") && strings.EqualFold(s, strings.TrimSuffix(prefix, "/")) {
|
||||
return ""
|
||||
}
|
||||
|
||||
offset := 0
|
||||
for prefix != "" {
|
||||
if offset >= len(s) {
|
||||
return s
|
||||
}
|
||||
sRune, sSize := utf8.DecodeRuneInString(s[offset:])
|
||||
prefixRune, prefixSize := utf8.DecodeRuneInString(prefix)
|
||||
if !strings.EqualFold(string(sRune), string(prefixRune)) {
|
||||
return s
|
||||
}
|
||||
offset += sSize
|
||||
prefix = prefix[prefixSize:]
|
||||
}
|
||||
return s[offset:]
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
|
||||
cursor := startCursor
|
||||
var res *files.ListFolderLongpollResult
|
||||
@@ -1727,7 +1601,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
Timeout: timeout,
|
||||
}
|
||||
|
||||
res, err = f.svc.ListFolderLongpollContext(ctx, &args)
|
||||
res, err = f.svc.ListFolderLongpoll(&args)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1750,7 +1624,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
Cursor: cursor,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
changeList, err = f.srv.ListFolderContinueContext(ctx, &arg)
|
||||
changeList, err = f.srv.ListFolderContinue(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1763,18 +1637,17 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
switch info := entry.(type) {
|
||||
case *files.FolderMetadata:
|
||||
entryType = fs.EntryDirectory
|
||||
entryPath = info.PathDisplay
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
case *files.FileMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = info.PathDisplay
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
case *files.DeletedMetadata:
|
||||
entryType = fs.EntryObject
|
||||
entryPath = info.PathDisplay
|
||||
entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
|
||||
default:
|
||||
fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
|
||||
continue
|
||||
}
|
||||
entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
|
||||
|
||||
if entryPath != "" {
|
||||
notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
|
||||
@@ -1886,7 +1759,7 @@ func (o *Object) Size() int64 {
|
||||
return o.bytes
|
||||
}
|
||||
|
||||
func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPath bool) {
|
||||
func (o *Object) setMetadataForExport(info *files.FileMetadata) {
|
||||
o.bytes = -1
|
||||
o.hash = ""
|
||||
|
||||
@@ -1907,10 +1780,8 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
|
||||
o.exportType = exportExportable
|
||||
// get rid of any paper extension, if present
|
||||
o.remote = strings.TrimSuffix(o.remote, paperExtension)
|
||||
if !remoteIsExportPath {
|
||||
// add the export extension
|
||||
o.remote += "." + string(exportExt)
|
||||
}
|
||||
// add the export extension
|
||||
o.remote += "." + string(exportExt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1918,23 +1789,19 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPa
|
||||
//
|
||||
// This isn't a complete set of metadata and has an inaccurate date
|
||||
func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
|
||||
return o.setMetadataFromEntryWithExportPath(info, false)
|
||||
}
|
||||
|
||||
func (o *Object) setMetadataFromEntryWithExportPath(info *files.FileMetadata, remoteIsExportPath bool) error {
|
||||
o.id = info.Id
|
||||
o.bytes = int64(info.Size)
|
||||
o.modTime = time.Time(info.ClientModified)
|
||||
o.modTime = info.ClientModified
|
||||
o.hash = info.ContentHash
|
||||
|
||||
if !info.IsDownloadable {
|
||||
o.setMetadataForExport(info, remoteIsExportPath)
|
||||
o.setMetadataForExport(info)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reads the entry for a file from dropbox
|
||||
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, bool, error) {
|
||||
func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
|
||||
return o.fs.getFileMetadata(ctx, o.remotePath())
|
||||
}
|
||||
|
||||
@@ -1944,11 +1811,11 @@ func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
|
||||
if !o.modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
entry, remoteIsExportPath, err := o.readEntry(ctx)
|
||||
entry, err := o.readEntry(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return o.setMetadataFromEntryWithExportPath(entry, remoteIsExportPath)
|
||||
return o.setMetadataFromEntry(entry)
|
||||
}
|
||||
|
||||
// Returns the remote path for the object
|
||||
@@ -2002,7 +1869,7 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
|
||||
arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
|
||||
var exportResult *files.ExportResult
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
exportResult, in, err = o.fs.srv.ExportContext(ctx, &arg)
|
||||
exportResult, in, err = o.fs.srv.Export(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2024,7 +1891,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
Url: o.url,
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.sharing.GetSharedLinkFileContext(ctx, &arg)
|
||||
_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2044,7 +1911,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
ExtraHeaders: headers,
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, in, err = o.fs.srv.DownloadContext(ctx, &arg)
|
||||
_, in, err = o.fs.srv.Download(&arg)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -2068,7 +1935,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
// start upload
|
||||
var res *files.UploadSessionStartResult
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
res, err = o.fs.srv.UploadSessionStartContext(ctx, &files.UploadSessionStartArg{}, nil)
|
||||
res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, nil)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -2083,16 +1950,12 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
|
||||
// write chunks
|
||||
in := readers.NewCountingReader(in0)
|
||||
buf := make([]byte, int(chunkSize))
|
||||
cursor := files.UploadSessionCursor{
|
||||
SessionId: res.SessionId,
|
||||
Offset: 0,
|
||||
}
|
||||
appendArg := files.UploadSessionAppendArg{
|
||||
Cursor: &cursor,
|
||||
// A known-size upload which fits in a single chunk can close the
|
||||
// session with its only append, saving an empty append request
|
||||
Close: size >= 0 && size <= chunkSize,
|
||||
}
|
||||
appendArg := files.UploadSessionAppendArg{Cursor: &cursor}
|
||||
for currentChunk := 1; ; currentChunk++ {
|
||||
cursor.Offset = in.BytesRead()
|
||||
|
||||
@@ -2102,21 +1965,14 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
|
||||
}
|
||||
|
||||
// Buffer the chunk in memory from the global pool for retries
|
||||
rw := multipart.NewRW()
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return nil, err
|
||||
}
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
|
||||
skip := int64(0)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// seek to the start in case this is a retry
|
||||
if _, err = rw.Seek(skip, io.SeekStart); err != nil {
|
||||
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
|
||||
err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
|
||||
// after session is started, we retry everything
|
||||
if err != nil {
|
||||
// Check for incorrect offset error and retry with new offset
|
||||
@@ -2128,10 +1984,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
|
||||
if skip < 0 {
|
||||
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
|
||||
} else if skip == n {
|
||||
} else if skip == chunkSize {
|
||||
fs.Debugf(o, "%s: chunk received OK - continuing", what)
|
||||
return false, nil
|
||||
} else if skip > n {
|
||||
} else if skip > chunkSize {
|
||||
// This error should never happen
|
||||
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
|
||||
}
|
||||
@@ -2141,27 +1997,11 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
}
|
||||
}
|
||||
// Don't waste the low level retries if the context has
|
||||
// been cancelled
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
return err != nil, err
|
||||
})
|
||||
closeErr := rw.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
if size >= 0 {
|
||||
// Check for sources which truncate early
|
||||
expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
|
||||
if in.BytesRead() < expected {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
}
|
||||
if appendArg.Close {
|
||||
break
|
||||
}
|
||||
@@ -2191,7 +2031,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
entry, err = o.fs.srv.UploadSessionFinishContext(ctx, args, nil)
|
||||
entry, err = o.fs.srv.UploadSessionFinish(args, nil)
|
||||
if retry, err := shouldRetryExclude(ctx, err); !retry {
|
||||
return retry, err
|
||||
}
|
||||
@@ -2245,7 +2085,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
commitInfo := files.NewCommitInfo(o.fs.opt.Enc.FromStandardPath(o.remotePath()))
|
||||
commitInfo.Mode.Tag = "overwrite"
|
||||
// The Dropbox API only accepts timestamps in UTC with second precision.
|
||||
clientModified := dropbox.DBXTime(src.ModTime(ctx).UTC().Round(time.Second))
|
||||
clientModified := src.ModTime(ctx).UTC().Round(time.Second)
|
||||
commitInfo.ClientModified = &clientModified
|
||||
// Don't attempt to create filenames that are too long
|
||||
if cErr := checkPathLength(commitInfo.Path); cErr != nil {
|
||||
@@ -2259,7 +2099,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
|
||||
} else {
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
entry, err = o.fs.srv.UploadContext(ctx, &files.UploadArg{CommitInfo: *commitInfo}, in)
|
||||
entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
@@ -2271,7 +2111,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// This will only happen if we are uploading async batches
|
||||
if entry == nil {
|
||||
o.bytes = size
|
||||
o.modTime = time.Time(*commitInfo.ClientModified)
|
||||
o.modTime = *commitInfo.ClientModified
|
||||
o.hash = "" // we don't have this
|
||||
return nil
|
||||
}
|
||||
@@ -2284,7 +2124,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
|
||||
return errNotSupportedInSharedMode
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = o.fs.srv.DeleteV2Context(ctx, &files.DeleteArg{
|
||||
_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
|
||||
Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
|
||||
})
|
||||
return shouldRetry(ctx, err)
|
||||
|
||||
@@ -2,155 +2,17 @@ package dropbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/batcher"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type paperMetadataClient struct {
|
||||
files.ContextClient
|
||||
info *files.FileMetadata
|
||||
}
|
||||
|
||||
func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
|
||||
if arg.Path == "document" {
|
||||
return c.info, nil
|
||||
}
|
||||
return nil, files.GetMetadataAPIError{
|
||||
APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
|
||||
EndpointError: &files.GetMetadataError{
|
||||
Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
|
||||
Path: &files.LookupError{
|
||||
Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalGetMetadataCancellation(t *testing.T) {
|
||||
requestStarted := make(chan struct{})
|
||||
releaseRequest := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(requestStarted)
|
||||
<-releaseRequest
|
||||
}))
|
||||
defer server.Close()
|
||||
defer close(releaseRequest)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
f := &Fs{
|
||||
srv: files.NewContext(dropbox.Config{
|
||||
Client: server.Client(),
|
||||
URLGenerator: func(hostType string, namespace string, route string) string {
|
||||
return server.URL
|
||||
},
|
||||
}),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
|
||||
}
|
||||
|
||||
result := make(chan getMetadataResult, 1)
|
||||
go func() {
|
||||
result <- f.getMetadata(ctx, "/file")
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-requestStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dropbox request did not start")
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case res := <-result:
|
||||
require.ErrorIs(t, res.err, context.Canceled)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Dropbox request did not observe cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
type changeNotifyClient struct {
|
||||
files.ContextClient
|
||||
entries []files.IsMetadata
|
||||
}
|
||||
|
||||
func (c changeNotifyClient) ListFolderLongpollContext(context.Context, *files.ListFolderLongpollArg) (*files.ListFolderLongpollResult, error) {
|
||||
return files.NewListFolderLongpollResult(true), nil
|
||||
}
|
||||
|
||||
func (c changeNotifyClient) ListFolderContinueContext(context.Context, *files.ListFolderContinueArg) (*files.ListFolderResult, error) {
|
||||
return files.NewListFolderResult(c.entries, "next", false), nil
|
||||
}
|
||||
|
||||
func TestTrimPrefixFold(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
path string
|
||||
prefix string
|
||||
want string
|
||||
}{
|
||||
{name: "exact casing", path: "/Docs/Sub", prefix: "/Docs/", want: "Sub"},
|
||||
{name: "different casing", path: "/docs/Sub", prefix: "/Docs/", want: "Sub"},
|
||||
{name: "different UTF-8 lengths", path: "/K/Sub", prefix: "/K/", want: "Sub"},
|
||||
{name: "exact root", path: "/docs", prefix: "/Docs/", want: ""},
|
||||
{name: "sibling boundary", path: "/Docs2/File", prefix: "/Docs/", want: "/Docs2/File"},
|
||||
{name: "root remote", path: "/File", prefix: "/", want: "File"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, trimPrefixFold(test.path, test.prefix))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangeNotifyTrimsRootCaseInsensitively(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := changeNotifyClient{entries: []files.IsMetadata{
|
||||
&files.FolderMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub"}},
|
||||
&files.FileMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub/File.TXT"}},
|
||||
&files.DeletedMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Gone.md"}},
|
||||
}}
|
||||
f := &Fs{
|
||||
ci: fs.GetConfig(ctx),
|
||||
srv: client,
|
||||
svc: client,
|
||||
slashRootSlash: "/Docs/",
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
|
||||
}
|
||||
|
||||
type notification struct {
|
||||
path string
|
||||
entryType fs.EntryType
|
||||
}
|
||||
var notifications []notification
|
||||
cursor, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
|
||||
notifications = append(notifications, notification{path: path, entryType: entryType})
|
||||
}, "start")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "next", cursor)
|
||||
assert.Equal(t, []notification{
|
||||
{path: "Sub", entryType: fs.EntryDirectory},
|
||||
{path: "Sub/File.TXT", entryType: fs.EntryObject},
|
||||
{path: "Gone.md", entryType: fs.EntryObject},
|
||||
}, notifications)
|
||||
}
|
||||
|
||||
func TestInternalCheckPathLength(t *testing.T) {
|
||||
rep := func(n int, r rune) (out string) {
|
||||
rs := make([]rune, n)
|
||||
@@ -188,170 +50,6 @@ func TestInternalCheckPathLength(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInternalSharedFolderName(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
root string
|
||||
want string
|
||||
}{
|
||||
{root: "", want: ""},
|
||||
{root: "SharedFolder", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
|
||||
{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
|
||||
} {
|
||||
assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPaperExportRemote(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
info := &files.FileMetadata{
|
||||
ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
|
||||
}
|
||||
f := &Fs{
|
||||
exportExts: []exportExtension{"md"},
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault()),
|
||||
srv: paperMetadataClient{info: info},
|
||||
}
|
||||
|
||||
direct, err := f.NewObject(ctx, "document.md")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md", direct.Remote())
|
||||
|
||||
listed, err := f.newObjectWithInfo(ctx, "document.md", info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md.md", listed.Remote())
|
||||
|
||||
legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "document.md", legacy.Remote())
|
||||
}
|
||||
|
||||
// uploadSessionClient is a mock files.ContextClient which records the
|
||||
// chunked upload calls made to it
|
||||
type uploadSessionClient struct {
|
||||
files.ContextClient
|
||||
appends int // number of UploadSessionAppendV2Context calls
|
||||
maxAppends int // fail the append after this many calls to stop runaway loops
|
||||
bytesWritten int64 // bytes received by UploadSessionAppendV2Context
|
||||
finishCalled bool // set if UploadSessionFinishContext was called
|
||||
appended func() // if set, called after each successful append
|
||||
}
|
||||
|
||||
var errTooManyAppends = errors.New("too many appends - upload looping?")
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
|
||||
return &files.UploadSessionStartResult{SessionId: "session"}, nil
|
||||
}
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
|
||||
// the real client fails the request if the context is cancelled
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
c.appends++
|
||||
if c.appends > c.maxAppends {
|
||||
return errTooManyAppends
|
||||
}
|
||||
n, err := io.Copy(io.Discard, content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.bytesWritten += n
|
||||
if c.appended != nil {
|
||||
c.appended()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
|
||||
c.finishCalled = true
|
||||
return &files.FileMetadata{}, nil
|
||||
}
|
||||
|
||||
// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
|
||||
func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
|
||||
ctx := context.Background()
|
||||
f := &Fs{
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
|
||||
srv: srv,
|
||||
}
|
||||
f.opt.ChunkSize = chunkSize
|
||||
batcherOptions := defaultBatcherOptions
|
||||
batcherOptions.Mode = "off"
|
||||
var err error
|
||||
f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
|
||||
// endlessReader supplies bytes forever
|
||||
type endlessReader struct{}
|
||||
|
||||
func (endlessReader) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = 'x'
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestUploadChunkedEarlyEOF(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("MultiChunk", func(t *testing.T) {
|
||||
// The declared size spans 4 chunks but the source ends after 1.5
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
||||
})
|
||||
|
||||
t.Run("SingleChunk", func(t *testing.T) {
|
||||
// The declared size fits in one chunk but the source ends early
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 500)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
assert.False(t, client.finishCalled, "must not commit a truncated upload")
|
||||
})
|
||||
|
||||
t.Run("Complete", func(t *testing.T) {
|
||||
// A source which supplies exactly the declared size uploads OK
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.True(t, client.finishCalled)
|
||||
assert.Equal(t, int64(250), client.bytesWritten)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUploadChunkedCancel(t *testing.T) {
|
||||
// Cancelling the context must stop the upload even though every
|
||||
// append is succeeding
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
client := &uploadSessionClient{maxAppends: 8}
|
||||
client.appended = func() {
|
||||
if client.appends == 2 {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
f := newUploadTestFs(t, client, 100)
|
||||
o := &Object{fs: f, remote: "test.bin"}
|
||||
_, err := o.uploadChunked(ctx, endlessReader{}, files.NewCommitInfo("/test.bin"), -1)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
assert.False(t, client.finishCalled)
|
||||
}
|
||||
|
||||
func (f *Fs) importPaperForTest(t *testing.T) {
|
||||
content := `# test doc
|
||||
|
||||
@@ -366,7 +64,7 @@ Lorem ipsum __dolor__ sit amet
|
||||
var err error
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
reader := strings.NewReader(content)
|
||||
_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
|
||||
_, err = f.srv.PaperCreate(&arg, reader)
|
||||
return shouldRetry(context.Background(), err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -402,136 +100,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// newSharingTestFs builds a minimal *Fs whose sharing client talks to a
|
||||
// local mock server instead of the real Dropbox API.
|
||||
func newSharingTestFs(t *testing.T, handler http.HandlerFunc) *Fs {
|
||||
t.Helper()
|
||||
ts := httptest.NewServer(handler)
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
cfg := dropbox.Config{
|
||||
Client: &http.Client{Transport: http.DefaultTransport},
|
||||
URLGenerator: func(hostType, namespace, route string) string {
|
||||
return fmt.Sprintf("%s/2/%s/%s", ts.URL, namespace, route)
|
||||
},
|
||||
}
|
||||
|
||||
return &Fs{
|
||||
opt: Options{
|
||||
Enc: encoder.Base |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeDel |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8,
|
||||
},
|
||||
sharing: sharing.NewContext(cfg),
|
||||
pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond))),
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseDBXTime(t *testing.T, s string) *dropbox.DBXTime {
|
||||
t.Helper()
|
||||
tm, err := time.Parse(time.RFC3339, s)
|
||||
require.NoError(t, err)
|
||||
return (*dropbox.DBXTime)(&tm)
|
||||
}
|
||||
|
||||
func receivedFilesHandler(t *testing.T, rawName string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.URL.Path, "list_received_files") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
resp := sharing.ListFilesResult{
|
||||
Entries: []*sharing.SharedFileMetadata{{
|
||||
Id: "id:123",
|
||||
Name: rawName,
|
||||
PreviewUrl: "https://dropbox.com/preview/123",
|
||||
Policy: &sharing.FolderPolicy{},
|
||||
TimeInvited: mustParseDBXTime(t, "2024-01-01T00:00:00Z"),
|
||||
}},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListReceivedFilesDecodesName confirms received-file names are decoded
|
||||
// via Enc.ToStandardName before being returned, matching listSharedFolders'
|
||||
// handling of shared-folder names.
|
||||
func TestListReceivedFilesDecodesName(t *testing.T) {
|
||||
const rawName = "report.txt␠"
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
|
||||
|
||||
wantName := f.opt.Enc.ToStandardName(rawName)
|
||||
require.NotEqual(t, rawName, wantName)
|
||||
|
||||
var gotNames []string
|
||||
err := f.listReceivedFiles(context.Background(), func(entry fs.DirEntry) error {
|
||||
gotNames = append(gotNames, entry.Remote())
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, gotNames, 1)
|
||||
assert.Equal(t, wantName, gotNames[0])
|
||||
}
|
||||
|
||||
// TestFindSharedFileResolvesDecodedName confirms findSharedFile can look up
|
||||
// a received file by its standard, decoded rclone-visible name.
|
||||
func TestFindSharedFileResolvesDecodedName(t *testing.T) {
|
||||
const rawName = "report.txt␠"
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
|
||||
|
||||
encodedName := f.opt.Enc.ToStandardName(rawName)
|
||||
require.NotEqual(t, rawName, encodedName)
|
||||
|
||||
o, err := f.findSharedFile(context.Background(), encodedName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, encodedName, o.remote)
|
||||
}
|
||||
|
||||
// sharedFoldersHandler serves a single shared folder from list_folders.
|
||||
func sharedFoldersHandler(name, id string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.Contains(r.URL.Path, "list_folders") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
resp := sharing.ListFoldersResult{
|
||||
Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
}
|
||||
|
||||
// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
|
||||
// lookup must match regardless of the case of the requested name.
|
||||
func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
|
||||
f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))
|
||||
|
||||
for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
|
||||
id, err := f.findSharedFolder(context.Background(), name)
|
||||
require.NoError(t, err, name)
|
||||
assert.Equal(t, "folder-id", id, name)
|
||||
}
|
||||
|
||||
_, err := f.findSharedFolder(context.Background(), "no-such-folder")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
}
|
||||
|
||||
// The Dropbox backend advertises CaseInsensitive: true, so a received file
|
||||
// lookup must match regardless of the case of the requested name.
|
||||
func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
|
||||
f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))
|
||||
|
||||
for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
|
||||
o, err := f.findSharedFile(context.Background(), name)
|
||||
require.NoError(t, err, name)
|
||||
assert.Equal(t, "TestFile.txt", o.remote, name)
|
||||
}
|
||||
|
||||
_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
+17
-15
@@ -222,36 +222,38 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fldMap := map[string]bool{}
|
||||
for _, folder := range result.Result.Folders {
|
||||
fldMap[folder.FldID.String()] = true
|
||||
if f.root == "" && dir == "" && strings.Contains(folder.Path, "/") {
|
||||
continue
|
||||
}
|
||||
|
||||
folderPath := strings.Trim(folder.Path, "/")
|
||||
root := strings.Trim(f.root, "/")
|
||||
|
||||
remotePathWithoutRoot := folderPath
|
||||
|
||||
if root != "" {
|
||||
if remotePathWithoutRoot == root {
|
||||
continue
|
||||
}
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, root+"/")
|
||||
paths := strings.Split(folder.Path, fullPath+"/")
|
||||
remote := paths[0]
|
||||
if len(paths) > 1 {
|
||||
remote = paths[1]
|
||||
}
|
||||
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
|
||||
|
||||
if remotePathWithoutRoot == "" {
|
||||
if strings.Contains(remote, "/") {
|
||||
continue
|
||||
}
|
||||
|
||||
pathsWithoutRoot := strings.Split(folder.Path, "/"+f.root+"/")
|
||||
remotePathWithoutRoot := pathsWithoutRoot[0]
|
||||
if len(pathsWithoutRoot) > 1 {
|
||||
remotePathWithoutRoot = pathsWithoutRoot[1]
|
||||
}
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
|
||||
entries = append(entries, fs.NewDir(remotePathWithoutRoot, time.Now()))
|
||||
}
|
||||
|
||||
for _, file := range result.Result.Files {
|
||||
if _, ok := fldMap[file.FldID.String()]; ok {
|
||||
continue
|
||||
}
|
||||
remote := path.Join(dir, file.Name)
|
||||
// trim leading slashes
|
||||
remote = strings.TrimPrefix(remote, "/")
|
||||
|
||||
obj := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
|
||||
@@ -16,16 +16,13 @@ import (
|
||||
|
||||
// multipartInit starts a new multipart upload and returns server details.
|
||||
func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
|
||||
encodedFolderPath := f.fromStandardPath(folderPath)
|
||||
encodedFilename := f.fromStandardPath(filename)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/multipart/init",
|
||||
Parameters: url.Values{
|
||||
"key": {f.opt.Key},
|
||||
"filename": {encodedFilename},
|
||||
"folder_path": {encodedFolderPath},
|
||||
"filename": {filename},
|
||||
"folder_path": {folderPath},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -10,11 +11,9 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
rclonemultipart "github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -26,9 +25,7 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
}
|
||||
|
||||
if dir != "" {
|
||||
if _, err := f.createFolder(ctx, dir); err != nil {
|
||||
return fmt.Errorf("failed to create multipart folder: %w", err)
|
||||
}
|
||||
_ = f.Mkdir(ctx, dir)
|
||||
}
|
||||
|
||||
folder := strings.Trim(dir, "/")
|
||||
@@ -48,32 +45,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
server := initResp.Result.Server
|
||||
objectPath := initResp.Result.ObjectPath
|
||||
|
||||
chunkSize := int64(f.opt.ChunkSize)
|
||||
if chunkSize <= 0 {
|
||||
return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)
|
||||
}
|
||||
for partNo := 1; ; partNo++ {
|
||||
// Buffer the part in memory from the global pool so it can be
|
||||
// re-sent on retry
|
||||
rw := rclonemultipart.NewRW()
|
||||
n, err := io.CopyN(rw, in, chunkSize)
|
||||
if err != nil && err != io.EOF {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("read failed: %w", err)
|
||||
}
|
||||
chunkSize := int(f.opt.ChunkSize)
|
||||
buf := make([]byte, 0, chunkSize)
|
||||
tmp := make([]byte, 1024*1024)
|
||||
partNo := 1
|
||||
|
||||
for {
|
||||
n, errRead := in.Read(tmp)
|
||||
if n > 0 {
|
||||
uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)
|
||||
if uploadErr != nil {
|
||||
_ = rw.Close()
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)
|
||||
buf = append(buf, tmp[:n]...)
|
||||
|
||||
// If buffer reached chunkSize, upload a full part
|
||||
if len(buf) >= chunkSize {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
partNo++
|
||||
buf = buf[:0]
|
||||
}
|
||||
}
|
||||
if closeErr := rw.Close(); closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if err == io.EOF {
|
||||
|
||||
if errRead == io.EOF {
|
||||
break
|
||||
}
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read failed: %w", errRead)
|
||||
}
|
||||
}
|
||||
|
||||
if len(buf) > 0 {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
|
||||
@@ -84,40 +89,31 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadPart sends a single multipart chunk of size bytes to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: server,
|
||||
Parameters: url.Values{
|
||||
"partNumber": {strconv.Itoa(partNo)},
|
||||
"uploadId": {uploadID},
|
||||
},
|
||||
Body: r,
|
||||
ContentLength: &size,
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-RC-Upload-Id": uploadID,
|
||||
"X-RC-Part-No": strconv.Itoa(partNo),
|
||||
"X-Sess-ID": sessID,
|
||||
"X-Object-Path": objectPath,
|
||||
},
|
||||
NoResponse: true,
|
||||
// uploadPart sends a single multipart chunk to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
|
||||
url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
// rewind the pooled buffer so each attempt sends the whole chunk
|
||||
_, err := r.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)
|
||||
}
|
||||
return shouldRetry(err), err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
req.Header.Set("X-RC-Upload-Id", uploadID)
|
||||
req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
|
||||
req.Header.Set("X-Sess-ID", sessID)
|
||||
req.Header.Set("X-Object-Path", objectPath)
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("uploadPart failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFile uploads a file to FileLu
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// SetUploadChunkSize changes the configured chunk size, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
return old, nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff changes the configured upload cutoff, returning the old value.
|
||||
//
|
||||
// It is only called by the integration tests while no transfer is in progress.
|
||||
func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
var old fs.SizeSuffix
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
|
||||
return old, nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -142,8 +141,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
// Update updates the object with new data
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
size := src.Size()
|
||||
counter := readers.NewCountingReader(in)
|
||||
in = counter
|
||||
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err := o.fs.uploadFile(ctx, in, o.remote)
|
||||
@@ -158,15 +155,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
if removeErr := o.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
o.size = size
|
||||
o.modTime = src.ModTime(ctx)
|
||||
return nil
|
||||
|
||||
@@ -12,8 +12,5 @@ func TestIntegration(t *testing.T) {
|
||||
RemoteName: "TestFileLu:",
|
||||
NilObject: nil,
|
||||
SkipInvalidUTF8: true,
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: 1,
|
||||
},
|
||||
})
|
||||
}
|
||||
+4
-38
@@ -21,11 +21,9 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -49,7 +47,7 @@ func init() {
|
||||
},
|
||||
{
|
||||
Name: "api_key",
|
||||
Help: `API Key for your Filen account
|
||||
Help: `API Key for your Filen account
|
||||
|
||||
Get this using the Filen CLI export-api-key command
|
||||
You can download the Filen CLI from https://github.com/FilenCloudDienste/filen-cli`,
|
||||
@@ -163,7 +161,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
filen: filen,
|
||||
Enc: opt.Encoder,
|
||||
concurrency: opt.UploadConcurrency,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
|
||||
fileSystem.features = (&fs.Features{
|
||||
@@ -197,13 +194,6 @@ type Options struct {
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
}
|
||||
|
||||
// defaults copied from box
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Fs represents a virtual filesystem mounted on a specific root folder
|
||||
type Fs struct {
|
||||
name string
|
||||
@@ -212,7 +202,6 @@ type Fs struct {
|
||||
Enc encoder.MultiEncoder
|
||||
features *fs.Features
|
||||
concurrency int
|
||||
pacer *fs.Pacer
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
@@ -362,7 +351,6 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
type chunkWriter struct {
|
||||
sdk.FileUpload
|
||||
filen *sdk.Filen
|
||||
pacer *fs.Pacer
|
||||
bucketAndRegion chan client.V3UploadResponse
|
||||
chunkSize int64
|
||||
|
||||
@@ -374,13 +362,6 @@ type chunkWriter struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
|
||||
func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
|
||||
realChunkNumber := int64(chunkNumber) * (cw.chunkSize) / sdk.ChunkSize
|
||||
chunk := make([]byte, sdk.ChunkSize, sdk.ChunkSize+cw.EncryptionKey.Cipher.Overhead())
|
||||
@@ -433,15 +414,7 @@ func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader i
|
||||
if err != nil {
|
||||
return totalWritten, err
|
||||
}
|
||||
var resp *client.V3UploadResponse
|
||||
err = cw.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
resp, err := cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
|
||||
if err != nil {
|
||||
return totalWritten, err
|
||||
}
|
||||
@@ -521,7 +494,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, &chunkWriter{
|
||||
FileUpload: *fu,
|
||||
filen: f.filen,
|
||||
pacer: f.pacer,
|
||||
chunkSize: chunkSize,
|
||||
bucketAndRegion: make(chan client.V3UploadResponse, 1),
|
||||
knownChunks: make(map[int64][]byte),
|
||||
@@ -710,10 +682,7 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
// The server stores modtimes with millisecond precision so round
|
||||
// here too to keep the in-memory modtime identical to the one a
|
||||
// fresh listing returns.
|
||||
o.file.LastModified = t.Round(time.Millisecond)
|
||||
o.file.LastModified = t
|
||||
return o.fs.filen.UpdateMeta(ctx, o.file)
|
||||
}
|
||||
|
||||
@@ -755,10 +724,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Logf(option, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
// The server stores modtimes with millisecond precision so round
|
||||
// here too to keep the in-memory modtime identical to the one a
|
||||
// fresh listing returns.
|
||||
newModTime := src.ModTime(ctx).Round(time.Millisecond)
|
||||
newModTime := src.ModTime(ctx)
|
||||
newIncomplete, err := o.file.NewFromBase(o.fs.filen.FileEncryptionVersion)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"github.com/Files-com/files-sdk-go/v3/file"
|
||||
file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
|
||||
"github.com/Files-com/files-sdk-go/v3/folder"
|
||||
files_sdk_lib "github.com/Files-com/files-sdk-go/v3/lib"
|
||||
"github.com/Files-com/files-sdk-go/v3/session"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
@@ -177,13 +176,6 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Errors from the upload storage servers are of this type
|
||||
var httpErr files_sdk_lib.ResponseError
|
||||
if errors.As(err, &httpErr) && slices.Contains(retryErrorCodes, httpErr.StatusCode) {
|
||||
fs.Debugf(nil, "Retrying HTTP error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
|
||||
@@ -353,7 +345,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
|
||||
for it.Next() {
|
||||
item := new(it.File())
|
||||
item := ptr(it.File())
|
||||
remote := f.opt.Enc.ToStandardPath(item.DisplayName)
|
||||
remote = path.Join(dir, remote)
|
||||
if remote == dir {
|
||||
@@ -424,7 +416,7 @@ func (f *Fs) mkdir(ctx context.Context, path string) error {
|
||||
|
||||
params := files_sdk.FolderCreateParams{
|
||||
Path: path,
|
||||
MkdirParents: new(true),
|
||||
MkdirParents: ptr(true),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -466,7 +458,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
|
||||
params := files_sdk.FileDeleteParams{
|
||||
Path: path,
|
||||
Recursive: new(!check),
|
||||
Recursive: ptr(!check),
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -537,7 +529,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.
|
||||
params := files_sdk.FileCopyParams{
|
||||
Path: srcPath,
|
||||
Destination: dstPath,
|
||||
Overwrite: new(true),
|
||||
Overwrite: ptr(true),
|
||||
}
|
||||
|
||||
var action files_sdk.FileAction
|
||||
@@ -680,7 +672,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
Paths: []string{f.absPath(remote)},
|
||||
}
|
||||
if expire < fs.DurationOff {
|
||||
params.ExpiresAt = new(time.Now().Add(time.Duration(expire)))
|
||||
params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
|
||||
}
|
||||
|
||||
var bundle files_sdk.Bundle
|
||||
@@ -842,6 +834,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
func isFolderNotEmpty(err error) bool {
|
||||
var re files_sdk.ResponseError
|
||||
ok := errors.As(err, &re)
|
||||
@@ -861,7 +858,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
file.UploadWithProvidedMtime(src.ModTime(ctx)),
|
||||
}
|
||||
|
||||
err := o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
err := o.fs.fileClient.Upload(uploadOpts...)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -869,24 +866,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return err
|
||||
}
|
||||
|
||||
// The server computes the MD5 asynchronously after upload so
|
||||
// retry reading the metadata for a short time until it appears.
|
||||
const maxTries = 10
|
||||
for tries := 1; ; tries++ {
|
||||
err = o.readMetaData(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if o.md5 != "" || tries >= maxTries {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(tries) * 100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return o.readMetaData(ctx)
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
|
||||
Loaded 100 of 858 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user