Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
23e9066b4f | ||
|
|
693ca3215f |
No files matched your search
@@ -2,9 +2,6 @@
|
||||
go.mod text eol=lf
|
||||
go.sum text eol=lf
|
||||
|
||||
# Embedded templates must render the same on every OS
|
||||
lib/http/templates/** text eol=lf
|
||||
|
||||
# Ignore generated files in GitHub language statistics and diffs
|
||||
/MANUAL.* linguist-generated=true
|
||||
/rclone.1 linguist-generated=true
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
# CODEOWNERS for rclone
|
||||
#
|
||||
# Maps paths to the maintainers auto-requested for review when a pull request
|
||||
# touches matching files. Patterns are evaluated top to bottom and the LAST
|
||||
# matching pattern wins, so the catch-all below is overridden by the more
|
||||
# specific subsystem/backend lines that follow.
|
||||
#
|
||||
# Every owner listed here has write access to the repo (GitHub ignores owners
|
||||
# without it). Assignments are derived from `git log` history per directory:
|
||||
# each area lists its dominant contributor(s) alongside @ncw, the project lead
|
||||
# and global fallback.
|
||||
|
||||
# Global fallback - everything not matched below
|
||||
* @ncw
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Core subsystems
|
||||
# -----------------------------------------------------------------------------
|
||||
/fs/ @ncw
|
||||
/vfs/ @ncw
|
||||
/lib/ @ncw
|
||||
/cmd/ @ncw
|
||||
/librclone/ @ncw
|
||||
/fstest/ @ncw
|
||||
|
||||
/lib/encoder/ @ncw @klauspost
|
||||
/fs/sync/ @ncw @nielash
|
||||
/fs/operations/ @ncw @nielash
|
||||
/cmd/bisync/ @nielash @ivandeex @ncw
|
||||
/cmd/serve/ @albertony @ncw
|
||||
/cmd/selfupdate/ @ivandeex @ncw
|
||||
/cmd/mount2/ @ncw @darthShadow
|
||||
/cmd/gitannex/ @dmcardle @ncw
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Backends - keyed to the dominant / specialist maintainer per `git log`
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# albertony
|
||||
/backend/jottacloud/ @albertony @buengese @ncw
|
||||
/backend/local/ @ncw @albertony @nielash
|
||||
/backend/netstorage/ @albertony @ncw
|
||||
/backend/hdfs/ @albertony @ncw
|
||||
|
||||
# buengese
|
||||
/backend/yandex/ @ncw @buengese
|
||||
/backend/zoho/ @buengese @ncw
|
||||
/backend/fichier/ @ncw @buengese
|
||||
/backend/compress/ @ncw @buengese
|
||||
|
||||
# ivandeex
|
||||
/backend/chunker/ @ncw @ivandeex
|
||||
/backend/mailru/ @ncw @ivandeex
|
||||
/backend/ftp/ @ncw @ivandeex
|
||||
/backend/sia/ @ncw @ivandeex
|
||||
|
||||
# Other maintainer-owned backends (see MAINTAINERS.md)
|
||||
/backend/cache/ @ncw @remusb
|
||||
/backend/union/ @ncw @Max-Sum @albertony
|
||||
/backend/filescom/ @ncw @childish-sambino
|
||||
|
||||
# Microsoft Azure
|
||||
/backend/azureblob/ @ncw @sandeepkru @darthShadow
|
||||
|
||||
# Single-maintainer specialist backends
|
||||
/backend/onedrive/ @ncw @Cnly @nielash
|
||||
/backend/iclouddrive/ @lostb1t @ncw
|
||||
/backend/pikpak/ @wiserain @ncw
|
||||
/backend/protondrive/ @henrybear327 @ncw
|
||||
/backend/seafile/ @ncw @creativeprojects
|
||||
/backend/filen/ @Enduriel @ncw
|
||||
/backend/storj/ @ncw @calebcase
|
||||
/backend/combine/ @ncw @nielash
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Repo plumbing
|
||||
# -----------------------------------------------------------------------------
|
||||
/.github/ @ncw
|
||||
@@ -0,0 +1,31 @@
|
||||
<!--
|
||||
|
||||
Welcome :-) We understand you are having a problem with rclone; we want to help you with that!
|
||||
|
||||
If you've just got a question or aren't sure if you've found a bug then please use the rclone forum:
|
||||
|
||||
https://forum.rclone.org/
|
||||
|
||||
instead of filing an issue for a quick response.
|
||||
|
||||
If you are reporting a bug or asking for a new feature then please use one of the templates here:
|
||||
|
||||
https://github.com/rclone/rclone/issues/new
|
||||
|
||||
otherwise fill in the form below.
|
||||
|
||||
Thank you
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
|
||||
#### Output of `rclone version`
|
||||
|
||||
|
||||
|
||||
#### Describe the issue
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Report a problem with rclone
|
||||
---
|
||||
|
||||
<!--
|
||||
|
||||
We understand you are having a problem with rclone; we want to help you with that!
|
||||
|
||||
**STOP and READ**
|
||||
**YOUR POST WILL BE REMOVED IF IT IS LOW QUALITY**:
|
||||
Please show the effort you've put into solving the problem and please be specific.
|
||||
People are volunteering their time to help! Low effort posts are not likely to get good answers!
|
||||
|
||||
If you think you might have found a bug, try to replicate it with the latest beta (or stable).
|
||||
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
|
||||
|
||||
If you can still replicate it or just got a question then please use the rclone forum:
|
||||
|
||||
https://forum.rclone.org/
|
||||
|
||||
for a quick response instead of filing an issue on this repo.
|
||||
|
||||
If nothing else helps, then please fill in the info below which helps us help you.
|
||||
|
||||
**DO NOT REDACT** any information except passwords/keys/personal info.
|
||||
|
||||
You should use 3 backticks to begin and end your paste to make it readable.
|
||||
|
||||
Make sure to include a log obtained with '-vv'.
|
||||
|
||||
You can also use '-vv --log-file bug.log' and a service such as https://pastebin.com or https://gist.github.com/
|
||||
|
||||
Thank you
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
|
||||
#### What is the problem you are having with rclone?
|
||||
|
||||
|
||||
|
||||
#### What is your rclone version (output from `rclone version`)
|
||||
|
||||
|
||||
|
||||
#### Which OS you are using and how many bits (e.g. Windows 7, 64 bit)
|
||||
|
||||
|
||||
|
||||
#### Which cloud storage system are you using? (e.g. Google Drive)
|
||||
|
||||
|
||||
|
||||
#### The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
|
||||
|
||||
|
||||
|
||||
#### A log from the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your bug report. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest a new feature or enhancement for rclone
|
||||
---
|
||||
|
||||
<!--
|
||||
|
||||
Welcome :-)
|
||||
|
||||
So you've got an idea to improve rclone? We love that!
|
||||
You'll be glad to hear we've incorporated hundreds of ideas from contributors already.
|
||||
|
||||
Probably the latest beta (or stable) release has your feature, so try to update your rclone.
|
||||
The update instructions are available at https://rclone.org/commands/rclone_selfupdate/
|
||||
|
||||
If it still isn't there, here is a checklist of things to do:
|
||||
|
||||
1. Search the old issues for your idea and +1 or comment on an existing issue if possible.
|
||||
2. Discuss on the forum: https://forum.rclone.org/
|
||||
3. Make a feature request issue (this is the right place!).
|
||||
4. Be prepared to get involved making the feature :-)
|
||||
|
||||
Looking forward to your great idea!
|
||||
|
||||
The Rclone Developers
|
||||
|
||||
-->
|
||||
|
||||
#### The associated forum post URL from `https://forum.rclone.org`
|
||||
|
||||
|
||||
|
||||
#### What is your current rclone version (output from `rclone version`)?
|
||||
|
||||
|
||||
|
||||
#### What problem are you are trying to solve?
|
||||
|
||||
|
||||
|
||||
#### How do you think rclone should be changed to solve that?
|
||||
|
||||
|
||||
|
||||
<!--- Please keep the note below for others who read your feature request. -->
|
||||
|
||||
#### How to use GitHub
|
||||
|
||||
* Please use the 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to show that you are affected by the same issue.
|
||||
* Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
|
||||
* Subscribe to receive notifications on status change and new comments.
|
||||
@@ -1,124 +0,0 @@
|
||||
name: Bug report
|
||||
description: Report a reproducible problem with rclone
|
||||
labels: ["bug"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
We want to help you with rclone - thank you for taking the time to fill this in!
|
||||
|
||||
**This tracker is for reproducible bugs only.** If you have a question, are
|
||||
not sure whether something is a bug, or want help with a configuration, please
|
||||
use the [rclone forum](https://forum.rclone.org/) instead - you will get a much
|
||||
quicker answer there.
|
||||
|
||||
Before filing, please **test with the [latest beta or stable](https://rclone.org/commands/rclone_selfupdate/)**
|
||||
(downloads at <https://beta.rclone.org/>) as your problem may already be fixed.
|
||||
|
||||
**Do not redact** any information except passwords, keys and personal info.
|
||||
- type: checkboxes
|
||||
id: acknowledgements
|
||||
attributes:
|
||||
label: Before you start
|
||||
description: Please confirm the following. Issues that skip these steps may be closed.
|
||||
options:
|
||||
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues for this problem.
|
||||
required: true
|
||||
- label: I have tested with the latest beta or stable release and can still reproduce the problem.
|
||||
required: true
|
||||
- label: This is a reproducible bug, not a usage question (questions belong on the forum).
|
||||
required: true
|
||||
- type: input
|
||||
id: forum-url
|
||||
attributes:
|
||||
label: Associated forum post URL
|
||||
description: If you discussed this on the forum first (encouraged), paste the link here.
|
||||
placeholder: https://forum.rclone.org/t/...
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: What is the problem you are having with rclone?
|
||||
description: Describe the problem and the exact steps to reproduce it.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: rclone version
|
||||
description: The full output of `rclone version`.
|
||||
placeholder: |
|
||||
rclone v1.75.0
|
||||
- os/version: ubuntu 24.04 (64 bit)
|
||||
- os/kernel: 6.8.0-111-generic (x86_64)
|
||||
- os/type: linux
|
||||
- os/arch: amd64
|
||||
- go/version: go1.26.4
|
||||
- go/linking: dynamic
|
||||
- go/tags: none
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating system
|
||||
description: Choose the operating system you are using with rclone.
|
||||
options:
|
||||
- Windows
|
||||
- macOS
|
||||
- Linux
|
||||
- FreeBSD
|
||||
- NetBSD
|
||||
- OpenBSD
|
||||
- Plan 9
|
||||
- Solaris
|
||||
- Android
|
||||
- iOS
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: backend
|
||||
attributes:
|
||||
label: Which cloud storage system are you using?
|
||||
description: e.g. Google Drive, S3, Dropbox, local disk. List all involved.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: config
|
||||
attributes:
|
||||
label: Output of `rclone config redacted`
|
||||
description: |
|
||||
Run `rclone config redacted` (or `rclone config redacted <remote>` for a
|
||||
single remote). This command automatically replaces passwords and tokens
|
||||
with `XXX`, so it should be safe to paste but please double-check before posting.
|
||||
render: ini
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: command
|
||||
attributes:
|
||||
label: The command you were trying to run
|
||||
description: e.g. `rclone copy /tmp remote:tmp`
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: log
|
||||
attributes:
|
||||
label: A log from the command with the `-vv` flag
|
||||
description: |
|
||||
e.g. output from `rclone -vv copy /tmp remote:tmp`. For long logs (> 100 lines) use
|
||||
`-vv --log-file bug.log` and attach it here or send to a service such as
|
||||
<https://gist.github.com/> or <https://pastebin.com/> and link it here.
|
||||
render: shell
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: extra
|
||||
attributes:
|
||||
label: Anything else?
|
||||
description: Anything else that might help us, e.g. a minimal way to replicate the problem.
|
||||
validations:
|
||||
required: false
|
||||
@@ -1,8 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Rclone Forum - questions & support
|
||||
- name: Rclone Forum Community Support
|
||||
url: https://forum.rclone.org/
|
||||
about: Have a question, need help, or not sure if it's a bug? Please ask here for a quick answer.
|
||||
- name: Rclone Documentation
|
||||
url: https://rclone.org/docs/
|
||||
about: The docs answer many "how do I..." questions, including per-backend guides.
|
||||
about: Please ask and answer questions here.
|
||||
@@ -1,68 +0,0 @@
|
||||
name: Feature request
|
||||
description: Suggest a new feature or enhancement for rclone
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
So you've got an idea to improve rclone? We love that - we've already
|
||||
incorporated hundreds of ideas from contributors!
|
||||
|
||||
Please **search first**: your idea may already exist in the latest
|
||||
[beta or stable](https://rclone.org/commands/rclone_selfupdate/), or already
|
||||
be requested. It is often worth [discussing on the forum](https://forum.rclone.org/)
|
||||
to refine the idea or checking to see if rclone can do it already before filing it here.
|
||||
- type: checkboxes
|
||||
id: acknowledgements
|
||||
attributes:
|
||||
label: Before you start
|
||||
options:
|
||||
- label: I have searched the [forum](https://forum.rclone.org/) and the existing issues and this hasn't already been requested.
|
||||
required: true
|
||||
- label: I have checked the latest beta and this feature doesn't already exist.
|
||||
required: true
|
||||
- type: input
|
||||
id: forum-url
|
||||
attributes:
|
||||
label: Associated forum post URL
|
||||
description: If you discussed this on the forum first (encouraged), paste the link here.
|
||||
placeholder: https://forum.rclone.org/t/...
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: version
|
||||
attributes:
|
||||
label: rclone version
|
||||
description: The full output of `rclone version`, not just the version number.
|
||||
placeholder: |
|
||||
rclone v1.75.0
|
||||
- os/version: ubuntu 24.04 (64 bit)
|
||||
- os/kernel: 6.8.0-111-generic (x86_64)
|
||||
- os/type: linux
|
||||
- os/arch: amd64
|
||||
- go/version: go1.26.4
|
||||
- go/linking: dynamic
|
||||
- go/tags: none
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: What problem are you trying to solve?
|
||||
description: Describe the use case and what is difficult or impossible today.
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: How do you think rclone should be changed to solve that?
|
||||
validations:
|
||||
required: true
|
||||
- type: checkboxes
|
||||
id: involvement
|
||||
attributes:
|
||||
label: Getting involved
|
||||
description: Features are much more likely to happen if someone helps make them.
|
||||
options:
|
||||
- label: I'm willing to help implement, test or fund this feature.
|
||||
required: false
|
||||
@@ -1,54 +1,29 @@
|
||||
<!--
|
||||
Thank you for contributing to rclone!
|
||||
Thank you very much for contributing code or documentation to rclone! Please
|
||||
fill out the following questions to make it easier for us to review your
|
||||
changes.
|
||||
|
||||
Please discuss anything more than a trivial fix in an issue FIRST - see the
|
||||
"Linked issue" section below. Then fill in the questions to help us review.
|
||||
You do not need to check all the boxes below all at once, feel free to take
|
||||
your time and add more commits. If you're done and ready for review, please
|
||||
check the last box.
|
||||
-->
|
||||
|
||||
#### What does this change do?
|
||||
|
||||
<!-- Describe the change here. -->
|
||||
|
||||
#### Linked issue
|
||||
#### What is the purpose of this change?
|
||||
|
||||
<!--
|
||||
Put `Fixes #1234` here if this closes an issue, or `#1234` to link without closing.
|
||||
|
||||
IMPORTANT: Anything beyond a trivial fix (typo, doc tweak, small obvious bug fix)
|
||||
should be discussed and agreed in an issue BEFORE you open a pull request.
|
||||
Pull requests for larger changes that have not been discussed in an issue first
|
||||
may be closed. This saves everyone's time if the change needs a different approach
|
||||
or isn't a good fit.
|
||||
Describe the changes here
|
||||
-->
|
||||
|
||||
Fixes #
|
||||
|
||||
#### For new or changed backends
|
||||
#### Was the change discussed in an issue or in the forum before?
|
||||
|
||||
<!--
|
||||
Backend pull requests are often sent without the integration tests having been run.
|
||||
We cannot merge a backend change until it passes the integration tests.
|
||||
|
||||
To be merged, a backend change REQUIRES:
|
||||
1. A clean run of `go run ./fstest/test_all -backends <remote>` (summarise the
|
||||
result below or paste a screenshot of the test_all output in the web browser).
|
||||
2. A test account so the maintainers can run the integration tests daily against
|
||||
the backend on the integration test server.
|
||||
|
||||
If the tests aren't quite passing yet and you need help getting them
|
||||
to pass, then submit the PR and we can help getting you over the line.
|
||||
|
||||
See https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#writing-a-new-backend
|
||||
and https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests
|
||||
Link issues and relevant forum posts here.
|
||||
-->
|
||||
|
||||
#### Checklist
|
||||
|
||||
- [ ] This change is trivial **OR** it has been discussed and agreed in the linked issue.
|
||||
- [ ] I have read the [contribution guidelines](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#submitting-a-new-feature-or-bug-fix).
|
||||
- [ ] **(If I used AI tools to help write this code)** I have read and understood the [AI-assisted contributions guidance](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#ai-assisted-contributions), and I have tested and take ownership of this change myself.
|
||||
- [ ] I have added tests for all changes in this PR if appropriate.
|
||||
- [ ] I have added documentation for the changes if appropriate.
|
||||
- [ ] All commit messages are in [house style](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#commit-messages).
|
||||
- [ ] **(Backend changes only)** `test_all` passes for this backend and if submitting a new backend can provide a test account for the integration tester - see [CONTRIBUTING.md](https://github.com/rclone/rclone/blob/master/CONTRIBUTING.md#integration-tests).
|
||||
- [ ] This Pull Request is ready for review.
|
||||
- [ ] I'm done, this Pull Request is ready for review :-)
|
||||
@@ -4,5 +4,3 @@ updates:
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
+25
-148
@@ -19,9 +19,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
@@ -32,12 +29,12 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.24']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '>=1.25.0-rc.1'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +45,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '>=1.25.0-rc.1'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '>=1.25.0-rc.1'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +61,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.27.2'
|
||||
go: '>=1.25.0-rc.1'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.27.2'
|
||||
go: '>=1.25.0-rc.1'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,14 +78,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.27.2'
|
||||
go: '>=1.25.0-rc.1'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.26
|
||||
- job_name: go1.24
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.9'
|
||||
go: '1.24'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -97,58 +94,16 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
check-latest: true
|
||||
# Caching is handled explicitly below. setup-go's built-in cache
|
||||
# keys only on go.sum, so all matrix jobs collide on one key and
|
||||
# only the first to finish saves - which loses the cross-compile
|
||||
# build cache.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
# The module cache holds downloaded source and is identical across
|
||||
# jobs (it depends only on go.sum, not on OS, arch or Go version), so
|
||||
# it is shared via a single key to avoid storing the same ~600 MiB in
|
||||
# every job's cache and overflowing the repo cache limit.
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
# The build cache holds compiled artifacts which are specific to the
|
||||
# OS, arch and Go version, so it is kept per job.
|
||||
- name: Build cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
@@ -165,11 +120,6 @@ jobs:
|
||||
sudo chown root:$USER /etc/fuse.conf
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
|
||||
# The podman static bundle in runner image ubuntu24/20260726.254 and
|
||||
# later ships a non-setuid fusermount3 in /usr/local/bin which
|
||||
# shadows the setuid one from the fuse3 package and breaks FUSE
|
||||
# mounts with "Operation not permitted", so remove it.
|
||||
sudo rm -f /usr/local/bin/fusermount /usr/local/bin/fusermount3
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
|
||||
- name: Install Libraries on macOS
|
||||
@@ -179,7 +129,6 @@ jobs:
|
||||
unset HOMEBREW_NO_INSTALL_FROM_API
|
||||
brew untap --force homebrew/core
|
||||
brew untap --force homebrew/cask
|
||||
brew untap --force aws/tap
|
||||
brew update
|
||||
brew install --cask macfuse
|
||||
brew install git-annex git-annex-remote-rclone
|
||||
@@ -253,15 +202,6 @@ jobs:
|
||||
# Deploy binaries if enabled in config && not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
lint:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -276,44 +216,36 @@ jobs:
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
go-version: '1.24'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Cache
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
~/.cache/golangci-lint
|
||||
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
|
||||
- name: Code quality test (Linux)
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
uses: golangci/golangci-lint-action@v6
|
||||
with:
|
||||
version: latest
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (Windows)
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
uses: golangci/golangci-lint-action@v6
|
||||
env:
|
||||
GOOS: "windows"
|
||||
with:
|
||||
@@ -321,7 +253,7 @@ jobs:
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (macOS)
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
uses: golangci/golangci-lint-action@v6
|
||||
env:
|
||||
GOOS: "darwin"
|
||||
with:
|
||||
@@ -329,7 +261,7 @@ jobs:
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (FreeBSD)
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
uses: golangci/golangci-lint-action@v6
|
||||
env:
|
||||
GOOS: "freebsd"
|
||||
with:
|
||||
@@ -337,7 +269,7 @@ jobs:
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (OpenBSD)
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
uses: golangci/golangci-lint-action@v6
|
||||
env:
|
||||
GOOS: "openbsd"
|
||||
with:
|
||||
@@ -358,24 +290,12 @@ jobs:
|
||||
MAINTAINERS.md
|
||||
README.md
|
||||
RELEASE.md
|
||||
CODE_OF_CONDUCT.md
|
||||
librclone\README.md
|
||||
backend\s3\README.md
|
||||
docs/content/{_index,authors,bugs,changelog,docs,downloads,faq,filtering,gui,install,licence,overview,privacy}.md
|
||||
docs/content/{authors,bugs,changelog,docs,downloads,faq,filtering,gui,install,licence,overview,privacy}.md
|
||||
|
||||
- name: Scan edits of autogenerated files
|
||||
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
android:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -383,50 +303,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Upgrade together with NDK version
|
||||
- name: Set up Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v7
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '~1.27.2'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Build cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
go-version: '>=1.25.0-rc.1'
|
||||
|
||||
- name: Set global environment variables
|
||||
run: |
|
||||
@@ -502,12 +388,3 @@ jobs:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# Upload artifacts if not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
@@ -18,10 +18,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build-image:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
|
||||
@@ -56,7 +52,7 @@ jobs:
|
||||
df -h .
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -96,7 +92,7 @@ jobs:
|
||||
# There's no way around this, because "ImageOS" is only available to
|
||||
# processes, but the setup-go action uses it in its key.
|
||||
id: imageos
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
@@ -104,7 +100,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
|
||||
with:
|
||||
@@ -126,14 +122,14 @@ jobs:
|
||||
type=raw,value=beta,enable={{is_default_branch}}
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Load Go Build Cache for Docker
|
||||
id: go-cache
|
||||
uses: actions/cache@v6
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
@@ -152,7 +148,7 @@ jobs:
|
||||
skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
@@ -163,7 +159,7 @@ jobs:
|
||||
|
||||
- name: Build and Publish Image Digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
file: Dockerfile
|
||||
context: .
|
||||
@@ -187,7 +183,7 @@ jobs:
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload Image Digest
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ env.PLATFORM }}
|
||||
path: /tmp/digests/*
|
||||
@@ -202,7 +198,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Download Image Digests
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v5
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-*
|
||||
@@ -214,7 +210,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@v5
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
|
||||
with:
|
||||
@@ -265,16 +261,16 @@ jobs:
|
||||
env.write(f"ANNOTATIONS={annotations_string}\n")
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
|
||||
@@ -14,9 +14,6 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build_docker_volume_plugin:
|
||||
if: inputs.manual || github.repository == 'rclone/rclone'
|
||||
@@ -33,7 +30,7 @@ jobs:
|
||||
sudo rm -rf /usr/share/dotnet || true
|
||||
df -h .
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@v7
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and publish docker plugin
|
||||
|
||||
@@ -4,9 +4,6 @@ on:
|
||||
issues:
|
||||
types: [labeled]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -3,9 +3,6 @@ on:
|
||||
release:
|
||||
types: [released]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+130
-151
@@ -1,165 +1,144 @@
|
||||
version: "2"
|
||||
# golangci-lint configuration options
|
||||
|
||||
linters:
|
||||
# Configure the linter set. To avoid unexpected results the implicit default
|
||||
# set is ignored and all the ones to use are explicitly enabled.
|
||||
default: none
|
||||
enable:
|
||||
# Default
|
||||
- errcheck
|
||||
- govet
|
||||
- ineffassign
|
||||
- staticcheck
|
||||
- unused
|
||||
# Additional
|
||||
- gocritic
|
||||
- misspell
|
||||
#- prealloc # TODO
|
||||
- revive
|
||||
- unconvert
|
||||
exclusions:
|
||||
rules:
|
||||
- linters:
|
||||
- revive
|
||||
text: 'var-naming: avoid meaningless package names'
|
||||
- linters:
|
||||
- revive
|
||||
text: 'var-naming: avoid package names that conflict with Go standard library package names'
|
||||
# Configure checks. Mostly using defaults but with some commented exceptions.
|
||||
settings:
|
||||
govet:
|
||||
enable-all: true
|
||||
disable:
|
||||
- fieldalignment
|
||||
- shadow
|
||||
staticcheck:
|
||||
# With staticcheck there is only one setting, so to extend the implicit
|
||||
# default value it must be explicitly included.
|
||||
checks:
|
||||
# Default
|
||||
- all
|
||||
- -ST1000
|
||||
- -ST1003
|
||||
- -ST1016
|
||||
- -ST1020
|
||||
- -ST1021
|
||||
- -ST1022
|
||||
# Disable quickfix checks
|
||||
- -QF*
|
||||
# Disabled because its dataflow analysis (new in staticcheck
|
||||
# 0.8.0) makes linting large packages more than 10x slower,
|
||||
# which takes the CI lint job past its timeout, and it
|
||||
# produces false positives (eg on operations.Delete).
|
||||
- -SA4023
|
||||
gocritic:
|
||||
# With gocritic there are different settings, but since enabled-checks
|
||||
# and disabled-checks cannot both be set, for full customization the
|
||||
# alternative is to disable all defaults and explicitly enable the ones
|
||||
# to use.
|
||||
disable-all: true
|
||||
enabled-checks:
|
||||
#- appendAssign # Skip default
|
||||
- argOrder
|
||||
- assignOp
|
||||
- badCall
|
||||
- badCond
|
||||
#- captLocal # Skip default
|
||||
- caseOrder
|
||||
- codegenComment
|
||||
#- commentFormatting # Skip default
|
||||
- defaultCaseOrder
|
||||
- deprecatedComment
|
||||
- dupArg
|
||||
- dupBranchBody
|
||||
- dupCase
|
||||
- dupSubExpr
|
||||
- elseif
|
||||
#- exitAfterDefer # Skip default
|
||||
- flagDeref
|
||||
- flagName
|
||||
#- ifElseChain # Skip default
|
||||
- mapKey
|
||||
- newDeref
|
||||
- offBy1
|
||||
- regexpMust
|
||||
- ruleguard # Enable additional check that are not enabled by default
|
||||
#- singleCaseSwitch # Skip default
|
||||
- sloppyLen
|
||||
- sloppyTypeAssert
|
||||
- switchTrue
|
||||
- typeSwitchVar
|
||||
- underef
|
||||
- unlambda
|
||||
- unslice
|
||||
- valSwap
|
||||
- wrapperFunc
|
||||
settings:
|
||||
ruleguard:
|
||||
rules: ${base-path}/bin/rules.go
|
||||
revive:
|
||||
# With revive there is in reality only one setting, and when at least one
|
||||
# rule are specified then only these rules will be considered, defaults
|
||||
# and all others are then implicitly disabled, so must explicitly enable
|
||||
# all rules to be used.
|
||||
rules:
|
||||
- name: blank-imports
|
||||
disabled: false
|
||||
- name: context-as-argument
|
||||
disabled: false
|
||||
- name: context-keys-type
|
||||
disabled: false
|
||||
- name: dot-imports
|
||||
disabled: false
|
||||
#- name: empty-block # Skip default
|
||||
# disabled: true
|
||||
- name: error-naming
|
||||
disabled: false
|
||||
- name: error-return
|
||||
disabled: false
|
||||
- name: error-strings
|
||||
disabled: false
|
||||
- name: errorf
|
||||
disabled: false
|
||||
- name: exported
|
||||
disabled: false
|
||||
#- name: increment-decrement # Skip default
|
||||
# disabled: true
|
||||
- name: indent-error-flow
|
||||
disabled: false
|
||||
- name: package-comments
|
||||
disabled: false
|
||||
- name: range
|
||||
disabled: false
|
||||
- name: receiver-naming
|
||||
disabled: false
|
||||
#- name: redefines-builtin-id # Skip default
|
||||
# disabled: true
|
||||
#- name: superfluous-else # Skip default
|
||||
# disabled: true
|
||||
- name: time-naming
|
||||
disabled: false
|
||||
- name: unexported-return
|
||||
disabled: false
|
||||
#- name: unreachable-code # Skip default
|
||||
# disabled: true
|
||||
#- name: unused-parameter # Skip default
|
||||
# disabled: true
|
||||
- name: var-declaration
|
||||
disabled: false
|
||||
- name: var-naming
|
||||
disabled: false
|
||||
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- goimports
|
||||
- revive
|
||||
- ineffassign
|
||||
- govet
|
||||
- unconvert
|
||||
- staticcheck
|
||||
- gosimple
|
||||
- stylecheck
|
||||
- unused
|
||||
- misspell
|
||||
- gocritic
|
||||
#- prealloc
|
||||
#- maligned
|
||||
disable-all: true
|
||||
|
||||
issues:
|
||||
# Enable some lints excluded by default
|
||||
exclude-use-default: false
|
||||
|
||||
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
|
||||
max-issues-per-linter: 0
|
||||
|
||||
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
|
||||
max-same-issues: 0
|
||||
|
||||
exclude-rules:
|
||||
|
||||
- linters:
|
||||
- staticcheck
|
||||
text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
|
||||
|
||||
# don't disable the revive messages about comments on exported functions
|
||||
include:
|
||||
- EXC0012
|
||||
- EXC0013
|
||||
- EXC0014
|
||||
- EXC0015
|
||||
|
||||
run:
|
||||
# Timeout for total work, e.g. 30s, 5m, 5m30s. Default is 0 (disabled).
|
||||
# timeout for analysis, e.g. 30s, 5m, default is 1m
|
||||
timeout: 10m
|
||||
|
||||
linters-settings:
|
||||
revive:
|
||||
# setting rules seems to disable all the rules, so re-enable them here
|
||||
rules:
|
||||
- name: blank-imports
|
||||
disabled: false
|
||||
- name: context-as-argument
|
||||
disabled: false
|
||||
- name: context-keys-type
|
||||
disabled: false
|
||||
- name: dot-imports
|
||||
disabled: false
|
||||
- name: empty-block
|
||||
disabled: true
|
||||
- name: error-naming
|
||||
disabled: false
|
||||
- name: error-return
|
||||
disabled: false
|
||||
- name: error-strings
|
||||
disabled: false
|
||||
- name: errorf
|
||||
disabled: false
|
||||
- name: exported
|
||||
disabled: false
|
||||
- name: increment-decrement
|
||||
disabled: true
|
||||
- name: indent-error-flow
|
||||
disabled: false
|
||||
- name: package-comments
|
||||
disabled: false
|
||||
- name: range
|
||||
disabled: false
|
||||
- name: receiver-naming
|
||||
disabled: false
|
||||
- name: redefines-builtin-id
|
||||
disabled: true
|
||||
- name: superfluous-else
|
||||
disabled: true
|
||||
- name: time-naming
|
||||
disabled: false
|
||||
- name: unexported-return
|
||||
disabled: false
|
||||
- name: unreachable-code
|
||||
disabled: true
|
||||
- name: unused-parameter
|
||||
disabled: true
|
||||
- name: var-declaration
|
||||
disabled: false
|
||||
- name: var-naming
|
||||
disabled: false
|
||||
stylecheck:
|
||||
# Only enable the checks performed by the staticcheck stand-alone tool,
|
||||
# as documented here: https://staticcheck.io/docs/configuration/options/#checks
|
||||
checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]
|
||||
gocritic:
|
||||
# Enable all default checks with some exceptions and some additions (commented).
|
||||
# Cannot use both enabled-checks and disabled-checks, so must specify all to be used.
|
||||
disable-all: true
|
||||
enabled-checks:
|
||||
#- appendAssign # Enabled by default
|
||||
- argOrder
|
||||
- assignOp
|
||||
- badCall
|
||||
- badCond
|
||||
#- captLocal # Enabled by default
|
||||
- caseOrder
|
||||
- codegenComment
|
||||
#- commentFormatting # Enabled by default
|
||||
- defaultCaseOrder
|
||||
- deprecatedComment
|
||||
- dupArg
|
||||
- dupBranchBody
|
||||
- dupCase
|
||||
- dupSubExpr
|
||||
- elseif
|
||||
#- exitAfterDefer # Enabled by default
|
||||
- flagDeref
|
||||
- flagName
|
||||
#- ifElseChain # Enabled by default
|
||||
- mapKey
|
||||
- newDeref
|
||||
- offBy1
|
||||
- regexpMust
|
||||
- ruleguard # Not enabled by default
|
||||
#- singleCaseSwitch # Enabled by default
|
||||
- sloppyLen
|
||||
- sloppyTypeAssert
|
||||
- switchTrue
|
||||
- typeSwitchVar
|
||||
- underef
|
||||
- unlambda
|
||||
- unslice
|
||||
- valSwap
|
||||
- wrapperFunc
|
||||
settings:
|
||||
ruleguard:
|
||||
rules: "${configDir}/bin/rules.go"
|
||||
+1
-30
@@ -1,6 +1,6 @@
|
||||
default: true
|
||||
|
||||
# Use specific styles, to be consistent across all documents.
|
||||
# Use specific styles, to be consistent accross all documents.
|
||||
# Default is to accept any as long as it is consistent within the same document.
|
||||
heading-style: # MD003
|
||||
style: atx
|
||||
@@ -41,32 +41,3 @@ single-title: # MD025
|
||||
# Markdown files we must use whatever works in the final HTML generated docs.
|
||||
# Suppress Markdownlint warning: Link fragments should be valid.
|
||||
link-fragments: false # MD051
|
||||
|
||||
# Restrict the languages and language identifiers to use for code blocks.
|
||||
# We only want those supported by both Hugo and GitHub. These are documented
|
||||
# here:
|
||||
# https://gohugo.io/content-management/syntax-highlighting/#languages
|
||||
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
|
||||
# In addition, we only want to allow identifiers (aliases) that correspond to
|
||||
# the same language in Hugo and GitHub, and preferably also VSCode and other
|
||||
# commonly used tools, to avoid confusion. An example of this is that "shell"
|
||||
# by some are considered an identifier for shell scripts, i.e. an alias for
|
||||
# "sh", while others consider it an identifier for shell sessions, i.e. an
|
||||
# alias for "console". Although Hugo and GitHub in this case are consistent and
|
||||
# have chosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# avoids the confusion entirely.
|
||||
fenced-code-language: # MD040
|
||||
allowed_languages:
|
||||
- text
|
||||
- console
|
||||
- sh
|
||||
- bat
|
||||
- ini
|
||||
- json
|
||||
- yaml
|
||||
- go
|
||||
- python
|
||||
- c++
|
||||
- c#
|
||||
- java
|
||||
- powershell
|
||||
@@ -1,136 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
This file provides guidance to AI coding agents (e.g. Claude Code, Codex, Cursor, Gemini CLI, and similar tools) when working with code in this repository.
|
||||
|
||||
Rclone welcomes AI-assisted contributions, but the expectation is that you, the human submitter, understand every line you propose and have compiled and tested it against real rclone code - not just generated it. See the "AI-assisted contributions" section of [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request.
|
||||
|
||||
## Project Overview
|
||||
|
||||
Rclone is a command-line program to sync files and directories to and from cloud storage providers. It's written in Go and supports 70+ backends (cloud storage systems). Think "rsync for cloud storage".
|
||||
|
||||
## General Notes
|
||||
|
||||
**We take backwards compatibility very seriously.** PRs should not change the observable behaviour of existing commands, flags or rc API without very good reason. Rclone does not try to preserve a stable Go API but try not to change it gratuitously.
|
||||
|
||||
Rclone operates with a lot of different backends, so **compatibility is key**. It is the backend integration tests which guarantee that compatibility. Changes should consider both known and unknown backends and should take care not to break functionality of existing installations.
|
||||
|
||||
The core parts of rclone under `fs` and `vfs` need to work with all backends and **backend specific hacks won't be merged**. Fixes likely need to go in the relevant backend or if new behaviour is really needed, a new Feature flag needs to be added.
|
||||
|
||||
**Changes should be kept to the minimum.** Work hard to make the most elegant, smallest change you can. Do not refactor or re-order code unless necessary as this makes review more challenging. Re-use existing test scaffolding, existing or library routines (e.g. `lib`) where possible.
|
||||
|
||||
Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
|
||||
|
||||
Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
|
||||
|
||||
## Build and Test Commands
|
||||
|
||||
```bash
|
||||
# Build rclone (simple)
|
||||
go build
|
||||
|
||||
# Build with version info (preferred)
|
||||
make
|
||||
|
||||
# Run all unit tests (no cloud credentials needed)
|
||||
make quicktest
|
||||
# or equivalently:
|
||||
RCLONE_CONFIG="/notfound" go test ./...
|
||||
|
||||
# Run tests for a specific package
|
||||
cd backend/memory && go test -v
|
||||
# or from root:
|
||||
go test -v ./backend/memory/
|
||||
|
||||
# Run a single test
|
||||
go test -v -run TestIntegration/FsCheckWrap ./backend/memory/
|
||||
|
||||
# Run tests with race detector
|
||||
make racequicktest
|
||||
|
||||
# Lint (requires golangci-lint)
|
||||
golangci-lint run ./...
|
||||
|
||||
# Run backend integration tests (requires configured TestRemote remote)
|
||||
cd backend/drive && go test -v
|
||||
# Run sync/operations integration tests against a remote
|
||||
cd fs/sync && go test -v -remote TestDrive:
|
||||
cd fs/operations && go test -v -remote TestDrive:
|
||||
|
||||
# Run integration tests via test framework
|
||||
go run ./fstest/test_all -backends drive
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Entry Point and Plugin Registration
|
||||
|
||||
`rclone.go` is the main entry point. It imports `backend/all` and `cmd/all` which use Go's `init()` pattern to register all backends and commands. Each backend calls `fs.Register()` with a `fs.RegInfo` struct during init.
|
||||
|
||||
### Core Interfaces (`fs/`)
|
||||
|
||||
The `fs` package defines the core abstractions:
|
||||
- **`fs.Fs`** (`fs/types.go`): The filesystem interface every backend must implement (List, NewObject, Put, Mkdir, Rmdir).
|
||||
- **`fs.Object`** (`fs/types.go`): Interface for a file/object (Open, Update, Remove, SetModTime).
|
||||
- **`fs.Features`** (`fs/features.go`): Optional capabilities a backend can declare (Purge, Copy, Move, DirMove, etc.). Backends set function pointers for operations they support; nil means not supported.
|
||||
- **`fs.RegInfo`** (`fs/registry.go`): Registration metadata for a backend including its name, config options, and NewFs constructor.
|
||||
|
||||
### Backend Structure (`backend/`)
|
||||
|
||||
Each backend is a single Go package (e.g., `backend/s3/`, `backend/drive/`). Key conventions:
|
||||
- Main implementation in a single file (e.g., `s3.go`) - **do not** split into `fs.go`/`object.go`.
|
||||
- API types go in a separate `api/types.go` file.
|
||||
- Test file (e.g., `s3_test.go`) uses `fstests.Run()` from `fstest/fstests` for standardized integration tests.
|
||||
- Register in `backend/all/all.go` via blank import.
|
||||
- HTTP-based backends should use `lib/rest` for HTTP calls and `fs/fshttp` for the HTTP client.
|
||||
- Use `lib/dircache` for directory-ID-based remotes, `lib/oauthutil` for OAuth, `lib/pacer` for rate limiting.
|
||||
|
||||
### Command Structure (`cmd/`)
|
||||
|
||||
Each command is a package under `cmd/` registered in `cmd/all/all.go` via blank import. Commands use cobra via `cmd.Main()`.
|
||||
|
||||
### Key Subsystems
|
||||
|
||||
- **`fs/operations/`**: Core file operations (Copy, Move, Delete, etc.)
|
||||
- **`fs/sync/`**: Directory sync logic
|
||||
- **`fs/march/`**: Parallel directory tree walker used by sync
|
||||
- **`fs/filter/`**: Include/exclude filtering
|
||||
- **`fs/accounting/`**: Transfer statistics and bandwidth limiting
|
||||
- **`fs/config/`**: Configuration file management
|
||||
- **`vfs/`**: Virtual filesystem layer (used by mount, serve)
|
||||
- **`librclone/`**: C-compatible library interface for embedding rclone
|
||||
- **`fstest/`**: Integration test framework; `fstest/fstests/` has the generic backend test suite
|
||||
|
||||
## Commit Message Convention
|
||||
|
||||
Prefix with the directory of the change, then a colon: `drive: add team drive support - fixes #885`. For cross-cutting changes use a broader prefix like `fs` or `operations`.
|
||||
|
||||
Make the first line of your commit message a summary of the change that a user (not a developer) of rclone would like to read. So write `drive: fix server side copy of big files` instead of `drive: no longer set the MimeType in Move or Copy`. This is important because these lines go into the change log which is read by users.
|
||||
|
||||
## Code Commenting Style
|
||||
|
||||
Comments describe the code as it is now, for a future reader who has no knowledge of the change that introduced it.
|
||||
|
||||
Every exported type, function, field, and constant has a **godoc comment** that starts with its name and is phrased as a present-tense statement of what it is or does (`// Mkdir makes the directory (container, bucket)`)
|
||||
|
||||
**Document the contract** callers need - preconditions, what's returned, which sentinel errors are returned and when, and any "shouldn't return an error if it already exists" style caveats - rather than the implementation.
|
||||
|
||||
**Keep comments terse**: a single line for most things, with extra paragraphs (separated by blank `//` lines) reserved for genuine subtlety.
|
||||
|
||||
Inline comments inside function bodies should **explain why** - a non-obvious API quirk, a workaround, a gotcha, or an ordering constraint - and may cite an external reference (forum thread, vendor docs, RFC) when that's what makes the behaviour non-obvious; skip comments that merely restate what the code plainly does.
|
||||
|
||||
Use `FIXME` and `TODO` for known shortcomings.
|
||||
|
||||
**Do not write comments that narrate the change itself** or compare against the previous behaviour (no "now we also handle...", "changed to...", "previously this returned...", or references to bug/PR numbers in the code) - that context belongs in the commit message, not in source that will outlive the change.
|
||||
|
||||
Code comments should only refer to **current** behaviour - and shouldn't describe behaviour that is trivially deducible from reading the code.
|
||||
|
||||
## Linting Configuration
|
||||
|
||||
Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck, govet, ineffassign, staticcheck, unused, gocritic, misspell, revive, unconvert. The `goimports` formatter is also enabled.
|
||||
|
||||
## Documentation
|
||||
|
||||
- Backend option docs come from `Help:` fields in the Go source Options structs, not from markdown files.
|
||||
- Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
|
||||
- Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
|
||||
- Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
|
||||
@@ -1,6 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
Guidance for this repository lives in AGENTS.md, shared with all AI coding
|
||||
agents. Claude Code reads it via the import below.
|
||||
|
||||
@AGENTS.md
|
||||
@@ -1,80 +0,0 @@
|
||||
# Rclone Code of Conduct
|
||||
|
||||
Like the technical community as a whole, the Rclone team and community
|
||||
is made up of a mixture of professionals and volunteers from all over
|
||||
the world, working on every aspect of the mission - including
|
||||
mentorship, teaching, and connecting people.
|
||||
|
||||
Diversity is one of our huge strengths, but it can also lead to
|
||||
communication issues and unhappiness. To that end, we have a few
|
||||
ground rules that we ask people to adhere to. This code applies
|
||||
equally to founders, mentors and those seeking help and guidance.
|
||||
|
||||
This isn't an exhaustive list of things that you can't do. Rather,
|
||||
take it in the spirit in which it's intended - a guide to make it
|
||||
easier to enrich all of us and the technical communities in which we
|
||||
participate.
|
||||
|
||||
This code of conduct applies to all spaces managed by the Rclone
|
||||
project or Rclone Services Ltd. This includes the issue tracker, the
|
||||
forum, the GitHub site, the wiki, any other online services or
|
||||
in-person events. In addition, violations of this code outside these
|
||||
spaces may affect a person's ability to participate within them.
|
||||
|
||||
- **Be friendly and patient.**
|
||||
- **Be welcoming.** We strive to be a community that welcomes and
|
||||
supports people of all backgrounds and identities. This includes,
|
||||
but is not limited to members of any race, ethnicity, culture,
|
||||
national origin, colour, immigration status, social and economic
|
||||
class, educational level, sex, sexual orientation, gender identity
|
||||
and expression, age, size, family status, political belief,
|
||||
religion, and mental and physical ability.
|
||||
- **Be considerate.** Your work will be used by other people, and you
|
||||
in turn will depend on the work of others. Any decision you take
|
||||
will affect users and colleagues, and you should take those
|
||||
consequences into account when making decisions. Remember that we're
|
||||
a world-wide community, so you might not be communicating in someone
|
||||
else's primary language.
|
||||
- **Be respectful.** Not all of us will agree all the time, but
|
||||
disagreement is no excuse for poor behavior and poor manners. We
|
||||
might all experience some frustration now and then, but we cannot
|
||||
allow that frustration to turn into a personal attack. It's
|
||||
important to remember that a community where people feel
|
||||
uncomfortable or threatened is not a productive one. Members of the
|
||||
Rclone community should be respectful when dealing with other
|
||||
members as well as with people outside the Rclone community.
|
||||
- **Be careful in the words that you choose.** We are a community of
|
||||
professionals, and we conduct ourselves professionally. Be kind to
|
||||
others. Do not insult or put down other participants. Harassment and
|
||||
other exclusionary behavior aren't acceptable. This includes, but is
|
||||
not limited to:
|
||||
- Violent threats or language directed against another person.
|
||||
- Discriminatory jokes and language.
|
||||
- Posting sexually explicit or violent material.
|
||||
- Posting (or threatening to post) other people's personally
|
||||
identifying information ("doxing").
|
||||
- Personal insults, especially those using racist or sexist terms.
|
||||
- Unwelcome sexual attention.
|
||||
- Advocating for, or encouraging, any of the above behavior.
|
||||
- Repeated harassment of others. In general, if someone asks you to
|
||||
stop, then stop.
|
||||
- **When we disagree, try to understand why.** Disagreements, both
|
||||
social and technical, happen all the time and Rclone is no
|
||||
exception. It is important that we resolve disagreements and
|
||||
differing views constructively. Remember that we're different. The
|
||||
strength of Rclone comes from its varied community, people from a
|
||||
wide range of backgrounds. Different people have different
|
||||
perspectives on issues. Being unable to understand why someone holds
|
||||
a viewpoint doesn't mean that they're wrong. Don't forget that it is
|
||||
human to err and blaming each other doesn't get us anywhere.
|
||||
Instead, focus on helping to resolve issues and learning from
|
||||
mistakes.
|
||||
|
||||
If you believe someone is violating the code of conduct, we ask that
|
||||
you report it by emailing [info@rclone.com](mailto:info@rclone.com).
|
||||
|
||||
Original text courtesy of the [Speak Up! project](http://web.archive.org/web/20141109123859/http://speakup.io/coc.html).
|
||||
|
||||
## Questions?
|
||||
|
||||
If you have questions, please feel free to [contact us](mailto:info@rclone.com).
|
||||
+77
-142
@@ -38,7 +38,7 @@ and [email](https://docs.github.com/en/github/setting-up-and-managing-your-githu
|
||||
Next open your terminal, change directory to your preferred folder and initialise
|
||||
your local rclone project:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git clone https://github.com/rclone/rclone.git
|
||||
cd rclone
|
||||
git remote rename origin upstream
|
||||
@@ -53,13 +53,13 @@ executed from the rclone folder created above.
|
||||
|
||||
Now [install Go](https://golang.org/doc/install) and verify your installation:
|
||||
|
||||
```console
|
||||
```sh
|
||||
go version
|
||||
```
|
||||
|
||||
Great, you can now compile and execute your own version of rclone:
|
||||
|
||||
```console
|
||||
```sh
|
||||
go build
|
||||
./rclone version
|
||||
```
|
||||
@@ -68,7 +68,7 @@ go build
|
||||
more accurate version number in the executable as well as enable you to specify
|
||||
more build options.) Finally make a branch to add your new feature
|
||||
|
||||
```console
|
||||
```sh
|
||||
git checkout -b my-new-feature
|
||||
```
|
||||
|
||||
@@ -80,7 +80,7 @@ and a quick view on the rclone [code organisation](#code-organisation).
|
||||
When ready - test the affected functionality and run the unit tests for the
|
||||
code you changed
|
||||
|
||||
```console
|
||||
```sh
|
||||
cd folder/with/changed/files
|
||||
go test -v
|
||||
```
|
||||
@@ -99,7 +99,7 @@ Make sure you
|
||||
|
||||
When you are done with that push your changes to GitHub:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git push -u origin my-new-feature
|
||||
```
|
||||
|
||||
@@ -113,43 +113,13 @@ GitHub.
|
||||
You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master)
|
||||
or [squash your commits](#squashing-your-commits).
|
||||
|
||||
## AI-assisted contributions
|
||||
|
||||
You are welcome to use AI coding assistants (Claude Code, Codex, Cursor, Gemini
|
||||
CLI, and similar) to help write your contribution. Rclone has an
|
||||
[AGENTS.md](AGENTS.md) file at the top of the repository describing the project's
|
||||
conventions; point your tool at it so the code it produces matches rclone's
|
||||
style.
|
||||
|
||||
However, the same standard applies to every pull request whether or not a tool
|
||||
was involved: **you are responsible for the code you submit.** Before you open a
|
||||
pull request please make sure that:
|
||||
|
||||
- You understand every line of the change and can explain why it is correct. If
|
||||
a reviewer asks a question about it, you should be able to answer without
|
||||
going back to the tool.
|
||||
- You have actually built and run it. At a minimum `go build` and
|
||||
`make quicktest` must pass, and for a backend change you should run the
|
||||
[backend tests](#backend-testing) against a real remote where you can.
|
||||
- The change is a genuine fix or feature that you have verified solves the
|
||||
problem, not a plausible-looking guess. Unverified, AI-generated pull requests
|
||||
that do not compile, do not pass the tests, invent APIs that do not exist, or
|
||||
do not actually do what the description claims waste maintainer time and are
|
||||
likely to be closed.
|
||||
- You have trimmed the comments. AI tools tend to add verbose comments that
|
||||
restate what the code plainly does or narrate the change being made; please
|
||||
cut these down to match the [code commenting style](AGENTS.md#code-commenting-style).
|
||||
|
||||
In short: an AI assistant is a tool to help *you* contribute, not a substitute
|
||||
for understanding and testing your own work.
|
||||
|
||||
## Using Git and GitHub
|
||||
|
||||
### Committing your changes
|
||||
|
||||
Follow the guideline for [commit messages](#commit-messages) and then:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git checkout my-new-feature # To switch to your branch
|
||||
git status # To see the new and changed files
|
||||
git add FILENAME # To select FILENAME for the commit
|
||||
@@ -160,7 +130,7 @@ git log # To verify the commit. Use q to quit the log
|
||||
|
||||
You can modify the message or changes in the latest commit using:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git commit --amend
|
||||
```
|
||||
|
||||
@@ -175,7 +145,7 @@ pushed to GitHub.
|
||||
|
||||
Your previously pushed commits are replaced by:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git push --force origin my-new-feature
|
||||
```
|
||||
|
||||
@@ -184,7 +154,7 @@ git push --force origin my-new-feature
|
||||
To base your changes on the latest version of the
|
||||
[rclone master](https://github.com/rclone/rclone/tree/master) (upstream):
|
||||
|
||||
```console
|
||||
```sh
|
||||
git checkout master
|
||||
git fetch upstream
|
||||
git merge --ff-only
|
||||
@@ -200,7 +170,7 @@ If you rebase commits that have been pushed to GitHub, then you will have to
|
||||
|
||||
To combine your commits into one commit:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git log # To count the commits to squash, e.g. the last 2
|
||||
git reset --soft HEAD~2 # To undo the 2 latest commits
|
||||
git status # To check everything is as expected
|
||||
@@ -208,13 +178,13 @@ git status # To check everything is as expected
|
||||
|
||||
If everything is fine, then make the new combined commit:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git commit # To commit the undone commits as one
|
||||
```
|
||||
|
||||
otherwise, you may roll back using:
|
||||
|
||||
```console
|
||||
```sh
|
||||
git reflog # To check that HEAD{1} is your previous state
|
||||
git reset --soft 'HEAD@{1}' # To roll back to your previous state
|
||||
```
|
||||
@@ -249,13 +219,13 @@ to check an error return).
|
||||
rclone's tests are run from the go testing framework, so at the top
|
||||
level you can run this to run all the tests.
|
||||
|
||||
```console
|
||||
```sh
|
||||
go test -v ./...
|
||||
```
|
||||
|
||||
You can also use `make`, if supported by your platform
|
||||
|
||||
```console
|
||||
```sh
|
||||
make quicktest
|
||||
```
|
||||
|
||||
@@ -276,7 +246,7 @@ need to make a remote called `TestDrive`.
|
||||
You can then run the unit tests in the drive directory. These tests
|
||||
are skipped if `TestDrive:` isn't defined.
|
||||
|
||||
```console
|
||||
```sh
|
||||
cd backend/drive
|
||||
go test -v
|
||||
```
|
||||
@@ -285,7 +255,7 @@ You can then run the integration tests which test all of rclone's
|
||||
operations. Normally these get run against the local file system,
|
||||
but they can be run against any of the remotes.
|
||||
|
||||
```console
|
||||
```sh
|
||||
cd fs/sync
|
||||
go test -v -remote TestDrive:
|
||||
go test -v -remote TestDrive: -fast-list
|
||||
@@ -298,8 +268,9 @@ If you want to use the integration test framework to run these tests
|
||||
altogether with an HTML report and test retries then from the
|
||||
project root:
|
||||
|
||||
```console
|
||||
go run ./fstest/test_all -backends drive
|
||||
```sh
|
||||
go install github.com/rclone/rclone/fstest/test_all
|
||||
test_all -backends drive
|
||||
```
|
||||
|
||||
### Full integration testing
|
||||
@@ -307,19 +278,19 @@ go run ./fstest/test_all -backends drive
|
||||
If you want to run all the integration tests against all the remotes,
|
||||
then change into the project root and run
|
||||
|
||||
```console
|
||||
```sh
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
The commands may require some extra go packages which you can install with
|
||||
|
||||
```console
|
||||
```sh
|
||||
make build_dep
|
||||
```
|
||||
|
||||
The full integration tests are run daily on the integration test server. You can
|
||||
find the results at <https://integration.rclone.org>
|
||||
find the results at <https://pub.rclone.org/integration-tests/>
|
||||
|
||||
## Code Organisation
|
||||
|
||||
@@ -378,13 +349,11 @@ If you are adding a new feature then please update the documentation.
|
||||
|
||||
The documentation sources are generally in Markdown format, in conformance
|
||||
with the CommonMark specification and compatible with GitHub Flavored
|
||||
Markdown (GFM). The markdown format and style is checked as part of the lint
|
||||
operation that runs automatically on pull requests, to enforce standards and
|
||||
consistency. This is based on the [markdownlint](https://github.com/DavidAnson/markdownlint)
|
||||
tool by David Anson, which can also be integrated into editors so you can
|
||||
perform the same checks while writing. It generally follows Ciro Santilli's
|
||||
[Markdown Style Guide](https://cirosantilli.com/markdown-style-guide), which
|
||||
is good source if you want to know more.
|
||||
Markdown (GFM). The markdown format is checked as part of the lint operation
|
||||
that runs automatically on pull requests, to enforce standards and consistency.
|
||||
This is based on the [markdownlint](https://github.com/DavidAnson/markdownlint)
|
||||
tool, which can also be integrated into editors so you can perform the same
|
||||
checks while writing.
|
||||
|
||||
HTML pages, served as website <rclone.org>, are generated from the Markdown,
|
||||
using [Hugo](https://gohugo.io). Note that when generating the HTML pages,
|
||||
@@ -413,7 +382,7 @@ If you add a new general flag (not for a backend), then document it in
|
||||
alphabetical order.
|
||||
|
||||
If you add a new backend option/flag, then it should be documented in
|
||||
the source file in the `Help:` field:
|
||||
the source file in the `Help:` field.
|
||||
|
||||
- Start with the most important information about the option,
|
||||
as a single sentence on a single line.
|
||||
@@ -435,30 +404,6 @@ the source file in the `Help:` field:
|
||||
as an unordered list, therefore a single line break is enough to
|
||||
create a new list item. Also, for enumeration texts like name of
|
||||
countries, it looks better without an ending period/full stop character.
|
||||
- You can run `make backenddocs` to verify the resulting Markdown.
|
||||
- This will update the autogenerated sections of the backend docs Markdown
|
||||
files under `docs/content`.
|
||||
- It requires you to have [Python](https://www.python.org) installed.
|
||||
- The `backenddocs` make target runs the Python script `bin/make_backend_docs.py`,
|
||||
and you can also run this directly, optionally with the name of a backend
|
||||
as argument to only update the docs for a specific backend.
|
||||
- **Do not** commit the updated Markdown files. This operation is run as part of
|
||||
the release process. Since any manual changes in the autogenerated sections
|
||||
of the Markdown files will then be lost, we have a pull request check that
|
||||
reports error for any changes within the autogenerated sections. Should you
|
||||
have done manual changes outside of the autogenerated sections they must be
|
||||
committed, of course.
|
||||
- You can run `make serve` to verify the resulting website.
|
||||
- This will build the website and serve it locally, so you can open it in
|
||||
your web browser and verify that the end result looks OK. Check specifically
|
||||
any added links, also in light of the note above regarding different algorithms
|
||||
for generated header anchors.
|
||||
- It requires you to have the [Hugo](https://gohugo.io) tool available.
|
||||
- The `serve` make target depends on the `website` target, which runs the
|
||||
`hugo` command from the `docs` directory to build the website, and then
|
||||
it serves the website locally with an embedded web server using a command
|
||||
`hugo server --logLevel info -w --disableFastRender --ignoreCache`, so you
|
||||
can run similar Hugo commands directly as well.
|
||||
|
||||
When writing documentation for an entirely new backend,
|
||||
see [backend documentation](#backend-documentation).
|
||||
@@ -475,11 +420,6 @@ for small changes in the docs which makes it very easy. Just remember the
|
||||
caveat when linking to header anchors, noted above, which means that GitHub's
|
||||
Markdown preview may not be an entirely reliable verification of the results.
|
||||
|
||||
After your changes have been merged, you can verify them on
|
||||
[tip.rclone.org](https://tip.rclone.org). This site is updated daily with the
|
||||
current state of the master branch at 07:00 UTC. The changes will be on the main
|
||||
[rclone.org](https://rclone.org) site once they have been included in a release.
|
||||
|
||||
## Making a release
|
||||
|
||||
There are separate instructions for making a release in the RELEASE.md
|
||||
@@ -538,7 +478,7 @@ To add a dependency `github.com/ncw/new_dependency` see the
|
||||
instructions below. These will fetch the dependency and add it to
|
||||
`go.mod` and `go.sum`.
|
||||
|
||||
```console
|
||||
```sh
|
||||
go get github.com/ncw/new_dependency
|
||||
```
|
||||
|
||||
@@ -552,7 +492,7 @@ and `go.sum` in the same commit as your other changes.
|
||||
|
||||
If you need to update a dependency then run
|
||||
|
||||
```console
|
||||
```sh
|
||||
go get golang.org/x/crypto
|
||||
```
|
||||
|
||||
@@ -630,37 +570,10 @@ remote or an fs.
|
||||
- **Remember** we have >50 backends to maintain so keeping them as similar as
|
||||
possible to each other is a high priority!
|
||||
|
||||
### Managing memory
|
||||
|
||||
Use rclone's memory pool for upload buffers rather than allocating them
|
||||
yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
|
||||
rclone's memory usage smaller.
|
||||
|
||||
- If the provider supports multipart or parallel chunk uploads, implement
|
||||
[fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
|
||||
and drive it with
|
||||
[multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
|
||||
This gives you pooled chunk buffers, concurrency, retries and
|
||||
`--multi-thread-streams` support for free. See s3/b2 backends for examples.
|
||||
- If you must buffer data yourself take the buffer from the global pool with
|
||||
[multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
|
||||
and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
|
||||
be rewound for a retry.
|
||||
- When using a pooled buffer `Close()` it on all paths so its pages go back to
|
||||
the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
|
||||
attempt, set the request's `ContentLength` explicitly, and if the buffer
|
||||
reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
|
||||
in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
|
||||
so the transport can't close it and free its pages before the retry.
|
||||
|
||||
### Unit tests
|
||||
|
||||
- Create a config entry called `TestRemote` for the unit tests to use
|
||||
- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
|
||||
- If your backend does chunked uploads, set `ChunkedUpload` in the
|
||||
`fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
|
||||
on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
|
||||
`FsPutRetry` tests can run the chunked path with small chunks.
|
||||
- Make sure all tests pass with `go test -v`
|
||||
|
||||
### Integration tests
|
||||
@@ -668,7 +581,8 @@ rclone's memory usage smaller.
|
||||
- Add your backend to `fstest/test_all/config.yaml`
|
||||
- Once you've done that then you can use the integration test framework from
|
||||
the project root:
|
||||
- `go run ./fstest/test_all -backends remote`
|
||||
- go install ./...
|
||||
- test_all -backends remote
|
||||
|
||||
Or if you want to run the integration tests manually:
|
||||
|
||||
@@ -680,32 +594,14 @@ Or if you want to run the integration tests manually:
|
||||
- If your remote defines `ListR` check with this also
|
||||
- `go test -v -remote TestRemote: -fast-list`
|
||||
|
||||
Before a new or changed backend can be merged we require:
|
||||
|
||||
- A clean run of `go run ./fstest/test_all -backends remote` (please include the
|
||||
result in your pull request).
|
||||
- A test account for the backend so the maintainers can add it to the
|
||||
[integration test server](https://integration.rclone.org) and keep the backend
|
||||
working as rclone evolves. A backend that we cannot test is likely to break and
|
||||
may be removed.
|
||||
|
||||
See the [testing](#testing) section for more information on integration tests.
|
||||
|
||||
### Backend documentation
|
||||
|
||||
Add your backend to the docs. Keep lists of remotes in alphabetical
|
||||
order of full name of remote (e.g. `drive` is ordered as `Google
|
||||
Drive`) but with the local file system last.
|
||||
|
||||
First add a data file about your backend in
|
||||
`docs/data/backends/remote.yaml` - this is used to build the overview
|
||||
tables and the tiering info.
|
||||
|
||||
- Create it with: `bin/manage_backends.py create docs/data/backends/remote.yaml`
|
||||
- Edit it to fill in the blanks. Look at the [tiers docs](https://rclone.org/tiers/).
|
||||
- Run this command to fill in the features: `bin/manage_backends.py features docs/data/backends/remote.yaml`
|
||||
|
||||
Next edit these files:
|
||||
Add your backend to the docs - you'll need to pick an icon for it from
|
||||
[fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in
|
||||
alphabetical order of full name of remote (e.g. `drive` is ordered as
|
||||
`Google Drive`) but with the local file system last.
|
||||
|
||||
- `README.md` - main GitHub page
|
||||
- `docs/content/remote.md` - main docs page (note the backend options are
|
||||
@@ -713,6 +609,8 @@ Next edit these files:
|
||||
- make sure this has the `autogenerated options` comments in (see your
|
||||
reference backend docs)
|
||||
- update them in your backend with `bin/make_backend_docs.py remote`
|
||||
- `docs/content/overview.md` - overview docs - add an entry into the Features
|
||||
table and the Optional Features table.
|
||||
- `docs/content/docs.md` - list of remotes in config section
|
||||
- `docs/content/_index.md` - front page of rclone.org
|
||||
- `docs/layouts/chrome/navbar.html` - add it to the website navigation
|
||||
@@ -723,7 +621,44 @@ in the web browser and the links (internal and external) all work.
|
||||
|
||||
## Adding a new s3 provider
|
||||
|
||||
[Please see the guide in the S3 backend directory](backend/s3/README.md).
|
||||
It is quite easy to add a new S3 provider to rclone.
|
||||
|
||||
You'll need to modify the following files
|
||||
|
||||
- `backend/s3/s3.go`
|
||||
- Add the provider to `providerOption` at the top of the file
|
||||
- Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
|
||||
- Exclude your provider from generic config questions (eg `region` and `endpoint).
|
||||
- Add the provider to the `setQuirks` function - see the documentation there.
|
||||
- `docs/content/s3.md`
|
||||
- Add the provider at the top of the page.
|
||||
- Add a section about the provider linked from there.
|
||||
- Make sure this is in alphabetical order in the `Providers` section.
|
||||
- Add a transcript of a trial `rclone config` session
|
||||
- Edit the transcript to remove things which might change in subsequent versions
|
||||
- **Do not** alter or add to the autogenerated parts of `s3.md`
|
||||
- **Do not** run `make backenddocs` or `bin/make_backend_docs.py s3`
|
||||
- `README.md` - this is the home page in github
|
||||
- Add the provider and a link to the section you wrote in `docs/contents/s3.md`
|
||||
- `docs/content/_index.md` - this is the home page of rclone.org
|
||||
- Add the provider and a link to the section you wrote in `docs/contents/s3.md`
|
||||
|
||||
When adding the provider, endpoints, quirks, docs etc keep them in
|
||||
alphabetical order by `Provider` name, but with `AWS` first and
|
||||
`Other` last.
|
||||
|
||||
Once you've written the docs, run `make serve` and check they look OK
|
||||
in the web browser and the links (internal and external) all work.
|
||||
|
||||
Once you've written the code, test `rclone config` works to your
|
||||
satisfaction, and check the integration tests work `go test -v -remote
|
||||
NewS3Provider:`. You may need to adjust the quirks to get them to
|
||||
pass. Some providers just can't pass the tests with control characters
|
||||
in the names so if these fail and the provider doesn't support
|
||||
`urlEncodeListings` in the quirks then ignore them. Note that the
|
||||
`SetTier` test may also fail on non AWS providers.
|
||||
|
||||
For an example of adding an s3 provider see [eb3082a1](https://github.com/rclone/rclone/commit/eb3082a1ebdb76d5625f14cedec3f5154a5e7b10).
|
||||
|
||||
## Writing a plugin
|
||||
|
||||
|
||||
+1
-5
@@ -35,21 +35,17 @@ RUN echo "**** Print Version Binary ****" && \
|
||||
# Begin final image
|
||||
FROM alpine:latest
|
||||
|
||||
ARG RCLONE_UID=1009
|
||||
ARG RCLONE_GID=1009
|
||||
|
||||
RUN echo "**** Install Dependencies ****" && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
fuse3 \
|
||||
mailcap \
|
||||
tzdata && \
|
||||
echo "Enable user_allow_other in fuse" && \
|
||||
echo "user_allow_other" >> /etc/fuse.conf
|
||||
|
||||
COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
|
||||
|
||||
RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
|
||||
RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
|
||||
|
||||
ENTRYPOINT [ "rclone" ]
|
||||
|
||||
|
||||
+21
-28
@@ -2,34 +2,27 @@
|
||||
|
||||
Current active maintainers of rclone are:
|
||||
|
||||
| Name | GitHub ID |
|
||||
| :--------------- | :---------------- |
|
||||
| Nick Craig-Wood | @ncw |
|
||||
| Stefan Breunig | @breunigs |
|
||||
| Ishuah Kariuki | @ishuah |
|
||||
| Remus Bunduc | @remusb |
|
||||
| Fabian Möller | @B4dM4n |
|
||||
| Alex Chen | @Cnly |
|
||||
| Sandeep Ummadi | @sandeepkru |
|
||||
| Sebastian Bünger | @buengese |
|
||||
| Ivan Andreev | @ivandeex |
|
||||
| Max Sum | @Max-Sum |
|
||||
| Fred | @creativeprojects |
|
||||
| Caleb Case | @calebcase |
|
||||
| wiserain | @wiserain |
|
||||
| albertony | @albertony |
|
||||
| Chun-Hung Tseng | @henrybear327 |
|
||||
| Hideo Aoyama | @boukendesho |
|
||||
| nielash | @nielash |
|
||||
| Dan McArdle | @dmcardle |
|
||||
| Sam Harrison | @childish-sambino |
|
||||
| Enduriel | @Enduriel |
|
||||
|
||||
Per-area ownership of code (which backend or command each maintainer looks
|
||||
after) is defined in [.github/CODEOWNERS](.github/CODEOWNERS), which GitHub uses
|
||||
to auto-request reviews. Two responsibilities don't map to a code path: @ncw
|
||||
looks after overall project health, and @boukendesho maintains the snap
|
||||
packaging.
|
||||
| Name | GitHub ID | Specific Responsibilities |
|
||||
| :--------------- | :---------------- | :-------------------------- |
|
||||
| Nick Craig-Wood | @ncw | overall project health |
|
||||
| Stefan Breunig | @breunigs | |
|
||||
| Ishuah Kariuki | @ishuah | |
|
||||
| Remus Bunduc | @remusb | cache backend |
|
||||
| Fabian Möller | @B4dM4n | |
|
||||
| Alex Chen | @Cnly | onedrive backend |
|
||||
| Sandeep Ummadi | @sandeepkru | azureblob backend |
|
||||
| Sebastian Bünger | @buengese | jottacloud, yandex & compress backends |
|
||||
| Ivan Andreev | @ivandeex | chunker & mailru backends |
|
||||
| Max Sum | @Max-Sum | union backend |
|
||||
| Fred | @creativeprojects | seafile backend |
|
||||
| Caleb Case | @calebcase | storj backend |
|
||||
| wiserain | @wiserain | pikpak backend |
|
||||
| albertony | @albertony | |
|
||||
| Chun-Hung Tseng | @henrybear327 | Proton Drive Backend |
|
||||
| Hideo Aoyama | @boukendesho | snap packaging |
|
||||
| nielash | @nielash | bisync |
|
||||
| Dan McArdle | @dmcardle | gitannex |
|
||||
| Sam Harrison | @childish-sambino | filescom |
|
||||
|
||||
## This is a work in progress draft
|
||||
|
||||
|
||||
Generated
+16670
-49050
File diff suppressed because it is too large.
Load diff
Generated
+5284
-16802
File diff suppressed because it is too large.
Load diff
@@ -38,9 +38,6 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
|
||||
BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
|
||||
BETA_UPLOAD_ROOT := beta.rclone.org:
|
||||
BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
|
||||
# Update the beta site's directory listings after an upload - add
|
||||
# --changed for the uploaded directories
|
||||
BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
# Pass in GOTAGS=xyz on the make command line to set build tags
|
||||
ifdef GOTAGS
|
||||
BUILDTAGS=-tags "$(GOTAGS)"
|
||||
@@ -48,7 +45,7 @@ LINTTAGS=--build-tags "$(GOTAGS)"
|
||||
endif
|
||||
LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
|
||||
|
||||
.PHONY: rclone test_all vars version fetch-gui fetch-gui-and-commit
|
||||
.PHONY: rclone test_all vars version
|
||||
|
||||
rclone:
|
||||
ifeq ($(GO_OS),windows)
|
||||
@@ -62,12 +59,6 @@ endif
|
||||
cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
|
||||
mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
|
||||
|
||||
fetch-gui:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh
|
||||
|
||||
fetch-gui-and-commit:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh --commit
|
||||
|
||||
test_all:
|
||||
go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
|
||||
|
||||
@@ -96,15 +87,11 @@ test: rclone test_all
|
||||
@echo "Written logs in test_all.log"
|
||||
|
||||
# Quick test
|
||||
#
|
||||
# The timeout is raised above the go test default of 10m as the
|
||||
# cmd/gitannex end to end tests can take longer than that on slow CI
|
||||
# runners.
|
||||
quicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
|
||||
|
||||
racequicktest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
|
||||
|
||||
compiletest:
|
||||
RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
|
||||
@@ -113,7 +100,6 @@ compiletest:
|
||||
check: rclone
|
||||
@echo "-- START CODE QUALITY REPORT -------------------------------"
|
||||
@golangci-lint run $(LINTTAGS) ./...
|
||||
@bin/markdown-lint
|
||||
@echo "-- END CODE QUALITY REPORT ---------------------------------"
|
||||
|
||||
# Get the build dependencies
|
||||
@@ -127,40 +113,21 @@ release_dep_linux:
|
||||
# Update dependencies
|
||||
showupdates:
|
||||
@echo "*** Direct dependencies that could be updated ***"
|
||||
@go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
|
||||
@GO111MODULE=on go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
|
||||
|
||||
# Update direct dependencies only
|
||||
updatedirect:
|
||||
go get $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
go mod tidy
|
||||
|
||||
# Update direct dependencies only but won't update the `go` line in go.mod
|
||||
updatedirectnoupgrade:
|
||||
@GO_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
for mod in $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all); do \
|
||||
cp go.mod go.mod.bak && \
|
||||
cp go.sum go.sum.bak && \
|
||||
go get $$mod && \
|
||||
NEW_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
if [ "$$NEW_VERSION" != "$$GO_VERSION" ]; then \
|
||||
echo "SKIPPING $$mod (requires go $$NEW_VERSION)"; \
|
||||
cp go.mod.bak go.mod; \
|
||||
cp go.sum.bak go.sum; \
|
||||
else \
|
||||
echo "updated $$mod"; \
|
||||
fi; \
|
||||
done && \
|
||||
rm -f go.mod.bak go.sum.bak && \
|
||||
go mod tidy
|
||||
GO111MODULE=on go get -d $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
GO111MODULE=on go mod tidy
|
||||
|
||||
# Update direct and indirect dependencies and test dependencies
|
||||
update:
|
||||
go get -u -t ./...
|
||||
go mod tidy
|
||||
GO111MODULE=on go get -d -u -t ./...
|
||||
GO111MODULE=on go mod tidy
|
||||
|
||||
# Tidy the module dependencies
|
||||
tidy:
|
||||
go mod tidy
|
||||
GO111MODULE=on go mod tidy
|
||||
|
||||
doc: rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
|
||||
|
||||
@@ -177,12 +144,9 @@ MANUAL.txt: MANUAL.md
|
||||
pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
|
||||
|
||||
commanddocs: rclone
|
||||
go generate ./lib/transform
|
||||
go generate ./cmd/bisync
|
||||
-@rmdir -p '$$HOME/.config/rclone'
|
||||
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
|
||||
@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
|
||||
go run bin/make_bisync_docs.go ./docs/content/
|
||||
|
||||
backenddocs: rclone bin/make_backend_docs.py
|
||||
-@rmdir -p '$$HOME/.config/rclone'
|
||||
@@ -237,7 +201,6 @@ check_sign:
|
||||
upload:
|
||||
rclone -P copy build/ downloads.rclone.org:/$(TAG)
|
||||
rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
|
||||
rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
|
||||
|
||||
upload_github:
|
||||
./bin/upload-github $(TAG)
|
||||
@@ -250,12 +213,6 @@ beta:
|
||||
rclone -v copy build/ pub.rclone.org:/$(TAG)
|
||||
@echo Beta release ready at https://pub.rclone.org/$(TAG)/
|
||||
|
||||
privatebeta:
|
||||
go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) -include '^(darwin|windows|linux)/(arm64|amd64)$$' $(TAG)
|
||||
rclone -Pv copy build/ private-downloads:/beta/$(TAG)
|
||||
@echo Private beta release ready at private-downloads:/beta/$(TAG)/
|
||||
rclone link private-downloads:/beta/$(TAG)
|
||||
|
||||
log_since_last_release:
|
||||
git log $(LAST_TAG)..
|
||||
|
||||
@@ -270,7 +227,6 @@ ci_upload:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
|
||||
endif
|
||||
./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
|
||||
@echo Beta release ready at $(BETA_URL)/testbuilds
|
||||
|
||||
ci_beta:
|
||||
@@ -280,7 +236,6 @@ ci_beta:
|
||||
ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
|
||||
rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
|
||||
endif
|
||||
rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
|
||||
@echo Beta release ready at $(BETA_URL)
|
||||
|
||||
# Fetch the binary builds from GitHub actions
|
||||
@@ -302,26 +257,21 @@ retag:
|
||||
@echo "Version is $(VERSION)"
|
||||
git tag -f -s -m "Version $(VERSION)" $(VERSION)
|
||||
|
||||
# Start development of the next version after a release: the next
|
||||
# minor version on master or the next patch version on a stable branch.
|
||||
startdev:
|
||||
@branch=$$(git rev-parse --abbrev-ref HEAD); \
|
||||
case "$$branch" in \
|
||||
master) next="$(NEXT_VERSION)" ;; \
|
||||
v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
|
||||
*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
|
||||
echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
|
||||
fi; \
|
||||
echo "Version is $(VERSION)"; \
|
||||
echo "Next version is $$next"; \
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
|
||||
echo -n "$$next" > docs/layouts/partials/version.html; \
|
||||
echo "$$next" > VERSION; \
|
||||
git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next version is $(NEXT_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
startstable: startdev
|
||||
startstable:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_PATCH_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
winzip:
|
||||
zip -9 rclone-$(TAG).zip rclone.exe
|
||||
|
||||
@@ -28,27 +28,20 @@ directories to and from different cloud storage providers.
|
||||
- Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
|
||||
- Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
|
||||
- ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
|
||||
- Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
|
||||
- Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
|
||||
- Box [:page_facing_up:](https://rclone.org/box/)
|
||||
- Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
|
||||
- China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
|
||||
- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
|
||||
- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
|
||||
- Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
|
||||
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
|
||||
- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
|
||||
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
|
||||
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
|
||||
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
|
||||
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
- Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
|
||||
- Exaba [:page_facing_up:](https://rclone.org/s3/#exaba)
|
||||
- Fastly Object Storage [:page_facing_up:](https://rclone.org/s3/#fastly)
|
||||
- Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
|
||||
- FileLu [:page_facing_up:](https://rclone.org/filelu/)
|
||||
- Filen [:page_facing_up:](https://rclone.org/filen/)
|
||||
- Files.com [:page_facing_up:](https://rclone.org/filescom/)
|
||||
- FlashBlade [:page_facing_up:](https://rclone.org/s3/#pure-storage-flashblade)
|
||||
- FTP [:page_facing_up:](https://rclone.org/ftp/)
|
||||
@@ -57,21 +50,15 @@ directories to and from different cloud storage providers.
|
||||
- Google Drive [:page_facing_up:](https://rclone.org/drive/)
|
||||
- Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
|
||||
- HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
|
||||
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
|
||||
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
|
||||
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
|
||||
- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp)
|
||||
- HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
- Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
|
||||
- Huawei Drive [:page_facing_up:](https://rclone.org/huaweidrive/)
|
||||
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
|
||||
- ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
|
||||
- Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
|
||||
- Internxt [:page_facing_up:](https://rclone.org/internxt/)
|
||||
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
|
||||
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
|
||||
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
|
||||
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
|
||||
- Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
|
||||
@@ -107,34 +94,26 @@ directories to and from different cloud storage providers.
|
||||
- Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)
|
||||
- QingStor [:page_facing_up:](https://rclone.org/qingstor/)
|
||||
- Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
|
||||
- Rabata Cloud Storage [:page_facing_up:](https://rclone.org/s3/#Rabata)
|
||||
- Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
|
||||
- Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
|
||||
- RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
|
||||
- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
|
||||
- rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
|
||||
- Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
|
||||
- Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)
|
||||
- Seafile [:page_facing_up:](https://rclone.org/seafile/)
|
||||
- Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
|
||||
- SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
|
||||
- Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
|
||||
- Servercore Object Storage [:page_facing_up:](https://rclone.org/s3/#servercore)
|
||||
- SFTP [:page_facing_up:](https://rclone.org/sftp/)
|
||||
- Shade [:page_facing_up:](https://rclone.org/shade/)
|
||||
- SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
|
||||
- Spectra Logic [:page_facing_up:](https://rclone.org/s3/#spectralogic)
|
||||
- StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
|
||||
- Storj [:page_facing_up:](https://rclone.org/storj/)
|
||||
- SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
|
||||
- Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
|
||||
- Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
|
||||
- Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
|
||||
- US3 Object Storage [:page_facing_up:](https://rclone.org/s3/#us3)
|
||||
- Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
|
||||
- WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
- Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
|
||||
- Zadara Object Storage [:page_facing_up:](https://rclone.org/s3/#zadara)
|
||||
- Zero Services (ZERO-Z3) [:page_facing_up:](https://rclone.org/s3/#zero-z3)
|
||||
- Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
|
||||
- Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
|
||||
- The local filesystem [:page_facing_up:](https://rclone.org/local/)
|
||||
@@ -146,7 +125,6 @@ Please see [the full list of all storage providers and their features](https://r
|
||||
These backends adapt or modify other storage providers
|
||||
|
||||
- Alias: rename existing remotes [:page_facing_up:](https://rclone.org/alias/)
|
||||
- Archive: read archive files [:page_facing_up:](https://rclone.org/archive/)
|
||||
- Cache: cache remotes (DEPRECATED) [:page_facing_up:](https://rclone.org/cache/)
|
||||
- Chunker: split large files [:page_facing_up:](https://rclone.org/chunker/)
|
||||
- Combine: combine multiple remotes into a directory tree [:page_facing_up:](https://rclone.org/combine/)
|
||||
|
||||
+50
-159
@@ -6,65 +6,41 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
- [gh the github cli](https://github.com/cli/cli) for uploading packages
|
||||
- pandoc for making the html and man pages
|
||||
- hugo for building the website
|
||||
- golangci-lint and docker for `make check` (markdownlint runs in docker)
|
||||
- a gpg key for signing the tag and the checksums
|
||||
- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
|
||||
`www.rclone.org:` and `test-rclone-org:`
|
||||
- the sponsor logos - see [Sponsor logos](#sponsor-logos)
|
||||
|
||||
## Making a release
|
||||
|
||||
- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
|
||||
- git checkout master # see below for stable branch
|
||||
- git pull # IMPORTANT
|
||||
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
|
||||
- git status - make sure everything is checked in
|
||||
- Check GitHub actions build for master is Green
|
||||
- Check the integration tests at <https://integration.rclone.org>
|
||||
- cat VERSION # check this is the version being released
|
||||
- make tag # signed tag on HEAD + new changelog entry + docs
|
||||
- edit docs/content/changelog.md # entries already in point releases are
|
||||
removed automatically
|
||||
- make test # see integration test server or run locally
|
||||
- make tag
|
||||
- edit docs/content/changelog.md # make sure to remove duplicate logs from point
|
||||
releases
|
||||
- make tidy
|
||||
- make doc # again to pick up the changelog edits
|
||||
- git status - to check for new command docs in docs/content/commands - git add them
|
||||
- make doc
|
||||
- git status - to check for new man pages - git add them
|
||||
- git commit -a -v -m "Version v1.XX.0"
|
||||
- make check # if this fails, fix, git commit --amend and run it again
|
||||
- make retag # moves the tag onto the Version commit
|
||||
- make retag
|
||||
- git push origin # without --follow-tags so it doesn't push the tag if it fails
|
||||
- git push --follow-tags origin
|
||||
- \# Wait for the GitHub builds to complete then...
|
||||
- \# Don't commit anything until after make upload_github as these steps
|
||||
use the tag on HEAD to find the version
|
||||
- make fetch_binaries
|
||||
- make tarball
|
||||
- make vendorball
|
||||
- make sign_upload
|
||||
- make check_sign
|
||||
- make upload
|
||||
- make upload_test_website # check sponsors are correct/unchanged
|
||||
- make upload_website
|
||||
- make upload_github # this triggers the docker plugin and winget workflows
|
||||
- Check the docker image, docker plugin and winget GitHub actions succeeded
|
||||
- make startdev # works out the next version for master or a stable branch
|
||||
- git push
|
||||
- Disable the release freeze rule
|
||||
- Release any security advisories
|
||||
- request a CVE for each advisory if not done already
|
||||
- set the patched version on each advisory and publish it
|
||||
- when the CVEs are assigned replace `CVE-PENDING` in the changelog on
|
||||
master and the stable branch then `make upload_website`
|
||||
- \# announce with forum post, twitter post, and post to announcements list
|
||||
- make upload_github
|
||||
- make startdev # make startstable for stable branch
|
||||
- \# announce with forum post, twitter post, patreon post
|
||||
|
||||
## Update dependencies
|
||||
|
||||
Early in the next release cycle update the dependencies.
|
||||
|
||||
- Active Pins
|
||||
- `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
|
||||
- See: [#9539](https://github.com/rclone/rclone/issues/9539)
|
||||
- Review Active Pins in go.mod and remove if possible
|
||||
- Review any pinned packages in go.mod and remove if possible
|
||||
- `make updatedirect`
|
||||
- `make GOTAGS=cmount`
|
||||
- `make compiletest`
|
||||
@@ -77,18 +53,32 @@ If the `make updatedirect` upgrades the version of go in the `go.mod`
|
||||
go 1.22.0
|
||||
```
|
||||
|
||||
then so this instead `make updatedirectnoupgrade`. This will try each
|
||||
upgrade individually. Consider making an upstream bug report for any
|
||||
`SKIPPING` lines as these only support the latest Go version not the
|
||||
previous version too.
|
||||
then go to manual mode. `go1.22` here is the lowest supported version
|
||||
in the `go.mod`.
|
||||
|
||||
If `make updatedirect` or `make updatedirectnoupgrade` added a
|
||||
`toolchain` directive then remove it. We don't want to force a
|
||||
toolchain on our users. Linux packagers are often using a version of
|
||||
Go that is a few versions out of date.
|
||||
If `make updatedirect` added a `toolchain` directive then remove it.
|
||||
We don't want to force a toolchain on our users. Linux packagers are
|
||||
often using a version of Go that is a few versions out of date.
|
||||
|
||||
```sh
|
||||
go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
|
||||
go get -d $(cat /tmp/potential-upgrades)
|
||||
go mod tidy -go=1.22 -compat=1.22
|
||||
```
|
||||
|
||||
If the `go mod tidy` fails use the output from it to remove the
|
||||
package which can't be upgraded from `/tmp/potential-upgrades` when
|
||||
done
|
||||
|
||||
```sh
|
||||
git co go.mod go.sum
|
||||
```
|
||||
|
||||
And try again.
|
||||
|
||||
Optionally upgrade the direct and indirect dependencies. This is very
|
||||
likely to fail if `make updatedirectnoupgrade` was used.
|
||||
likely to fail if the manual method was used abve - in that case
|
||||
ignore it as it is too time consuming to fix.
|
||||
|
||||
- `make update`
|
||||
- `make GOTAGS=cmount`
|
||||
@@ -106,89 +96,23 @@ build.
|
||||
Once it compiles locally, push it on a test branch and commit fixes
|
||||
until the tests pass.
|
||||
|
||||
### Pseudo versions
|
||||
|
||||
Go makes pseudo versions for untagged repos and repos not at a tag.
|
||||
The pseudo versions on repos that have been tagged before do not get
|
||||
updated automatically so need manually checking. These can be found with
|
||||
|
||||
```console
|
||||
grep -E '[0-9]{14}-[0-9a-f]{12}' go.mod | grep -v indirect | grep -v 'v0\.0\.0'
|
||||
```
|
||||
|
||||
These will need to be updated manually using the `go get ...@branch`
|
||||
syntax.
|
||||
|
||||
### Major versions
|
||||
|
||||
The above procedure will not upgrade major versions, so v2 to v3.
|
||||
However this tool can show which major versions might need to be
|
||||
upgraded:
|
||||
|
||||
```console
|
||||
```sh
|
||||
go run github.com/icholy/gomajor@latest list -major
|
||||
```
|
||||
|
||||
Expect API breakage when updating major versions.
|
||||
|
||||
## Updating Go
|
||||
|
||||
When a new Go stable is released update to it. We support the current
|
||||
stable Go and the previous release which is in line with the rest of
|
||||
the Go ecosystem.
|
||||
|
||||
These files will need editing:
|
||||
|
||||
- `.github/workflows/build.yml` - change current and previous Go versions
|
||||
- `docs/content/install.md` - change minimum Go version required
|
||||
- `fs/versioncheck.go` - update minimum Go version required
|
||||
- `go.mod` - update minimum Go version required
|
||||
|
||||
Check it builds
|
||||
|
||||
- `make GOTAGS=cmount`
|
||||
- `make compiletest`
|
||||
|
||||
Assuming `go1.XX` is current and `go1.YY` is previous version:
|
||||
|
||||
Use `git grep go1.YY` and `git grep go1.YY` to look for opportunities
|
||||
to remove build tags we no longer need.
|
||||
|
||||
Commit with message like this:
|
||||
|
||||
```text
|
||||
build: update to go1.YY and make go1.YY the minimum required version
|
||||
```
|
||||
|
||||
Send to CI and if it passes, merge.
|
||||
|
||||
### gofix
|
||||
|
||||
Updating the minimum required version of Go is a good opportunity to
|
||||
run the `go fix` command to modernize Go usage.
|
||||
|
||||
This needs to be run for all architectures.
|
||||
|
||||
```console
|
||||
GOOS=linux go fix -tags cmount ./...
|
||||
GOOS=freebsd go fix -tags cmount ./...
|
||||
GOOS=windows go fix -tags cmount ./...
|
||||
GOOS=darwin go fix -tags cmount ./...
|
||||
```
|
||||
|
||||
Examine the diff carefully.
|
||||
|
||||
Commit with message
|
||||
|
||||
```text
|
||||
build: modernize Go code with go fix for go1.YY
|
||||
```
|
||||
|
||||
## Tidy beta
|
||||
|
||||
At some point after the release run
|
||||
|
||||
```console
|
||||
```sh
|
||||
bin/tidy-beta v1.55
|
||||
```
|
||||
|
||||
@@ -208,53 +132,20 @@ First make the release branch. If this is a second point release then
|
||||
this will be done already.
|
||||
|
||||
- git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
|
||||
- make startdev # sets the version to ${BASE_TAG}.1
|
||||
- git push -u origin ${BASE_TAG}-stable
|
||||
- make startstable
|
||||
|
||||
Now
|
||||
|
||||
- Enable release freeze rule
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- make fetch-gui-and-commit # update the GUI on master first
|
||||
- git push
|
||||
- git co ${BASE_TAG}-stable
|
||||
- git pull
|
||||
- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
|
||||
- `git cherry-pick -x` any fixes, including the GUI update if there was one
|
||||
- for security releases consider updating the Go patch version in
|
||||
`.github/workflows/build.yml` and dependencies with CVE fixes
|
||||
- fixes may depend on commits which weren't picked so push to CI
|
||||
and check it is green before tagging
|
||||
- Do the steps in "Making a release" above, except
|
||||
- stay on ${BASE_TAG}-stable - don't checkout master or run
|
||||
make fetch-gui-and-commit
|
||||
- check the GitHub actions build for ${BASE_TAG}-stable not master
|
||||
- run the integration tests on ${BASE_TAG}-stable as described below
|
||||
- the release freeze rule is already enabled - leave it until the end
|
||||
- git cherry-pick any fixes
|
||||
- make startstable
|
||||
- Do the steps as above
|
||||
- git co master
|
||||
- git pull --rebase
|
||||
- `#` cherry pick the changes to the changelog - check the diff to make sure it
|
||||
is correct
|
||||
- git checkout ${BASE_TAG}-stable docs/content/changelog.md
|
||||
- git diff --cached # check only the ${NEW_TAG} entry has changed
|
||||
- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
|
||||
- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
|
||||
- git push
|
||||
- Disable release freeze rule
|
||||
|
||||
Point releases are only made for the latest release. Once v1.(XX+1).0
|
||||
is out the ${BASE_TAG}-stable branch is abandoned.
|
||||
|
||||
### Running the integration tests on the stable branch
|
||||
|
||||
The integration tests at <https://integration.rclone.org> test master.
|
||||
To test the stable branch push it to the testing server and run the
|
||||
tests there in `screen` as they take a while.
|
||||
|
||||
```console
|
||||
git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
|
||||
ssh rclone@rclone-testing
|
||||
screen
|
||||
cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
|
||||
```
|
||||
|
||||
## Sponsor logos
|
||||
|
||||
@@ -268,7 +159,7 @@ which is a private repo containing artwork from sponsors.
|
||||
|
||||
Create an update website branch based off the last release
|
||||
|
||||
```console
|
||||
```sh
|
||||
git co -b update-website
|
||||
```
|
||||
|
||||
@@ -276,19 +167,19 @@ If the branch already exists, double check there are no commits that need saving
|
||||
|
||||
Now reset the branch to the last release
|
||||
|
||||
```console
|
||||
```sh
|
||||
git reset --hard v1.64.0
|
||||
```
|
||||
|
||||
Create the changes, check them in, test with `make serve` then
|
||||
|
||||
```console
|
||||
```sh
|
||||
make upload_test_website
|
||||
```
|
||||
|
||||
Check out <https://test.rclone.org> and when happy
|
||||
|
||||
```console
|
||||
```sh
|
||||
make upload_website
|
||||
```
|
||||
|
||||
@@ -298,14 +189,14 @@ Cherry pick any changes back to master and the stable branch if it is active.
|
||||
|
||||
To do a basic build of rclone's docker image to debug builds locally:
|
||||
|
||||
```console
|
||||
```sh
|
||||
docker buildx build --load -t rclone/rclone:testing --progress=plain .
|
||||
docker run --rm rclone/rclone:testing version
|
||||
```
|
||||
|
||||
To test the multipatform build
|
||||
|
||||
```console
|
||||
```sh
|
||||
docker buildx build -t rclone/rclone:testing --progress=plain --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 .
|
||||
```
|
||||
|
||||
@@ -313,6 +204,6 @@ To make a full build then set the tags correctly and add `--push`
|
||||
|
||||
Note that you can't only build one architecture - you need to build them all.
|
||||
|
||||
```console
|
||||
```sh
|
||||
docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
|
||||
```
|
||||
+1
-7
@@ -4,7 +4,6 @@ package all
|
||||
import (
|
||||
// Active file systems
|
||||
_ "github.com/rclone/rclone/backend/alias"
|
||||
_ "github.com/rclone/rclone/backend/archive"
|
||||
_ "github.com/rclone/rclone/backend/azureblob"
|
||||
_ "github.com/rclone/rclone/backend/azurefiles"
|
||||
_ "github.com/rclone/rclone/backend/b2"
|
||||
@@ -16,14 +15,11 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/compress"
|
||||
_ "github.com/rclone/rclone/backend/crypt"
|
||||
_ "github.com/rclone/rclone/backend/doi"
|
||||
_ "github.com/rclone/rclone/backend/dosya"
|
||||
_ "github.com/rclone/rclone/backend/drime"
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/dropbox"
|
||||
_ "github.com/rclone/rclone/backend/fichier"
|
||||
_ "github.com/rclone/rclone/backend/filefabric"
|
||||
_ "github.com/rclone/rclone/backend/filelu"
|
||||
_ "github.com/rclone/rclone/backend/filen"
|
||||
_ "github.com/rclone/rclone/backend/filescom"
|
||||
_ "github.com/rclone/rclone/backend/ftp"
|
||||
_ "github.com/rclone/rclone/backend/gofile"
|
||||
@@ -33,11 +29,9 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/hdfs"
|
||||
_ "github.com/rclone/rclone/backend/hidrive"
|
||||
_ "github.com/rclone/rclone/backend/http"
|
||||
_ "github.com/rclone/rclone/backend/huaweidrive"
|
||||
_ "github.com/rclone/rclone/backend/iclouddrive"
|
||||
_ "github.com/rclone/rclone/backend/imagekit"
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
_ "github.com/rclone/rclone/backend/internxt"
|
||||
_ "github.com/rclone/rclone/backend/jottacloud"
|
||||
_ "github.com/rclone/rclone/backend/koofr"
|
||||
_ "github.com/rclone/rclone/backend/linkbox"
|
||||
@@ -60,7 +54,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
_ "github.com/rclone/rclone/backend/seafile"
|
||||
_ "github.com/rclone/rclone/backend/sftp"
|
||||
_ "github.com/rclone/rclone/backend/shade"
|
||||
_ "github.com/rclone/rclone/backend/sharefile"
|
||||
_ "github.com/rclone/rclone/backend/sia"
|
||||
_ "github.com/rclone/rclone/backend/smb"
|
||||
@@ -69,6 +62,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/swift"
|
||||
_ "github.com/rclone/rclone/backend/ulozto"
|
||||
_ "github.com/rclone/rclone/backend/union"
|
||||
_ "github.com/rclone/rclone/backend/uptobox"
|
||||
_ "github.com/rclone/rclone/backend/webdav"
|
||||
_ "github.com/rclone/rclone/backend/yandex"
|
||||
_ "github.com/rclone/rclone/backend/zoho"
|
||||
|
||||
@@ -1,715 +0,0 @@
|
||||
//go:build !plan9
|
||||
|
||||
// Package archive implements a backend to access archive files in a remote
|
||||
package archive
|
||||
|
||||
// FIXME factor common code between backends out - eg VFS initialization
|
||||
|
||||
// FIXME can we generalize the VFS handle caching and use it in zip backend
|
||||
|
||||
// Factor more stuff out if possible
|
||||
|
||||
// Odd stats which are probably coming from the VFS
|
||||
// * tensorflow.sqfs: 0% /3.074Gi, 204.426Ki/s, 4h22m46s
|
||||
|
||||
// FIXME this will perform poorly for unpacking as the VFS Reader is bad
|
||||
// at multiple streams - need cache mode setting?
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
// Import all the required archivers here
|
||||
_ "github.com/rclone/rclone/backend/archive/squashfs"
|
||||
_ "github.com/rclone/rclone/backend/archive/zip"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fspath"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fsi := &fs.RegInfo{
|
||||
Name: "archive",
|
||||
Description: "Read archives",
|
||||
NewFs: NewFs,
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
Help: `Any metadata supported by the underlying remote is read and written.`,
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: "remote",
|
||||
Help: `Remote to wrap to read archives from.
|
||||
|
||||
Normally should contain a ':' and a path, e.g. "myremote:path/to/dir",
|
||||
"myremote:bucket" or "myremote:".
|
||||
|
||||
If this is left empty, then the archive backend will use the root as
|
||||
the remote.
|
||||
|
||||
This means that you can use :archive:remote:path and it will be
|
||||
equivalent to setting remote="remote:path".
|
||||
`,
|
||||
Required: false,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Remote string `config:"remote"`
|
||||
}
|
||||
|
||||
// Fs represents a archive of upstreams
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
features *fs.Features // optional features
|
||||
opt Options // options for this Fs
|
||||
root string // the path we are working on
|
||||
f fs.Fs // remote we are wrapping
|
||||
wrapper fs.Fs // fs that wraps us
|
||||
|
||||
mu sync.Mutex // protects the below
|
||||
archives map[string]*archive // the archives we have, by path
|
||||
}
|
||||
|
||||
// A single open archive
|
||||
type archive struct {
|
||||
archiver archiver.Archiver // archiver responsible
|
||||
remote string // path to the archive
|
||||
prefix string // prefix to add on to listings
|
||||
root string // root of the archive to remove from listings
|
||||
mu sync.Mutex // protects the following variables
|
||||
f fs.Fs // the archive Fs, may be nil
|
||||
}
|
||||
|
||||
// If remote is an archive then return it otherwise return nil
|
||||
func findArchive(remote string) *archive {
|
||||
// FIXME use something faster than linear search?
|
||||
for _, archiver := range archiver.Archivers {
|
||||
if strings.HasSuffix(remote, archiver.Extension) {
|
||||
return &archive{
|
||||
archiver: archiver,
|
||||
remote: remote,
|
||||
prefix: remote,
|
||||
root: "",
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Find an archive buried in remote
|
||||
func subArchive(remote string) *archive {
|
||||
archive := findArchive(remote)
|
||||
if archive != nil {
|
||||
return archive
|
||||
}
|
||||
parent := path.Dir(remote)
|
||||
if parent == "/" || parent == "." {
|
||||
return nil
|
||||
}
|
||||
return subArchive(parent)
|
||||
}
|
||||
|
||||
// If remote is an archive then return it otherwise return nil
|
||||
func (f *Fs) findArchive(remote string) (archive *archive) {
|
||||
archive = findArchive(remote)
|
||||
if archive != nil {
|
||||
f.mu.Lock()
|
||||
f.archives[remote] = archive
|
||||
f.mu.Unlock()
|
||||
}
|
||||
return archive
|
||||
}
|
||||
|
||||
// Instantiate archive if it hasn't been instantiated yet
|
||||
//
|
||||
// This is done lazily so that we can list a directory full of
|
||||
// archives without opening them all.
|
||||
func (a *archive) init(ctx context.Context, f fs.Fs) (fs.Fs, error) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.f != nil {
|
||||
return a.f, nil
|
||||
}
|
||||
newFs, err := a.archiver.New(ctx, f, a.remote, a.prefix, a.root)
|
||||
if err != nil && err != fs.ErrorIsFile {
|
||||
return nil, fmt.Errorf("failed to create archive %q: %w", a.remote, err)
|
||||
}
|
||||
a.f = newFs
|
||||
return a.f, nil
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path.
|
||||
//
|
||||
// The returned Fs is the actual Fs, referenced by remote in the config
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) {
|
||||
// defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err)
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err = configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := opt.Remote
|
||||
origRoot := root
|
||||
|
||||
// If remote is empty, use the root instead
|
||||
if remote == "" {
|
||||
remote = root
|
||||
root = ""
|
||||
}
|
||||
isDirectory := strings.HasSuffix(remote, "/")
|
||||
remote = strings.TrimRight(remote, "/")
|
||||
if remote == "" {
|
||||
remote = "/"
|
||||
}
|
||||
if strings.HasPrefix(remote, name+":") {
|
||||
return nil, errors.New("can't point archive remote at itself - check the value of the upstreams setting")
|
||||
}
|
||||
|
||||
_ = isDirectory
|
||||
|
||||
foundArchive := subArchive(remote)
|
||||
if foundArchive != nil {
|
||||
fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
|
||||
// Archive path, in canonical form so that it compares equal
|
||||
// to the cleaned entry names inside the archive
|
||||
foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
|
||||
if foundArchive.root == "." {
|
||||
foundArchive.root = ""
|
||||
}
|
||||
// Path to the archive
|
||||
archiveRemote := remote[:len(foundArchive.remote)]
|
||||
// Remote is archive leaf name
|
||||
foundArchive.remote = path.Base(archiveRemote)
|
||||
foundArchive.prefix = ""
|
||||
// Point remote to archive file
|
||||
remote = archiveRemote
|
||||
}
|
||||
|
||||
// Make sure to remove trailing . referring to the current dir
|
||||
if path.Base(root) == "." {
|
||||
root = strings.TrimSuffix(root, ".")
|
||||
}
|
||||
remotePath := fspath.JoinRootPath(remote, root)
|
||||
wrappedFs, err := cache.Get(ctx, remotePath)
|
||||
if err != fs.ErrorIsFile && err != nil {
|
||||
return nil, fmt.Errorf("failed to make remote %q to wrap: %w", remote, err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
//root: path.Join(remotePath, root),
|
||||
root: origRoot,
|
||||
opt: *opt,
|
||||
f: wrappedFs,
|
||||
archives: make(map[string]*archive),
|
||||
}
|
||||
cache.PinUntilFinalized(f.f, f)
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: true,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: true,
|
||||
WriteMimeType: true,
|
||||
CanHaveEmptyDirectories: true,
|
||||
BucketBased: true,
|
||||
SetTier: true,
|
||||
GetTier: true,
|
||||
ReadMetadata: true,
|
||||
WriteMetadata: true,
|
||||
UserMetadata: true,
|
||||
PartialUploads: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
if foundArchive != nil {
|
||||
fs.Debugf(f, "Root is an archive")
|
||||
if err != fs.ErrorIsFile {
|
||||
return nil, fmt.Errorf("expecting to find a file at %q", remote)
|
||||
}
|
||||
return foundArchive.init(ctx, f.f)
|
||||
}
|
||||
// Correct root if definitely pointing to a file
|
||||
if err == fs.ErrorIsFile {
|
||||
f.root = path.Dir(f.root)
|
||||
if f.root == "." || f.root == "/" {
|
||||
f.root = ""
|
||||
}
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String converts this Fs to a string
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("archive root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// Rmdir removes the root directory of the Fs object
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return f.f.Rmdir(ctx, dir)
|
||||
}
|
||||
|
||||
// Hashes returns hash.HashNone to indicate remote hashing is unavailable
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return f.f.Hashes()
|
||||
}
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return f.f.Mkdir(ctx, dir)
|
||||
}
|
||||
|
||||
// Purge all files in the directory
|
||||
//
|
||||
// Implement this if you have a way of deleting all the files
|
||||
// quicker than just running Remove() on the result of List()
|
||||
//
|
||||
// Return an error if it doesn't exist
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
do := f.f.Features().Purge
|
||||
if do == nil {
|
||||
return fs.ErrorCantPurge
|
||||
}
|
||||
return do(ctx, dir)
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side copy operations.
|
||||
//
|
||||
// This is stored with the remote path given.
|
||||
//
|
||||
// It returns the destination Object and a possible error.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantCopy
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
do := f.f.Features().Copy
|
||||
if do == nil {
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
// FIXME
|
||||
// o, ok := src.(*Object)
|
||||
// if !ok {
|
||||
// return nil, fs.ErrorCantCopy
|
||||
// }
|
||||
return do(ctx, src, remote)
|
||||
}
|
||||
|
||||
// Move src to this remote using server-side move operations.
|
||||
//
|
||||
// This is stored with the remote path given.
|
||||
//
|
||||
// It returns the destination Object and a possible error.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantMove
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
do := f.f.Features().Move
|
||||
if do == nil {
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
// FIXME
|
||||
// o, ok := src.(*Object)
|
||||
// if !ok {
|
||||
// return nil, fs.ErrorCantMove
|
||||
// }
|
||||
return do(ctx, src, remote)
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
// using server-side move operations.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantDirMove
|
||||
//
|
||||
// If destination exists then return fs.ErrorDirExists
|
||||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
|
||||
do := f.f.Features().DirMove
|
||||
if do == nil {
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
return do(ctx, srcFs.f, srcRemote, dstRemote)
|
||||
}
|
||||
|
||||
// ChangeNotify calls the passed function with a path
|
||||
// that has had changes. If the implementation
|
||||
// uses polling, it should adhere to the given interval.
|
||||
// At least one value will be written to the channel,
|
||||
// specifying the initial value and updated values might
|
||||
// follow. A 0 Duration should pause the polling.
|
||||
// The ChangeNotify implementation must empty the channel
|
||||
// regularly. When the channel gets closed, the implementation
|
||||
// should stop polling and release resources.
|
||||
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), ch <-chan time.Duration) {
|
||||
do := f.f.Features().ChangeNotify
|
||||
if do == nil {
|
||||
return
|
||||
}
|
||||
wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
|
||||
// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
|
||||
notifyFunc(path, entryType)
|
||||
}
|
||||
do(ctx, wrappedNotifyFunc, ch)
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing
|
||||
// as an optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
do := f.f.Features().DirCacheFlush
|
||||
if do != nil {
|
||||
do()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
|
||||
var o fs.Object
|
||||
var err error
|
||||
if stream {
|
||||
o, err = f.f.Features().PutStream(ctx, in, src, options...)
|
||||
} else {
|
||||
o, err = f.f.Put(ctx, in, src, options...)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
o, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return o, o.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, false, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// PutStream uploads to the remote path with the modTime given of indeterminate size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
o, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return o, o.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, true, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.f.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// Find the Fs for the directory
|
||||
func (f *Fs) findFs(ctx context.Context, dir string) (subFs fs.Fs, err error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
subFs = f.f
|
||||
|
||||
// FIXME should do this with a better datastructure like a prefix tree
|
||||
// FIXME want to find the longest first otherwise nesting won't work
|
||||
dirSlash := dir + "/"
|
||||
for archiverRemote, archive := range f.archives {
|
||||
subRemote := archiverRemote + "/"
|
||||
if strings.HasPrefix(dirSlash, subRemote) {
|
||||
subFs, err = archive.init(ctx, f.f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return subFs, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
|
||||
|
||||
subFs, err := f.findFs(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
subEntries, err := subFs.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = subEntries[:0]
|
||||
skipped := 0
|
||||
for _, entry := range subEntries {
|
||||
remote := entry.Remote()
|
||||
// Only pass on direct children of dir - anything else
|
||||
// could escape the archive's namespace
|
||||
if !isDirectChild(dir, remote) {
|
||||
fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
// Can only unarchive files
|
||||
if o, ok := entry.(fs.Object); ok {
|
||||
archive := f.findArchive(remote)
|
||||
if archive != nil {
|
||||
// Overwrite entry with directory
|
||||
entry = fs.NewDir(remote, o.ModTime(ctx))
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// isDirectChild reports whether remote names an entry directly in dir
|
||||
// ("" being the root), with no ".." or other components in between.
|
||||
func isDirectChild(dir, remote string) bool {
|
||||
if dir != "" {
|
||||
var ok bool
|
||||
remote, ok = strings.CutPrefix(remote, dir+"/")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return sanitize.Leaf(remote) == nil
|
||||
}
|
||||
|
||||
// NewObject creates a new remote archive file object
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
dir := path.Dir(remote)
|
||||
if dir == "/" || dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
|
||||
subFs, err := f.findFs(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
o, err := subFs.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Don't trust the archiver to have returned the object asked for
|
||||
if o.Remote() != remote {
|
||||
fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Precision is the greatest precision of all the archivers
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Shutdown the backend, closing any background tasks and any
|
||||
// cached connections.
|
||||
func (f *Fs) Shutdown(ctx context.Context) error {
|
||||
if do := f.f.Features().Shutdown; do != nil {
|
||||
return do(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
do := f.f.Features().PublicLink
|
||||
if do == nil {
|
||||
return "", errors.New("PublicLink not supported")
|
||||
}
|
||||
return do(ctx, remote, expire, unlink)
|
||||
}
|
||||
|
||||
// PutUnchecked in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
//
|
||||
// May create duplicates or return errors if src already
|
||||
// exists.
|
||||
func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
do := f.f.Features().PutUnchecked
|
||||
if do == nil {
|
||||
return nil, errors.New("can't PutUnchecked")
|
||||
}
|
||||
o, err := do(ctx, in, src, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// MergeDirs merges the contents of all the directories passed
|
||||
// in into the first one and rmdirs the other directories.
|
||||
func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
if len(dirs) == 0 {
|
||||
return nil
|
||||
}
|
||||
do := f.f.Features().MergeDirs
|
||||
if do == nil {
|
||||
return errors.New("MergeDirs not supported")
|
||||
}
|
||||
return do(ctx, dirs)
|
||||
}
|
||||
|
||||
// CleanUp the trash in the Fs
|
||||
//
|
||||
// Implement this if you have a way of emptying the trash or
|
||||
// otherwise cleaning up old versions of files.
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.f.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// OpenWriterAt opens with a handle for random access writes
|
||||
//
|
||||
// Pass in the remote desired and the size if known.
|
||||
//
|
||||
// It truncates any existing object
|
||||
func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
|
||||
do := f.f.Features().OpenWriterAt
|
||||
if do == nil {
|
||||
return nil, fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx, remote, size)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// OpenChunkWriter returns the chunk size and a ChunkWriter
|
||||
//
|
||||
// Pass in the remote and the src object
|
||||
// You can also use options to hint at the desired chunk size
|
||||
func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
|
||||
do := f.f.Features().OpenChunkWriter
|
||||
if do == nil {
|
||||
return info, nil, fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx, remote, src, options...)
|
||||
}
|
||||
|
||||
// UserInfo returns info about the connected user
|
||||
func (f *Fs) UserInfo(ctx context.Context) (map[string]string, error) {
|
||||
do := f.f.Features().UserInfo
|
||||
if do == nil {
|
||||
return nil, fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// Disconnect the current user
|
||||
func (f *Fs) Disconnect(ctx context.Context) error {
|
||||
do := f.f.Features().Disconnect
|
||||
if do == nil {
|
||||
return fs.ErrorNotImplemented
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PutStreamer = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ fs.ChangeNotifier = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.Shutdowner = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.PutUncheckeder = (*Fs)(nil)
|
||||
_ fs.MergeDirser = (*Fs)(nil)
|
||||
_ fs.CleanUpper = (*Fs)(nil)
|
||||
_ fs.OpenWriterAter = (*Fs)(nil)
|
||||
_ fs.OpenChunkWriter = (*Fs)(nil)
|
||||
_ fs.UserInfoer = (*Fs)(nil)
|
||||
_ fs.Disconnecter = (*Fs)(nil)
|
||||
// FIXME _ fs.FullObject = (*Object)(nil)
|
||||
)
|
||||
@@ -1,519 +0,0 @@
|
||||
//go:build !plan9
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// FIXME need to test Open with seek
|
||||
|
||||
// run - run a shell command
|
||||
func run(t *testing.T, args ...string) {
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
fs.Debugf(nil, "run args = %v", args)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf(`
|
||||
----------------------------
|
||||
Failed to run %v: %v
|
||||
Command output was:
|
||||
%s
|
||||
----------------------------
|
||||
`, args, err, out)
|
||||
}
|
||||
}
|
||||
|
||||
// check the dst and src are identical
|
||||
func checkTree(ctx context.Context, name string, t *testing.T, dstArchive, src string, expectedCount int) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fs.Debugf(nil, "check %q vs %q", dstArchive, src)
|
||||
Farchive, err := cache.Get(ctx, dstArchive)
|
||||
if err != fs.ErrorIsFile {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
Fsrc, err := cache.Get(ctx, src)
|
||||
if err != fs.ErrorIsFile {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
var matches bytes.Buffer
|
||||
opt := operations.CheckOpt{
|
||||
Fdst: Farchive,
|
||||
Fsrc: Fsrc,
|
||||
Match: &matches,
|
||||
}
|
||||
|
||||
for _, action := range []string{"Check", "Download"} {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
matches.Reset()
|
||||
if action == "Download" {
|
||||
assert.NoError(t, operations.CheckDownload(ctx, &opt))
|
||||
} else {
|
||||
assert.NoError(t, operations.Check(ctx, &opt))
|
||||
}
|
||||
if expectedCount > 0 {
|
||||
assert.Equal(t, expectedCount, strings.Count(matches.String(), "\n"))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("NewObject", func(t *testing.T) {
|
||||
// Check we can run NewObject on all files and read them
|
||||
assert.NoError(t, operations.ListFn(ctx, Fsrc, func(srcObj fs.Object) {
|
||||
if t.Failed() {
|
||||
return
|
||||
}
|
||||
remote := srcObj.Remote()
|
||||
archiveObj, err := Farchive.NewObject(ctx, remote)
|
||||
require.NoError(t, err, remote)
|
||||
assert.Equal(t, remote, archiveObj.Remote(), remote)
|
||||
|
||||
// Test that the contents are the same
|
||||
archiveBuf := fstests.ReadObject(ctx, t, archiveObj, -1)
|
||||
srcBuf := fstests.ReadObject(ctx, t, srcObj, -1)
|
||||
assert.Equal(t, srcBuf, archiveBuf)
|
||||
|
||||
if len(srcBuf) < 81 {
|
||||
return
|
||||
}
|
||||
|
||||
// Tests that Open works with SeekOption
|
||||
assert.Equal(t, srcBuf[50:], fstests.ReadObject(ctx, t, archiveObj, -1, &fs.SeekOption{Offset: 50}), "contents differ after seek")
|
||||
|
||||
// Tests that Open works with RangeOption
|
||||
for _, test := range []struct {
|
||||
ro fs.RangeOption
|
||||
wantStart, wantEnd int
|
||||
}{
|
||||
{fs.RangeOption{Start: 5, End: 15}, 5, 16},
|
||||
{fs.RangeOption{Start: 80, End: -1}, 80, len(srcBuf)},
|
||||
{fs.RangeOption{Start: 81, End: 100000}, 81, len(srcBuf)},
|
||||
{fs.RangeOption{Start: -1, End: 20}, len(srcBuf) - 20, len(srcBuf)}, // if start is omitted this means get the final bytes
|
||||
// {fs.RangeOption{Start: -1, End: -1}, 0, len(srcBuf)}, - this seems to work but the RFC doesn't define it
|
||||
} {
|
||||
got := fstests.ReadObject(ctx, t, archiveObj, -1, &test.ro)
|
||||
foundAt := strings.Index(srcBuf, got)
|
||||
help := fmt.Sprintf("%#v failed want [%d:%d] got [%d:%d]", test.ro, test.wantStart, test.wantEnd, foundAt, foundAt+len(got))
|
||||
assert.Equal(t, srcBuf[test.wantStart:test.wantEnd], got, help)
|
||||
}
|
||||
|
||||
// Test that the modtimes are correct
|
||||
fstest.AssertTimeEqualWithPrecision(t, remote, srcObj.ModTime(ctx), archiveObj.ModTime(ctx), Farchive.Precision())
|
||||
|
||||
// Test that the sizes are correct
|
||||
assert.Equal(t, srcObj.Size(), archiveObj.Size())
|
||||
|
||||
// Test that Strings are OK
|
||||
assert.Equal(t, srcObj.String(), archiveObj.String())
|
||||
}))
|
||||
})
|
||||
|
||||
// t.Logf("Fdst ------------- %v", Fdst)
|
||||
// operations.List(ctx, Fdst, os.Stdout)
|
||||
// t.Logf("Fsrc ------------- %v", Fsrc)
|
||||
// operations.List(ctx, Fsrc, os.Stdout)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// test creating and reading back some archives
|
||||
//
|
||||
// Note that this uses rclone and zip as external binaries.
|
||||
func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T, output, input string)) {
|
||||
ctx := context.Background()
|
||||
checkFiles := 1000
|
||||
|
||||
// create random test input files
|
||||
inputRoot := t.TempDir()
|
||||
input := filepath.Join(inputRoot, archiveName)
|
||||
require.NoError(t, os.Mkdir(input, 0777))
|
||||
run(t, "rclone", "test", "makefiles", "--files", strconv.Itoa(checkFiles), "--ascii", input)
|
||||
|
||||
// Create the archive
|
||||
output := t.TempDir()
|
||||
zipFile := path.Join(output, archiveName)
|
||||
archiveFn(t, zipFile, input)
|
||||
|
||||
// Check the archive itself
|
||||
checkTree(ctx, "Archive", t, ":archive:"+zipFile, input, checkFiles)
|
||||
|
||||
// Now check a subdirectory
|
||||
fis, err := os.ReadDir(input)
|
||||
require.NoError(t, err)
|
||||
subDir := "NOT FOUND"
|
||||
aFile := "NOT FOUND"
|
||||
for _, fi := range fis {
|
||||
if fi.IsDir() {
|
||||
subDir = fi.Name()
|
||||
} else {
|
||||
aFile = fi.Name()
|
||||
}
|
||||
}
|
||||
checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
|
||||
|
||||
// Now check a single file
|
||||
//
|
||||
// The filter below applies to both sides, so check first that the
|
||||
// archive exposes only that file without it
|
||||
Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
|
||||
require.NoError(t, err)
|
||||
entries, err := Fsingle.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
_, err = Fsingle.List(ctx, subDir)
|
||||
assert.Equal(t, fs.ErrorDirNotFound, err)
|
||||
_, err = Fsingle.NewObject(ctx, subDir)
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
fiCtx, fi := filter.AddConfig(ctx)
|
||||
require.NoError(t, fi.AddRule("+ "+aFile))
|
||||
require.NoError(t, fi.AddRule("- *"))
|
||||
checkTree(fiCtx, "SingleFile", t, ":archive:"+zipFile+"/"+aFile, filepath.Join(input, aFile), 0)
|
||||
|
||||
// Now check the level above
|
||||
checkTree(ctx, "Root", t, ":archive:"+output, inputRoot, checkFiles)
|
||||
// run(t, "cp", "-a", inputRoot, output, "/tmp/test-"+archiveName)
|
||||
}
|
||||
|
||||
// Make sure we have the executable named
|
||||
func skipIfNoExe(t *testing.T, exeName string) {
|
||||
_, err := exec.LookPath(exeName)
|
||||
if err != nil {
|
||||
t.Skipf("%s executable not installed", exeName)
|
||||
}
|
||||
}
|
||||
|
||||
// Test creating and reading back some archives
|
||||
//
|
||||
// Note that this uses rclone and zip as external binaries.
|
||||
func TestArchiveZip(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "zip")
|
||||
skipIfNoExe(t, "rclone")
|
||||
testArchive(t, "test.zip", func(t *testing.T, output, input string) {
|
||||
oldcwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Chdir(input))
|
||||
defer func() {
|
||||
require.NoError(t, os.Chdir(oldcwd))
|
||||
}()
|
||||
run(t, "zip", "-9r", output, ".")
|
||||
})
|
||||
}
|
||||
|
||||
// Test creating and reading back some archives
|
||||
//
|
||||
// Note that this uses rclone and squashfs as external binaries.
|
||||
func TestArchiveSquashfs(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "mksquashfs")
|
||||
skipIfNoExe(t, "rclone")
|
||||
testArchive(t, "test.sqfs", func(t *testing.T, output, input string) {
|
||||
run(t, "mksquashfs", input, output)
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveSquashfsIssue9004 lists and reads squashfs images that exercise
|
||||
// two layouts go-diskfs used to choke on (fixed in go-diskfs v1.9.4):
|
||||
//
|
||||
// - 1.sqfs: a single empty directory, so the image has no fragment table
|
||||
// (its fragment-table start holds the "not present" sentinel).
|
||||
// - 2.sqfs: a small tree whose superblock has the NO_XATTRS flag set while
|
||||
// inodes still carry a (non-sentinel) xattr index - the shape squashfs-
|
||||
// tools-ng can emit. Built by packing a two-file tree with xattrs via
|
||||
// `gensquashfs -x`, then setting the NO_XATTRS superblock flag; the tree
|
||||
// content is trivial placeholder data.
|
||||
//
|
||||
// Both images used to fail to list. Regression test for #9004.
|
||||
func TestArchiveSquashfsIssue9004(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
testdata, err := filepath.Abs(filepath.Join("squashfs", "testdata"))
|
||||
require.NoError(t, err)
|
||||
|
||||
archiveFor := func(t *testing.T, name string) fs.Fs {
|
||||
f, err := cache.Get(ctx, ":archive:"+filepath.Join(testdata, name))
|
||||
require.NoError(t, err)
|
||||
return f
|
||||
}
|
||||
|
||||
t.Run("EmptyDir", func(t *testing.T) {
|
||||
// 1.sqfs is a single empty directory - it must list without error.
|
||||
entries, err := archiveFor(t, "1.sqfs").List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, len(entries))
|
||||
})
|
||||
|
||||
t.Run("NoXattrTree", func(t *testing.T) {
|
||||
f := archiveFor(t, "2.sqfs")
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, path.Base(e.Remote()))
|
||||
}
|
||||
assert.Contains(t, names, "alpha")
|
||||
assert.Contains(t, names, "beta")
|
||||
|
||||
// A file in the tree must be readable with its real content.
|
||||
obj, err := f.NewObject(ctx, "beta/sample.xml")
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, obj.Size(), int64(0))
|
||||
rc, err := obj.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
data, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, int(obj.Size()), len(data))
|
||||
assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
|
||||
})
|
||||
}
|
||||
|
||||
// TestArchiveUncleanRoot checks that a path into an archive which isn't
|
||||
// in canonical form (with "./" or doubled slashes) still finds its
|
||||
// directory.
|
||||
func TestArchiveUncleanRoot(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create("sub/dir/a.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
zipPath := filepath.Join(t.TempDir(), "test.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
|
||||
for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
|
||||
t.Run(root, func(t *testing.T) {
|
||||
f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
|
||||
require.NoError(t, err)
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "a.txt", entries[0].Remote())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// escapingObject is an object whose remote is not where it was asked for.
|
||||
type escapingObject struct {
|
||||
fs.Object
|
||||
remote string
|
||||
}
|
||||
|
||||
func (o *escapingObject) Remote() string { return o.remote }
|
||||
func (o *escapingObject) String() string { return o.remote }
|
||||
|
||||
// escapingFs stands in for a badly behaved archiver which exposes entry
|
||||
// names outside the directory being listed.
|
||||
type escapingFs struct {
|
||||
fs.Fs
|
||||
prefix string
|
||||
}
|
||||
|
||||
func (f *escapingFs) Name() string { return "escaping" }
|
||||
func (f *escapingFs) Root() string { return "" }
|
||||
func (f *escapingFs) String() string { return "escaping" }
|
||||
func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
|
||||
|
||||
func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return fs.DirEntries{
|
||||
&escapingObject{remote: path.Join(dir, "good.txt")},
|
||||
fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
&escapingObject{remote: path.Join(dir, "../escape.txt")},
|
||||
&escapingObject{remote: "../../escape.txt"},
|
||||
&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
|
||||
fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
return &escapingObject{remote: "../escape.txt"}, nil
|
||||
}
|
||||
|
||||
// TestArchiveEscapingArchiver checks that the archive backend does not
|
||||
// pass on entries from an archiver which escape the directory being
|
||||
// listed, whatever the archiver does.
|
||||
func TestArchiveEscapingArchiver(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
return &escapingFs{prefix: prefix}, nil
|
||||
},
|
||||
Extension: ".escaping",
|
||||
})
|
||||
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
|
||||
f, err := cache.Get(ctx, ":archive:"+dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Archives are discovered when their parent directory is listed
|
||||
_, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "test.escaping")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
|
||||
|
||||
_, err = f.NewObject(ctx, "test.escaping/file.txt")
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
|
||||
}
|
||||
|
||||
// TestIsDirectChild checks the guard which decides whether an entry
|
||||
// returned by an archiver belongs directly in the directory listed.
|
||||
func TestIsDirectChild(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
dir, remote string
|
||||
want bool
|
||||
}{
|
||||
{"", "a.txt", true},
|
||||
{"", "/a.txt", false},
|
||||
{"", "a.txt/", false},
|
||||
{"", "../a.txt", false},
|
||||
{"", "sub/a.txt", false},
|
||||
{"d", "d/a.txt", true},
|
||||
{"d", "d", false},
|
||||
{"d", "d/", false},
|
||||
{"d", "d//a.txt", false},
|
||||
{"d", "d/../a.txt", false},
|
||||
{"d", "dd/a.txt", false},
|
||||
{"d", "a.txt", false},
|
||||
} {
|
||||
assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
|
||||
}
|
||||
}
|
||||
|
||||
// A symlink in a squashfs image is skipped unless -l/--links is in use,
|
||||
// as it is on the other backends. With the flag it is exposed under its
|
||||
// name with the link suffix and reads back as its target.
|
||||
//
|
||||
// Note that this uses mksquashfs as an external binary.
|
||||
func TestArchiveSquashfsSymlink(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
skipIfNoExe(t, "mksquashfs")
|
||||
ctx := context.Background()
|
||||
|
||||
dir := t.TempDir()
|
||||
input := filepath.Join(dir, "input")
|
||||
require.NoError(t, os.Mkdir(input, 0777))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
|
||||
require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
|
||||
// a regular file which happens to carry the link suffix must stay
|
||||
// readable under the flag, not be taken for a symlink
|
||||
require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
|
||||
image := filepath.Join(dir, "test.sqfs")
|
||||
run(t, "mksquashfs", input, image)
|
||||
|
||||
remotes := func(t *testing.T, f fs.Fs) []string {
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
names = append(names, entry.Remote())
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
read := func(t *testing.T, f fs.Fs, remote string) string {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
require.NoError(t, err)
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
contents, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, o.Size(), int64(len(contents)))
|
||||
return string(contents)
|
||||
}
|
||||
|
||||
f, err := fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = fs.NewFs(ctx, ":archive:"+image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
|
||||
|
||||
assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
|
||||
assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
|
||||
}
|
||||
|
||||
// A hostile archive must not be able to write outside the destination.
|
||||
// A symlink pointing above the root with a member inside it is the
|
||||
// classic way to try it, and the refusal comes from the local backend,
|
||||
// so pin it from end to end here rather than trusting it stays.
|
||||
func TestArchiveZipSymlinkEscape(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
hdr := &zip.FileHeader{Name: "a"}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err := zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("../outside"))
|
||||
require.NoError(t, err)
|
||||
w, err = zw.Create("a/pwned.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("pwned"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
zipPath := filepath.Join(dir, "evil.zip")
|
||||
require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
|
||||
dst := filepath.Join(dir, "dst")
|
||||
require.NoError(t, os.Mkdir(dst, 0777))
|
||||
|
||||
fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
|
||||
require.NoError(t, err)
|
||||
fdst, err := fs.NewFs(ctx, dst)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
|
||||
|
||||
_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
|
||||
assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
//go:build !plan9
|
||||
|
||||
// Test Archive filesystem interface
|
||||
package archive_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
var (
|
||||
unimplementableFsMethods = []string{"ListR", "ListP", "MkdirMetadata", "DirSetModTime"}
|
||||
// In these tests we receive objects from the underlying remote which don't implement these methods
|
||||
unimplementableObjectMethods = []string{"GetTier", "ID", "Metadata", "MimeType", "SetTier", "UnWrap", "SetMetadata"}
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
if *fstest.RemoteName == "" {
|
||||
t.Skip("Skipping as -remote not set")
|
||||
}
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: *fstest.RemoteName,
|
||||
UnimplementableFsMethods: unimplementableFsMethods,
|
||||
UnimplementableObjectMethods: unimplementableObjectMethods,
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocal(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
remote := t.TempDir()
|
||||
name := "TestArchiveLocal"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "archive"},
|
||||
{Name: name, Key: "remote", Value: remote},
|
||||
},
|
||||
QuickTestOK: true,
|
||||
UnimplementableFsMethods: unimplementableFsMethods,
|
||||
UnimplementableObjectMethods: unimplementableObjectMethods,
|
||||
})
|
||||
}
|
||||
|
||||
func TestMemory(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
remote := ":memory:"
|
||||
name := "TestArchiveMemory"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "archive"},
|
||||
{Name: name, Key: "remote", Value: remote},
|
||||
},
|
||||
QuickTestOK: true,
|
||||
UnimplementableFsMethods: unimplementableFsMethods,
|
||||
UnimplementableObjectMethods: unimplementableObjectMethods,
|
||||
})
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
// Build for archive for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9
|
||||
|
||||
// Package archive implements a backend to access archive files in a remote
|
||||
package archive
|
||||
@@ -1,29 +0,0 @@
|
||||
// Package archiver registers all the archivers
|
||||
package archiver
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
// Archiver describes an archive package
|
||||
//
|
||||
// Entry names inside an archive are attacker controlled. An archiver
|
||||
// must not expose an entry whose remote escapes the archive's own
|
||||
// namespace (for example one with a ".." component), typically by
|
||||
// validating names with lib/sanitize.
|
||||
type Archiver struct {
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
New func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error)
|
||||
Extension string
|
||||
}
|
||||
|
||||
// Archivers is a slice of all registered archivers
|
||||
var Archivers []Archiver
|
||||
|
||||
// Register adds the archivers provided to the list of known archivers
|
||||
func Register(as ...Archiver) {
|
||||
Archivers = append(Archivers, as...)
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
// Package base is a base archive Fs
|
||||
package base
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
)
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
f fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
features *fs.Features // optional features
|
||||
vfs *vfs.VFS
|
||||
node vfs.Node // archive object
|
||||
remote string // remote of the archive object
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
}
|
||||
|
||||
var errNotImplemented = errors.New("internal error: method not implemented in archiver")
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs, error) {
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
VFS := vfs.New(ctx, wrappedFs, nil)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
f: wrappedFs,
|
||||
name: path.Join(fs.ConfigString(wrappedFs), remote),
|
||||
vfs: VFS,
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
|
||||
// FIXME
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
//
|
||||
// FIXME some of these need to be forced on - CanHaveEmptyDirectories
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: false, // MimeTypes not supported with gzip
|
||||
WriteMimeType: false,
|
||||
BucketBased: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
|
||||
return nil, vfs.EROFS
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.None)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// Object describes an object to be read from the raw zip file
|
||||
type Object struct {
|
||||
f *Fs
|
||||
remote string
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.f
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.Remote()
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return -1
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Storable raturns a boolean indicating if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
return nil, errNotImplemented
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.UnWrapper = (*Fs)(nil)
|
||||
_ fs.Wrapper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
)
|
||||
@@ -1,170 +0,0 @@
|
||||
package squashfs
|
||||
|
||||
// Could just be using bare object Open with RangeRequest which
|
||||
// would transfer the minimum amount of data but may be slower.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/diskfs/go-diskfs/backend"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
)
|
||||
|
||||
// Cache file handles for accessing the file
|
||||
type cache struct {
|
||||
node vfs.Node
|
||||
fhsMu sync.Mutex
|
||||
fhs []cacheHandle
|
||||
}
|
||||
|
||||
// A cached file handle
|
||||
type cacheHandle struct {
|
||||
offset int64
|
||||
fh vfs.Handle
|
||||
}
|
||||
|
||||
// Make a new cache
|
||||
func newCache(node vfs.Node) *cache {
|
||||
return &cache{
|
||||
node: node,
|
||||
}
|
||||
}
|
||||
|
||||
// Get a vfs.Handle from the pool or open one
|
||||
//
|
||||
// This tries to find an open file handle which doesn't require seeking.
|
||||
func (c *cache) open(off int64) (fh vfs.Handle, err error) {
|
||||
c.fhsMu.Lock()
|
||||
defer c.fhsMu.Unlock()
|
||||
|
||||
if len(c.fhs) > 0 {
|
||||
// Look for exact match first
|
||||
for i, cfh := range c.fhs {
|
||||
if cfh.offset == off {
|
||||
// fs.Debugf(nil, "CACHE MATCH")
|
||||
c.fhs = append(c.fhs[:i], c.fhs[i+1:]...)
|
||||
return cfh.fh, nil
|
||||
|
||||
}
|
||||
}
|
||||
// fs.Debugf(nil, "CACHE MISS")
|
||||
// Just take the first one if not found
|
||||
cfh := c.fhs[0]
|
||||
c.fhs = c.fhs[1:]
|
||||
return cfh.fh, nil
|
||||
}
|
||||
|
||||
fh, err = c.node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open squashfs archive: %w", err)
|
||||
}
|
||||
|
||||
return fh, nil
|
||||
}
|
||||
|
||||
// Close a vfs.Handle or return it to the pool
|
||||
//
|
||||
// off should be the offset the file handle would read from without seeking
|
||||
func (c *cache) close(fh vfs.Handle, off int64) {
|
||||
c.fhsMu.Lock()
|
||||
defer c.fhsMu.Unlock()
|
||||
|
||||
c.fhs = append(c.fhs, cacheHandle{
|
||||
offset: off,
|
||||
fh: fh,
|
||||
})
|
||||
}
|
||||
|
||||
// ReadAt reads len(p) bytes into p starting at offset off in the underlying
|
||||
// input source. It returns the number of bytes read (0 <= n <= len(p)) and any
|
||||
// error encountered.
|
||||
//
|
||||
// When ReadAt returns n < len(p), it returns a non-nil error explaining why
|
||||
// more bytes were not returned. In this respect, ReadAt is stricter than Read.
|
||||
//
|
||||
// Even if ReadAt returns n < len(p), it may use all of p as scratch
|
||||
// space during the call. If some data is available but not len(p) bytes,
|
||||
// ReadAt blocks until either all the data is available or an error occurs.
|
||||
// In this respect ReadAt is different from Read.
|
||||
//
|
||||
// If the n = len(p) bytes returned by ReadAt are at the end of the input
|
||||
// source, ReadAt may return either err == EOF or err == nil.
|
||||
//
|
||||
// If ReadAt is reading from an input source with a seek offset, ReadAt should
|
||||
// not affect nor be affected by the underlying seek offset.
|
||||
//
|
||||
// Clients of ReadAt can execute parallel ReadAt calls on the same input
|
||||
// source.
|
||||
//
|
||||
// Implementations must not retain p.
|
||||
func (c *cache) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
fh, err := c.open(off)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
defer func() {
|
||||
c.close(fh, off+int64(len(p)))
|
||||
}()
|
||||
// fs.Debugf(nil, "ReadAt(p[%d], off=%d, fh=%p)", len(p), off, fh)
|
||||
return fh.ReadAt(p, off)
|
||||
}
|
||||
|
||||
var errCacheNotImplemented = errors.New("internal error: squashfs cache doesn't implement method")
|
||||
|
||||
// WriteAt method dummy stub to satisfy interface
|
||||
func (c *cache) WriteAt(p []byte, off int64) (n int, err error) {
|
||||
return 0, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Seek method dummy stub to satisfy interface
|
||||
func (c *cache) Seek(offset int64, whence int) (int64, error) {
|
||||
return 0, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Read method dummy stub to satisfy interface
|
||||
func (c *cache) Read(p []byte) (n int, err error) {
|
||||
return 0, errCacheNotImplemented
|
||||
}
|
||||
|
||||
func (c *cache) Stat() (fs.FileInfo, error) {
|
||||
return nil, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Close the file
|
||||
func (c *cache) Close() (err error) {
|
||||
c.fhsMu.Lock()
|
||||
defer c.fhsMu.Unlock()
|
||||
|
||||
// Close any open file handles
|
||||
for i := range c.fhs {
|
||||
fh := &c.fhs[i]
|
||||
newErr := fh.fh.Close()
|
||||
if err == nil {
|
||||
err = newErr
|
||||
}
|
||||
}
|
||||
c.fhs = nil
|
||||
return err
|
||||
}
|
||||
|
||||
// Sys returns OS-specific file for ioctl calls via fd
|
||||
func (c *cache) Sys() (*os.File, error) {
|
||||
return nil, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Writable returns file for read-write operations
|
||||
func (c *cache) Writable() (backend.WritableFile, error) {
|
||||
return nil, errCacheNotImplemented
|
||||
}
|
||||
|
||||
// Path returns the path to the underlying storage - not applicable here
|
||||
func (c *cache) Path() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// check interfaces
|
||||
var _ backend.Storage = (*cache)(nil)
|
||||
@@ -1,587 +0,0 @@
|
||||
// Package squashfs implements a squashfs archiver for the archive backend
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/diskfs/go-diskfs/filesystem/squashfs"
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
|
||||
func init() {
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: New,
|
||||
Extension: ".sqfs",
|
||||
})
|
||||
}
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
f fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
features *fs.Features // optional features
|
||||
vfs *vfs.VFS
|
||||
sqfs *squashfs.FileSystem // interface to the squashfs
|
||||
c *cache
|
||||
node vfs.Node // squashfs file object - set if reading
|
||||
remote string // remote of the squashfs file object
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
file string // remote of the single file root points at, if any
|
||||
links bool // read symlinks as .rclonelink files
|
||||
}
|
||||
|
||||
// recoverParsePanic converts a panic raised while parsing a squashfs image
|
||||
// into an error assigned through err.
|
||||
func recoverParsePanic(err *error) {
|
||||
if r := recover(); r != nil {
|
||||
*err = fmt.Errorf("invalid or corrupt squashfs image: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// recoveringReadCloser wraps a reader returned by the go-diskfs parser so
|
||||
// panics raised while reading become errors.
|
||||
//
|
||||
// The parser reads file data lazily, so an image which opens cleanly can
|
||||
// still panic on the first Read.
|
||||
type recoveringReadCloser struct {
|
||||
io.ReadCloser
|
||||
}
|
||||
|
||||
func (r recoveringReadCloser) Read(p []byte) (n int, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
return r.ReadCloser.Read(p)
|
||||
}
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ fs.Fs, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
}
|
||||
|
||||
c := newCache(node)
|
||||
|
||||
// FIXME blocksize
|
||||
sqfs, err := squashfs.Read(c, node.Size(), 0, 1024*1024)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read squashfs: %w", err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
f: wrappedFs,
|
||||
name: path.Join(fs.ConfigString(wrappedFs), remote),
|
||||
vfs: VFS,
|
||||
node: node,
|
||||
sqfs: sqfs,
|
||||
c: c,
|
||||
remote: remote,
|
||||
root: strings.Trim(root, "/"),
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
if prefix == "" {
|
||||
f.prefixSlash = ""
|
||||
}
|
||||
|
||||
singleObject := false
|
||||
|
||||
// Find the directory the root points to
|
||||
if f.root != "" && !strings.HasSuffix(root, "/") {
|
||||
native, err := f.toNative("")
|
||||
if err == nil {
|
||||
native = strings.TrimRight(native, "/")
|
||||
_, err := f.newObjectNative(native)
|
||||
if err == nil {
|
||||
// If it pointed to a file, find the directory above
|
||||
// and remember the file so that only it is listed
|
||||
f.root = path.Dir(f.root)
|
||||
if f.root == "." || f.root == "/" {
|
||||
f.root = ""
|
||||
}
|
||||
f.file = f.prefixSlash + path.Base(native)
|
||||
singleObject = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
//
|
||||
// FIXME some of these need to be forced on - CanHaveEmptyDirectories
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: false, // MimeTypes not supported with gsquashfs
|
||||
WriteMimeType: false,
|
||||
BucketBased: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
if singleObject {
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
// Include the file so the fs cache can tell this Fs apart from
|
||||
// the one for the directory containing it
|
||||
return path.Join(f.root, f.file)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("Squashfs %q", f.name)
|
||||
}
|
||||
|
||||
// This turns a remote into a native path in the squashfs starting with a /
|
||||
func (f *Fs) toNative(remote string) (string, error) {
|
||||
native := strings.Trim(remote, "/")
|
||||
if f.prefix == "" {
|
||||
native = "/" + native
|
||||
} else if native == f.prefix {
|
||||
native = "/"
|
||||
} else if !strings.HasPrefix(native, f.prefixSlash) {
|
||||
return "", fmt.Errorf("internal error: %q doesn't start with prefix %q", native, f.prefixSlash)
|
||||
} else {
|
||||
native = native[len(f.prefix):]
|
||||
}
|
||||
if f.root != "" {
|
||||
native = "/" + f.root + native
|
||||
}
|
||||
return native, nil
|
||||
}
|
||||
|
||||
// This turns a native path with a leading / into a path suitable for
|
||||
// the squashfs library which requires io/fs.ValidPath paths with no
|
||||
// leading / and "." for the root directory.
|
||||
func toIOFS(native string) string {
|
||||
native = strings.Trim(native, "/")
|
||||
if native == "" {
|
||||
return "."
|
||||
}
|
||||
return native
|
||||
}
|
||||
|
||||
// Turn a (nativeDir, leaf) into a remote
|
||||
func (f *Fs) fromNative(nativeDir string, leaf string) string {
|
||||
// fs.Debugf(nil, "nativeDir = %q, leaf = %q, root=%q", nativeDir, leaf, f.root)
|
||||
dir := nativeDir
|
||||
if f.root != "" {
|
||||
dir = strings.TrimPrefix(dir, "/"+f.root)
|
||||
}
|
||||
remote := f.prefixSlash + strings.Trim(path.Join(dir, leaf), "/")
|
||||
// fs.Debugf(nil, "dir = %q, remote=%q", dir, remote)
|
||||
return remote
|
||||
}
|
||||
|
||||
// Convert a FileInfo into an Object from native dir
|
||||
func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()),
|
||||
size: item.Size(),
|
||||
modTime: item.ModTime(),
|
||||
}
|
||||
}
|
||||
|
||||
// Convert a symlink entry into an Object holding its target
|
||||
func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
|
||||
size: int64(len(target)),
|
||||
modTime: item.ModTime(),
|
||||
isLink: true,
|
||||
link: target,
|
||||
}
|
||||
}
|
||||
|
||||
// readlink returns the target of the symlink the info describes.
|
||||
//
|
||||
// A symlink may carry an empty target, so the caller decides what is a
|
||||
// symlink from the mode rather than from the target being set.
|
||||
func readlink(info os.FileInfo) (string, error) {
|
||||
st, ok := info.Sys().(*squashfs.StatT)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
|
||||
}
|
||||
return st.LinkTarget, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
||||
defer recoverParsePanic(&err)
|
||||
|
||||
if f.file != "" && dir != "" {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
nativeDir, err := f.toNative(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items, err := f.sqfs.ReadDir(toIOFS(nativeDir))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read squashfs: couldn't read directory: %w", err)
|
||||
}
|
||||
|
||||
entries = make(fs.DirEntries, 0, len(items))
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, item := range items {
|
||||
// fs.Debugf(item.Name(), "entry = %#v", item)
|
||||
// Skip entry names that aren't safe
|
||||
if err := sanitize.Leaf(item.Name()); err != nil {
|
||||
fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
var entry fs.DirEntry
|
||||
if item.IsDir() {
|
||||
remote := f.fromNative(nativeDir, item.Name())
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = fs.NewDir(remote, info.ModTime())
|
||||
} else if item.Type().IsRegular() {
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
entry = f.objectFromFileInfo(nativeDir, info)
|
||||
} else if item.Type()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
info, err := item.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
|
||||
}
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry = f.objectFromLink(nativeDir, info, target)
|
||||
} else {
|
||||
fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
|
||||
continue
|
||||
}
|
||||
if f.file != "" && entry.Remote() != f.file {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
// Log once per listing rather than per entry so a crafted image
|
||||
// can't flood the log
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
|
||||
}
|
||||
|
||||
// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// newObjectNative finds the object at the native path passed in
|
||||
func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
// get the path and filename
|
||||
dir, leaf := path.Split(nativePath)
|
||||
dir = strings.TrimRight(dir, "/")
|
||||
leaf = strings.Trim(leaf, "/")
|
||||
|
||||
// FIXME need to detect directory not found
|
||||
fis, err := f.sqfs.ReadDir(toIOFS(dir))
|
||||
if err != nil {
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
// Name the entry as List does, so that the two agree on
|
||||
// which names exist
|
||||
isLink := fi.Type()&os.ModeSymlink != 0
|
||||
if isLink && !f.links {
|
||||
continue
|
||||
}
|
||||
name := fi.Name()
|
||||
if isLink {
|
||||
name += fs.LinkSuffix
|
||||
}
|
||||
if name == leaf {
|
||||
if fi.IsDir() {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
info, err := fi.Info()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
|
||||
}
|
||||
if isLink {
|
||||
target, err := readlink(info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o = f.objectFromLink(dir, info, target)
|
||||
break
|
||||
}
|
||||
o = f.objectFromFileInfo(dir, info)
|
||||
break
|
||||
}
|
||||
}
|
||||
if o == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
|
||||
|
||||
if f.file != "" && remote != f.file {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
nativePath, err := f.toNative(remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.newObjectNative(nativePath)
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
|
||||
return nil, vfs.EROFS
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.None)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// Object describes an object to be read from the raw squashfs file
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
isLink bool // whether this is a symlink
|
||||
link string // target of the symlink
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.Remote()
|
||||
}
|
||||
|
||||
// Turn a squashfs path into a full path for the parent Fs
|
||||
// func (o *Object) path(remote string) string {
|
||||
// return path.Join(o.fs.prefix, remote)
|
||||
// }
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return o.size
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
return o.modTime
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Storable raturns a boolean indicating if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
defer recoverParsePanic(&err)
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.SeekOption:
|
||||
offset = x.Offset
|
||||
case *fs.RangeOption:
|
||||
offset, limit = x.Decode(o.Size())
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if o.isLink {
|
||||
// Clamp offset into range to avoid panic
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
if offset > int64(len(o.link)) {
|
||||
offset = int64(len(o.link))
|
||||
}
|
||||
return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
|
||||
}
|
||||
|
||||
remote, err := o.fs.toNative(o.remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fs.Debugf(o, "Opening %q", remote)
|
||||
fh, err := o.fs.sqfs.OpenFile(toIOFS(remote), os.O_RDONLY)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// discard data from start as necessary
|
||||
if offset > 0 {
|
||||
_, err = fh.Seek(offset, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
rc = recoveringReadCloser{fh}
|
||||
|
||||
// If limited then don't return everything
|
||||
if limit >= 0 {
|
||||
fs.Debugf(nil, "limit=%d, offset=%d, options=%v", limit, offset, options)
|
||||
return readers.NewLimitedReadCloser(rc, limit), nil
|
||||
}
|
||||
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.UnWrapper = (*Fs)(nil)
|
||||
_ fs.Wrapper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
)
|
||||
@@ -1,32 +0,0 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
fscache "github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A backslash is an ordinary character in a file name on the systems
|
||||
// squashfs images are made on, so an entry containing one must be
|
||||
// listed and openable like any other file.
|
||||
func TestBackslashInName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
localFs, err := fscache.Get(ctx, "testdata")
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, "backslash.sqfs", "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, `back\slash.txt`, entries[0].Remote())
|
||||
|
||||
o, err := f.NewObject(ctx, `back\slash.txt`)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(6), o.Size())
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package squashfs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// panickingReader panics on Read, standing in for the go-diskfs parser
|
||||
// hitting an unchecked slice while reading a malformed image.
|
||||
type panickingReader struct{}
|
||||
|
||||
func (panickingReader) Read([]byte) (int, error) { panic("boom") }
|
||||
func (panickingReader) Close() error { return nil }
|
||||
|
||||
// Test that a panic raised while reading file data becomes an error. The
|
||||
// parser reads lazily, so an image can open cleanly and only panic here.
|
||||
func TestRecoveringReadCloser(t *testing.T) {
|
||||
var rc io.ReadCloser = recoveringReadCloser{panickingReader{}}
|
||||
|
||||
_, err := rc.Read(make([]byte, 16))
|
||||
assert.ErrorContains(t, err, "invalid or corrupt squashfs image")
|
||||
assert.ErrorContains(t, err, "boom")
|
||||
}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -1,89 +0,0 @@
|
||||
//go:build !plan9
|
||||
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// squashfsSuperblock builds a minimal 96-byte squashfs superblock with the
|
||||
// given attacker-controllable fields. It mirrors the on-disk layout the
|
||||
// go-diskfs parser reads, and is only used to construct deliberately malformed
|
||||
// images for the tests below.
|
||||
func squashfsSuperblock(blockSize uint32, blockLog, compression uint16, rootInode, inodeTableStart uint64) []byte {
|
||||
const u64max = ^uint64(0)
|
||||
b := make([]byte, 96)
|
||||
binary.LittleEndian.PutUint32(b[0:], 0x73717368) // "hsqs" magic
|
||||
binary.LittleEndian.PutUint32(b[4:], 1) // inode count
|
||||
binary.LittleEndian.PutUint32(b[8:], 0) // modtime
|
||||
binary.LittleEndian.PutUint32(b[12:], blockSize)
|
||||
binary.LittleEndian.PutUint32(b[16:], 0) // fragment count
|
||||
binary.LittleEndian.PutUint16(b[20:], compression)
|
||||
binary.LittleEndian.PutUint16(b[22:], blockLog)
|
||||
binary.LittleEndian.PutUint16(b[24:], 0x0210) // no fragments/xattrs
|
||||
binary.LittleEndian.PutUint16(b[26:], 0) // id count
|
||||
binary.LittleEndian.PutUint16(b[28:], 4) // version major
|
||||
binary.LittleEndian.PutUint16(b[30:], 0) // version minor
|
||||
binary.LittleEndian.PutUint64(b[32:], rootInode)
|
||||
binary.LittleEndian.PutUint64(b[40:], 96) // bytes used
|
||||
binary.LittleEndian.PutUint64(b[48:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[56:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[64:], inodeTableStart)
|
||||
binary.LittleEndian.PutUint64(b[72:], inodeTableStart)
|
||||
binary.LittleEndian.PutUint64(b[80:], u64max)
|
||||
binary.LittleEndian.PutUint64(b[88:], u64max)
|
||||
return b
|
||||
}
|
||||
|
||||
// TestArchiveSquashfsMalformed feeds the archive backend squashfs images whose
|
||||
// superblock/metadata make the go-diskfs parser panic. See GHSA-6jcg-q3wp-x2f4.
|
||||
func TestArchiveSquashfsMalformed(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
|
||||
// Variant 1: zero block size reaches an integer division in the parser.
|
||||
zeroBlock := squashfsSuperblock(0, 0, 1, 0, 96)
|
||||
|
||||
// Variant 2: an out-of-range root inode byte offset reaches an unchecked
|
||||
// slice against an empty, uncompressed metadata block.
|
||||
oob := append(squashfsSuperblock(65536, 16, 1, 12336, 96), 0x00, 0x80)
|
||||
binary.LittleEndian.PutUint64(oob[40:], uint64(len(oob)))
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
image []byte
|
||||
}{
|
||||
{"ZeroBlockSize", zeroBlock},
|
||||
{"OutOfRangeInodeOffset", oob},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// A directory per image: the fs cache keys on the parent
|
||||
// directory, so sharing one would hand the second subtest a
|
||||
// cached listing which does not contain its image.
|
||||
path := filepath.Join(t.TempDir(), tc.name+".sqfs")
|
||||
require.NoError(t, os.WriteFile(path, tc.image, 0o644))
|
||||
|
||||
// The whole point is that this returns rather than panicking and
|
||||
// killing the test binary (and, in production, the serve daemon).
|
||||
f, err := cache.Get(ctx, ":archive:"+path)
|
||||
if err == nil {
|
||||
// The image parsed, so the panic must surface from a later
|
||||
// lazily-parsed operation instead.
|
||||
_, err = f.List(ctx, "")
|
||||
}
|
||||
require.Error(t, err, "a malformed squashfs image must error, not panic")
|
||||
assert.Contains(t, err.Error(), "invalid or corrupt squashfs image",
|
||||
"error should come from the recover, not an unrelated failure")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,419 +0,0 @@
|
||||
// Package zip implements a zip archiver for the archive backend
|
||||
package zip
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/archive/archiver"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/dirtree"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/sanitize"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
)
|
||||
|
||||
func init() {
|
||||
archiver.Register(archiver.Archiver{
|
||||
New: New,
|
||||
Extension: ".zip",
|
||||
})
|
||||
}
|
||||
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
f fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
features *fs.Features // optional features
|
||||
vfs *vfs.VFS
|
||||
node vfs.Node // zip file object - set if reading
|
||||
remote string // remote of the zip file object
|
||||
prefix string // position for objects
|
||||
prefixSlash string // position for objects with a slash on
|
||||
root string // position to read from within the archive
|
||||
links bool // read symlinks as .rclonelink files
|
||||
dt dirtree.DirTree // read from zipfile
|
||||
}
|
||||
|
||||
// New constructs an Fs from the (wrappedFs, remote) with the objects
|
||||
// prefix with prefix and rooted at root
|
||||
func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.Fs, error) {
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
}
|
||||
|
||||
f := &Fs{
|
||||
f: wrappedFs,
|
||||
name: path.Join(fs.ConfigString(wrappedFs), remote),
|
||||
vfs: VFS,
|
||||
node: node,
|
||||
remote: remote,
|
||||
root: root,
|
||||
links: fs.GetConfig(ctx).Links,
|
||||
prefix: prefix,
|
||||
prefixSlash: prefix + "/",
|
||||
}
|
||||
|
||||
// Read the contents of the zip file
|
||||
singleObject, err := f.readZip()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open zip file: %w", err)
|
||||
}
|
||||
|
||||
// FIXME
|
||||
// the features here are ones we could support, and they are
|
||||
// ANDed with the ones from wrappedFs
|
||||
//
|
||||
// FIXME some of these need to be forced on - CanHaveEmptyDirectories
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
DuplicateFiles: false,
|
||||
ReadMimeType: false, // MimeTypes not supported with gzip
|
||||
WriteMimeType: false,
|
||||
BucketBased: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
if singleObject {
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("Zip %q", f.name)
|
||||
}
|
||||
|
||||
// readZip the zip file into f
|
||||
//
|
||||
// Returns singleObject=true if f.root points to a file
|
||||
func (f *Fs) readZip() (singleObject bool, err error) {
|
||||
if f.node == nil {
|
||||
return singleObject, fs.ErrorDirNotFound
|
||||
}
|
||||
size := f.node.Size()
|
||||
if size < 0 {
|
||||
return singleObject, errors.New("can't read from zip file with unknown size")
|
||||
}
|
||||
r, err := f.node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return singleObject, fmt.Errorf("failed to open zip file: %w", err)
|
||||
}
|
||||
zr, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return singleObject, fmt.Errorf("failed to read zip file: %w", err)
|
||||
}
|
||||
dt := dirtree.New()
|
||||
skipped := 0
|
||||
skippedLinks := 0
|
||||
for _, file := range zr.File {
|
||||
// Skip entries whose name escapes the archive's own namespace
|
||||
remote, err := sanitize.Path(file.Name)
|
||||
if err != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
isDir := strings.HasSuffix(file.Name, "/")
|
||||
// A file entry whose last component is "", "." or ".." names
|
||||
// a directory rather than a file, so skip
|
||||
if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !isDir && file.Mode()&os.ModeSymlink != 0 {
|
||||
if !f.links {
|
||||
skippedLinks++
|
||||
continue
|
||||
}
|
||||
// The member holds the link target, so it reads back
|
||||
// like a .rclonelink file
|
||||
remote += fs.LinkSuffix
|
||||
}
|
||||
remote = path.Join(f.prefix, remote)
|
||||
if f.root != "" {
|
||||
// Ignore all files outside the root, requiring a path
|
||||
// boundary so that root "foo" does not also match a
|
||||
// sibling entry such as "foobar"
|
||||
if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
|
||||
continue
|
||||
}
|
||||
if remote == f.root {
|
||||
remote = ""
|
||||
} else {
|
||||
remote = strings.TrimPrefix(remote, f.root+"/")
|
||||
}
|
||||
}
|
||||
if isDir {
|
||||
dir := fs.NewDir(remote, file.Modified)
|
||||
dt.AddDir(dir)
|
||||
} else {
|
||||
if remote == "" {
|
||||
// A file at the root is the archive member f.root points at
|
||||
remote = path.Base(f.root)
|
||||
singleObject = true
|
||||
dt = dirtree.New()
|
||||
}
|
||||
o := &Object{
|
||||
f: f,
|
||||
remote: remote,
|
||||
fh: &file.FileHeader,
|
||||
file: file,
|
||||
}
|
||||
dt.Add(o)
|
||||
if singleObject {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if skipped > 0 {
|
||||
fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
|
||||
}
|
||||
if skippedLinks > 0 {
|
||||
fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
|
||||
}
|
||||
dt.CheckParents("")
|
||||
dt.Sort()
|
||||
f.dt = dt
|
||||
//fs.Debugf(nil, "dt = %v", dt)
|
||||
return singleObject, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries. The
|
||||
// entries can be returned in any order but should be for a
|
||||
// complete directory.
|
||||
//
|
||||
// dir should be "" to list the root, and should not have
|
||||
// trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
|
||||
// _, err = f.strip(dir)
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
entries, ok := f.dt[dir]
|
||||
if !ok {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
|
||||
// Return a copy as callers may filter the listing in place
|
||||
return slices.Clone(entries), nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
|
||||
defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
|
||||
if f.dt == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
_, entry := f.dt.Find(remote)
|
||||
if entry == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
o, ok := entry.(*Object)
|
||||
if !ok {
|
||||
return nil, fs.ErrorNotAFile
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Put in to the remote path with the modTime given of the given size
|
||||
//
|
||||
// May create the object even if it returns an error - if so
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
|
||||
return nil, vfs.EROFS
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.Set(hash.CRC32)
|
||||
}
|
||||
|
||||
// UnWrap returns the Fs that this Fs is wrapping
|
||||
func (f *Fs) UnWrap() fs.Fs {
|
||||
return f.f
|
||||
}
|
||||
|
||||
// WrapFs returns the Fs that is wrapping this Fs
|
||||
func (f *Fs) WrapFs() fs.Fs {
|
||||
return f.wrapper
|
||||
}
|
||||
|
||||
// SetWrapper sets the Fs that is wrapping this Fs
|
||||
func (f *Fs) SetWrapper(wrapper fs.Fs) {
|
||||
f.wrapper = wrapper
|
||||
}
|
||||
|
||||
// Object describes an object to be read from the raw zip file
|
||||
type Object struct {
|
||||
f *Fs
|
||||
remote string
|
||||
fh *zip.FileHeader
|
||||
file *zip.File
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.f
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *Object) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.Remote()
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return int64(o.fh.UncompressedSize64)
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
//
|
||||
// It attempts to read the objects mtime and if that isn't present the
|
||||
// LastModified returned in the http headers
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
return o.fh.Modified
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the local fs object
|
||||
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Storable raturns a boolean indicating if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
|
||||
if ht == hash.CRC32 {
|
||||
// FIXME return empty CRC if writing
|
||||
if o.f.dt == nil {
|
||||
return "", nil
|
||||
}
|
||||
return fmt.Sprintf("%08x", o.fh.CRC32), nil
|
||||
}
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.ReadCloser, err error) {
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
case *fs.SeekOption:
|
||||
offset = x.Offset
|
||||
case *fs.RangeOption:
|
||||
offset, limit = x.Decode(o.Size())
|
||||
default:
|
||||
if option.Mandatory() {
|
||||
fs.Logf(o, "Unsupported mandatory option: %v", option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rc, err = o.file.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// discard data from start as necessary
|
||||
if offset > 0 {
|
||||
_, err = io.CopyN(io.Discard, rc, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// If limited then don't return everything
|
||||
if limit >= 0 {
|
||||
return readers.NewLimitedReadCloser(rc, limit), nil
|
||||
}
|
||||
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// Update in to the object with the modTime given of the given size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return vfs.EROFS
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.UnWrapper = (*Fs)(nil)
|
||||
_ fs.Wrapper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
)
|
||||
@@ -1,227 +0,0 @@
|
||||
package zip
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/cache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// writeZip builds a zip file with the given entry names (a name ending
|
||||
// in "/" is a directory) in dir and returns its leaf name.
|
||||
func writeZip(t *testing.T, dir, name string, names ...string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, n := range names {
|
||||
w, err := zw.Create(n)
|
||||
require.NoError(t, err)
|
||||
if !strings.HasSuffix(n, "/") {
|
||||
_, err = w.Write([]byte("data for " + n))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// writeZipSymlink builds a zip holding file and a symlink entry link
|
||||
// pointing at target, and returns its leaf name.
|
||||
func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.Create(file)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte("data for " + file))
|
||||
require.NoError(t, err)
|
||||
hdr := &zip.FileHeader{Name: link}
|
||||
hdr.SetMode(os.ModeSymlink | 0777)
|
||||
w, err = zw.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(target))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
|
||||
return name
|
||||
}
|
||||
|
||||
// allRemotes returns every Object remote in the mounted archive's dirtree.
|
||||
func allRemotes(t *testing.T, f fs.Fs) []string {
|
||||
zf, ok := f.(*Fs)
|
||||
require.True(t, ok)
|
||||
var remotes []string
|
||||
for _, entries := range zf.dt {
|
||||
for _, entry := range entries {
|
||||
if o, ok := entry.(*Object); ok {
|
||||
remotes = append(remotes, o.Remote())
|
||||
}
|
||||
}
|
||||
}
|
||||
// dt iteration order is nondeterministic, so sort for stable comparison.
|
||||
sort.Strings(remotes)
|
||||
return remotes
|
||||
}
|
||||
|
||||
// A malicious zip whose entry names escape the archive's own namespace
|
||||
// must not be exposed by the zip backend (Zip Slip).
|
||||
func TestReadZipSlip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "evil.zip",
|
||||
"good.txt",
|
||||
"../../etc/cron.d/evil",
|
||||
"../escape.txt",
|
||||
"sub/../../above.txt",
|
||||
`..\..\evil.exe`,
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
|
||||
// The one benign entry must survive.
|
||||
assert.Contains(t, remotes, "good.txt")
|
||||
|
||||
// No escaping entry may be exposed.
|
||||
for _, remote := range remotes {
|
||||
assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
|
||||
"escaping remote exposed: %q", remote)
|
||||
}
|
||||
assert.Equal(t, []string{"good.txt"}, remotes)
|
||||
}
|
||||
|
||||
// Mounting with a non-empty root must only expose entries within that
|
||||
// root directory, not sibling directories that merely share a name
|
||||
// prefix (root "foo" must not match "foobar").
|
||||
func TestReadZipRootBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip",
|
||||
"foo/a.txt",
|
||||
"foobar/b.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "foo")
|
||||
require.NoError(t, err)
|
||||
|
||||
remotes := allRemotes(t, f)
|
||||
assert.Equal(t, []string{"a.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A file entry whose name refers to a directory (".", "/", "./", "" or
|
||||
// "sub/.") must be skipped, not turn the archive or the directory
|
||||
// mounted as its root into a single file which hides every other
|
||||
// entry, or become an object named for the archive itself when the
|
||||
// archive is found by listing its directory.
|
||||
func TestReadZipDirNamedFileEntry(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "dot.zip",
|
||||
".",
|
||||
"/",
|
||||
"./",
|
||||
"",
|
||||
"good.txt",
|
||||
"sub/",
|
||||
"sub/.",
|
||||
"sub/good.txt",
|
||||
)
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, test := range []struct {
|
||||
prefix, root string
|
||||
want []string
|
||||
}{
|
||||
{"", "", []string{"good.txt", "sub/good.txt"}},
|
||||
{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
|
||||
{"", "sub", []string{"good.txt"}},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
|
||||
f, err := New(ctx, localFs, name, test.prefix, test.root)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.want, allRemotes(t, f))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Listings are served from a cache which must survive callers
|
||||
// filtering the returned slice in place, as fs/list does.
|
||||
func TestListCacheNotAliased(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 3)
|
||||
// Compact in place, dropping the first entry
|
||||
copy(entries, entries[1:])
|
||||
|
||||
entries, err = f.List(ctx, "")
|
||||
require.NoError(t, err)
|
||||
var remotes []string
|
||||
for _, entry := range entries {
|
||||
remotes = append(remotes, entry.Remote())
|
||||
}
|
||||
assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
|
||||
}
|
||||
|
||||
// A symlink stored in the archive is skipped unless -l/--links is in
|
||||
// use, as it is on the other backends. With the flag it is exposed
|
||||
// under its name with the link suffix and reads back as its target.
|
||||
func TestReadZipSymlink(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
|
||||
|
||||
localFs, err := cache.Get(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err := New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
|
||||
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.Links = true
|
||||
|
||||
f, err = New(ctx, localFs, name, "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
|
||||
|
||||
o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len("file.txt")), o.Size())
|
||||
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
target, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, "file.txt", string(target))
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
// Parallel listing support for azureblob, built on the Apache Arrow
|
||||
// listing format.
|
||||
//
|
||||
// The Arrow listing exposes the server-side startFrom (inclusive) and endBefore
|
||||
// (exclusive) name-range parameters, which let a listing be sharded by
|
||||
// blob-name range and listed concurrently.
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
|
||||
// response, used to tell whether the service honoured an Arrow request.
|
||||
const arrowContentType = "application/vnd.apache.arrow.stream"
|
||||
|
||||
// listArrowParallel lists (containerName, directory) by splitting the blob name
|
||||
// keyspace into shards and listing them concurrently. opts must already request
|
||||
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
|
||||
// returns the number of raw items seen. directory/prefix are already normalised
|
||||
// by the caller.
|
||||
func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
|
||||
// Split the keyspace by a single-case character ladder (see
|
||||
// arrowLadderBoundaries for why single-case matters).
|
||||
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
|
||||
if len(points) < 1 {
|
||||
// Couldn't usefully split - list sequentially.
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
|
||||
// Build half-open [startFrom, endBefore) shards from the sorted boundary
|
||||
// points p0..pk: ["", p0), [p0, p1), ..., [pk, "").
|
||||
type shard struct{ startFrom, endBefore string }
|
||||
shards := make([]shard, 0, len(points)+1)
|
||||
prev := ""
|
||||
for _, p := range points {
|
||||
shards = append(shards, shard{prev, p})
|
||||
prev = p
|
||||
}
|
||||
shards = append(shards, shard{prev, ""})
|
||||
|
||||
// fn (and the list.Helper behind it) is not concurrency-safe, so serialise
|
||||
// delivery. Only the cheap callback is locked, not the HTTP/decode work.
|
||||
var fnMu sync.Mutex
|
||||
safeFn := func(remote string, item *container.BlobItem, isDir bool) error {
|
||||
fnMu.Lock()
|
||||
defer fnMu.Unlock()
|
||||
return fn(remote, item, isDir)
|
||||
}
|
||||
|
||||
// Schedule every shard; SetLimit bounds concurrency, so more shards than
|
||||
// workers gives free load balancing as workers pick up the next shard.
|
||||
var total atomic.Int64
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(f.opt.ListParallelism)
|
||||
for _, sh := range shards {
|
||||
if gCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
// Clone opts per shard so concurrent shards don't share StartFrom/EndBefore.
|
||||
shardOpts := *opts
|
||||
if sh.startFrom != "" {
|
||||
startFrom := sh.startFrom
|
||||
shardOpts.StartFrom = &startFrom
|
||||
} else {
|
||||
shardOpts.StartFrom = nil
|
||||
}
|
||||
if sh.endBefore != "" {
|
||||
endBefore := sh.endBefore
|
||||
shardOpts.EndBefore = &endBefore
|
||||
} else {
|
||||
shardOpts.EndBefore = nil
|
||||
}
|
||||
n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
|
||||
total.Add(int64(n))
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := g.Wait()
|
||||
found := int(total.Load())
|
||||
if isEndBeforeUnsupported(err) {
|
||||
// The account can't honour the shards' endBefore bounds, so parallel
|
||||
// listing can't be used on it.
|
||||
f.arrowXMLFallback.Store(true)
|
||||
if found == 0 {
|
||||
// No shard delivered anything yet (the normal case - Arrow listing
|
||||
// enablement is per account so every shard fails on its first
|
||||
// page) so it is safe to relist sequentially.
|
||||
fs.Debugf(f, "Parallel Arrow listing not supported by this account (%v) - listing sequentially", err)
|
||||
return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
|
||||
}
|
||||
// Some entries were already delivered so a rerun would duplicate them.
|
||||
return found, fmt.Errorf("parallel listing aborted after partial results (retry, or unset list_parallelism): %w", err)
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
|
||||
// isEndBeforeUnsupported reports whether err means the service can't honour
|
||||
// the endBefore listing bound, so sharded parallel listing must not be used.
|
||||
//
|
||||
// The service rejects the endBefore parameter with 400
|
||||
// OperationNotSupportedWithFeatureMissing "The requested operation is not
|
||||
// allowed as EndBefore parameter support is missing on this account" -
|
||||
// endBefore is only supported on the Arrow listing path of accounts with
|
||||
// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
|
||||
// accept an XML page with endBefore set, but the service never sends one.)
|
||||
func isEndBeforeUnsupported(err error) bool {
|
||||
var respErr *azcore.ResponseError
|
||||
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
|
||||
}
|
||||
|
||||
// arrowShardTarget is the number of ladder boundaries to use for the given
|
||||
// parallelism (a few per worker for load balancing).
|
||||
func arrowShardTarget(parallelism int) int {
|
||||
return parallelism * 4
|
||||
}
|
||||
|
||||
// arrowLadderBoundaries returns up to target sorted ENCODED-space boundaries of
|
||||
// the form directory+c, used to split the keyspace into shards.
|
||||
//
|
||||
// The partition is always correct (no dup/gap) regardless of the alphabet: keys
|
||||
// whose next byte is outside the alphabet simply fall into an adjacent shard;
|
||||
// the alphabet only affects balance. A single-case (digit+lowercase) alphabet
|
||||
// is deliberate: the boundaries are then monotonic under BOTH byte order (the
|
||||
// listing order) AND the service's case-insensitive startFrom/endBefore
|
||||
// validation, so server-side endBefore is accepted. (A mixed-case ladder gives
|
||||
// pairs like [".../Z", ".../a") which are byte-ascending but rejected 400 as
|
||||
// "endbefore precedes startfrom".) directory is already encoded; the alphabet
|
||||
// is ASCII (encoder-invariant) and ascending, so the result is sorted.
|
||||
func arrowLadderBoundaries(directory string, target int) []string {
|
||||
const alphabet = "0123456789abcdefghijklmnopqrstuvwxyz"
|
||||
if target < 1 {
|
||||
return nil
|
||||
}
|
||||
if target > len(alphabet) {
|
||||
target = len(alphabet)
|
||||
}
|
||||
points := make([]string, 0, target)
|
||||
last := -1
|
||||
for i := 0; i < target; i++ {
|
||||
idx := i * len(alphabet) / target
|
||||
if idx == last {
|
||||
continue
|
||||
}
|
||||
last = idx
|
||||
points = append(points, directory+string(alphabet[idx]))
|
||||
}
|
||||
return points
|
||||
}
|
||||
@@ -1,219 +0,0 @@
|
||||
//go:build !plan9 && !js
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testArrowList exercises the experimental Apache Arrow listing.
|
||||
//
|
||||
// It is skipped automatically if Arrow listing is not enabled on the test
|
||||
// account (the server returns XML, or 409 on HNS accounts).
|
||||
func (f *Fs) testArrowList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
containerName, _ := f.split("probe")
|
||||
if containerName == "" {
|
||||
t.Skip("Arrow list test needs a container in the remote root")
|
||||
}
|
||||
|
||||
// withArrow runs fn with UseArrowList temporarily set.
|
||||
withArrow := func(on bool, fn func()) {
|
||||
old := f.opt.UseArrowList
|
||||
f.opt.UseArrowList = on
|
||||
defer func() { f.opt.UseArrowList = old }()
|
||||
fn()
|
||||
}
|
||||
|
||||
// withParallel runs fn with ListParallelism temporarily set.
|
||||
withParallel := func(n int, fn func()) {
|
||||
old := f.opt.ListParallelism
|
||||
f.opt.ListParallelism = n
|
||||
defer func() { f.opt.ListParallelism = old }()
|
||||
fn()
|
||||
}
|
||||
|
||||
// Create a set of known objects under a unique directory: flat files plus
|
||||
// one in a subdirectory (so we exercise both the flat and delimited paths).
|
||||
dir := "arrow-list-" + random.String(8)
|
||||
// Includes upper-case and digit first-characters: these byte-sort between
|
||||
// the digit and lowercase ladder boundaries, so they verify the server's
|
||||
// endBefore filtering is byte-ordered (not case-insensitive, which would
|
||||
// drop them from a shard and make parallel != sequential).
|
||||
objects := []string{"a.txt", "b.txt", "c.txt", "d.txt", "e.txt", "sub/x.txt", "Foo.txt", "Zoo.txt", "0num.txt", "9end.txt"}
|
||||
wantSizes := map[string]int64{}
|
||||
created := make([]string, 0, len(objects))
|
||||
for i, n := range objects {
|
||||
contents := random.String(10 + i)
|
||||
remote := dir + "/" + n
|
||||
item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
|
||||
_ = fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
created = append(created, remote)
|
||||
wantSizes[remote] = int64(len(contents))
|
||||
}
|
||||
defer func() {
|
||||
for _, remote := range created {
|
||||
if o, err := f.NewObject(ctx, remote); err == nil {
|
||||
_ = o.Remove(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// listing returns remote->size for dir (dirs reported as size -1).
|
||||
listing := func(recurse bool) (map[string]int64, error) {
|
||||
got := map[string]int64{}
|
||||
add := func(entries fs.DirEntries) error {
|
||||
for _, e := range entries {
|
||||
if o, ok := e.(fs.Object); ok {
|
||||
got[e.Remote()] = o.Size()
|
||||
} else {
|
||||
got[e.Remote()] = -1
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if recurse {
|
||||
return got, f.ListR(ctx, dir, add)
|
||||
}
|
||||
entries, err := f.List(ctx, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = add(entries)
|
||||
return got, nil
|
||||
}
|
||||
|
||||
// Probe: is Arrow available? If not, skip cleanly.
|
||||
var probeErr error
|
||||
withArrow(true, func() { _, probeErr = listing(true) })
|
||||
if probeErr != nil {
|
||||
if strings.Contains(probeErr.Error(), "OperationNotSupportedWithFeatureMissing") || strings.Contains(probeErr.Error(), "409") {
|
||||
t.Skipf("Arrow listing not available on this account: %v", probeErr)
|
||||
}
|
||||
require.NoError(t, probeErr)
|
||||
}
|
||||
|
||||
// Arrow listing must equal XML listing, recursively and per-directory
|
||||
// (including the "sub" subdirectory).
|
||||
t.Run("EqualsXML", func(t *testing.T) {
|
||||
for _, recurse := range []bool{true, false} {
|
||||
var xml, arrow map[string]int64
|
||||
var err error
|
||||
withArrow(false, func() { xml, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
withArrow(true, func() { arrow, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, xml, arrow, "recurse=%v: Arrow listing differs from XML", recurse)
|
||||
}
|
||||
|
||||
// Recursive Arrow listing contains every object with the right size.
|
||||
var arrow map[string]int64
|
||||
withArrow(true, func() { arrow, _ = listing(true) })
|
||||
for remote, size := range wantSizes {
|
||||
assert.Equal(t, size, arrow[remote], remote)
|
||||
}
|
||||
|
||||
// Single-directory Arrow listing reports "sub" as a directory.
|
||||
var entries fs.DirEntries
|
||||
withArrow(true, func() {
|
||||
var err error
|
||||
entries, err = f.List(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
foundSubDir := false
|
||||
for _, e := range entries {
|
||||
if e.Remote() == dir+"/sub" {
|
||||
_, isObj := e.(fs.Object)
|
||||
foundSubDir = !isObj
|
||||
}
|
||||
}
|
||||
assert.True(t, foundSubDir, "expected %q to be listed as a directory", dir+"/sub")
|
||||
})
|
||||
|
||||
// Parallel listing must equal sequential listing, recursively and
|
||||
// per-directory, including the "sub" subdirectory.
|
||||
t.Run("ParallelEqualsSequential", func(t *testing.T) {
|
||||
for _, recurse := range []bool{true, false} {
|
||||
var seq, par map[string]int64
|
||||
var err error
|
||||
withArrow(true, func() {
|
||||
withParallel(0, func() { seq, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
withParallel(8, func() { par, err = listing(recurse) })
|
||||
require.NoError(t, err)
|
||||
})
|
||||
assert.Equal(t, seq, par, "recurse=%v: parallel listing differs from sequential", recurse)
|
||||
}
|
||||
})
|
||||
|
||||
// A tiny list_chunk forces each shard to span multiple Arrow pages, so this
|
||||
// exercises pagination (startFrom+endBefore+marker resent together) inside a
|
||||
// shard. The "sub/" boundary puts a.txt..e.txt in one shard => >1 page at
|
||||
// chunk size 2.
|
||||
t.Run("ParallelMultiPage", func(t *testing.T) {
|
||||
oldChunk := f.opt.ListChunkSize
|
||||
f.opt.ListChunkSize = 2
|
||||
defer func() { f.opt.ListChunkSize = oldChunk }()
|
||||
var seq, par map[string]int64
|
||||
var err error
|
||||
withArrow(true, func() {
|
||||
withParallel(0, func() { seq, err = listing(true) })
|
||||
require.NoError(t, err)
|
||||
withParallel(8, func() { par, err = listing(true) })
|
||||
require.NoError(t, err)
|
||||
})
|
||||
assert.Equal(t, seq, par)
|
||||
for remote, size := range wantSizes {
|
||||
assert.Equal(t, size, par[remote], remote)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestArrowLadderBoundaries(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
directory string
|
||||
target int
|
||||
}{
|
||||
{"", 8},
|
||||
{"dir/", 16},
|
||||
{"dir/", 100}, // capped at alphabet size
|
||||
{"dir/", 1},
|
||||
} {
|
||||
points := arrowLadderBoundaries(tc.directory, tc.target)
|
||||
require.NotEmpty(t, points)
|
||||
assert.LessOrEqual(t, len(points), 62)
|
||||
for i, p := range points {
|
||||
assert.True(t, strings.HasPrefix(p, tc.directory), "boundary %q lacks prefix %q", p, tc.directory)
|
||||
if i > 0 {
|
||||
assert.Less(t, points[i-1], p, "boundaries not strictly ascending")
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.Nil(t, arrowLadderBoundaries("dir/", 0))
|
||||
// alphabet is single-case (digit+lowercase) so boundaries are monotonic
|
||||
// under both byte order and case-insensitive order
|
||||
pts := arrowLadderBoundaries("dir/", 36)
|
||||
for _, p := range pts {
|
||||
assert.Equal(t, p, strings.ToLower(p), "boundary must be lowercase: %q", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEndBeforeUnsupported(t *testing.T) {
|
||||
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
|
||||
assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
|
||||
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
|
||||
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
|
||||
assert.False(t, isEndBeforeUnsupported(nil))
|
||||
}
|
||||
@@ -1,595 +0,0 @@
|
||||
// Package auth supplies the authentication and client creation for the azure SDK
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
)
|
||||
|
||||
const (
|
||||
// Default storage account, key and blob endpoint for emulator support,
|
||||
// though it is a base64 key checked in here, it is publicly available secret.
|
||||
emulatorAccount = "devstoreaccount1"
|
||||
emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
||||
emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
|
||||
)
|
||||
|
||||
// ConfigOptions is the common authentication options for azure
|
||||
var ConfigOptions = []fs.Option{{
|
||||
Name: "account",
|
||||
Help: `Azure Storage Account Name.
|
||||
|
||||
Set this to the Azure Storage Account Name in use.
|
||||
|
||||
Leave blank to use SAS URL or Emulator, otherwise it needs to be set.
|
||||
|
||||
If this is blank and if env_auth is set it will be read from the
|
||||
environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible.
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "env_auth",
|
||||
Help: `Read credentials from runtime (environment variables, CLI or MSI).
|
||||
|
||||
See the [authentication docs](/azureblob#authentication) for full info.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: `Storage Account Shared Key.
|
||||
|
||||
Leave blank to use SAS URL or Emulator.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sas_url",
|
||||
Help: `SAS URL for container level access only.
|
||||
|
||||
Leave blank if using account/key or Emulator.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "connection_string",
|
||||
Help: `Storage Connection String.
|
||||
|
||||
Connection string for the storage. Leave blank if using other auth methods.
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "tenant",
|
||||
Help: `ID of the service principal's tenant. Also called its directory ID.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_id",
|
||||
Help: `The ID of the client in use.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_secret",
|
||||
Help: `One of the service principal's client secrets
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_certificate_path",
|
||||
Help: `Path to a PEM or PKCS12 certificate file including the private key.
|
||||
|
||||
Set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
}, {
|
||||
Name: "client_certificate_password",
|
||||
Help: `Password for the certificate file (optional).
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
|
||||
And the certificate has a password.
|
||||
`,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "client_send_certificate_chain",
|
||||
Help: `Send the certificate chain when using certificate auth.
|
||||
|
||||
Specifies whether an authentication request will include an x5c header
|
||||
to support subject name / issuer based authentication. When set to
|
||||
true, authentication requests include the x5c header.
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "username",
|
||||
Help: `User name (usually an email address)
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "password",
|
||||
Help: `The user's password
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
IsPassword: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "service_principal_file",
|
||||
Help: `Path to file containing credentials for use with a service principal.
|
||||
|
||||
Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
|
||||
|
||||
$ az ad sp create-for-rbac --name "<name>" \
|
||||
--role "Storage Blob Data Owner" \
|
||||
--scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
|
||||
> azure-principal.json
|
||||
|
||||
See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
|
||||
|
||||
It may be more convenient to put the credentials directly into the
|
||||
rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + `
|
||||
keys instead of setting ` + "`service_principal_file`" + `.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "disable_instance_discovery",
|
||||
Help: `Skip requesting Microsoft Entra instance metadata
|
||||
|
||||
This should be set true only by applications authenticating in
|
||||
disconnected clouds, or private clouds such as Azure Stack.
|
||||
|
||||
It determines whether rclone requests Microsoft Entra instance
|
||||
metadata from ` + "`https://login.microsoft.com/`" + ` before
|
||||
authenticating.
|
||||
|
||||
Setting this to true will skip this request, making you responsible
|
||||
for ensuring the configured authority is valid and trustworthy.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_msi",
|
||||
Help: `Use a managed service identity to authenticate (only works in Azure).
|
||||
|
||||
When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/)
|
||||
to authenticate to Azure Storage instead of a SAS token or account key.
|
||||
|
||||
If the VM(SS) on which this program is running has a system-assigned identity, it will
|
||||
be used by default. If the resource has no system-assigned but exactly one user-assigned identity,
|
||||
the user-assigned identity will be used by default. If the resource has multiple user-assigned
|
||||
identities, the identity to use must be explicitly specified using exactly one of the msi_object_id,
|
||||
msi_client_id, or msi_mi_res_id parameters.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "msi_object_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_client_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_mi_res_id",
|
||||
Help: "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "use_emulator",
|
||||
Help: "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_az",
|
||||
Help: `Use Azure CLI tool az for authentication
|
||||
|
||||
Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
as the sole means of authentication.
|
||||
|
||||
Setting this can be useful if you wish to use the az CLI on a host with
|
||||
a System Managed Identity that you do not want to use.
|
||||
|
||||
Don't set env_auth at the same time.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service.\n\nLeave blank normally.",
|
||||
Advanced: true,
|
||||
}}
|
||||
|
||||
// Options defines the common auth configuration for azure backends
|
||||
type Options struct {
|
||||
Account string `config:"account"`
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
Key string `config:"key"`
|
||||
SASURL string `config:"sas_url"`
|
||||
ConnectionString string `config:"connection_string"`
|
||||
Tenant string `config:"tenant"`
|
||||
ClientID string `config:"client_id"`
|
||||
ClientSecret string `config:"client_secret"`
|
||||
ClientCertificatePath string `config:"client_certificate_path"`
|
||||
ClientCertificatePassword string `config:"client_certificate_password"`
|
||||
ClientSendCertificateChain bool `config:"client_send_certificate_chain"`
|
||||
Username string `config:"username"`
|
||||
Password string `config:"password"`
|
||||
ServicePrincipalFile string `config:"service_principal_file"`
|
||||
DisableInstanceDiscovery bool `config:"disable_instance_discovery"`
|
||||
UseMSI bool `config:"use_msi"`
|
||||
MSIObjectID string `config:"msi_object_id"`
|
||||
MSIClientID string `config:"msi_client_id"`
|
||||
MSIResourceID string `config:"msi_mi_res_id"`
|
||||
UseEmulator bool `config:"use_emulator"`
|
||||
UseAZ bool `config:"use_az"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
}
|
||||
|
||||
type servicePrincipalCredentials struct {
|
||||
AppID string `json:"appId"`
|
||||
Password string `json:"password"`
|
||||
Tenant string `json:"tenant"`
|
||||
}
|
||||
|
||||
// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli.
|
||||
func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) {
|
||||
var spCredentials servicePrincipalCredentials
|
||||
if err := json.Unmarshal(credentialsData, &spCredentials); err != nil {
|
||||
return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
|
||||
}
|
||||
// TODO: support certificate credentials
|
||||
// Validate all fields present
|
||||
if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" {
|
||||
return nil, fmt.Errorf("missing fields in credentials file")
|
||||
}
|
||||
return &spCredentials, nil
|
||||
}
|
||||
|
||||
// Wrap the http.Transport to satisfy the Transporter interface
|
||||
type transporter struct {
|
||||
http.RoundTripper
|
||||
}
|
||||
|
||||
// Make a new transporter
|
||||
func newTransporter(ctx context.Context) transporter {
|
||||
// Set the User-Agent to include the Microsoft Partner Network prefix
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.UserAgent = "APN/1.0 rclone/1.0 rclone/" + strings.TrimPrefix(fs.Version, "v")
|
||||
return transporter{
|
||||
RoundTripper: fshttp.NewTransport(ctx),
|
||||
}
|
||||
}
|
||||
|
||||
// Do sends the HTTP request and returns the HTTP response or error.
|
||||
func (tr transporter) Do(req *http.Request) (*http.Response, error) {
|
||||
return tr.RoundTripper.RoundTrip(req)
|
||||
}
|
||||
|
||||
// Transporter returns the policy.Transporter rclone uses for Azure SDK
|
||||
// clients (an fshttp based transport with the APN user agent), for callers
|
||||
// which build their own azcore pipelines.
|
||||
func Transporter(ctx context.Context) policy.Transporter {
|
||||
return newTransporter(ctx)
|
||||
}
|
||||
|
||||
// NewClientOpts should be passed to configure NewClient
|
||||
type NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct {
|
||||
DefaultBaseURL string // Base URL, eg blob.core.windows.net
|
||||
Blob bool // set if this is blob storage
|
||||
RootContainer string // Container that rclone is looking at
|
||||
NewClient func(serviceURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error)
|
||||
NewClientFromConnectionString func(connectionString string, options *ClientOptions) (*Client, error)
|
||||
NewClientWithNoCredential func(serviceURL string, options *ClientOptions) (*Client, error)
|
||||
NewClientWithSharedKeyCredential func(serviceURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error)
|
||||
NewSharedKeyCredential func(accountName, accountKey string) (*SharedKeyCredential, error)
|
||||
SetClientOptions func(options *ClientOptions, policyClientOptions policy.ClientOptions)
|
||||
}
|
||||
|
||||
// NewClientResult is returned from NewClient
|
||||
type NewClientResult[Client any] struct {
|
||||
Client *Client // Client to access the Service
|
||||
Cred azcore.TokenCredential // how to generate tokens (may be nil)
|
||||
UsingSharedKeyCred bool // set if using shared key credentials
|
||||
Anonymous bool // true if anonymous authentication was used
|
||||
Container string // Container that SAS URL points to
|
||||
}
|
||||
|
||||
// NewClient creates a service client from the rclone options
|
||||
func NewClient[Client, ClientOptions, SharedKeyCredential any](ctx context.Context, conf NewClientOpts[Client, ClientOptions, SharedKeyCredential], opt *Options) (r NewClientResult[Client], err error) {
|
||||
var sharedKeyCred *SharedKeyCredential
|
||||
|
||||
// Client options specifying our own transport
|
||||
policyClientOptions := policy.ClientOptions{
|
||||
Transport: newTransporter(ctx),
|
||||
}
|
||||
// Can't do this with generics (yet)
|
||||
// clientOpt := service.ClientOptions{
|
||||
// ClientOptions: policyClientOptions,
|
||||
// }
|
||||
// So call back to user
|
||||
var clientOpt ClientOptions
|
||||
conf.SetClientOptions(&clientOpt, policyClientOptions)
|
||||
|
||||
// Here we auth by setting one of cred, sharedKeyCred, client or anonymous
|
||||
switch {
|
||||
case opt.EnvAuth:
|
||||
// Read account from environment if needed
|
||||
if opt.Account == "" {
|
||||
opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME")
|
||||
}
|
||||
// Read credentials from the environment
|
||||
options := azidentity.DefaultAzureCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
|
||||
}
|
||||
r.Cred, err = azidentity.NewDefaultAzureCredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create azure environment credential failed: %w", err)
|
||||
}
|
||||
case opt.UseEmulator:
|
||||
if opt.Account == "" {
|
||||
opt.Account = emulatorAccount
|
||||
}
|
||||
if opt.Key == "" {
|
||||
opt.Key = emulatorAccountKey
|
||||
}
|
||||
if opt.Endpoint == "" {
|
||||
opt.Endpoint = emulatorBlobEndpoint
|
||||
}
|
||||
if conf.NewSharedKeyCredential == nil {
|
||||
return r, errors.New("emulator use not supported")
|
||||
}
|
||||
sharedKeyCred, err = conf.NewSharedKeyCredential(opt.Account, opt.Key)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create new shared key credential for emulator failed: %w", err)
|
||||
}
|
||||
case opt.Account != "" && opt.Key != "":
|
||||
if conf.NewSharedKeyCredential == nil {
|
||||
return r, errors.New("shared key credentials not supported")
|
||||
}
|
||||
sharedKeyCred, err = conf.NewSharedKeyCredential(opt.Account, opt.Key)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create new shared key credential failed: %w", err)
|
||||
}
|
||||
case opt.SASURL != "":
|
||||
parts, err := sas.ParseURL(opt.SASURL)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to parse SAS URL: %w", err)
|
||||
}
|
||||
endpoint := opt.SASURL
|
||||
r.Container = parts.ContainerName
|
||||
// Check if we have container level SAS or account level SAS
|
||||
if conf.Blob && r.Container != "" {
|
||||
// Container level SAS
|
||||
if conf.RootContainer != "" && r.Container != conf.RootContainer {
|
||||
return r, fmt.Errorf("container name in SAS URL (%q) and container provided in command (%q) do not match", r.Container, conf.RootContainer)
|
||||
}
|
||||
// Rewrite the endpoint string to be without the container
|
||||
parts.ContainerName = ""
|
||||
endpoint = parts.String()
|
||||
}
|
||||
r.Client, err = conf.NewClientWithNoCredential(endpoint, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("unable to create SAS URL client: %w", err)
|
||||
}
|
||||
case opt.ConnectionString != "":
|
||||
r.Client, err = conf.NewClientFromConnectionString(opt.ConnectionString, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("unable to create connection string client: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "":
|
||||
// Service principal with client secret
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
r.Cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "":
|
||||
// Service principal with certificate
|
||||
//
|
||||
// Read the certificate
|
||||
data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath))
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error reading client certificate file: %w", err)
|
||||
}
|
||||
// NewClientCertificateCredential requires at least one *x509.Certificate, and a
|
||||
// crypto.PrivateKey.
|
||||
//
|
||||
// ParseCertificates returns these given certificate data in PEM or PKCS12 format.
|
||||
// It handles common scenarios but has limitations, for example it doesn't load PEM
|
||||
// encrypted private keys.
|
||||
var password []byte
|
||||
if opt.ClientCertificatePassword != "" {
|
||||
pw, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
password = []byte(pw)
|
||||
}
|
||||
certs, key, err := azidentity.ParseCertificates(data, password)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to parse client certificate file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientCertificateCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
SendCertificateChain: opt.ClientSendCertificateChain,
|
||||
}
|
||||
r.Cred, err = azidentity.NewClientCertificateCredential(
|
||||
opt.Tenant, opt.ClientID, certs, key, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
|
||||
// User with username and password
|
||||
//nolint:staticcheck // this is deprecated due to Azure policy
|
||||
options := azidentity.UsernamePasswordCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
password, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("user password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
r.Cred, err = azidentity.NewUsernamePasswordCredential(
|
||||
opt.Tenant, opt.ClientID, opt.Username, password, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("authenticate user with password failed: %w", err)
|
||||
}
|
||||
case opt.ServicePrincipalFile != "":
|
||||
// Loading service principal credentials from file.
|
||||
loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error opening service principal credentials file: %w", err)
|
||||
}
|
||||
parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error parsing service principal credentials file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
r.Cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.UseMSI:
|
||||
// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
|
||||
// Validate and ensure exactly one is set. (To do: better validation.)
|
||||
var b2i = map[bool]int{false: 0, true: 1}
|
||||
set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""]
|
||||
if set > 1 {
|
||||
return r, errors.New("more than one user-assigned identity ID is set")
|
||||
}
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
switch {
|
||||
case opt.MSIClientID != "":
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
case opt.MSIObjectID != "":
|
||||
// FIXME this doesn't appear to be in the new SDK?
|
||||
return r, fmt.Errorf("MSI object ID is currently unsupported")
|
||||
case opt.MSIResourceID != "":
|
||||
options.ID = azidentity.ResourceID(opt.MSIResourceID)
|
||||
}
|
||||
r.Cred, err = azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
r.Cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
case opt.UseAZ:
|
||||
var options = azidentity.AzureCLICredentialOptions{}
|
||||
r.Cred, err = azidentity.NewAzureCLICredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
|
||||
}
|
||||
case opt.Account != "":
|
||||
// Anonymous access
|
||||
r.Anonymous = true
|
||||
default:
|
||||
return r, errors.New("no authentication method configured")
|
||||
}
|
||||
|
||||
// Make the client if not already created
|
||||
if r.Client == nil {
|
||||
// Work out what the endpoint is if it is still unset
|
||||
if opt.Endpoint == "" {
|
||||
if opt.Account == "" {
|
||||
return r, fmt.Errorf("account must be set: can't make service URL")
|
||||
}
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, conf.DefaultBaseURL))
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to make azure storage URL from account: %w", err)
|
||||
}
|
||||
opt.Endpoint = u.String()
|
||||
}
|
||||
if sharedKeyCred != nil {
|
||||
// Shared key cred
|
||||
r.Client, err = conf.NewClientWithSharedKeyCredential(opt.Endpoint, sharedKeyCred, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create client with shared key failed: %w", err)
|
||||
}
|
||||
r.UsingSharedKeyCred = true
|
||||
} else if r.Cred != nil {
|
||||
// Azidentity cred
|
||||
r.Client, err = conf.NewClient(opt.Endpoint, r.Cred, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create client failed: %w", err)
|
||||
}
|
||||
} else if r.Anonymous {
|
||||
// Anonymous public access
|
||||
r.Client, err = conf.NewClientWithNoCredential(opt.Endpoint, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create public client failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.Client == nil {
|
||||
return r, fmt.Errorf("internal error: auth failed to make credentials or client")
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
+654
-915
File diff suppressed because it is too large.
Load diff
@@ -1,24 +1,15 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js && !wasm
|
||||
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
@@ -54,58 +45,6 @@ func TestBlockIDCreator(t *testing.T) {
|
||||
assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
|
||||
}
|
||||
|
||||
func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
|
||||
pInt64 := func(i int64) *int64 { return &i }
|
||||
pString := func(s string) *string { return &s }
|
||||
newTestObject := func() *Object {
|
||||
return &Object{
|
||||
fs: &Fs{},
|
||||
remote: "test.bin",
|
||||
size: -2, // sentinel to check it gets set
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("WholeBlob", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(12345)
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(12345), o.size)
|
||||
})
|
||||
|
||||
t.Run("RangeRequest", func(t *testing.T) {
|
||||
// On a ranged download Content-Length is the chunk length so the
|
||||
// size must come from the total in Content-Range
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("bytes 67108864-134217727/169721004032")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(169721004032), o.size)
|
||||
})
|
||||
|
||||
t.Run("BadContentRange", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
info.ContentLength = pInt64(67108864)
|
||||
info.ContentRange = pString("potato")
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(67108864), o.size)
|
||||
})
|
||||
|
||||
t.Run("UnknownLength", func(t *testing.T) {
|
||||
o := newTestObject()
|
||||
info := blob.DownloadStreamResponse{}
|
||||
require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
|
||||
assert.Equal(t, int64(-1), o.size)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCopySASTimingConstants(t *testing.T) {
|
||||
require.Greater(t, sasCopyStartSkew, time.Duration(0))
|
||||
require.Greater(t, sasCopyValidity, sasCopyStartSkew)
|
||||
}
|
||||
|
||||
func (f *Fs) testFeatures(t *testing.T) {
|
||||
// Check first feature flags are set on this remote
|
||||
enabled := f.Features().SetTier
|
||||
@@ -206,488 +145,7 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
|
||||
require.NoError(t, dst.Remove(ctx))
|
||||
}
|
||||
|
||||
func gz(t *testing.T, s string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
_, err := zw.Write([]byte(s))
|
||||
require.NoError(t, err)
|
||||
err = zw.Close()
|
||||
require.NoError(t, err)
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func md5sum(t *testing.T, s string) string {
|
||||
hash := md5.Sum([]byte(s))
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
func (f *Fs) testGzipEncoding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
original := random.String(1000)
|
||||
contents := gz(t, original)
|
||||
|
||||
item := fstest.NewItem("test-gzip", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
metadata := fs.Metadata{
|
||||
"content-encoding": "gzip",
|
||||
"content-type": "text/plain",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", metadata)
|
||||
defer func() {
|
||||
assert.NoError(t, obj.Remove(ctx))
|
||||
}()
|
||||
o := obj.(*Object)
|
||||
|
||||
// Test that the gzipped file we uploaded can be
|
||||
// downloaded with and without decompression
|
||||
checkDownload := func(wantContents string, wantSize int64, wantHash string) {
|
||||
gotContents := fstests.ReadObject(ctx, t, o, -1)
|
||||
assert.Equal(t, wantContents, gotContents)
|
||||
assert.Equal(t, wantSize, o.Size())
|
||||
gotHash, err := o.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantHash, gotHash)
|
||||
}
|
||||
|
||||
t.Run("NoDecompress", func(t *testing.T) {
|
||||
checkDownload(contents, int64(len(contents)), md5sum(t, contents))
|
||||
})
|
||||
t.Run("Decompress", func(t *testing.T) {
|
||||
f.opt.Decompress = true
|
||||
defer func() {
|
||||
f.opt.Decompress = false
|
||||
}()
|
||||
checkDownload(original, -1, "")
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Features", f.testFeatures)
|
||||
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
||||
t.Run("Metadata", f.testMetadataPaths)
|
||||
t.Run("GzipEncoding", f.testGzipEncoding)
|
||||
t.Run("ArrowList", f.testArrowList)
|
||||
}
|
||||
|
||||
// helper to read blob properties for an object
|
||||
func getProps(ctx context.Context, t *testing.T, o fs.Object) *blob.GetPropertiesResponse {
|
||||
ao := o.(*Object)
|
||||
props, err := ao.readMetaDataAlways(ctx)
|
||||
require.NoError(t, err)
|
||||
return props
|
||||
}
|
||||
|
||||
// helper to assert select headers and user metadata
|
||||
func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, want map[string]string, wantUserMeta map[string]string) {
|
||||
// Headers
|
||||
get := func(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return *p
|
||||
}
|
||||
if v, ok := want["content-type"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentType), "content-type")
|
||||
}
|
||||
if v, ok := want["cache-control"]; ok {
|
||||
assert.Equal(t, v, get(props.CacheControl), "cache-control")
|
||||
}
|
||||
if v, ok := want["content-disposition"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentDisposition), "content-disposition")
|
||||
}
|
||||
if v, ok := want["content-encoding"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentEncoding), "content-encoding")
|
||||
}
|
||||
if v, ok := want["content-language"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentLanguage), "content-language")
|
||||
}
|
||||
// User metadata (case-insensitive keys from service)
|
||||
norm := make(map[string]*string, len(props.Metadata))
|
||||
for kk, vv := range props.Metadata {
|
||||
norm[strings.ToLower(kk)] = vv
|
||||
}
|
||||
for k, v := range wantUserMeta {
|
||||
pv, ok := norm[strings.ToLower(k)]
|
||||
if assert.True(t, ok, fmt.Sprintf("missing user metadata key %q", k)) {
|
||||
if pv == nil {
|
||||
assert.Equal(t, v, "", k)
|
||||
} else {
|
||||
assert.Equal(t, v, *pv, k)
|
||||
}
|
||||
} else {
|
||||
// Log available keys for diagnostics
|
||||
keys := make([]string, 0, len(props.Metadata))
|
||||
for kk := range props.Metadata {
|
||||
keys = append(keys, kk)
|
||||
}
|
||||
t.Logf("available user metadata keys: %v", keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// helper to read blob tags for an object
|
||||
// skipIfHNS skips the test if the storage account has a hierarchical
|
||||
// namespace as these don't support blob tags
|
||||
func skipIfHNS(t *testing.T, f *Fs) {
|
||||
if f.hns {
|
||||
t.Skip("blob tags not supported with hierarchical namespace")
|
||||
}
|
||||
}
|
||||
|
||||
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
||||
ao := o.(*Object)
|
||||
blb := ao.getBlobSVC()
|
||||
resp, err := blb.GetTags(ctx, nil)
|
||||
require.NoError(t, err)
|
||||
out := make(map[string]string)
|
||||
for _, tag := range resp.BlobTagSet {
|
||||
if tag.Key != nil {
|
||||
k := *tag.Key
|
||||
v := ""
|
||||
if tag.Value != nil {
|
||||
v = *tag.Value
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Test metadata across different write paths
|
||||
func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping in short mode")
|
||||
}
|
||||
|
||||
// Common expected metadata and headers
|
||||
baseMeta := fs.Metadata{
|
||||
"cache-control": "no-cache",
|
||||
"content-disposition": "inline",
|
||||
"content-language": "en-US",
|
||||
// Note: Don't set content-encoding here to avoid download decoding differences
|
||||
// We will set a custom user metadata key
|
||||
"potato": "royal",
|
||||
// and modtime
|
||||
"mtime": fstest.Time("2009-05-06T04:05:06.499999999Z").Format(time.RFC3339Nano),
|
||||
}
|
||||
|
||||
// Singlepart upload
|
||||
t.Run("PutSinglepart", func(t *testing.T) {
|
||||
// size less than chunk size
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("meta-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
// override content-type via metadata mapping
|
||||
meta := fs.Metadata{}
|
||||
meta.Merge(baseMeta)
|
||||
meta["content-type"] = "text/plain"
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
props := getProps(ctx, t, obj)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "text/plain",
|
||||
"cache-control": "no-cache",
|
||||
"content-disposition": "inline",
|
||||
"content-language": "en-US",
|
||||
}, map[string]string{
|
||||
"potato": "royal",
|
||||
})
|
||||
_ = http.StatusOK // keep import for parity but don't inspect RawResponse
|
||||
})
|
||||
|
||||
// Multipart upload
|
||||
t.Run("PutMultipart", func(t *testing.T) {
|
||||
// size greater than chunk size to force multipart
|
||||
contents := random.String(int(f.opt.ChunkSize + 1024))
|
||||
item := fstest.NewItem("meta-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{}
|
||||
meta.Merge(baseMeta)
|
||||
meta["content-type"] = "application/json"
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
props := getProps(ctx, t, obj)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "application/json",
|
||||
"cache-control": "no-cache",
|
||||
"content-disposition": "inline",
|
||||
"content-language": "en-US",
|
||||
}, map[string]string{
|
||||
"potato": "royal",
|
||||
})
|
||||
|
||||
// Tags: Singlepart upload
|
||||
t.Run("PutSinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
"x-ms-tags": "env=dev,team=sync",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
tags := getTagsMap(ctx, t, obj)
|
||||
assert.Equal(t, "dev", tags["env"])
|
||||
assert.Equal(t, "sync", tags["team"])
|
||||
})
|
||||
|
||||
// Tags: Multipart upload
|
||||
t.Run("PutMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
contents := random.String(int(f.opt.ChunkSize + 2048))
|
||||
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
"x-ms-tags": "project=alpha,release=2025-08",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "application/octet-stream", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
tags := getTagsMap(ctx, t, obj)
|
||||
assert.Equal(t, "alpha", tags["project"])
|
||||
assert.Equal(t, "2025-08", tags["release"])
|
||||
})
|
||||
})
|
||||
|
||||
// Singlepart copy with metadata-set mapping; omit content-type to exercise fallback
|
||||
t.Run("CopySinglepart", func(t *testing.T) {
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("meta-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
// set mapping via MetadataSet
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"cache-control": "private, max-age=60",
|
||||
"content-disposition": "attachment; filename=foo.txt",
|
||||
"content-language": "fr",
|
||||
// no content-type: should fallback to source
|
||||
"potato": "maris",
|
||||
}
|
||||
|
||||
// do copy
|
||||
dstName := "meta-copy-single-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
// content-type should fallback to source (text/plain)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "text/plain",
|
||||
"cache-control": "private, max-age=60",
|
||||
"content-disposition": "attachment; filename=foo.txt",
|
||||
"content-language": "fr",
|
||||
}, map[string]string{
|
||||
"potato": "maris",
|
||||
})
|
||||
// mtime should be populated on copy when --metadata is used
|
||||
// and should equal the source ModTime (RFC3339Nano)
|
||||
// Read user metadata (case-insensitive)
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
// parse and compare times ignoring formatting differences
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// CopySinglepart with only --metadata (no MetadataSet) must inject mtime and preserve src content-type
|
||||
t.Run("CopySinglepart_MetadataOnly", func(t *testing.T) {
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("meta-copy-single-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
|
||||
dstName := "meta-copy-single-only-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "text/plain",
|
||||
}, map[string]string{})
|
||||
// Assert mtime injected
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// Multipart copy with metadata-set mapping; omit content-type to exercise fallback
|
||||
t.Run("CopyMultipart", func(t *testing.T) {
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 1024))
|
||||
srcItem := fstest.NewItem("meta-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
// set mapping via MetadataSet
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"cache-control": "max-age=0, no-cache",
|
||||
// omit content-type to trigger fallback
|
||||
"content-language": "de",
|
||||
"potato": "desiree",
|
||||
}
|
||||
|
||||
dstName := "meta-copy-multi-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
// content-type should fallback to source (application/octet-stream)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "application/octet-stream",
|
||||
"cache-control": "max-age=0, no-cache",
|
||||
"content-language": "de",
|
||||
}, map[string]string{
|
||||
"potato": "desiree",
|
||||
})
|
||||
// mtime should be populated on copy when --metadata is used
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// CopyMultipart with only --metadata must inject mtime and preserve src content-type
|
||||
t.Run("CopyMultipart_MetadataOnly", func(t *testing.T) {
|
||||
contents := random.String(int(f.opt.CopyCutoff + 2048))
|
||||
srcItem := fstest.NewItem("meta-copy-multi-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
|
||||
dstName := "meta-copy-multi-only-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "application/octet-stream",
|
||||
}, map[string]string{})
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// Tags: Singlepart copy
|
||||
t.Run("CopySinglepartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
// set mapping via MetadataSet including tags
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"x-ms-tags": "copy=single,mode=test",
|
||||
}
|
||||
|
||||
dstName := "tags-copy-single-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
tags := getTagsMap(ctx2, t, dst)
|
||||
assert.Equal(t, "single", tags["copy"])
|
||||
assert.Equal(t, "test", tags["mode"])
|
||||
})
|
||||
|
||||
// Tags: Multipart copy
|
||||
t.Run("CopyMultipartTags", func(t *testing.T) {
|
||||
skipIfHNS(t, f)
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
||||
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"x-ms-tags": "copy=multi,mode=test",
|
||||
}
|
||||
|
||||
dstName := "tags-copy-multi-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
tags := getTagsMap(ctx2, t, dst)
|
||||
assert.Equal(t, "multi", tags["copy"])
|
||||
assert.Equal(t, "test", tags["mode"])
|
||||
})
|
||||
|
||||
// Negative: invalid x-ms-tags must error
|
||||
t.Run("InvalidXMsTags", func(t *testing.T) {
|
||||
contents := random.String(32)
|
||||
item := fstest.NewItem("tags-invalid.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
// construct ObjectInfo with invalid x-ms-tags
|
||||
buf := strings.NewReader(contents)
|
||||
// Build obj info with metadata
|
||||
meta := fs.Metadata{
|
||||
"x-ms-tags": "badpair-without-equals",
|
||||
}
|
||||
// force metadata on
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
obji := object.NewStaticObjectInfo(item.Path, item.ModTime, int64(len(contents)), true, nil, nil)
|
||||
obji = obji.WithMetadata(meta).WithMimeType("text/plain")
|
||||
_, err := f.Put(ctx2, buf, obji)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid tag")
|
||||
})
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !solaris && !js && !wasm
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || solaris || js || wasm
|
||||
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
package azureblob
|
||||
+510
-122
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
// Package azurefiles provides an interface to Microsoft Azure Files
|
||||
package azurefiles
|
||||
@@ -29,28 +29,35 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/fileerror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/service"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/share"
|
||||
"github.com/rclone/rclone/backend/azureblob/auth"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
)
|
||||
|
||||
@@ -58,9 +65,6 @@ const (
|
||||
maxFileSize = 4 * fs.Tebi
|
||||
defaultChunkSize = 4 * fs.Mebi
|
||||
storageDefaultBaseURL = "file.core.windows.net"
|
||||
// smbTimePrecision is the precision the server stores
|
||||
// modtimes with (100 ns FILETIME ticks)
|
||||
smbTimePrecision = 100 * time.Nanosecond
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -68,12 +72,199 @@ func init() {
|
||||
Name: "azurefiles",
|
||||
Description: "Microsoft Azure Files",
|
||||
NewFs: NewFs,
|
||||
Options: slices.Concat(auth.ConfigOptions, []fs.Option{{
|
||||
Options: []fs.Option{{
|
||||
Name: "account",
|
||||
Help: `Azure Storage Account Name.
|
||||
|
||||
Set this to the Azure Storage Account Name in use.
|
||||
|
||||
Leave blank to use SAS URL or connection string, otherwise it needs to be set.
|
||||
|
||||
If this is blank and if env_auth is set it will be read from the
|
||||
environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible.
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "share_name",
|
||||
Help: `Azure Files Share Name.
|
||||
|
||||
This is required and is the name of the share to access.
|
||||
`,
|
||||
}, {
|
||||
Name: "env_auth",
|
||||
Help: `Read credentials from runtime (environment variables, CLI or MSI).
|
||||
|
||||
See the [authentication docs](/azurefiles#authentication) for full info.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: `Storage Account Shared Key.
|
||||
|
||||
Leave blank to use SAS URL or connection string.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sas_url",
|
||||
Help: `SAS URL.
|
||||
|
||||
Leave blank if using account/key or connection string.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "connection_string",
|
||||
Help: `Azure Files Connection String.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "tenant",
|
||||
Help: `ID of the service principal's tenant. Also called its directory ID.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_id",
|
||||
Help: `The ID of the client in use.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_secret",
|
||||
Help: `One of the service principal's client secrets
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_certificate_path",
|
||||
Help: `Path to a PEM or PKCS12 certificate file including the private key.
|
||||
|
||||
Set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
}, {
|
||||
Name: "client_certificate_password",
|
||||
Help: `Password for the certificate file (optional).
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
|
||||
And the certificate has a password.
|
||||
`,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "client_send_certificate_chain",
|
||||
Help: `Send the certificate chain when using certificate auth.
|
||||
|
||||
Specifies whether an authentication request will include an x5c header
|
||||
to support subject name / issuer based authentication. When set to
|
||||
true, authentication requests include the x5c header.
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "username",
|
||||
Help: `User name (usually an email address)
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "password",
|
||||
Help: `The user's password
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
IsPassword: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "service_principal_file",
|
||||
Help: `Path to file containing credentials for use with a service principal.
|
||||
|
||||
Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
|
||||
|
||||
$ az ad sp create-for-rbac --name "<name>" \
|
||||
--role "Storage Files Data Owner" \
|
||||
--scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
|
||||
> azure-principal.json
|
||||
|
||||
See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to files data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
|
||||
|
||||
**NB** this section needs updating for Azure Files - pull requests appreciated!
|
||||
|
||||
It may be more convenient to put the credentials directly into the
|
||||
rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + `
|
||||
keys instead of setting ` + "`service_principal_file`" + `.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_msi",
|
||||
Help: `Use a managed service identity to authenticate (only works in Azure).
|
||||
|
||||
When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/)
|
||||
to authenticate to Azure Storage instead of a SAS token or account key.
|
||||
|
||||
If the VM(SS) on which this program is running has a system-assigned identity, it will
|
||||
be used by default. If the resource has no system-assigned but exactly one user-assigned identity,
|
||||
the user-assigned identity will be used by default. If the resource has multiple user-assigned
|
||||
identities, the identity to use must be explicitly specified using exactly one of the msi_object_id,
|
||||
msi_client_id, or msi_mi_res_id parameters.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "msi_object_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_client_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_mi_res_id",
|
||||
Help: "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "disable_instance_discovery",
|
||||
Help: `Skip requesting Microsoft Entra instance metadata
|
||||
This should be set true only by applications authenticating in
|
||||
disconnected clouds, or private clouds such as Azure Stack.
|
||||
It determines whether rclone requests Microsoft Entra instance
|
||||
metadata from ` + "`https://login.microsoft.com/`" + ` before
|
||||
authenticating.
|
||||
Setting this to true will skip this request, making you responsible
|
||||
for ensuring the configured authority is valid and trustworthy.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_az",
|
||||
Help: `Use Azure CLI tool az for authentication
|
||||
Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
as the sole means of authentication.
|
||||
Setting this can be useful if you wish to use the az CLI on a host with
|
||||
a System Managed Identity that you do not want to use.
|
||||
Don't set env_auth at the same time.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service.\n\nLeave blank normally.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Upload chunk size.
|
||||
@@ -131,18 +322,38 @@ You will need this much free space in the share as the file will be this size te
|
||||
encoder.EncodeInvalidUtf8 |
|
||||
encoder.EncodeCtl | encoder.EncodeDel |
|
||||
encoder.EncodeDot | encoder.EncodeRightPeriod),
|
||||
}}),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
auth.Options
|
||||
ShareName string `config:"share_name"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
MaxStreamSize fs.SizeSuffix `config:"max_stream_size"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
Account string `config:"account"`
|
||||
ShareName string `config:"share_name"`
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
Key string `config:"key"`
|
||||
SASURL string `config:"sas_url"`
|
||||
ConnectionString string `config:"connection_string"`
|
||||
Tenant string `config:"tenant"`
|
||||
ClientID string `config:"client_id"`
|
||||
ClientSecret string `config:"client_secret"`
|
||||
ClientCertificatePath string `config:"client_certificate_path"`
|
||||
ClientCertificatePassword string `config:"client_certificate_password"`
|
||||
ClientSendCertificateChain bool `config:"client_send_certificate_chain"`
|
||||
Username string `config:"username"`
|
||||
Password string `config:"password"`
|
||||
ServicePrincipalFile string `config:"service_principal_file"`
|
||||
DisableInstanceDiscovery bool `config:"disable_instance_discovery"`
|
||||
UseMSI bool `config:"use_msi"`
|
||||
MSIObjectID string `config:"msi_object_id"`
|
||||
MSIClientID string `config:"msi_client_id"`
|
||||
MSIResourceID string `config:"msi_mi_res_id"`
|
||||
UseAZ bool `config:"use_az"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
MaxStreamSize fs.SizeSuffix `config:"max_stream_size"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a root directory inside a share. The root directory can be ""
|
||||
@@ -165,33 +376,266 @@ type Object struct {
|
||||
contentType string // content type if known
|
||||
}
|
||||
|
||||
// Wrap the http.Transport to satisfy the Transporter interface
|
||||
type transporter struct {
|
||||
http.RoundTripper
|
||||
}
|
||||
|
||||
// Make a new transporter
|
||||
func newTransporter(ctx context.Context) transporter {
|
||||
return transporter{
|
||||
RoundTripper: fshttp.NewTransport(ctx),
|
||||
}
|
||||
}
|
||||
|
||||
// Do sends the HTTP request and returns the HTTP response or error.
|
||||
func (tr transporter) Do(req *http.Request) (*http.Response, error) {
|
||||
return tr.RoundTripper.RoundTrip(req)
|
||||
}
|
||||
|
||||
type servicePrincipalCredentials struct {
|
||||
AppID string `json:"appId"`
|
||||
Password string `json:"password"`
|
||||
Tenant string `json:"tenant"`
|
||||
}
|
||||
|
||||
// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli.
|
||||
func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) {
|
||||
var spCredentials servicePrincipalCredentials
|
||||
if err := json.Unmarshal(credentialsData, &spCredentials); err != nil {
|
||||
return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
|
||||
}
|
||||
// TODO: support certificate credentials
|
||||
// Validate all fields present
|
||||
if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" {
|
||||
return nil, fmt.Errorf("missing fields in credentials file")
|
||||
}
|
||||
return &spCredentials, nil
|
||||
}
|
||||
|
||||
// Factored out from NewFs so that it can be tested with opt *Options and without m configmap.Mapper
|
||||
func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.Fs, error) {
|
||||
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
|
||||
DefaultBaseURL: storageDefaultBaseURL,
|
||||
NewClient: service.NewClient,
|
||||
NewClientFromConnectionString: service.NewClientFromConnectionString,
|
||||
NewClientWithNoCredential: service.NewClientWithNoCredential,
|
||||
NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential,
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
options.ClientOptions = policyClientOptions
|
||||
// FileRequestIntent is required for TokenCredential (OAuth)
|
||||
// auth; the SDK omits it for shared-key / SAS.
|
||||
backup := service.ShareTokenIntentBackup
|
||||
options.FileRequestIntent = &backup
|
||||
},
|
||||
// Client options specifying our own transport
|
||||
policyClientOptions := policy.ClientOptions{
|
||||
Transport: newTransporter(ctx),
|
||||
}
|
||||
res, err := auth.NewClient(ctx, conf, &opt.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
backup := service.ShareTokenIntentBackup
|
||||
clientOpt := service.ClientOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
FileRequestIntent: &backup,
|
||||
}
|
||||
// f.svc = res.Client
|
||||
// f.cred = res.Cred
|
||||
// f.sharedKeyCred = res.SharedKeyCred
|
||||
// f.anonymous = res.Anonymous
|
||||
|
||||
shareClient := res.Client.NewShareClient(opt.ShareName)
|
||||
// Here we auth by setting one of cred, sharedKeyCred or f.client
|
||||
var (
|
||||
cred azcore.TokenCredential
|
||||
sharedKeyCred *service.SharedKeyCredential
|
||||
client *service.Client
|
||||
err error
|
||||
)
|
||||
switch {
|
||||
case opt.EnvAuth:
|
||||
// Read account from environment if needed
|
||||
if opt.Account == "" {
|
||||
opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME")
|
||||
}
|
||||
// Read credentials from the environment
|
||||
options := azidentity.DefaultAzureCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
|
||||
}
|
||||
cred, err = azidentity.NewDefaultAzureCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create azure environment credential failed: %w", err)
|
||||
}
|
||||
case opt.Account != "" && opt.Key != "":
|
||||
sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create new shared key credential failed: %w", err)
|
||||
}
|
||||
case opt.UseAZ:
|
||||
options := azidentity.AzureCLICredentialOptions{}
|
||||
cred, err = azidentity.NewAzureCLICredential(&options)
|
||||
fmt.Println(cred)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
|
||||
}
|
||||
case opt.SASURL != "":
|
||||
client, err = service.NewClientWithNoCredential(opt.SASURL, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create SAS URL client: %w", err)
|
||||
}
|
||||
case opt.ConnectionString != "":
|
||||
client, err = service.NewClientFromConnectionString(opt.ConnectionString, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create connection string client: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "":
|
||||
// Service principal with client secret
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "":
|
||||
// Service principal with certificate
|
||||
//
|
||||
// Read the certificate
|
||||
data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading client certificate file: %w", err)
|
||||
}
|
||||
// NewClientCertificateCredential requires at least one *x509.Certificate, and a
|
||||
// crypto.PrivateKey.
|
||||
//
|
||||
// ParseCertificates returns these given certificate data in PEM or PKCS12 format.
|
||||
// It handles common scenarios but has limitations, for example it doesn't load PEM
|
||||
// encrypted private keys.
|
||||
var password []byte
|
||||
if opt.ClientCertificatePassword != "" {
|
||||
pw, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
password = []byte(pw)
|
||||
}
|
||||
certs, key, err := azidentity.ParseCertificates(data, password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse client certificate file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientCertificateCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
SendCertificateChain: opt.ClientSendCertificateChain,
|
||||
}
|
||||
cred, err = azidentity.NewClientCertificateCredential(
|
||||
opt.Tenant, opt.ClientID, certs, key, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
|
||||
// User with username and password
|
||||
//nolint:staticcheck // this is deprecated due to Azure policy
|
||||
options := azidentity.UsernamePasswordCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
password, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("user password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
cred, err = azidentity.NewUsernamePasswordCredential(
|
||||
opt.Tenant, opt.ClientID, opt.Username, password, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("authenticate user with password failed: %w", err)
|
||||
}
|
||||
case opt.ServicePrincipalFile != "":
|
||||
// Loading service principal credentials from file.
|
||||
loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
|
||||
}
|
||||
parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing service principal credentials file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.UseMSI:
|
||||
// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
|
||||
// Validate and ensure exactly one is set. (To do: better validation.)
|
||||
b2i := map[bool]int{false: 0, true: 1}
|
||||
set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""]
|
||||
if set > 1 {
|
||||
return nil, errors.New("more than one user-assigned identity ID is set")
|
||||
}
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
switch {
|
||||
case opt.MSIClientID != "":
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
case opt.MSIObjectID != "":
|
||||
// FIXME this doesn't appear to be in the new SDK?
|
||||
return nil, fmt.Errorf("MSI object ID is currently unsupported")
|
||||
case opt.MSIResourceID != "":
|
||||
options.ID = azidentity.ResourceID(opt.MSIResourceID)
|
||||
}
|
||||
cred, err = azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, errors.New("no authentication method configured")
|
||||
}
|
||||
|
||||
// Make the client if not already created
|
||||
if client == nil {
|
||||
// Work out what the endpoint is if it is still unset
|
||||
if opt.Endpoint == "" {
|
||||
if opt.Account == "" {
|
||||
return nil, fmt.Errorf("account must be set: can't make service URL")
|
||||
}
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, storageDefaultBaseURL))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to make azure storage URL from account: %w", err)
|
||||
}
|
||||
opt.Endpoint = u.String()
|
||||
}
|
||||
if sharedKeyCred != nil {
|
||||
// Shared key cred
|
||||
client, err = service.NewClientWithSharedKeyCredential(opt.Endpoint, sharedKeyCred, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create client with shared key failed: %w", err)
|
||||
}
|
||||
} else if cred != nil {
|
||||
// Azidentity cred
|
||||
client, err = service.NewClient(opt.Endpoint, cred, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create client failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("internal error: auth failed to make credentials or client")
|
||||
}
|
||||
|
||||
shareClient := client.NewShareClient(opt.ShareName)
|
||||
svc := shareClient.NewRootDirectoryClient()
|
||||
f := &Fs{
|
||||
shareClient: shareClient,
|
||||
@@ -253,10 +697,10 @@ func (f *Fs) Features() *fs.Features {
|
||||
|
||||
// Precision return the precision of this Fs
|
||||
//
|
||||
// The SMB last write time properties are ISO 8601 with 100 ns (FILETIME) precision
|
||||
// One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds
|
||||
// Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
return smbTimePrecision
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
@@ -399,32 +843,15 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return list.WithListP(ctx, dir, f)
|
||||
}
|
||||
|
||||
// ListP lists the objects and directories of the Fs starting
|
||||
// from dir non recursively into out.
|
||||
//
|
||||
// dir should be "" to start from the root, and should not
|
||||
// have trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
//
|
||||
// It should call callback for each tranche of entries read.
|
||||
// These need not be returned in any particular order. If
|
||||
// callback returns an error then the listing will stop
|
||||
// immediately.
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
||||
list := list.NewHelper(callback)
|
||||
var entries fs.DirEntries
|
||||
subDirClient := f.dirClient(dir)
|
||||
|
||||
// Checking whether directory exists
|
||||
_, err := subDirClient.GetProperties(ctx, nil)
|
||||
if fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ResourceNotFound) {
|
||||
return fs.ErrorDirNotFound
|
||||
return entries, fs.ErrorDirNotFound
|
||||
} else if err != nil {
|
||||
return err
|
||||
return entries, err
|
||||
}
|
||||
|
||||
opt := &directory.ListFilesAndDirectoriesOptions{
|
||||
@@ -436,7 +863,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
for pager.More() {
|
||||
resp, err := pager.NextPage(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return entries, err
|
||||
}
|
||||
for _, directory := range resp.Segment.Directories {
|
||||
// Name *string `xml:"Name"`
|
||||
@@ -462,10 +889,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
if directory.Properties.ContentLength != nil {
|
||||
entry.SetSize(*directory.Properties.ContentLength)
|
||||
}
|
||||
err = list.Add(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
for _, file := range resp.Segment.Files {
|
||||
leaf := f.opt.Enc.ToStandardPath(*file.Name)
|
||||
@@ -479,13 +903,10 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
if file.Properties.LastWriteTime != nil {
|
||||
entry.modTime = *file.Properties.LastWriteTime
|
||||
}
|
||||
err = list.Add(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
}
|
||||
return list.Flush()
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -601,10 +1022,7 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to set modTime: %w", err)
|
||||
}
|
||||
// Truncate to the precision the server stores the modtime with
|
||||
// to keep the in-memory modtime identical to the one a fresh
|
||||
// listing returns.
|
||||
o.modTime = t.Truncate(smbTimePrecision)
|
||||
o.modTime = t
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -650,6 +1068,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Returns a pointer to t - useful for returning pointers to constants
|
||||
func ptr[T any](t T) *T {
|
||||
return &t
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// Update the object with the contents of the io.Reader, modTime, size and MD5 hash
|
||||
@@ -691,9 +1114,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
// Count the bytes read so short reads from the source can be detected
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
// Measure the size if it is unknown
|
||||
if sizeUnknown {
|
||||
counter = readers.NewCountingReader(in)
|
||||
in = counter
|
||||
}
|
||||
|
||||
// Check we have a source MD5 hash...
|
||||
if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
|
||||
@@ -728,21 +1153,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if sizeUnknown {
|
||||
// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
|
||||
size = int64(counter.BytesRead())
|
||||
} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
|
||||
// The file was created at the declared size so a short source
|
||||
// leaves the remainder padded with zeroes
|
||||
if isNewlyCreated {
|
||||
if _, delErr := fc.Delete(ctx, nil); delErr != nil {
|
||||
fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
|
||||
}
|
||||
} else if bytesRead < size {
|
||||
// Truncate the file to the bytes actually received so it
|
||||
// is not left padded to the declared size with zeroes
|
||||
if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
|
||||
fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
|
||||
}
|
||||
if hashUnknown {
|
||||
md5Hash = hasher.Sum(nil)
|
||||
@@ -783,12 +1193,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return fmt.Errorf("update: failed to set properties: %w", err)
|
||||
}
|
||||
|
||||
// Make sure Object is in sync, truncating the modtime to the
|
||||
// precision the server stores it with so it is identical to the
|
||||
// one a fresh listing returns
|
||||
// Make sure Object is in sync
|
||||
o.size = size
|
||||
o.md5 = md5Hash
|
||||
o.modTime = modTime.Truncate(smbTimePrecision)
|
||||
o.modTime = modTime
|
||||
o.contentType = contentType
|
||||
return nil
|
||||
}
|
||||
@@ -813,8 +1221,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
|
||||
}
|
||||
opt := file.RenameOptions{
|
||||
IgnoreReadOnly: new(true),
|
||||
ReplaceIfExists: new(true),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
ReplaceIfExists: ptr(true),
|
||||
}
|
||||
dstAbsPath := f.absPath(remote)
|
||||
fc := srcObj.fileClient()
|
||||
@@ -859,8 +1267,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
opt := directory.RenameOptions{
|
||||
IgnoreReadOnly: new(false),
|
||||
ReplaceIfExists: new(false),
|
||||
IgnoreReadOnly: ptr(false),
|
||||
ReplaceIfExists: ptr(false),
|
||||
}
|
||||
dstAbsPath := dstFs.absPath(dstRemote)
|
||||
dirClient := srcFs.dirClient(srcRemote)
|
||||
@@ -899,35 +1307,16 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
ChangeTime: file.SourceCopyFileChangeTime{},
|
||||
CreationTime: file.SourceCopyFileCreationTime{},
|
||||
LastWriteTime: file.SourceCopyFileLastWriteTime{},
|
||||
PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: new(true),
|
||||
PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
|
||||
IgnoreReadOnly: ptr(true),
|
||||
},
|
||||
}
|
||||
srcURL := srcObj.fileClient().URL()
|
||||
fc := f.fileClient(remote)
|
||||
startCopy, err := fc.StartCopyFromURL(ctx, srcURL, &opt)
|
||||
_, err = fc.StartCopyFromURL(ctx, srcURL, &opt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Copy failed: %w", err)
|
||||
}
|
||||
|
||||
// Poll for completion if necessary
|
||||
//
|
||||
// The for loop is never executed for same storage account copies.
|
||||
copyStatus := startCopy.CopyStatus
|
||||
var properties file.GetPropertiesResponse
|
||||
pollTime := 100 * time.Millisecond
|
||||
|
||||
for copyStatus != nil && string(*copyStatus) == string(file.CopyStatusTypePending) {
|
||||
time.Sleep(pollTime)
|
||||
|
||||
properties, err = fc.GetProperties(ctx, &file.GetPropertiesOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copyStatus = properties.CopyStatus
|
||||
pollTime = min(2*pollTime, time.Second)
|
||||
}
|
||||
|
||||
dstObj, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Copy: NewObject failed: %w", err)
|
||||
@@ -1042,7 +1431,6 @@ var (
|
||||
_ fs.DirMover = &Fs{}
|
||||
_ fs.Copier = &Fs{}
|
||||
_ fs.OpenWriterAter = &Fs{}
|
||||
_ fs.ListPer = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
_ fs.MimeTyper = &Object{}
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package azurefiles
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/azureblob/auth"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
@@ -29,28 +28,22 @@ func (f *Fs) InternalTestAuth(t *testing.T) {
|
||||
{
|
||||
name: "ConnectionString",
|
||||
options: &Options{
|
||||
ShareName: shareName,
|
||||
Options: auth.Options{
|
||||
ConnectionString: "",
|
||||
},
|
||||
ShareName: shareName,
|
||||
ConnectionString: "",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "AccountAndKey",
|
||||
options: &Options{
|
||||
ShareName: shareName,
|
||||
Options: auth.Options{
|
||||
Account: "",
|
||||
Key: "",
|
||||
},
|
||||
Account: "",
|
||||
Key: "",
|
||||
}},
|
||||
{
|
||||
name: "SASUrl",
|
||||
options: &Options{
|
||||
ShareName: shareName,
|
||||
Options: auth.Options{
|
||||
SASURL: "",
|
||||
},
|
||||
SASURL: "",
|
||||
}},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package azurefiles
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for azurefiles for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || js || wasm
|
||||
|
||||
// Package azurefiles provides an interface to Microsoft Azure Files
|
||||
package azurefiles
|
||||
+22
-44
@@ -48,14 +48,6 @@ type LifecycleRule struct {
|
||||
FileNamePrefix string `json:"fileNamePrefix"`
|
||||
}
|
||||
|
||||
// ServerSideEncryption is a configuration object for B2 Server-Side Encryption
|
||||
type ServerSideEncryption struct {
|
||||
Mode string `json:"mode"`
|
||||
Algorithm string `json:"algorithm"` // Encryption algorithm to use
|
||||
CustomerKey string `json:"customerKey"` // User provided Base64 encoded key that is used by the server to encrypt files
|
||||
CustomerKeyMd5 string `json:"customerKeyMd5"` // An MD5 hash of the decoded key
|
||||
}
|
||||
|
||||
// Timestamp is a UTC time when this file was uploaded. It is a base
|
||||
// 10 number of milliseconds since midnight, January 1, 1970 UTC. This
|
||||
// fits in a 64 bit integer such as the type "long" in the programming
|
||||
@@ -133,32 +125,23 @@ type File struct {
|
||||
Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.
|
||||
}
|
||||
|
||||
// StorageAPI is as returned from the b2_authorize_account call
|
||||
type StorageAPI struct {
|
||||
// AuthorizeAccountResponse is as returned from the b2_authorize_account call
|
||||
type AuthorizeAccountResponse struct {
|
||||
AbsoluteMinimumPartSize int `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file.
|
||||
AccountID string `json:"accountId"` // The identifier for the account.
|
||||
Allowed struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
|
||||
Buckets []struct { // When present, access is restricted to one or more buckets.
|
||||
ID string `json:"id"` // ID of bucket
|
||||
Name string `json:"name"` // When present, name of bucket - may be empty
|
||||
} `json:"buckets"`
|
||||
Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has for every bucket.
|
||||
BucketID string `json:"bucketId"` // When present, access is restricted to one bucket.
|
||||
BucketName string `json:"bucketName"` // When present, name of bucket - may be empty
|
||||
Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.
|
||||
NamePrefix any `json:"namePrefix"` // When present, access is restricted to files whose names start with the prefix
|
||||
} `json:"allowed"`
|
||||
APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files.
|
||||
AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
|
||||
DownloadURL string `json:"downloadUrl"` // The base URL to use for downloading files.
|
||||
MinimumPartSize int `json:"minimumPartSize"` // DEPRECATED: This field will always have the same value as recommendedPartSize. Use recommendedPartSize instead.
|
||||
RecommendedPartSize int `json:"recommendedPartSize"` // The recommended size for each part of a large file. We recommend using this part size for optimal upload performance.
|
||||
}
|
||||
|
||||
// AuthorizeAccountResponse is as returned from the b2_authorize_account call
|
||||
type AuthorizeAccountResponse struct {
|
||||
AccountID string `json:"accountId"` // The identifier for the account.
|
||||
AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
|
||||
APIs struct { // Supported APIs for this account / key. These are API-dependent JSON objects.
|
||||
Storage StorageAPI `json:"storageApi"`
|
||||
} `json:"apiInfo"`
|
||||
}
|
||||
|
||||
// ListBucketsRequest is parameters for b2_list_buckets call
|
||||
type ListBucketsRequest struct {
|
||||
AccountID string `json:"accountId"` // The identifier for the account.
|
||||
@@ -278,11 +261,10 @@ type GetFileInfoRequest struct {
|
||||
//
|
||||
// Example: { "src_last_modified_millis" : "1452802803026", "large_file_sha1" : "a3195dc1e7b46a2ff5da4b3c179175b75671e80d", "color": "blue" }
|
||||
type StartLargeFileRequest struct {
|
||||
BucketID string `json:"bucketId"` // The ID of the bucket that the file will go in.
|
||||
Name string `json:"fileName"` // The name of the file. See Files for requirements on file names.
|
||||
ContentType string `json:"contentType"` // The MIME type of the content of the file, which will be returned in the Content-Type header when downloading the file. Use the Content-Type b2/x-auto to automatically set the stored Content-Type post upload. In the case where a file extension is absent or the lookup fails, the Content-Type is set to application/octet-stream.
|
||||
Info map[string]string `json:"fileInfo"` // A JSON object holding the name/value pairs for the custom file info.
|
||||
ServerSideEncryption *ServerSideEncryption `json:"serverSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption
|
||||
BucketID string `json:"bucketId"` //The ID of the bucket that the file will go in.
|
||||
Name string `json:"fileName"` // The name of the file. See Files for requirements on file names.
|
||||
ContentType string `json:"contentType"` // The MIME type of the content of the file, which will be returned in the Content-Type header when downloading the file. Use the Content-Type b2/x-auto to automatically set the stored Content-Type post upload. In the case where a file extension is absent or the lookup fails, the Content-Type is set to application/octet-stream.
|
||||
Info map[string]string `json:"fileInfo"` // A JSON object holding the name/value pairs for the custom file info.
|
||||
}
|
||||
|
||||
// StartLargeFileResponse is the response to StartLargeFileRequest
|
||||
@@ -343,25 +325,21 @@ type CancelLargeFileResponse struct {
|
||||
|
||||
// CopyFileRequest is as passed to b2_copy_file
|
||||
type CopyFileRequest struct {
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
Name string `json:"fileName"` // The name of the new file being created.
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
MetadataDirective string `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE
|
||||
ContentType string `json:"contentType,omitempty"` // The MIME type of the content of the file (REPLACE only)
|
||||
Info map[string]string `json:"fileInfo,omitempty"` // This field stores the metadata that will be stored with the file. (REPLACE only)
|
||||
DestBucketID string `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
|
||||
SourceServerSideEncryption *ServerSideEncryption `json:"sourceServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the source file
|
||||
DestinationServerSideEncryption *ServerSideEncryption `json:"destinationServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the destination file
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
Name string `json:"fileName"` // The name of the new file being created.
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
MetadataDirective string `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE
|
||||
ContentType string `json:"contentType,omitempty"` // The MIME type of the content of the file (REPLACE only)
|
||||
Info map[string]string `json:"fileInfo,omitempty"` // This field stores the metadata that will be stored with the file. (REPLACE only)
|
||||
DestBucketID string `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
|
||||
}
|
||||
|
||||
// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
|
||||
type CopyPartRequest struct {
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
LargeFileID string `json:"largeFileId"` // The ID of the large file the part will belong to, as returned by b2_start_large_file.
|
||||
PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1)
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
SourceServerSideEncryption *ServerSideEncryption `json:"sourceServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the source file
|
||||
DestinationServerSideEncryption *ServerSideEncryption `json:"destinationServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the destination file
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
LargeFileID string `json:"largeFileId"` // The ID of the large file the part will belong to, as returned by b2_start_large_file.
|
||||
PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1)
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
}
|
||||
|
||||
// UpdateBucketRequest describes a request to modify a B2 bucket
|
||||
|
||||
+84
-301
@@ -8,9 +8,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -55,9 +53,6 @@ const (
|
||||
nameHeader = "X-Bz-File-Name"
|
||||
timestampHeader = "X-Bz-Upload-Timestamp"
|
||||
retryAfterHeader = "Retry-After"
|
||||
sseAlgorithmHeader = "X-Bz-Server-Side-Encryption-Customer-Algorithm"
|
||||
sseKeyHeader = "X-Bz-Server-Side-Encryption-Customer-Key"
|
||||
sseMd5Header = "X-Bz-Server-Side-Encryption-Customer-Key-Md5"
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 5 * time.Minute
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
@@ -72,7 +67,7 @@ const (
|
||||
|
||||
// Globals
|
||||
var (
|
||||
errNotWithVersions = errors.New("can't modify files in --b2-versions mode")
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
|
||||
)
|
||||
|
||||
@@ -191,11 +186,6 @@ If the custom endpoint rewrites the requests for authentication,
|
||||
e.g., in Cloudflare Workers, this header needs to be handled properly.
|
||||
Leave blank if you want to use the endpoint provided by Backblaze.
|
||||
|
||||
Rclone will not follow a redirect from this endpoint which downgrades
|
||||
HTTPS to HTTP, and will not send the "Authorization" header or the
|
||||
SSE-C key headers once a redirect has left the host and port of this
|
||||
endpoint.
|
||||
|
||||
The URL provided here SHOULD have the protocol and SHOULD NOT have
|
||||
a trailing slash or specify the /file/bucket subpath as rclone will
|
||||
request files with "{download_url}/file/{bucket_name}/{path}".
|
||||
@@ -262,51 +252,6 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}, {
|
||||
Name: "sse_customer_algorithm",
|
||||
Help: "If using SSE-C, the server-side encryption algorithm used when storing this object in B2.",
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}, {
|
||||
Value: "AES256",
|
||||
Help: "Advanced Encryption Standard (256 bits key length)",
|
||||
}},
|
||||
}, {
|
||||
Name: "sse_customer_key",
|
||||
Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
|
||||
|
||||
Alternatively you can provide --sse-customer-key-base64.`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}},
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sse_customer_key_base64",
|
||||
Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
|
||||
|
||||
Alternatively you can provide --sse-customer-key.`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}},
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sse_customer_key_md5",
|
||||
Help: `If using SSE-C you may provide the secret encryption key MD5 checksum (optional).
|
||||
|
||||
If you leave it blank, this is calculated automatically from the sse_customer_key provided.
|
||||
`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}},
|
||||
Sensitive: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -329,10 +274,6 @@ type Options struct {
|
||||
DownloadAuthorizationDuration fs.Duration `config:"download_auth_duration"`
|
||||
Lifecycle int `config:"lifecycle"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
SSECustomerAlgorithm string `config:"sse_customer_algorithm"`
|
||||
SSECustomerKey string `config:"sse_customer_key"`
|
||||
SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`
|
||||
SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`
|
||||
}
|
||||
|
||||
// Fs represents a remote b2 server
|
||||
@@ -534,25 +475,6 @@ func (f *Fs) setRoot(root string) {
|
||||
f.rootBucket, f.rootDirectory = bucket.Split(f.root)
|
||||
}
|
||||
|
||||
// redirectSecretHeaders are the request headers carrying secrets that
|
||||
// must not be forwarded when a redirect leaves the original host.
|
||||
var redirectSecretHeaders = []string{
|
||||
"Authorization",
|
||||
sseAlgorithmHeader,
|
||||
sseKeyHeader,
|
||||
sseMd5Header,
|
||||
}
|
||||
|
||||
// newClient makes the http client used for all B2 requests.
|
||||
//
|
||||
// The download_url option sends the requests to a host the user does
|
||||
// not control so it strips headers on cross host redirects.
|
||||
func newClient(ctx context.Context) *http.Client {
|
||||
client := fshttp.NewClient(ctx)
|
||||
client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
|
||||
return client
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, bucket:path
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
@@ -582,30 +504,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if opt.Endpoint == "" {
|
||||
opt.Endpoint = defaultEndpoint
|
||||
}
|
||||
if opt.SSECustomerKey != "" && opt.SSECustomerKeyBase64 != "" {
|
||||
return nil, errors.New("b2: can't use both sse_customer_key and sse_customer_key_base64 at the same time")
|
||||
} else if opt.SSECustomerKeyBase64 != "" {
|
||||
// Decode the Base64-encoded key and store it in the SSECustomerKey field
|
||||
decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKeyBase64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("b2: Could not decode sse_customer_key_base64: %w", err)
|
||||
}
|
||||
opt.SSECustomerKey = string(decoded)
|
||||
} else {
|
||||
// Encode the raw key as Base64
|
||||
opt.SSECustomerKeyBase64 = base64.StdEncoding.EncodeToString([]byte(opt.SSECustomerKey))
|
||||
}
|
||||
if opt.SSECustomerKey != "" && opt.SSECustomerKeyMD5 == "" {
|
||||
// Calculate CustomerKeyMd5 if not supplied
|
||||
md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))
|
||||
opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
|
||||
}
|
||||
ci := fs.GetConfig(ctx)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
ci: ci,
|
||||
srv: rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
|
||||
srv: rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
|
||||
cache: bucket.NewCache(),
|
||||
_bucketID: make(map[string]string, 1),
|
||||
_bucketType: make(map[string]string, 1),
|
||||
@@ -631,29 +535,17 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to authorize account: %w", err)
|
||||
}
|
||||
// If this is a key limited to one or more buckets, one of them must exist
|
||||
// and be ours.
|
||||
if f.rootBucket != "" && len(f.info.APIs.Storage.Allowed.Buckets) != 0 {
|
||||
buckets := f.info.APIs.Storage.Allowed.Buckets
|
||||
var rootFound = false
|
||||
var rootID string
|
||||
for _, b := range buckets {
|
||||
allowedBucket := f.opt.Enc.ToStandardName(b.Name)
|
||||
if allowedBucket == "" {
|
||||
fs.Debugf(f, "bucket %q that application key is restricted to no longer exists", b.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
if allowedBucket == f.rootBucket {
|
||||
rootFound = true
|
||||
rootID = b.ID
|
||||
}
|
||||
// If this is a key limited to a single bucket, it must exist already
|
||||
if f.rootBucket != "" && f.info.Allowed.BucketID != "" {
|
||||
allowedBucket := f.opt.Enc.ToStandardName(f.info.Allowed.BucketName)
|
||||
if allowedBucket == "" {
|
||||
return nil, errors.New("bucket that application key is restricted to no longer exists")
|
||||
}
|
||||
if !rootFound {
|
||||
return nil, fmt.Errorf("you must use bucket(s) %q with this application key", buckets)
|
||||
if allowedBucket != f.rootBucket {
|
||||
return nil, fmt.Errorf("you must use bucket %q with this application key", allowedBucket)
|
||||
}
|
||||
f.cache.MarkOK(f.rootBucket)
|
||||
f.setBucketID(f.rootBucket, rootID)
|
||||
f.setBucketID(f.rootBucket, f.info.Allowed.BucketID)
|
||||
}
|
||||
if f.rootBucket != "" && f.rootDirectory != "" {
|
||||
// Check to see if the (bucket,directory) is actually an existing file
|
||||
@@ -679,7 +571,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
|
||||
defer f.authMu.Unlock()
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/b2api/v4/b2_authorize_account",
|
||||
Path: "/b2api/v1/b2_authorize_account",
|
||||
RootURL: f.opt.Endpoint,
|
||||
UserName: f.opt.Account,
|
||||
Password: f.opt.Key,
|
||||
@@ -692,13 +584,13 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to authenticate: %w", err)
|
||||
}
|
||||
f.srv.SetRoot(f.info.APIs.Storage.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
|
||||
f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasPermission returns if the current AuthorizationToken has the selected permission
|
||||
func (f *Fs) hasPermission(permission string) bool {
|
||||
return slices.Contains(f.info.APIs.Storage.Allowed.Capabilities, permission)
|
||||
return slices.Contains(f.info.Allowed.Capabilities, permission)
|
||||
}
|
||||
|
||||
// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
|
||||
@@ -955,7 +847,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *api.File
|
||||
}
|
||||
|
||||
// listDir lists a single directory
|
||||
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool, callback func(fs.DirEntry) error) (err error) {
|
||||
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) {
|
||||
last := ""
|
||||
err = f.list(ctx, bucket, directory, prefix, f.rootBucket == "", false, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory, &last)
|
||||
@@ -963,16 +855,16 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
|
||||
return err
|
||||
}
|
||||
if entry != nil {
|
||||
return callback(entry)
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
// bucket must be present if listing succeeded
|
||||
f.cache.MarkOK(bucket)
|
||||
return nil
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// listBuckets returns all the buckets to out
|
||||
@@ -998,46 +890,14 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
return list.WithListP(ctx, dir, f)
|
||||
}
|
||||
|
||||
// ListP lists the objects and directories of the Fs starting
|
||||
// from dir non recursively into out.
|
||||
//
|
||||
// dir should be "" to start from the root, and should not
|
||||
// have trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
//
|
||||
// It should call callback for each tranche of entries read.
|
||||
// These need not be returned in any particular order. If
|
||||
// callback returns an error then the listing will stop
|
||||
// immediately.
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
||||
list := list.NewHelper(callback)
|
||||
bucket, directory := f.split(dir)
|
||||
if bucket == "" {
|
||||
if directory != "" {
|
||||
return fs.ErrorListBucketRequired
|
||||
}
|
||||
entries, err := f.listBuckets(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
err = list.Add(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err := f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", list.Add)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, fs.ErrorListBucketRequired
|
||||
}
|
||||
return f.listBuckets(ctx)
|
||||
}
|
||||
return list.Flush()
|
||||
return f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "")
|
||||
}
|
||||
|
||||
// ListR lists the objects and directories of the Fs starting
|
||||
@@ -1103,83 +963,44 @@ type listBucketFn func(*api.Bucket) error
|
||||
|
||||
// listBucketsToFn lists the buckets to the function supplied
|
||||
func (f *Fs) listBucketsToFn(ctx context.Context, bucketName string, fn listBucketFn) error {
|
||||
responses := make([]api.ListBucketsResponse, len(f.info.APIs.Storage.Allowed.Buckets))[:0]
|
||||
|
||||
call := func(id string) error {
|
||||
var account = api.ListBucketsRequest{
|
||||
AccountID: f.info.AccountID,
|
||||
BucketID: id,
|
||||
}
|
||||
if bucketName != "" && account.BucketID == "" {
|
||||
account.BucketName = f.opt.Enc.FromStandardName(bucketName)
|
||||
}
|
||||
|
||||
var response api.ListBucketsResponse
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_list_buckets",
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
responses = append(responses, response)
|
||||
return nil
|
||||
var account = api.ListBucketsRequest{
|
||||
AccountID: f.info.AccountID,
|
||||
BucketID: f.info.Allowed.BucketID,
|
||||
}
|
||||
if bucketName != "" && account.BucketID == "" {
|
||||
account.BucketName = f.opt.Enc.FromStandardName(bucketName)
|
||||
}
|
||||
|
||||
for i := range f.info.APIs.Storage.Allowed.Buckets {
|
||||
b := &f.info.APIs.Storage.Allowed.Buckets[i]
|
||||
// Empty names indicate a bucket that no longer exists, this is non-fatal
|
||||
// for multi-bucket API keys.
|
||||
if b.Name == "" {
|
||||
continue
|
||||
}
|
||||
// When requesting a specific bucket skip over non-matching names
|
||||
if bucketName != "" && b.Name != bucketName {
|
||||
continue
|
||||
}
|
||||
|
||||
err := call(b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var response api.ListBucketsResponse
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_list_buckets",
|
||||
}
|
||||
|
||||
if len(f.info.APIs.Storage.Allowed.Buckets) == 0 {
|
||||
err := call("")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.bucketIDMutex.Lock()
|
||||
f.bucketTypeMutex.Lock()
|
||||
f._bucketID = make(map[string]string, 1)
|
||||
f._bucketType = make(map[string]string, 1)
|
||||
|
||||
for ri := range responses {
|
||||
response := &responses[ri]
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
|
||||
f.cache.MarkOK(bucket.Name)
|
||||
f._bucketID[bucket.Name] = bucket.ID
|
||||
f._bucketType[bucket.Name] = bucket.Type
|
||||
}
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
|
||||
f.cache.MarkOK(bucket.Name)
|
||||
f._bucketID[bucket.Name] = bucket.ID
|
||||
f._bucketType[bucket.Name] = bucket.Type
|
||||
}
|
||||
f.bucketTypeMutex.Unlock()
|
||||
f.bucketIDMutex.Unlock()
|
||||
for ri := range responses {
|
||||
response := &responses[ri]
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
err := fn(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
err = fn(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1582,16 +1403,6 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
|
||||
Name: f.opt.Enc.FromStandardPath(dstPath),
|
||||
DestBucketID: destBucketID,
|
||||
}
|
||||
if f.opt.SSECustomerKey != "" && f.opt.SSECustomerKeyMD5 != "" {
|
||||
serverSideEncryptionConfig := api.ServerSideEncryption{
|
||||
Mode: "SSE-C",
|
||||
Algorithm: f.opt.SSECustomerAlgorithm,
|
||||
CustomerKey: f.opt.SSECustomerKeyBase64,
|
||||
CustomerKeyMd5: f.opt.SSECustomerKeyMD5,
|
||||
}
|
||||
request.SourceServerSideEncryption = &serverSideEncryptionConfig
|
||||
request.DestinationServerSideEncryption = &serverSideEncryptionConfig
|
||||
}
|
||||
if newInfo == nil {
|
||||
request.MetadataDirective = "COPY"
|
||||
} else {
|
||||
@@ -1681,7 +1492,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
bucket, bucketPath := f.split(remote)
|
||||
var RootURL string
|
||||
if f.opt.DownloadURL == "" {
|
||||
RootURL = f.info.APIs.Storage.DownloadURL
|
||||
RootURL = f.info.DownloadURL
|
||||
} else {
|
||||
RootURL = f.opt.DownloadURL
|
||||
}
|
||||
@@ -2023,16 +1834,15 @@ var _ io.ReadCloser = &openFile{}
|
||||
|
||||
func (o *Object) getOrHead(ctx context.Context, method string, options []fs.OpenOption) (resp *http.Response, info *api.File, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: method,
|
||||
Options: options,
|
||||
NoResponse: method == "HEAD",
|
||||
ExtraHeaders: map[string]string{},
|
||||
Method: method,
|
||||
Options: options,
|
||||
NoResponse: method == "HEAD",
|
||||
}
|
||||
|
||||
// Use downloadUrl from backblaze if downloadUrl is not set
|
||||
// otherwise use the custom downloadUrl
|
||||
if o.fs.opt.DownloadURL == "" {
|
||||
opts.RootURL = o.fs.info.APIs.Storage.DownloadURL
|
||||
opts.RootURL = o.fs.info.DownloadURL
|
||||
} else {
|
||||
opts.RootURL = o.fs.opt.DownloadURL
|
||||
}
|
||||
@@ -2044,11 +1854,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
|
||||
bucket, bucketPath := o.split()
|
||||
opts.Path += "/file/" + urlEncode(o.fs.opt.Enc.FromStandardName(bucket)) + "/" + urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath))
|
||||
}
|
||||
if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
opts.ExtraHeaders[sseAlgorithmHeader] = o.fs.opt.SSECustomerAlgorithm
|
||||
opts.ExtraHeaders[sseKeyHeader] = o.fs.opt.SSECustomerKeyBase64
|
||||
opts.ExtraHeaders[sseMd5Header] = o.fs.opt.SSECustomerKeyMD5
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
@@ -2313,11 +2118,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
},
|
||||
ContentLength: &size,
|
||||
}
|
||||
if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
opts.ExtraHeaders[sseAlgorithmHeader] = o.fs.opt.SSECustomerAlgorithm
|
||||
opts.ExtraHeaders[sseKeyHeader] = o.fs.opt.SSECustomerKeyBase64
|
||||
opts.ExtraHeaders[sseMd5Header] = o.fs.opt.SSECustomerKeyMD5
|
||||
}
|
||||
var response api.FileInfo
|
||||
// Don't retry, return a retry error instead
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
@@ -2392,27 +2192,20 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
|
||||
if err != nil {
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: up.chunkSize,
|
||||
ChunkSize: int64(f.opt.ChunkSize),
|
||||
Concurrency: o.fs.opt.UploadConcurrency,
|
||||
//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
|
||||
}
|
||||
return info, up, nil
|
||||
up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
|
||||
return info, up, err
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
bucket, bucketPath := o.split()
|
||||
if o.fs.opt.Versions {
|
||||
t, path := api.RemoveVersion(bucketPath)
|
||||
if !t.IsZero() {
|
||||
return o.fs.deleteByID(ctx, o.id, path)
|
||||
}
|
||||
return errNotWithVersions
|
||||
}
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
return errNotWithVersionAt
|
||||
@@ -2435,36 +2228,32 @@ func (o *Object) ID() string {
|
||||
|
||||
var lifecycleHelp = fs.CommandHelp{
|
||||
Name: "lifecycle",
|
||||
Short: "Read or set the lifecycle for a bucket.",
|
||||
Short: "Read or set the lifecycle for a bucket",
|
||||
Long: `This command can be used to read or set the lifecycle for a bucket.
|
||||
|
||||
Usage Examples:
|
||||
|
||||
To show the current lifecycle rules:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend lifecycle b2:bucket
|
||||
` + "```" + `
|
||||
rclone backend lifecycle b2:bucket
|
||||
|
||||
This will dump something like this showing the lifecycle rules.
|
||||
|
||||
` + "```json" + `
|
||||
[
|
||||
{
|
||||
"daysFromHidingToDeleting": 1,
|
||||
"daysFromUploadingToHiding": null,
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": null,
|
||||
"fileNamePrefix": ""
|
||||
}
|
||||
]
|
||||
` + "```" + `
|
||||
[
|
||||
{
|
||||
"daysFromHidingToDeleting": 1,
|
||||
"daysFromUploadingToHiding": null,
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": null,
|
||||
"fileNamePrefix": ""
|
||||
}
|
||||
]
|
||||
|
||||
If there are no lifecycle rules (the default) then it will just return ` + "`[]`" + `.
|
||||
If there are no lifecycle rules (the default) then it will just return [].
|
||||
|
||||
To reset the current lifecycle rules:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=30
|
||||
rclone backend lifecycle b2:bucket -o daysFromUploadingToHiding=5 -o daysFromHidingToDeleting=1
|
||||
` + "```" + `
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=30
|
||||
rclone backend lifecycle b2:bucket -o daysFromUploadingToHiding=5 -o daysFromHidingToDeleting=1
|
||||
|
||||
This will run and then print the new lifecycle rules as above.
|
||||
|
||||
@@ -2476,17 +2265,14 @@ the daysFromHidingToDeleting to 1 day. You can enable hard_delete in
|
||||
the config also which will mean deletions won't cause versions but
|
||||
overwrites will still cause versions to be made.
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=1
|
||||
` + "```" + `
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=1
|
||||
|
||||
See: <https://www.backblaze.com/docs/cloud-storage-lifecycle-rules>`,
|
||||
See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
|
||||
`,
|
||||
Opts: map[string]string{
|
||||
"daysFromHidingToDeleting": `After a file has been hidden for this many days
|
||||
it is deleted. 0 is off.`,
|
||||
"daysFromUploadingToHiding": `This many days after uploading a file is hidden.`,
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": `Cancels any unfinished
|
||||
large file versions after this many days.`,
|
||||
"daysFromHidingToDeleting": "After a file has been hidden for this many days it is deleted. 0 is off.",
|
||||
"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": "Cancels any unfinished large file versions after this many days",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2569,14 +2355,13 @@ max-age, which defaults to 24 hours.
|
||||
Note that you can use --interactive/-i or --dry-run with this command to see what
|
||||
it would do.
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend cleanup b2:bucket/path/to/object
|
||||
rclone backend cleanup -o max-age=7w b2:bucket/path/to/object
|
||||
` + "```" + `
|
||||
rclone backend cleanup b2:bucket/path/to/object
|
||||
rclone backend cleanup -o max-age=7w b2:bucket/path/to/object
|
||||
|
||||
Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.`,
|
||||
Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
|
||||
`,
|
||||
Opts: map[string]string{
|
||||
"max-age": "Max age of upload to delete.",
|
||||
"max-age": "Max age of upload to delete",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2599,9 +2384,8 @@ var cleanupHiddenHelp = fs.CommandHelp{
|
||||
Note that you can use --interactive/-i or --dry-run with this command to see what
|
||||
it would do.
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend cleanup-hidden b2:bucket/path/to/dir
|
||||
` + "```",
|
||||
rclone backend cleanup-hidden b2:bucket/path/to/dir
|
||||
`,
|
||||
}
|
||||
|
||||
func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
|
||||
@@ -2644,7 +2428,6 @@ var (
|
||||
_ fs.PutStreamer = &Fs{}
|
||||
_ fs.CleanUpper = &Fs{}
|
||||
_ fs.ListRer = &Fs{}
|
||||
_ fs.ListPer = &Fs{}
|
||||
_ fs.PublicLinker = &Fs{}
|
||||
_ fs.OpenChunkWriter = &Fs{}
|
||||
_ fs.Commander = &Fs{}
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -619,60 +617,3 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
|
||||
// redirectTestHeaders is a deliberately literal copy of
|
||||
// redirectSecretHeaders so that a header dropped from the production
|
||||
// list fails TestRedirectSecretHeaders rather than silently losing
|
||||
// test coverage.
|
||||
var redirectTestHeaders = map[string]string{
|
||||
"Authorization": "secret-token",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Key": "secret-key",
|
||||
"X-Bz-Server-Side-Encryption-Customer-Key-Md5": "secret-md5",
|
||||
}
|
||||
|
||||
func TestRedirectSecretHeaders(t *testing.T) {
|
||||
want := make([]string, 0, len(redirectTestHeaders))
|
||||
for header := range redirectTestHeaders {
|
||||
want = append(want, header)
|
||||
}
|
||||
sort.Strings(want)
|
||||
got := append([]string(nil), redirectSecretHeaders...)
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
// redirectTestRequest makes a GET request to rawURL carrying the
|
||||
// secret headers plus a harmless marker header.
|
||||
func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
|
||||
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
|
||||
require.NoError(t, err)
|
||||
for header, value := range redirectTestHeaders {
|
||||
req.Header.Set(header, value)
|
||||
}
|
||||
req.Header.Set("X-Bz-Test-Mode", "marker")
|
||||
return req
|
||||
}
|
||||
|
||||
func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
|
||||
client := newClient(context.Background())
|
||||
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
for header := range redirectTestHeaders {
|
||||
assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
|
||||
}
|
||||
assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer initialServer.Close()
|
||||
|
||||
resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
}
|
||||
+1
-33
@@ -22,7 +22,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/transferaccounter"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -145,14 +144,6 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
|
||||
request.ContentType = newInfo.ContentType
|
||||
request.Info = newInfo.Info
|
||||
}
|
||||
if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
request.ServerSideEncryption = &api.ServerSideEncryption{
|
||||
Mode: "SSE-C",
|
||||
Algorithm: o.fs.opt.SSECustomerAlgorithm,
|
||||
CustomerKey: o.fs.opt.SSECustomerKeyBase64,
|
||||
CustomerKeyMd5: o.fs.opt.SSECustomerKeyMD5,
|
||||
}
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_start_large_file",
|
||||
@@ -304,12 +295,6 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
|
||||
ContentLength: &sizeWithHash,
|
||||
}
|
||||
|
||||
if up.o.fs.opt.SSECustomerKey != "" && up.o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
opts.ExtraHeaders[sseAlgorithmHeader] = up.o.fs.opt.SSECustomerAlgorithm
|
||||
opts.ExtraHeaders[sseKeyHeader] = up.o.fs.opt.SSECustomerKeyBase64
|
||||
opts.ExtraHeaders[sseMd5Header] = up.o.fs.opt.SSECustomerKeyMD5
|
||||
}
|
||||
|
||||
var response api.UploadPartResponse
|
||||
|
||||
resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
@@ -349,17 +334,6 @@ func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64)
|
||||
PartNumber: int64(part + 1),
|
||||
Range: fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
|
||||
}
|
||||
|
||||
if up.o.fs.opt.SSECustomerKey != "" && up.o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
serverSideEncryptionConfig := api.ServerSideEncryption{
|
||||
Mode: "SSE-C",
|
||||
Algorithm: up.o.fs.opt.SSECustomerAlgorithm,
|
||||
CustomerKey: up.o.fs.opt.SSECustomerKeyBase64,
|
||||
CustomerKeyMd5: up.o.fs.opt.SSECustomerKeyMD5,
|
||||
}
|
||||
request.SourceServerSideEncryption = &serverSideEncryptionConfig
|
||||
request.DestinationServerSideEncryption = &serverSideEncryptionConfig
|
||||
}
|
||||
var response api.UploadPartResponse
|
||||
resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
retry, err := up.f.shouldRetry(ctx, resp, err)
|
||||
@@ -500,12 +474,10 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
|
||||
defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
|
||||
fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
|
||||
var (
|
||||
account = transferaccounter.Get(ctx)
|
||||
g, gCtx = errgroup.WithContext(ctx)
|
||||
remaining = up.size
|
||||
)
|
||||
g.SetLimit(up.f.opt.UploadConcurrency)
|
||||
account.Start()
|
||||
for part := range up.parts {
|
||||
// Fail fast, in case an errgroup managed function returns an error
|
||||
// gCtx is cancelled. There is no point in copying all the other parts.
|
||||
@@ -517,11 +489,7 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
|
||||
|
||||
part := part // for the closure
|
||||
g.Go(func() (err error) {
|
||||
err = up.copyChunk(gCtx, part, reqSize)
|
||||
if err == nil {
|
||||
account.Add(reqSize)
|
||||
}
|
||||
return err
|
||||
return up.copyChunk(gCtx, part, reqSize)
|
||||
})
|
||||
remaining -= reqSize
|
||||
}
|
||||
|
||||
@@ -125,21 +125,10 @@ type FolderItems struct {
|
||||
Offset int `json:"offset"`
|
||||
Limit int `json:"limit"`
|
||||
NextMarker *string `json:"next_marker,omitempty"`
|
||||
// There is some confusion about how this is actually
|
||||
// returned. The []struct has worked for many years, but in
|
||||
// https://github.com/rclone/rclone/issues/8776 box was
|
||||
// returning it returned not as a list. We don't actually use
|
||||
// this so comment it out.
|
||||
//
|
||||
// Order struct {
|
||||
// By string `json:"by"`
|
||||
// Direction string `json:"direction"`
|
||||
// } `json:"order"`
|
||||
//
|
||||
// Order []struct {
|
||||
// By string `json:"by"`
|
||||
// Direction string `json:"direction"`
|
||||
// } `json:"order"`
|
||||
Order []struct {
|
||||
By string `json:"by"`
|
||||
Direction string `json:"direction"`
|
||||
} `json:"order"`
|
||||
}
|
||||
|
||||
// Parent defined the ID of the parent directory
|
||||
|
||||
+27
-77
@@ -37,7 +37,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
@@ -45,7 +44,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/youmark/pkcs8"
|
||||
)
|
||||
@@ -88,11 +86,13 @@ func init() {
|
||||
Description: "Box",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
jsonFile, ok := m.Get("box_config_file")
|
||||
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
|
||||
boxAccessToken, boxAccessTokenOk := m.Get("access_token")
|
||||
var err error
|
||||
// If using box config.json, use JWT auth
|
||||
if usesJWTAuth(m) {
|
||||
err = refreshJWTToken(ctx, name, m)
|
||||
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
|
||||
err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to configure token with jwt authentication: %w", err)
|
||||
}
|
||||
@@ -113,11 +113,6 @@ func init() {
|
||||
}, {
|
||||
Name: "box_config_file",
|
||||
Help: "Box App config.json location\n\nLeave blank normally." + env.ShellExpandHelp,
|
||||
}, {
|
||||
Name: "config_credentials",
|
||||
Help: "Box App config.json contents.\n\nLeave blank normally.",
|
||||
Hide: fs.OptionHideBoth,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "access_token",
|
||||
Help: "Box App Primary Access Token\n\nLeave blank normally.",
|
||||
@@ -188,17 +183,9 @@ See: https://developer.box.com/guides/authentication/jwt/as-user/
|
||||
})
|
||||
}
|
||||
|
||||
func usesJWTAuth(m configmap.Mapper) bool {
|
||||
jsonFile, okFile := m.Get("box_config_file")
|
||||
jsonFileCredentials, okCredentials := m.Get("config_credentials")
|
||||
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
|
||||
return (okFile || okCredentials) && boxSubTypeOk && (jsonFile != "" || jsonFileCredentials != "") && boxSubType != ""
|
||||
}
|
||||
|
||||
func refreshJWTToken(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
boxSubType, _ := m.Get("box_sub_type")
|
||||
|
||||
boxConfig, err := getBoxConfig(m)
|
||||
func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
|
||||
jsonFile = env.ShellExpand(jsonFile)
|
||||
boxConfig, err := getBoxConfig(jsonFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get box config: %w", err)
|
||||
}
|
||||
@@ -217,19 +204,12 @@ func refreshJWTToken(ctx context.Context, name string, m configmap.Mapper) error
|
||||
return err
|
||||
}
|
||||
|
||||
func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
|
||||
configFileCredentials, _ := m.Get("config_credentials")
|
||||
configFileBytes := []byte(configFileCredentials)
|
||||
|
||||
if configFileCredentials == "" {
|
||||
configFile, _ := m.Get("box_config_file")
|
||||
configFileBytes, err = os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("box: failed to read Box config: %w", err)
|
||||
}
|
||||
func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
|
||||
file, err := os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("box: failed to read Box config: %w", err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(configFileBytes, &boxConfig)
|
||||
err = json.Unmarshal(file, &boxConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("box: failed to parse Box config: %w", err)
|
||||
}
|
||||
@@ -504,12 +484,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.srv.SetHeader("as-user", f.opt.Impersonate)
|
||||
}
|
||||
|
||||
jsonFile, ok := m.Get("box_config_file")
|
||||
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
|
||||
|
||||
if ts != nil {
|
||||
// If using box config.json and JWT, renewing should just refresh the token and
|
||||
// should do so whether there are uploads pending or not.
|
||||
if usesJWTAuth(m) {
|
||||
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
|
||||
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
|
||||
err := refreshJWTToken(ctx, name, m)
|
||||
err := refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
|
||||
return err
|
||||
})
|
||||
f.tokenRenewer.Start()
|
||||
@@ -722,27 +705,9 @@ OUTER:
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
return list.WithListP(ctx, dir, f)
|
||||
}
|
||||
|
||||
// ListP lists the objects and directories of the Fs starting
|
||||
// from dir non recursively into out.
|
||||
//
|
||||
// dir should be "" to start from the root, and should not
|
||||
// have trailing slashes.
|
||||
//
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
//
|
||||
// It should call callback for each tranche of entries read.
|
||||
// These need not be returned in any particular order. If
|
||||
// callback returns an error then the listing will stop
|
||||
// immediately.
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
||||
list := list.NewHelper(callback)
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
var iErr error
|
||||
_, err = f.listAll(ctx, directoryID, false, false, true, func(info *api.Item) bool {
|
||||
@@ -752,22 +717,14 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
f.dirCache.Put(remote, info.ID)
|
||||
d := fs.NewDir(remote, info.ModTime()).SetID(info.ID)
|
||||
// FIXME more info from dir?
|
||||
err = list.Add(d)
|
||||
if err != nil {
|
||||
iErr = err
|
||||
return true
|
||||
}
|
||||
entries = append(entries, d)
|
||||
} else if info.Type == api.ItemTypeFile {
|
||||
o, err := f.newObjectWithInfo(ctx, remote, info)
|
||||
if err != nil {
|
||||
iErr = err
|
||||
return true
|
||||
}
|
||||
err = list.Add(o)
|
||||
if err != nil {
|
||||
iErr = err
|
||||
return true
|
||||
}
|
||||
entries = append(entries, o)
|
||||
}
|
||||
|
||||
// Cache some metadata for this Item to help us process events later
|
||||
@@ -783,12 +740,12 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
return false
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if iErr != nil {
|
||||
return iErr
|
||||
return nil, iErr
|
||||
}
|
||||
return list.Flush()
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Creates from the parameters passed in a half finished Object which
|
||||
@@ -1461,7 +1418,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
// Item in the cache has the same or newer SequenceID than
|
||||
// this event. Ignore this event, it must be old.
|
||||
f.itemMetaCacheMu.Unlock()
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
|
||||
fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1692,7 +1649,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// upload does a single non-multipart upload
|
||||
//
|
||||
// This is recommended for less than 50 MiB of content
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
|
||||
upload := api.UploadFile{
|
||||
Name: o.fs.opt.Enc.FromStandardName(leaf),
|
||||
ContentModifiedAt: api.Time(modTime),
|
||||
@@ -1704,10 +1661,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
|
||||
var resp *http.Response
|
||||
var result api.FolderItems
|
||||
counter := readers.NewCountingReader(in)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Body: counter,
|
||||
Body: in,
|
||||
MultipartMetadataName: "attributes",
|
||||
MultipartContentName: "contents",
|
||||
MultipartFileName: upload.Name,
|
||||
@@ -1727,11 +1683,6 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Check the source supplied the number of bytes it declared
|
||||
// otherwise a truncated file would be stored as a good upload.
|
||||
if size >= 0 && int64(counter.BytesRead()) != size {
|
||||
return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
if result.TotalCount != 1 || len(result.Entries) != 1 {
|
||||
return fmt.Errorf("failed to upload %v - not sure why", o)
|
||||
}
|
||||
@@ -1761,7 +1712,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// Upload with simple or multipart
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
|
||||
} else {
|
||||
err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
|
||||
}
|
||||
@@ -1790,7 +1741,6 @@ var (
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.CleanUpper = (*Fs)(nil)
|
||||
_ fs.ListPer = (*Fs)(nil)
|
||||
_ fs.Shutdowner = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
|
||||
+15
-23
@@ -3,6 +3,7 @@
|
||||
package box
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
@@ -19,7 +20,6 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -55,15 +55,9 @@ func sha1Digest(digest []byte) string {
|
||||
}
|
||||
|
||||
// uploadPart uploads a part in an upload session
|
||||
//
|
||||
// chunk must contain exactly the part's data and is read from the start
|
||||
// on each attempt.
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
hasher := sha1.New()
|
||||
chunkSize, err := io.Copy(hasher, chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
|
||||
chunkSize := int64(len(chunk))
|
||||
sha1sum := sha1.Sum(chunk)
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/files/upload_sessions/" + SessionID,
|
||||
@@ -73,16 +67,12 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
|
||||
ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Digest": sha1Digest(hasher.Sum(nil)),
|
||||
"Digest": sha1Digest(sha1sum[:]),
|
||||
},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
_, err = chunk.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
opts.Body = wrap(chunk)
|
||||
opts.Body = wrap(bytes.NewReader(chunk))
|
||||
resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
@@ -223,25 +213,27 @@ outer:
|
||||
|
||||
reqSize := min(remaining, chunkSize)
|
||||
|
||||
// Read the chunk into memory from the global pool, hashing
|
||||
// it for the whole file hash (must be done sequentially)
|
||||
rw := multipart.NewRW().Reserve(reqSize)
|
||||
_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
|
||||
// Make a block of memory
|
||||
buf := make([]byte, reqSize)
|
||||
|
||||
// Read the chunk
|
||||
_, err = io.ReadFull(in, buf)
|
||||
if err != nil {
|
||||
_ = rw.Close()
|
||||
err = fmt.Errorf("multipart upload failed to read source: %w", err)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Make the global hash (must be done sequentially)
|
||||
_, _ = hash.Write(buf)
|
||||
|
||||
// Transfer the chunk
|
||||
wg.Add(1)
|
||||
o.fs.uploadToken.Get()
|
||||
go func(part int, position int64) {
|
||||
defer wg.Done()
|
||||
defer o.fs.uploadToken.Put()
|
||||
defer func() { _ = rw.Close() }()
|
||||
fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
|
||||
partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("multipart upload failed to upload part: %w", err)
|
||||
select {
|
||||
|
||||
Vendored
+7
-12
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
// Package cache implements a virtual provider to cache existing remotes.
|
||||
package cache
|
||||
@@ -503,19 +503,15 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
|
||||
}
|
||||
|
||||
go func() {
|
||||
// Wait on the timer and the stop channel together so that a stop
|
||||
// signalled by StopBackgroundRunners is honoured immediately.
|
||||
timer := time.NewTimer(time.Duration(f.opt.ChunkCleanInterval))
|
||||
defer timer.Stop()
|
||||
for {
|
||||
time.Sleep(time.Duration(f.opt.ChunkCleanInterval))
|
||||
select {
|
||||
case <-f.cleanupChan:
|
||||
fs.Infof(f, "stopping cleanup")
|
||||
return
|
||||
case <-timer.C:
|
||||
default:
|
||||
fs.Debugf(f, "starting cleanup")
|
||||
f.CleanUpCache(false)
|
||||
timer.Reset(time.Duration(f.opt.ChunkCleanInterval))
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -567,10 +563,9 @@ Eg
|
||||
})
|
||||
|
||||
rc.Add(rc.Call{
|
||||
Path: "cache/stats",
|
||||
NoAuth: true,
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Path: "cache/stats",
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Help: `
|
||||
Show statistics for the cache remote.
|
||||
`,
|
||||
@@ -689,7 +684,7 @@ func (f *Fs) rcFetch(ctx context.Context, in rc.Params) (rc.Params, error) {
|
||||
start, end int64
|
||||
}
|
||||
parseChunks := func(ranges string) (crs []chunkRange, err error) {
|
||||
for part := range strings.SplitSeq(ranges, ",") {
|
||||
for _, part := range strings.Split(ranges, ",") {
|
||||
var start, end int64 = 0, math.MaxInt64
|
||||
switch ints := strings.Split(part, ":"); len(ints) {
|
||||
case 1:
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js && !race
|
||||
//go:build !plan9 && !js && !wasm && !race
|
||||
|
||||
package cache_test
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,6 +1,6 @@
|
||||
// Test Cache filesystem interface
|
||||
|
||||
//go:build !plan9 && !js && !race
|
||||
//go:build !plan9 && !js && !wasm && !race
|
||||
|
||||
package cache_test
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,7 +1,7 @@
|
||||
// Build for cache for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build plan9 || js
|
||||
//go:build plan9 || js || wasm
|
||||
|
||||
// Package cache implements a virtual provider to cache existing remotes.
|
||||
package cache
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js && !race
|
||||
//go:build !plan9 && !js && !wasm && !race
|
||||
|
||||
package cache_test
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package cache
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package cache
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package cache
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package cache
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package cache
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package cache
|
||||
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
//go:build !plan9 && !js
|
||||
//go:build !plan9 && !js && !wasm
|
||||
|
||||
package cache
|
||||
|
||||
|
||||
@@ -801,12 +801,9 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
|
||||
fs.Debugf(f, "skip orphan data chunk %q", remote)
|
||||
break
|
||||
}
|
||||
if mainObject == nil && !f.useMeta && !isSpecial {
|
||||
if mainObject == nil && !f.useMeta {
|
||||
// this is the "nometa" case
|
||||
// create dummy chunked object without metadata
|
||||
//
|
||||
// not for temporary or control chunks as they may be all
|
||||
// that is left of an interrupted upload
|
||||
mainObject = f.newObject(mainRemote, nil, nil)
|
||||
byRemote[mainRemote] = mainObject
|
||||
if !badEntry[mainRemote] {
|
||||
|
||||
@@ -43,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
|
||||
type settings map[string]any
|
||||
|
||||
func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
|
||||
fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
|
||||
fsName := strings.Split(f.Name(), "{")[0] // strip off hash
|
||||
configMap := configmap.Simple{}
|
||||
for key, val := range opts {
|
||||
configMap[key] = fmt.Sprintf("%v", val)
|
||||
@@ -627,22 +627,6 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
|
||||
_ = file.Remove(ctx)
|
||||
}
|
||||
|
||||
// testOrphanTempChunks checks that temporary chunks left behind by an
|
||||
// interrupted upload are not listed and don't break the listing
|
||||
func testOrphanTempChunks(t *testing.T, f *Fs) {
|
||||
const dir = "orphan"
|
||||
ctx := context.Background()
|
||||
defer func() {
|
||||
_ = operations.Purge(ctx, f.base, dir)
|
||||
}()
|
||||
|
||||
modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
|
||||
item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
|
||||
fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
|
||||
|
||||
fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
|
||||
}
|
||||
|
||||
func testMetadataInput(t *testing.T, f *Fs) {
|
||||
const minChunkForTest = 50
|
||||
if f.opt.ChunkSize < minChunkForTest {
|
||||
@@ -944,9 +928,6 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("BackwardsCompatibility", func(t *testing.T) {
|
||||
testBackwardsCompatibility(t, f)
|
||||
})
|
||||
t.Run("OrphanTempChunks", func(t *testing.T) {
|
||||
testOrphanTempChunks(t, f)
|
||||
})
|
||||
t.Run("ChunkerServerSideMove", func(t *testing.T) {
|
||||
testChunkerServerSideMove(t, f)
|
||||
})
|
||||
|
||||
@@ -34,13 +34,6 @@ import (
|
||||
"github.com/zeebo/blake3"
|
||||
)
|
||||
|
||||
// Pacer settings - tuned to match other rclone backends
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Cloudinary shouldn't have a trailing dot if there is no path
|
||||
func cldPathDir(somePath string) string {
|
||||
if somePath == "" || somePath == "." {
|
||||
@@ -196,7 +189,7 @@ func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (f
|
||||
root: root,
|
||||
opt: *opt,
|
||||
cld: cld,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
|
||||
srv: rest.NewClient(client),
|
||||
}
|
||||
|
||||
@@ -472,8 +465,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
if updateOptions, ok := option.(*api.UpdateOptions); ok {
|
||||
if updateOptions.PublicID != "" {
|
||||
updateObject = true
|
||||
params.Overwrite = new(true)
|
||||
params.Invalidate = new(true)
|
||||
params.Overwrite = SDKApi.Bool(true)
|
||||
params.Invalidate = SDKApi.Bool(true)
|
||||
params.PublicID = updateOptions.PublicID
|
||||
params.ResourceType = updateOptions.ResourceType
|
||||
params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)
|
||||
@@ -593,18 +586,21 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
// Cloudinary returns rate-limit windows as "Try again on YYYY-MM-DD HH:MM:SS UTC"
|
||||
// embedded in the error message - honour it as a retry-after.
|
||||
if err != nil {
|
||||
tryAgain := "Try again on "
|
||||
if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
|
||||
layout := "2006-01-02 15:04:05 UTC"
|
||||
dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
|
||||
if timestamp, err2 := time.Parse(layout, dateStr); err2 == nil {
|
||||
timestamp, err2 := time.Parse(layout, dateStr)
|
||||
if err2 == nil {
|
||||
return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
|
||||
}
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "Retrying API error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
|
||||
@@ -187,6 +187,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var mu sync.Mutex
|
||||
for _, upstream := range opt.Upstreams {
|
||||
upstream := upstream
|
||||
g.Go(func() (err error) {
|
||||
equal := strings.IndexRune(upstream, '=')
|
||||
if equal < 0 {
|
||||
@@ -240,22 +241,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
DirModTimeUpdatesOnWrite: true,
|
||||
PartialUploads: true,
|
||||
}).Fill(ctx, f)
|
||||
canMove, slowHash := true, false
|
||||
canMove := true
|
||||
for _, u := range f.upstreams {
|
||||
features = features.Mask(ctx, u.f) // Mask all upstream fs
|
||||
if !operations.CanServerSideMove(u.f) {
|
||||
canMove = false
|
||||
}
|
||||
slowHash = slowHash || u.f.Features().SlowHash
|
||||
}
|
||||
// We can move if all remotes support Move or Copy
|
||||
if canMove {
|
||||
features.Move = f.Move
|
||||
}
|
||||
|
||||
// If any of upstreams are SlowHash, propagate it
|
||||
features.SlowHash = slowHash
|
||||
|
||||
// Enable ListR when upstreams either support ListR or is local
|
||||
// But not when all upstreams are local
|
||||
if features.ListR == nil {
|
||||
@@ -369,6 +366,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
|
||||
func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream) error) error {
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
for _, u := range f.upstreams {
|
||||
u := u
|
||||
g.Go(func() (err error) {
|
||||
return fn(gCtx, u)
|
||||
})
|
||||
@@ -635,6 +633,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
var uChans []chan time.Duration
|
||||
|
||||
for _, u := range f.upstreams {
|
||||
u := u
|
||||
if do := u.f.Features().ChangeNotify; do != nil {
|
||||
ch := make(chan time.Duration)
|
||||
uChans = append(uChans, ch)
|
||||
@@ -1104,7 +1103,7 @@ func (o *Object) Remote() string {
|
||||
newPath, err := o.u.pathAdjustment.do(o.Object.String())
|
||||
if err != nil {
|
||||
fs.Errorf(o.Object, "Bad object: %v", err)
|
||||
return ""
|
||||
return err.Error()
|
||||
}
|
||||
return newPath
|
||||
}
|
||||
|
||||
+193
-191
@@ -2,8 +2,10 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
@@ -31,7 +33,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -45,7 +46,6 @@ const (
|
||||
minCompressionRatio = 1.1
|
||||
|
||||
gzFileExt = ".gz"
|
||||
zstdFileExt = ".zst"
|
||||
metaFileExt = ".json"
|
||||
uncompressedFileExt = ".bin"
|
||||
)
|
||||
@@ -54,7 +54,6 @@ const (
|
||||
const (
|
||||
Uncompressed = 0
|
||||
Gzip = 2
|
||||
Zstd = 4
|
||||
)
|
||||
|
||||
var nameRegexp = regexp.MustCompile(`^(.+?)\.([A-Za-z0-9-_]{11})$`)
|
||||
@@ -67,10 +66,6 @@ func init() {
|
||||
Value: "gzip",
|
||||
Help: "Standard gzip compression with fastest parameters.",
|
||||
},
|
||||
{
|
||||
Value: "zstd",
|
||||
Help: "Zstandard compression — fast modern algorithm offering adjustable speed-to-compression tradeoffs.",
|
||||
},
|
||||
}
|
||||
|
||||
// Register our remote
|
||||
@@ -92,23 +87,17 @@ func init() {
|
||||
Examples: compressionModeOptions,
|
||||
}, {
|
||||
Name: "level",
|
||||
Help: `GZIP (levels -2 to 9):
|
||||
- -2 — Huffman encoding only. Only use if you know what you're doing.
|
||||
- -1 (default) — recommended; equivalent to level 5.
|
||||
- 0 — turns off compression.
|
||||
- 1–9 — increase compression at the cost of speed. Going past 6 generally offers very little return.
|
||||
|
||||
ZSTD (levels 0 to 4):
|
||||
- 0 — turns off compression entirely.
|
||||
- 1 — fastest compression with the lowest ratio.
|
||||
- 2 (default) — good balance of speed and compression.
|
||||
- 3 — better compression, but uses about 2–3x more CPU than the default.
|
||||
- 4 — best possible compression ratio (highest CPU cost).
|
||||
|
||||
Notes:
|
||||
- Choose GZIP for wide compatibility; ZSTD for better speed/ratio tradeoffs.
|
||||
- Negative gzip levels: -2 = Huffman-only, -1 = default (≈ level 5).`,
|
||||
Required: true,
|
||||
Help: `GZIP compression level (-2 to 9).
|
||||
|
||||
Generally -1 (default, equivalent to 5) is recommended.
|
||||
Levels 1 to 9 increase compression at the cost of speed. Going past 6
|
||||
generally offers very little return.
|
||||
|
||||
Level -2 uses Huffman encoding only. Only use if you know what you
|
||||
are doing.
|
||||
Level 0 turns off compression.`,
|
||||
Default: sgzip.DefaultCompression,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "ram_cache_limit",
|
||||
Help: `Some remotes don't allow the upload of files with unknown size.
|
||||
@@ -123,47 +112,6 @@ this limit will be cached on disk.`,
|
||||
})
|
||||
}
|
||||
|
||||
// compressionModeHandler defines the interface for handling different compression modes
|
||||
type compressionModeHandler interface {
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
processFileNameGetFileExtension(compressionMode int) string
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error)
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
isCompressible(r io.Reader, compressionMode int) (bool, error)
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error)
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error)
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error)
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Remote string `config:"remote"`
|
||||
@@ -177,13 +125,12 @@ type Options struct {
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
mode int // compression mode id
|
||||
features *fs.Features // optional features
|
||||
modeHandler compressionModeHandler // compression mode handler
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
mode int // compression mode id
|
||||
features *fs.Features // optional features
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
@@ -220,28 +167,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
|
||||
return nil, fmt.Errorf("failed to make remote %s:%q to wrap: %w", wName, remotePath, err)
|
||||
}
|
||||
|
||||
compressionMode := compressionModeFromName(opt.CompressionMode)
|
||||
var modeHandler compressionModeHandler
|
||||
|
||||
switch compressionMode {
|
||||
case Gzip:
|
||||
modeHandler = &gzipModeHandler{}
|
||||
case Zstd:
|
||||
modeHandler = &zstdModeHandler{}
|
||||
case Uncompressed:
|
||||
modeHandler = &uncompressedModeHandler{}
|
||||
default:
|
||||
modeHandler = &unknownModeHandler{}
|
||||
}
|
||||
|
||||
// Create the wrapping fs
|
||||
f := &Fs{
|
||||
Fs: wrappedFs,
|
||||
name: name,
|
||||
root: rpath,
|
||||
opt: *opt,
|
||||
mode: compressionMode,
|
||||
modeHandler: modeHandler,
|
||||
Fs: wrappedFs,
|
||||
name: name,
|
||||
root: rpath,
|
||||
opt: *opt,
|
||||
mode: compressionModeFromName(opt.CompressionMode),
|
||||
}
|
||||
// Correct root if definitely pointing to a file
|
||||
if err == fs.ErrorIsFile {
|
||||
@@ -283,13 +215,10 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
|
||||
return f, err
|
||||
}
|
||||
|
||||
// compressionModeFromName converts a compression mode name to its int representation.
|
||||
func compressionModeFromName(name string) int {
|
||||
switch name {
|
||||
case "gzip":
|
||||
return Gzip
|
||||
case "zstd":
|
||||
return Zstd
|
||||
default:
|
||||
return Uncompressed
|
||||
}
|
||||
@@ -313,7 +242,7 @@ func base64ToInt64(str string) (int64, error) {
|
||||
|
||||
// Processes a file name for a compressed file. Returns the original file name, the extension, and the size of the original file.
|
||||
// Returns -2 for the original size if the file is uncompressed.
|
||||
func processFileName(compressedFileName string, modeHandler compressionModeHandler) (origFileName string, extension string, origSize int64, err error) {
|
||||
func processFileName(compressedFileName string) (origFileName string, extension string, origSize int64, err error) {
|
||||
// Separate the filename and size from the extension
|
||||
extensionPos := strings.LastIndex(compressedFileName, ".")
|
||||
if extensionPos == -1 {
|
||||
@@ -332,8 +261,7 @@ func processFileName(compressedFileName string, modeHandler compressionModeHandl
|
||||
if err != nil {
|
||||
return "", "", 0, errors.New("could not decode size")
|
||||
}
|
||||
ext := modeHandler.processFileNameGetFileExtension(compressionModeFromName(compressedFileName[extensionPos+1:]))
|
||||
return match[1], ext, size, nil
|
||||
return match[1], gzFileExt, size, nil
|
||||
}
|
||||
|
||||
// Generates the file name for a metadata file
|
||||
@@ -358,15 +286,11 @@ func unwrapMetadataFile(filename string) (string, bool) {
|
||||
|
||||
// makeDataName generates the file name for a data file with specified compression mode
|
||||
func makeDataName(remote string, size int64, mode int) (newRemote string) {
|
||||
switch mode {
|
||||
case Gzip:
|
||||
if mode != Uncompressed {
|
||||
newRemote = remote + "." + int64ToBase64(size) + gzFileExt
|
||||
case Zstd:
|
||||
newRemote = remote + "." + int64ToBase64(size) + zstdFileExt
|
||||
default:
|
||||
} else {
|
||||
newRemote = remote + uncompressedFileExt
|
||||
}
|
||||
|
||||
return newRemote
|
||||
}
|
||||
|
||||
@@ -380,7 +304,7 @@ func (f *Fs) dataName(remote string, size int64, compressed bool) (name string)
|
||||
|
||||
// addData parses an object and adds it to the DirEntries
|
||||
func (f *Fs) addData(entries *fs.DirEntries, o fs.Object) {
|
||||
origFileName, _, size, err := processFileName(o.Remote(), f.modeHandler)
|
||||
origFileName, _, size, err := processFileName(o.Remote())
|
||||
if err != nil {
|
||||
fs.Errorf(o, "Error on parsing file name: %v", err)
|
||||
return
|
||||
@@ -503,12 +427,8 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error decoding metadata: %w", err)
|
||||
}
|
||||
size, err := f.modeHandler.newObjectGetOriginalSize(meta)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading metadata: %w", err)
|
||||
}
|
||||
// Create our Object
|
||||
o, err := f.Fs.NewObject(ctx, makeDataName(remote, size, meta.Mode))
|
||||
o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -517,7 +437,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
// checkCompressAndType checks if an object is compressible and determines it's mime type
|
||||
// returns a multireader with the bytes that were read to determine mime type
|
||||
func checkCompressAndType(in io.Reader, compressionMode int, modeHandler compressionModeHandler) (newReader io.Reader, compressible bool, mimeType string, err error) {
|
||||
func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool, mimeType string, err error) {
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
buf := make([]byte, heuristicBytes)
|
||||
n, err := in.Read(buf)
|
||||
@@ -526,7 +446,7 @@ func checkCompressAndType(in io.Reader, compressionMode int, modeHandler compres
|
||||
return nil, false, "", err
|
||||
}
|
||||
mime := mimetype.Detect(buf)
|
||||
compressible, err = modeHandler.isCompressible(bytes.NewReader(buf), compressionMode)
|
||||
compressible, err = isCompressible(bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
}
|
||||
@@ -534,6 +454,26 @@ func checkCompressAndType(in io.Reader, compressionMode int, modeHandler compres
|
||||
return wrap(in), compressible, mime.String(), nil
|
||||
}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func isCompressible(r io.Reader) (bool, error) {
|
||||
var b bytes.Buffer
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := io.Copy(w, r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
// verifyObjectHash verifies the Objects hash
|
||||
func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.MultiHasher, ht hash.Type) error {
|
||||
srcHash := hasher.Sums()[ht]
|
||||
@@ -554,9 +494,9 @@ func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.Mul
|
||||
|
||||
type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
|
||||
|
||||
type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
type compressionResult struct {
|
||||
err error
|
||||
meta T
|
||||
meta sgzip.GzipMetadata
|
||||
}
|
||||
|
||||
// replicating some of operations.Rcat functionality because we want to support remotes without streaming
|
||||
@@ -564,19 +504,15 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
|
||||
|
||||
// cache small files in memory and do normal upload
|
||||
rw := multipart.NewRW()
|
||||
defer fs.CheckClose(rw, &err)
|
||||
_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
|
||||
if err == io.EOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, rw, src, options...)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
buf := make([]byte, f.opt.RAMCacheLimit)
|
||||
if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
|
||||
return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
|
||||
}
|
||||
|
||||
// Need to include what we already read
|
||||
in = &ReadCloserWrapper{
|
||||
Reader: io.MultiReader(rw, in),
|
||||
Reader: io.MultiReader(bytes.NewReader(buf), in),
|
||||
Closer: in,
|
||||
}
|
||||
|
||||
@@ -601,18 +537,106 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
|
||||
return nil, fmt.Errorf("failed to write temporary local file: %w", err)
|
||||
}
|
||||
if _, err = tempFile.Seek(0, 0); err != nil {
|
||||
return nil, fmt.Errorf("failed to seek temporary local file: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
finfo, err := tempFile.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stat temporary local file: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return f.Fs.Put(ctx, tempFile, object.NewStaticObjectInfo(dstFileName, modTime, finfo.Size(), false, nil, f.Fs))
|
||||
}
|
||||
|
||||
// Put a compressed version of a file. Returns a wrappable object and metadata.
|
||||
func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, mimeType string) (fs.Object, *ObjectMetadata, error) {
|
||||
return f.modeHandler.putCompress(ctx, f, in, src, options, mimeType)
|
||||
// Unwrap reader accounting
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
|
||||
// Add the metadata hasher
|
||||
metaHasher := md5.New()
|
||||
in = io.TeeReader(in, metaHasher)
|
||||
|
||||
// Compress the file
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
results := make(chan compressionResult)
|
||||
go func() {
|
||||
gz, err := sgzip.NewWriterLevel(pipeWriter, f.opt.CompressionLevel)
|
||||
if err != nil {
|
||||
results <- compressionResult{err: err, meta: sgzip.GzipMetadata{}}
|
||||
return
|
||||
}
|
||||
_, err = io.Copy(gz, in)
|
||||
gzErr := gz.Close()
|
||||
if gzErr != nil {
|
||||
fs.Errorf(nil, "Failed to close compress: %v", gzErr)
|
||||
if err == nil {
|
||||
err = gzErr
|
||||
}
|
||||
}
|
||||
closeErr := pipeWriter.Close()
|
||||
if closeErr != nil {
|
||||
fs.Errorf(nil, "Failed to close pipe: %v", closeErr)
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}
|
||||
results <- compressionResult{err: err, meta: gz.MetaData()}
|
||||
}()
|
||||
wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize)) // Probably no longer needed as sgzip has it's own buffering
|
||||
|
||||
// Find a hash the destination supports to compute a hash of
|
||||
// the compressed data.
|
||||
ht := f.Fs.Hashes().GetOne()
|
||||
var hasher *hash.MultiHasher
|
||||
var err error
|
||||
if ht != hash.None {
|
||||
// unwrap the accounting again
|
||||
wrappedIn, wrap = accounting.UnWrap(wrappedIn)
|
||||
hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// add the hasher and re-wrap the accounting
|
||||
wrappedIn = io.TeeReader(wrappedIn, hasher)
|
||||
wrappedIn = wrap(wrappedIn)
|
||||
}
|
||||
|
||||
// Transfer the data
|
||||
o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
|
||||
//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
|
||||
if err != nil {
|
||||
if o != nil {
|
||||
removeErr := o.Remove(ctx)
|
||||
if removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially transferred object: %v", err)
|
||||
}
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
// Check whether we got an error during compression
|
||||
result := <-results
|
||||
err = result.err
|
||||
if err != nil {
|
||||
if o != nil {
|
||||
removeErr := o.Remove(ctx)
|
||||
if removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially compressed object: %v", err)
|
||||
}
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Generate metadata
|
||||
meta := newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)
|
||||
|
||||
// Check the hashes of the compressed data if we were comparing them
|
||||
if ht != hash.None && hasher != nil {
|
||||
err = f.verifyObjectHash(ctx, o, hasher, ht)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return o, meta, nil
|
||||
}
|
||||
|
||||
// Put an uncompressed version of a file. Returns a wrappable object and metadata.
|
||||
@@ -656,8 +680,7 @@ func (f *Fs) putUncompress(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return f.modeHandler.putUncompressGetNewMetadata(o, Uncompressed, hex.EncodeToString(sum), mimeType, sum)
|
||||
return o, newMetadata(o.Size(), Uncompressed, sgzip.GzipMetadata{}, hex.EncodeToString(sum), mimeType), nil
|
||||
}
|
||||
|
||||
// This function will write a metadata struct to a metadata Object for an src. Returns a wrappable metadata object.
|
||||
@@ -703,16 +726,6 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check the source supplied the number of bytes it declared -
|
||||
// compressed data is stored under a name containing the
|
||||
// original size so a mismatch makes the object unreadable.
|
||||
if size := src.Size(); size >= 0 && meta.Size != size {
|
||||
if removeErr := dataObject.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
|
||||
|
||||
// meta data upload may fail. in this case we try to remove the original object
|
||||
@@ -738,7 +751,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
o, err := f.NewObject(ctx, src.Remote())
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// Get our file compressibility
|
||||
in, compressible, mimeType, err := checkCompressAndType(in, f.mode, f.modeHandler)
|
||||
in, compressible, mimeType, err := checkCompressAndType(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -758,7 +771,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
}
|
||||
found := err == nil
|
||||
|
||||
in, compressible, mimeType, err := checkCompressAndType(in, f.mode, f.modeHandler)
|
||||
in, compressible, mimeType, err := checkCompressAndType(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1077,41 +1090,11 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, duration fs.Duration
|
||||
|
||||
// ObjectMetadata describes the metadata for an Object.
|
||||
type ObjectMetadata struct {
|
||||
Mode int // Compression mode of the file.
|
||||
Size int64 // Size of the object.
|
||||
MD5 string // MD5 hash of the file.
|
||||
MimeType string // Mime type of the file
|
||||
CompressionMetadataGzip *sgzip.GzipMetadata // Metadata for Gzip compression
|
||||
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
|
||||
}
|
||||
|
||||
// validate checks metadata read from the wrapped remote before it is used to
|
||||
// construct a compressed reader.
|
||||
func (meta *ObjectMetadata) validate() error {
|
||||
if meta.Mode != Gzip {
|
||||
return nil
|
||||
}
|
||||
if meta.CompressionMetadataGzip == nil {
|
||||
return errors.New("missing gzip metadata")
|
||||
}
|
||||
if meta.CompressionMetadataGzip.BlockSize <= 0 {
|
||||
return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
|
||||
}
|
||||
if meta.Size != meta.CompressionMetadataGzip.Size {
|
||||
return errors.New("gzip metadata size does not match object size")
|
||||
}
|
||||
if meta.Size < 0 {
|
||||
return errors.New("invalid gzip object size")
|
||||
}
|
||||
blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
|
||||
blocks := meta.Size / blockSize
|
||||
if meta.Size%blockSize != 0 {
|
||||
blocks++
|
||||
}
|
||||
if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
|
||||
return errors.New("gzip block data is incomplete")
|
||||
}
|
||||
return nil
|
||||
Mode int // Compression mode of the file.
|
||||
Size int64 // Size of the object.
|
||||
MD5 string // MD5 hash of the file.
|
||||
MimeType string // Mime type of the file
|
||||
CompressionMetadata sgzip.GzipMetadata
|
||||
}
|
||||
|
||||
// Object with external metadata
|
||||
@@ -1124,6 +1107,17 @@ type Object struct {
|
||||
meta *ObjectMetadata // Metadata struct for this object (nil if not loaded)
|
||||
}
|
||||
|
||||
// This function generates a metadata object
|
||||
func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mimeType string) *ObjectMetadata {
|
||||
meta := new(ObjectMetadata)
|
||||
meta.Size = size
|
||||
meta.Mode = mode
|
||||
meta.CompressionMetadata = cmeta
|
||||
meta.MD5 = md5
|
||||
meta.MimeType = mimeType
|
||||
return meta
|
||||
}
|
||||
|
||||
// This function will read the metadata from a metadata object.
|
||||
func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {
|
||||
// Open our meradata object
|
||||
@@ -1137,9 +1131,6 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
|
||||
if err = jr.Decode(meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = meta.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
@@ -1157,15 +1148,6 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
return objErr
|
||||
}
|
||||
|
||||
// countingDiscard is an io.Writer which discards its input and counts
|
||||
// the bytes written
|
||||
type countingDiscard int64
|
||||
|
||||
func (c *countingDiscard) Write(p []byte) (int, error) {
|
||||
*c += countingDiscard(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
|
||||
type ReadCloserWrapper struct {
|
||||
io.Reader
|
||||
@@ -1183,7 +1165,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return o.mo, o.mo.Update(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
in, compressible, mimeType, err := checkCompressAndType(in, o.meta.Mode, o.f.modeHandler)
|
||||
in, compressible, mimeType, err := checkCompressAndType(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1296,7 +1278,7 @@ func (o *Object) String() string {
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
origFileName, _, _, err := processFileName(o.Object.Remote(), o.f.modeHandler)
|
||||
origFileName, _, _, err := processFileName(o.Object.Remote())
|
||||
if err != nil {
|
||||
fs.Errorf(o.f, "Could not get remote path for: %s", o.Object.Remote())
|
||||
return o.Object.Remote()
|
||||
@@ -1399,6 +1381,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
return o.Object.Open(ctx, options...)
|
||||
}
|
||||
// Get offset and limit from OpenOptions, pass the rest to the underlying remote
|
||||
var openOptions = []fs.OpenOption{&fs.SeekOption{Offset: 0}}
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
@@ -1406,12 +1389,31 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
offset = x.Offset
|
||||
case *fs.RangeOption:
|
||||
offset, limit = x.Decode(o.Size())
|
||||
default:
|
||||
openOptions = append(openOptions, option)
|
||||
}
|
||||
}
|
||||
// Get a chunkedreader for the wrapped object
|
||||
chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize, chunkStreams)
|
||||
var retCloser io.Closer = chunkedReader
|
||||
return o.f.modeHandler.openGetReadCloser(ctx, o, offset, limit, chunkedReader, retCloser, options...)
|
||||
// Get file handle
|
||||
var file io.Reader
|
||||
if offset != 0 {
|
||||
file, err = sgzip.NewReaderAt(chunkedReader, &o.meta.CompressionMetadata, offset)
|
||||
} else {
|
||||
file, err = sgzip.NewReader(chunkedReader)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fileReader io.Reader
|
||||
if limit != -1 {
|
||||
fileReader = io.LimitReader(file, limit)
|
||||
} else {
|
||||
fileReader = file
|
||||
}
|
||||
// Return a ReadCloser
|
||||
return ReadCloserWrapper{Reader: fileReader, Closer: chunkedReader}, nil
|
||||
}
|
||||
|
||||
// ObjectInfo describes a wrapped fs.ObjectInfo for being the source
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/buengese/sgzip"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
@@ -16,93 +14,6 @@ import (
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
func TestObjectMetadataValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
meta ObjectMetadata
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "nil gzip metadata",
|
||||
meta: ObjectMetadata{Mode: Gzip},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "zero block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 0,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "negative block size",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: -1,
|
||||
Size: 1024,
|
||||
BlockData: []uint32{0},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "mismatched sizes",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1024,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 2048,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "short block data",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1},
|
||||
},
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
meta: ObjectMetadata{
|
||||
Mode: Gzip,
|
||||
Size: 1025,
|
||||
CompressionMetadataGzip: &sgzip.GzipMetadata{
|
||||
BlockSize: 512,
|
||||
Size: 1025,
|
||||
BlockData: []uint32{0, 1, 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := test.meta.validate()
|
||||
if (err != nil) != test.want {
|
||||
t.Fatalf("validate() error = %v, want error: %v", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var defaultOpt = fstests.Opt{
|
||||
RemoteName: "TestCompress:",
|
||||
NilObject: (*Object)(nil),
|
||||
@@ -137,27 +48,7 @@ func TestRemoteGzip(t *testing.T) {
|
||||
opt.ExtraConfig = []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "compress"},
|
||||
{Name: name, Key: "remote", Value: tempdir},
|
||||
{Name: name, Key: "mode", Value: "gzip"},
|
||||
{Name: name, Key: "level", Value: "-1"},
|
||||
}
|
||||
opt.QuickTestOK = true
|
||||
fstests.Run(t, &opt)
|
||||
}
|
||||
|
||||
// TestRemoteZstd tests ZSTD compression
|
||||
func TestRemoteZstd(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
tempdir := filepath.Join(os.TempDir(), "rclone-compress-test-zstd")
|
||||
name := "TestCompressZstd"
|
||||
opt := defaultOpt
|
||||
opt.RemoteName = name + ":"
|
||||
opt.ExtraConfig = []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "compress"},
|
||||
{Name: name, Key: "remote", Value: tempdir},
|
||||
{Name: name, Key: "mode", Value: "zstd"},
|
||||
{Name: name, Key: "level", Value: "2"},
|
||||
{Name: name, Key: "compression_mode", Value: "gzip"},
|
||||
}
|
||||
opt.QuickTestOK = true
|
||||
fstests.Run(t, &opt)
|
||||
|
||||
@@ -1,206 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/buengese/sgzip"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
// gzipModeHandler implements compressionModeHandler for gzip
|
||||
type gzipModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var n int64
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err = io.Copy(w, r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (g *gzipModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
if meta.CompressionMetadataGzip == nil {
|
||||
return 0, errors.New("missing gzip metadata")
|
||||
}
|
||||
return meta.CompressionMetadataGzip.Size, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (g *gzipModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
var file io.Reader
|
||||
|
||||
if offset != 0 {
|
||||
file, err = sgzip.NewReaderAt(cr, o.meta.CompressionMetadataGzip, offset)
|
||||
} else {
|
||||
file, err = sgzip.NewReader(cr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fileReader io.Reader
|
||||
if limit != -1 {
|
||||
fileReader = io.LimitReader(file, limit)
|
||||
} else {
|
||||
fileReader = file
|
||||
}
|
||||
// Return a ReadCloser
|
||||
return ReadCloserWrapper{Reader: fileReader, Closer: closer}, nil
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (g *gzipModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
if compressionMode == Gzip {
|
||||
return gzFileExt
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (g *gzipModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
// Unwrap reader accounting
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
|
||||
// Add the metadata hasher
|
||||
metaHasher := md5.New()
|
||||
in = io.TeeReader(in, metaHasher)
|
||||
|
||||
// Compress the file
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
|
||||
resultsGzip := make(chan compressionResult[sgzip.GzipMetadata])
|
||||
go func() {
|
||||
gz, err := sgzip.NewWriterLevel(pipeWriter, f.opt.CompressionLevel)
|
||||
if err != nil {
|
||||
resultsGzip <- compressionResult[sgzip.GzipMetadata]{err: err, meta: sgzip.GzipMetadata{}}
|
||||
close(resultsGzip)
|
||||
return
|
||||
}
|
||||
_, err = io.Copy(gz, in)
|
||||
gzErr := gz.Close()
|
||||
if gzErr != nil && err == nil {
|
||||
err = gzErr
|
||||
}
|
||||
closeErr := pipeWriter.Close()
|
||||
if closeErr != nil && err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
resultsGzip <- compressionResult[sgzip.GzipMetadata]{err: err, meta: gz.MetaData()}
|
||||
close(resultsGzip)
|
||||
}()
|
||||
|
||||
wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize)) // Probably no longer needed as sgzip has it's own buffering
|
||||
|
||||
// Find a hash the destination supports to compute a hash of
|
||||
// the compressed data.
|
||||
ht := f.Fs.Hashes().GetOne()
|
||||
var hasher *hash.MultiHasher
|
||||
var err error
|
||||
if ht != hash.None {
|
||||
// unwrap the accounting again
|
||||
wrappedIn, wrap = accounting.UnWrap(wrappedIn)
|
||||
hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// add the hasher and re-wrap the accounting
|
||||
wrappedIn = io.TeeReader(wrappedIn, hasher)
|
||||
wrappedIn = wrap(wrappedIn)
|
||||
}
|
||||
|
||||
// Transfer the data
|
||||
o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
|
||||
if err != nil {
|
||||
if o != nil {
|
||||
if removeErr := o.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
// Check whether we got an error during compression
|
||||
result := <-resultsGzip
|
||||
if result.err != nil {
|
||||
if o != nil {
|
||||
if removeErr := o.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially compressed object: %v", removeErr)
|
||||
}
|
||||
}
|
||||
return nil, nil, result.err
|
||||
}
|
||||
|
||||
// Generate metadata
|
||||
meta := g.newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)
|
||||
|
||||
// Check the hashes of the compressed data if we were comparing them
|
||||
if ht != hash.None && hasher != nil {
|
||||
err = f.verifyObjectHash(ctx, o, hasher, ht)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return o, meta, nil
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (g *gzipModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return o, g.newMetadata(o.Size(), mode, sgzip.GzipMetadata{}, hex.EncodeToString(sum), mimeType), nil
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (g *gzipModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
meta, ok := cmeta.(sgzip.GzipMetadata)
|
||||
if !ok {
|
||||
panic("invalid cmeta type: expected sgzip.GzipMetadata")
|
||||
}
|
||||
|
||||
objMeta := new(ObjectMetadata)
|
||||
objMeta.Size = size
|
||||
objMeta.Mode = mode
|
||||
objMeta.CompressionMetadataGzip = &meta
|
||||
objMeta.CompressionMetadataZstd = nil
|
||||
objMeta.MD5 = md5
|
||||
objMeta.MimeType = mimeType
|
||||
|
||||
return objMeta
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putRecorder wraps an fs.Fs and records what rcat hands to Put and
|
||||
// PutStream. It reads each body twice when it is seekable to model a
|
||||
// backend retrying an upload.
|
||||
type putRecorder struct {
|
||||
fs.Fs
|
||||
t *testing.T
|
||||
stream bool // expose PutStream from the wrapped Fs
|
||||
size int64
|
||||
seekable bool
|
||||
body []byte
|
||||
streamed bool
|
||||
}
|
||||
|
||||
func (r *putRecorder) Features() *fs.Features {
|
||||
features := *r.Fs.Features()
|
||||
features.PutStream = nil
|
||||
if r.stream {
|
||||
features.PutStream = r.PutStream
|
||||
}
|
||||
return &features
|
||||
}
|
||||
|
||||
func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
|
||||
r.size = src.Size()
|
||||
body, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
if seeker, ok := in.(io.Seeker); ok {
|
||||
r.seekable = true
|
||||
_, err = seeker.Seek(0, io.SeekStart)
|
||||
require.NoError(r.t, err)
|
||||
again, err := io.ReadAll(in)
|
||||
require.NoError(r.t, err)
|
||||
assert.Equal(r.t, body, again, "retried body should be identical")
|
||||
}
|
||||
r.body = body
|
||||
return body
|
||||
}
|
||||
|
||||
func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
body := r.record(in, src)
|
||||
return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
|
||||
}
|
||||
|
||||
func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
r.streamed = true
|
||||
return r.Put(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// TestRcat checks the three upload paths rcat takes: small files buffered
|
||||
// in memory and uploaded with a known size and a seekable body, large
|
||||
// files streamed when the wrapped remote supports it, and large files
|
||||
// spooled to a temporary file when it doesn't. Pool pages taken for the
|
||||
// buffer must be returned on every path.
|
||||
func TestRcat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const limit = 1024
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
size int
|
||||
stream bool
|
||||
wantSize int64
|
||||
}{
|
||||
{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
|
||||
{name: "Empty", size: 0, stream: true, wantSize: 0},
|
||||
{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
|
||||
{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
|
||||
{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
|
||||
require.NoError(t, err)
|
||||
rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
|
||||
f := &Fs{Fs: rec}
|
||||
f.opt.RAMCacheLimit = limit
|
||||
content := []byte(random.String(test.size))
|
||||
inUse := pool.Global().InUse()
|
||||
|
||||
o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
|
||||
|
||||
assert.Equal(t, test.wantSize, rec.size)
|
||||
assert.Equal(t, content, rec.body)
|
||||
assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
|
||||
if test.size < limit {
|
||||
assert.True(t, rec.seekable, "small file body should be seekable")
|
||||
}
|
||||
rc, err := o.Open(ctx)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, content, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsCompressible checks the compressibility heuristic on data at
|
||||
// both ends of the scale for every compression mode which has one.
|
||||
func TestIsCompressible(t *testing.T) {
|
||||
compressible := bytes.Repeat([]byte("compress me "), 4096)
|
||||
incompressible := make([]byte, len(compressible))
|
||||
_, err := io.ReadFull(rand.Reader, incompressible)
|
||||
require.NoError(t, err)
|
||||
for name, handler := range map[string]compressionModeHandler{
|
||||
"gzip": &gzipModeHandler{},
|
||||
"zstd": &zstdModeHandler{},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,327 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
szstd "github.com/a1ex3/zstd-seekable-format-go/pkg"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
const szstdChunkSize int = 1 << 20 // 1 MiB chunk size
|
||||
|
||||
// SzstdMetadata holds metadata for szstd compressed files.
|
||||
type SzstdMetadata struct {
|
||||
BlockSize int // BlockSize is the size of the blocks in the zstd file
|
||||
Size int64 // Size is the uncompressed size of the file
|
||||
BlockData []uint32 // BlockData is the block data for the zstd file, used for seeking
|
||||
}
|
||||
|
||||
// SzstdWriter is a writer that compresses data in szstd format.
|
||||
type SzstdWriter struct {
|
||||
enc *zstd.Encoder
|
||||
w szstd.ConcurrentWriter
|
||||
metadata SzstdMetadata
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewWriterSzstd creates a new szstd writer with the specified options.
|
||||
// It initializes the szstd writer with a zstd encoder and returns a pointer to the SzstdWriter.
|
||||
// The writer can be used to write data in chunks, and it will automatically handle block sizes and metadata.
|
||||
func NewWriterSzstd(w io.Writer, opts ...zstd.EOption) (*SzstdWriter, error) {
|
||||
encoder, err := zstd.NewWriter(nil, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sw, err := szstd.NewWriter(w, encoder)
|
||||
if err != nil {
|
||||
if err := encoder.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &SzstdWriter{
|
||||
enc: encoder,
|
||||
w: sw,
|
||||
metadata: SzstdMetadata{
|
||||
BlockSize: szstdChunkSize,
|
||||
Size: 0,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Write writes data to the szstd writer in chunks of szstdChunkSize.
|
||||
// It handles the block size and metadata updates automatically.
|
||||
func (w *SzstdWriter) Write(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if w.metadata.BlockData == nil {
|
||||
numBlocks := (len(p) + w.metadata.BlockSize - 1) / w.metadata.BlockSize
|
||||
w.metadata.BlockData = make([]uint32, 1, numBlocks+1)
|
||||
w.metadata.BlockData[0] = 0
|
||||
}
|
||||
|
||||
start := 0
|
||||
total := len(p)
|
||||
|
||||
var writerFunc szstd.FrameSource = func() ([]byte, error) {
|
||||
if start >= total {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
end := min(start+w.metadata.BlockSize, total)
|
||||
chunk := p[start:end]
|
||||
size := end - start
|
||||
|
||||
w.mu.Lock()
|
||||
w.metadata.Size += int64(size)
|
||||
w.mu.Unlock()
|
||||
|
||||
start = end
|
||||
return chunk, nil
|
||||
}
|
||||
|
||||
// write sizes of compressed blocks in the callback
|
||||
err := w.w.WriteMany(context.Background(), writerFunc,
|
||||
szstd.WithWriteCallback(func(size uint32) {
|
||||
w.mu.Lock()
|
||||
lastOffset := w.metadata.BlockData[len(w.metadata.BlockData)-1]
|
||||
w.metadata.BlockData = append(w.metadata.BlockData, lastOffset+size)
|
||||
w.mu.Unlock()
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// Close closes the SzstdWriter and its underlying encoder.
|
||||
func (w *SzstdWriter) Close() error {
|
||||
if err := w.w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.enc.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetMetadata returns the metadata of the szstd writer.
|
||||
func (w *SzstdWriter) GetMetadata() SzstdMetadata {
|
||||
return w.metadata
|
||||
}
|
||||
|
||||
// SzstdReaderAt is a reader that allows random access in szstd compressed data.
|
||||
type SzstdReaderAt struct {
|
||||
r szstd.Reader
|
||||
decoder *zstd.Decoder
|
||||
metadata *SzstdMetadata
|
||||
pos int64
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewReaderAtSzstd creates a new SzstdReaderAt at the specified io.ReadSeeker.
|
||||
func NewReaderAtSzstd(rs io.ReadSeeker, meta *SzstdMetadata, offset int64, opts ...zstd.DOption) (*SzstdReaderAt, error) {
|
||||
decoder, err := zstd.NewReader(nil, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, err := szstd.NewReader(rs, decoder)
|
||||
if err != nil {
|
||||
decoder.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sr := &SzstdReaderAt{
|
||||
r: r,
|
||||
decoder: decoder,
|
||||
metadata: meta,
|
||||
pos: 0,
|
||||
}
|
||||
|
||||
// Set initial position to the provided offset
|
||||
if _, err := sr.Seek(offset, io.SeekStart); err != nil {
|
||||
if err := sr.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sr, nil
|
||||
}
|
||||
|
||||
// Seek sets the offset for the next Read.
|
||||
func (s *SzstdReaderAt) Seek(offset int64, whence int) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
pos, err := s.r.Seek(offset, whence)
|
||||
if err == nil {
|
||||
s.pos = pos
|
||||
}
|
||||
return pos, err
|
||||
}
|
||||
|
||||
func (s *SzstdReaderAt) Read(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
n, err := s.r.Read(p)
|
||||
if err == nil {
|
||||
s.pos += int64(n)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadAt reads data at the specified offset.
|
||||
func (s *SzstdReaderAt) ReadAt(p []byte, off int64) (int, error) {
|
||||
if off < 0 {
|
||||
return 0, errors.New("invalid offset")
|
||||
}
|
||||
if off >= s.metadata.Size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
endOff := min(off+int64(len(p)), s.metadata.Size)
|
||||
|
||||
// Find all blocks covered by the range
|
||||
type blockInfo struct {
|
||||
index int // Block index
|
||||
offsetInBlock int64 // Offset within the block for starting reading
|
||||
bytesToRead int64 // How many bytes to read from this block
|
||||
}
|
||||
|
||||
var blocks []blockInfo
|
||||
uncompressedOffset := int64(0)
|
||||
currentOff := off
|
||||
|
||||
for i := 0; i < len(s.metadata.BlockData)-1; i++ {
|
||||
blockUncompressedEnd := min(uncompressedOffset+int64(s.metadata.BlockSize), s.metadata.Size)
|
||||
|
||||
if currentOff < blockUncompressedEnd && endOff > uncompressedOffset {
|
||||
offsetInBlock := max(0, currentOff-uncompressedOffset)
|
||||
bytesToRead := min(blockUncompressedEnd-uncompressedOffset-offsetInBlock, endOff-currentOff)
|
||||
|
||||
blocks = append(blocks, blockInfo{
|
||||
index: i,
|
||||
offsetInBlock: offsetInBlock,
|
||||
bytesToRead: bytesToRead,
|
||||
})
|
||||
|
||||
currentOff += bytesToRead
|
||||
if currentOff >= endOff {
|
||||
break
|
||||
}
|
||||
}
|
||||
uncompressedOffset = blockUncompressedEnd
|
||||
}
|
||||
|
||||
if len(blocks) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// Parallel block decoding
|
||||
type decodeResult struct {
|
||||
index int
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
resultCh := make(chan decodeResult, len(blocks))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, runtime.NumCPU())
|
||||
|
||||
for _, block := range blocks {
|
||||
wg.Add(1)
|
||||
go func(block blockInfo) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
|
||||
startOffset := int64(s.metadata.BlockData[block.index])
|
||||
endOffset := int64(s.metadata.BlockData[block.index+1])
|
||||
compressedSize := endOffset - startOffset
|
||||
|
||||
compressed := make([]byte, compressedSize)
|
||||
_, err := s.r.ReadAt(compressed, startOffset)
|
||||
if err != nil && err != io.EOF {
|
||||
resultCh <- decodeResult{index: block.index, err: err}
|
||||
return
|
||||
}
|
||||
|
||||
decoded, err := s.decoder.DecodeAll(compressed, nil)
|
||||
if err != nil {
|
||||
resultCh <- decodeResult{index: block.index, err: err}
|
||||
return
|
||||
}
|
||||
|
||||
resultCh <- decodeResult{index: block.index, data: decoded, err: nil}
|
||||
}(block)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(resultCh)
|
||||
}()
|
||||
|
||||
// Collect results in block index order
|
||||
totalRead := 0
|
||||
results := make(map[int]decodeResult)
|
||||
expected := len(blocks)
|
||||
minIndex := blocks[0].index
|
||||
|
||||
for res := range resultCh {
|
||||
results[res.index] = res
|
||||
for {
|
||||
if result, ok := results[minIndex]; ok {
|
||||
if result.err != nil {
|
||||
return 0, result.err
|
||||
}
|
||||
// find the corresponding blockInfo
|
||||
var blk blockInfo
|
||||
for _, b := range blocks {
|
||||
if b.index == result.index {
|
||||
blk = b
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
start := blk.offsetInBlock
|
||||
end := start + blk.bytesToRead
|
||||
copy(p[totalRead:totalRead+int(blk.bytesToRead)], result.data[start:end])
|
||||
totalRead += int(blk.bytesToRead)
|
||||
minIndex++
|
||||
if minIndex-blocks[0].index >= len(blocks) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(results) == expected && minIndex-blocks[0].index >= len(blocks) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return totalRead, nil
|
||||
}
|
||||
|
||||
// Close closes the SzstdReaderAt and underlying decoder.
|
||||
func (s *SzstdReaderAt) Close() error {
|
||||
if err := s.r.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.decoder.Close()
|
||||
return nil
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
)
|
||||
|
||||
// uncompressedModeHandler implements compressionModeHandler for uncompressed files
|
||||
type uncompressedModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (u *uncompressedModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (u *uncompressedModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (u *uncompressedModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
return o.Object.Open(ctx, options...)
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (u *uncompressedModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (u *uncompressedModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unsupported compression mode %d", f.mode)
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (u *uncompressedModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unsupported compression mode %d", Uncompressed)
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (u *uncompressedModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
return nil
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
)
|
||||
|
||||
// unknownModeHandler implements compressionModeHandler for unknown compression types
|
||||
type unknownModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (unk *unknownModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
return false, fmt.Errorf("unknown compression mode %d", compressionMode)
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (unk *unknownModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (unk *unknownModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
return nil, fmt.Errorf("unknown compression mode %d", o.meta.Mode)
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (unk *unknownModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (unk *unknownModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unknown compression mode %d", f.mode)
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (unk *unknownModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unknown compression mode")
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (unk *unknownModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
return nil
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
// zstdModeHandler implements compressionModeHandler for zstd
|
||||
type zstdModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b countingDiscard
|
||||
var n int64
|
||||
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err = io.Copy(w, r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b)
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (z *zstdModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
if meta.CompressionMetadataZstd == nil {
|
||||
return 0, errors.New("missing zstd metadata")
|
||||
}
|
||||
return meta.CompressionMetadataZstd.Size, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (z *zstdModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
var file io.Reader
|
||||
|
||||
if offset != 0 {
|
||||
file, err = NewReaderAtSzstd(cr, o.meta.CompressionMetadataZstd, offset)
|
||||
} else {
|
||||
file, err = zstd.NewReader(cr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fileReader io.Reader
|
||||
if limit != -1 {
|
||||
fileReader = io.LimitReader(file, limit)
|
||||
} else {
|
||||
fileReader = file
|
||||
}
|
||||
// Return a ReadCloser
|
||||
return ReadCloserWrapper{Reader: fileReader, Closer: closer}, nil
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (z *zstdModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
if compressionMode == Zstd {
|
||||
return zstdFileExt
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (z *zstdModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
// Unwrap reader accounting
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
|
||||
// Add the metadata hasher
|
||||
metaHasher := md5.New()
|
||||
in = io.TeeReader(in, metaHasher)
|
||||
|
||||
// Compress the file
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
|
||||
resultsZstd := make(chan compressionResult[SzstdMetadata])
|
||||
go func() {
|
||||
writer, err := NewWriterSzstd(pipeWriter, zstd.WithEncoderLevel(zstd.EncoderLevel(f.opt.CompressionLevel)))
|
||||
if err != nil {
|
||||
resultsZstd <- compressionResult[SzstdMetadata]{err: err}
|
||||
close(resultsZstd)
|
||||
return
|
||||
}
|
||||
_, err = io.Copy(writer, in)
|
||||
if wErr := writer.Close(); wErr != nil && err == nil {
|
||||
err = wErr
|
||||
}
|
||||
if cErr := pipeWriter.Close(); cErr != nil && err == nil {
|
||||
err = cErr
|
||||
}
|
||||
|
||||
resultsZstd <- compressionResult[SzstdMetadata]{err: err, meta: writer.GetMetadata()}
|
||||
close(resultsZstd)
|
||||
}()
|
||||
|
||||
wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize))
|
||||
|
||||
ht := f.Fs.Hashes().GetOne()
|
||||
var hasher *hash.MultiHasher
|
||||
var err error
|
||||
if ht != hash.None {
|
||||
wrappedIn, wrap = accounting.UnWrap(wrappedIn)
|
||||
hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
wrappedIn = io.TeeReader(wrappedIn, hasher)
|
||||
wrappedIn = wrap(wrappedIn)
|
||||
}
|
||||
|
||||
o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
result := <-resultsZstd
|
||||
if result.err != nil {
|
||||
if o != nil {
|
||||
_ = o.Remove(ctx)
|
||||
}
|
||||
return nil, nil, result.err
|
||||
}
|
||||
|
||||
// Build metadata using uncompressed size for filename
|
||||
meta := z.newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)
|
||||
if ht != hash.None && hasher != nil {
|
||||
err = f.verifyObjectHash(ctx, o, hasher, ht)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return o, meta, nil
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (z *zstdModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return o, z.newMetadata(o.Size(), mode, SzstdMetadata{}, hex.EncodeToString(sum), mimeType), nil
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (z *zstdModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
meta, ok := cmeta.(SzstdMetadata)
|
||||
if !ok {
|
||||
panic("invalid cmeta type: expected SzstdMetadata")
|
||||
}
|
||||
|
||||
objMeta := new(ObjectMetadata)
|
||||
objMeta.Size = size
|
||||
objMeta.Mode = mode
|
||||
objMeta.CompressionMetadataGzip = nil
|
||||
objMeta.CompressionMetadataZstd = &meta
|
||||
objMeta.MD5 = md5
|
||||
objMeta.MimeType = mimeType
|
||||
|
||||
return objMeta
|
||||
}
|
||||
+18
-45
@@ -11,7 +11,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -118,8 +117,8 @@ func (mode NameEncryptionMode) String() (out string) {
|
||||
return out
|
||||
}
|
||||
|
||||
// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
|
||||
type FileNameEncoding interface {
|
||||
// fileNameEncoding are the encoding methods dealing with encrypted file names
|
||||
type fileNameEncoding interface {
|
||||
EncodeToString(src []byte) string
|
||||
DecodeString(s string) ([]byte, error)
|
||||
}
|
||||
@@ -154,7 +153,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
|
||||
}
|
||||
|
||||
// NewNameEncoding creates a NameEncoding from a string
|
||||
func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
|
||||
func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
|
||||
s = strings.ToLower(s)
|
||||
switch s {
|
||||
case "base32":
|
||||
@@ -176,7 +175,7 @@ type Cipher struct {
|
||||
nameTweak [nameCipherBlockSize]byte // used to tweak the name crypto
|
||||
block gocipher.Block
|
||||
mode NameEncryptionMode
|
||||
fileNameEnc FileNameEncoding
|
||||
fileNameEnc fileNameEncoding
|
||||
buffers sync.Pool // encrypt/decrypt buffers
|
||||
cryptoRand io.Reader // read crypto random numbers from here
|
||||
dirNameEncrypt bool
|
||||
@@ -185,7 +184,7 @@ type Cipher struct {
|
||||
}
|
||||
|
||||
// newCipher initialises the cipher. If salt is "" then it uses a built in salt val
|
||||
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
|
||||
func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
|
||||
c := &Cipher{
|
||||
mode: mode,
|
||||
fileNameEnc: enc,
|
||||
@@ -404,14 +403,14 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
if ciphertext == "" {
|
||||
return "", nil
|
||||
}
|
||||
before, after, ok := strings.Cut(ciphertext, ".")
|
||||
if !ok {
|
||||
pos := strings.Index(ciphertext, ".")
|
||||
if pos == -1 {
|
||||
return "", ErrorNotAnEncryptedFile
|
||||
} // No .
|
||||
num := before
|
||||
num := ciphertext[:pos]
|
||||
if num == "!" {
|
||||
// No rotation; probably original was not valid unicode
|
||||
return after, nil
|
||||
return ciphertext[pos+1:], nil
|
||||
}
|
||||
dir, err := strconv.Atoi(num)
|
||||
if err != nil {
|
||||
@@ -426,7 +425,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
var result bytes.Buffer
|
||||
|
||||
inQuote := false
|
||||
for _, runeValue := range after {
|
||||
for _, runeValue := range ciphertext[pos+1:] {
|
||||
switch {
|
||||
case inQuote:
|
||||
_, _ = result.WriteRune(runeValue)
|
||||
@@ -487,12 +486,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
}
|
||||
|
||||
// encryptFileName encrypts a file path
|
||||
//
|
||||
// If stripVersion is set, a version string (as used by --b2-versions)
|
||||
// on the last segment is kept in plain text. Set it for file names
|
||||
// only: directory names must encrypt identically whether passed alone
|
||||
// or as the parent of a file name.
|
||||
func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
|
||||
func (c *Cipher) encryptFileName(in string) string {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
// Skip directory name encryption if the user chose to
|
||||
@@ -505,7 +499,7 @@ func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
|
||||
// of the file name gets encrypted/obfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -535,24 +529,19 @@ func (c *Cipher) EncryptFileName(in string) string {
|
||||
if c.mode == NameEncryptionOff {
|
||||
return in + c.encryptedSuffix
|
||||
}
|
||||
return c.encryptFileName(in, true)
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// EncryptDirName encrypts a directory path
|
||||
//
|
||||
// Unlike EncryptFileName, a version string on the last segment is
|
||||
// encrypted along with the rest of the name.
|
||||
func (c *Cipher) EncryptDirName(in string) string {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in
|
||||
}
|
||||
return c.encryptFileName(in, false)
|
||||
return c.encryptFileName(in)
|
||||
}
|
||||
|
||||
// decryptFileName decrypts a file path
|
||||
//
|
||||
// stripVersion is as for encryptFileName.
|
||||
func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
|
||||
func (c *Cipher) decryptFileName(in string) (string, error) {
|
||||
segments := strings.Split(in, "/")
|
||||
for i := range segments {
|
||||
var err error
|
||||
@@ -566,7 +555,7 @@ func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
|
||||
// of the file name gets decrypted/deobfuscated
|
||||
hasVersion := false
|
||||
var t time.Time
|
||||
if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
if i == (len(segments)-1) && version.Match(segments[i]) {
|
||||
var s string
|
||||
t, s = version.Remove(segments[i])
|
||||
// version.Remove can fail, in which case it returns segments[i]
|
||||
@@ -612,31 +601,15 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
|
||||
// Leave the version string on, if it was there
|
||||
return decrypted, nil
|
||||
}
|
||||
return c.decryptFileName(in, true)
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// DecryptDirName decrypts a directory path
|
||||
//
|
||||
// Unlike DecryptFileName, a version string on the last segment is
|
||||
// expected to be part of the encrypted name. Directory names created
|
||||
// by rclone before v1.76 had a version-like suffix left in plain
|
||||
// text; these are still decrypted so that they appear in listings,
|
||||
// but they can't be opened or removed until renamed on the
|
||||
// underlying remote.
|
||||
func (c *Cipher) DecryptDirName(in string) (string, error) {
|
||||
if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
|
||||
return in, nil
|
||||
}
|
||||
out, err := c.decryptFileName(in, false)
|
||||
if err != nil && version.Match(path.Base(in)) {
|
||||
var legacyErr error
|
||||
out, legacyErr = c.decryptFileName(in, true)
|
||||
if legacyErr == nil {
|
||||
fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
return c.decryptFileName(in)
|
||||
}
|
||||
|
||||
// NameEncryptionMode returns the encryption mode in use for names
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -683,43 +682,6 @@ func TestNonStandardDecryptDirName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test directories whose name looks like it has a version string -
|
||||
// these encrypt verbatim, so that EncryptDirName agrees with the same
|
||||
// directory encrypted as the parent of a file name, whereas file names
|
||||
// keep the version string in plain text
|
||||
func TestVersionedDirName(t *testing.T) {
|
||||
const dir = "dir-v2001-02-03-040506-123"
|
||||
enc, err := NewNameEncoding("base32")
|
||||
require.NoError(t, err)
|
||||
for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
|
||||
c, err := newCipher(mode, "", "", true, enc)
|
||||
require.NoError(t, err)
|
||||
what := fmt.Sprintf("Testing mode=%v", mode)
|
||||
// Check EncryptDirName matches the parent of an encrypted file name
|
||||
encryptedDir := c.EncryptDirName(dir)
|
||||
encryptedFile := c.EncryptFileName(dir + "/file.txt")
|
||||
assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
|
||||
// Check the encrypted directory name round trips OK
|
||||
decryptedDir, err := c.DecryptDirName(encryptedDir)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// Check a file with the same name keeps its version string in plain text
|
||||
encryptedFile = c.EncryptFileName(dir)
|
||||
assert.NotEqual(t, encryptedDir, encryptedFile, what)
|
||||
assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
|
||||
decryptedFile, err := c.DecryptFileName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedFile, what)
|
||||
// Check a directory with a legacy encrypted name (the file form) still decrypts
|
||||
decryptedDir, err = c.DecryptDirName(encryptedFile)
|
||||
assert.NoError(t, err, what)
|
||||
assert.Equal(t, dir, decryptedDir, what)
|
||||
// but a name which is bad in both forms is still an error
|
||||
_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
|
||||
assert.Error(t, err, what)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptedSize(t *testing.T) {
|
||||
c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
|
||||
for _, test := range []struct {
|
||||
|
||||
+10
-17
@@ -923,30 +923,28 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
var commandHelp = []fs.CommandHelp{
|
||||
{
|
||||
Name: "encode",
|
||||
Short: "Encode the given filename(s).",
|
||||
Short: "Encode the given filename(s)",
|
||||
Long: `This encodes the filenames given as arguments returning a list of
|
||||
strings of the encoded results.
|
||||
|
||||
Usage examples:
|
||||
Usage Example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend encode crypt: file1 [file2...]
|
||||
rclone rc backend/command command=encode fs=crypt: file1 [file2...]
|
||||
` + "```",
|
||||
rclone backend encode crypt: file1 [file2...]
|
||||
rclone rc backend/command command=encode fs=crypt: file1 [file2...]
|
||||
`,
|
||||
},
|
||||
{
|
||||
Name: "decode",
|
||||
Short: "Decode the given filename(s).",
|
||||
Short: "Decode the given filename(s)",
|
||||
Long: `This decodes the filenames given as arguments returning a list of
|
||||
strings of the decoded results. It will return an error if any of the
|
||||
inputs are invalid.
|
||||
|
||||
Usage examples:
|
||||
Usage Example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend decode crypt: encryptedfile1 [encryptedfile2...]
|
||||
rclone rc backend/command command=decode fs=crypt: encryptedfile1 [encryptedfile2...]
|
||||
` + "```",
|
||||
rclone backend decode crypt: encryptedfile1 [encryptedfile2...]
|
||||
rclone rc backend/command command=decode fs=crypt: encryptedfile1 [encryptedfile2...]
|
||||
`,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1179,11 +1177,6 @@ func (o *ObjectInfo) Size() int64 {
|
||||
// Hash returns the selected checksum of the file
|
||||
// If no checksum is available it returns ""
|
||||
func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
|
||||
// If the data is unchanged then the hash of the source is the
|
||||
// hash of the object which will be uploaded
|
||||
if o.f.opt.NoDataEncryption {
|
||||
return o.ObjectInfo.Hash(ctx, hash)
|
||||
}
|
||||
var srcObj fs.Object
|
||||
var ok bool
|
||||
// Get the underlying object if there is one
|
||||
|
||||
@@ -84,10 +84,6 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
|
||||
|
||||
// Test ObjectInfo.Hash
|
||||
wantHash := md5.Sum(outBuf.Bytes())
|
||||
if f.opt.NoDataEncryption {
|
||||
// If the data isn't encrypted, the hash should be that of the plaintext
|
||||
wantHash = md5.Sum([]byte(contents))
|
||||
}
|
||||
gotHash, err := src.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
|
||||
|
||||
+12
-16
@@ -77,7 +77,7 @@ The DOI provider can be set when rclone does not automatically recognize a suppo
|
||||
Name: "doi_resolver_api_url",
|
||||
Help: `The URL of the DOI resolver API to use.
|
||||
|
||||
The DOI resolver can be set for testing or for cases when the canonical DOI resolver API cannot be used.
|
||||
The DOI resolver can be set for testing or for cases when the the canonical DOI resolver API cannot be used.
|
||||
|
||||
Defaults to "https://doi.org/api".`,
|
||||
Required: false,
|
||||
@@ -563,26 +563,21 @@ var commandHelp = []fs.CommandHelp{{
|
||||
Short: "Show metadata about the DOI.",
|
||||
Long: `This command returns a JSON object with some information about the DOI.
|
||||
|
||||
Usage example:
|
||||
rclone backend medatadata doi:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend metadata doi:
|
||||
` + "```" + `
|
||||
|
||||
It returns a JSON object representing metadata about the DOI.`,
|
||||
It returns a JSON object representing metadata about the DOI.
|
||||
`,
|
||||
}, {
|
||||
Name: "set",
|
||||
Short: "Set command for updating the config parameters.",
|
||||
Long: `This set command can be used to update the config parameters
|
||||
for a running doi backend.
|
||||
|
||||
Usage examples:
|
||||
Usage Examples:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend set doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: -o doi=NEW_DOI
|
||||
` + "```" + `
|
||||
rclone backend set doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: -o doi=NEW_DOI
|
||||
|
||||
The option keys are named as they are in the config file.
|
||||
|
||||
@@ -590,7 +585,8 @@ This rebuilds the connection to the doi backend when it is called with
|
||||
the new parameters. Only new parameters need be passed as the values
|
||||
will default to those currently in use.
|
||||
|
||||
It doesn't return anything.`,
|
||||
It doesn't return anything.
|
||||
`,
|
||||
}}
|
||||
|
||||
// Command the backend to run a named command
|
||||
@@ -602,7 +598,7 @@ It doesn't return anything.`,
|
||||
// The result should be capable of being JSON encoded
|
||||
// If it is a string or a []string it will be shown to the user
|
||||
// otherwise it will be JSON encoded and shown to the user like that
|
||||
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
|
||||
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
|
||||
switch name {
|
||||
case "metadata":
|
||||
return f.ShowMetadata(ctx)
|
||||
@@ -629,7 +625,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
}
|
||||
|
||||
// ShowMetadata returns some metadata about the corresponding DOI
|
||||
func (f *Fs) ShowMetadata(ctx context.Context) (metadata any, err error) {
|
||||
func (f *Fs) ShowMetadata(ctx context.Context) (metadata interface{}, err error) {
|
||||
doiURL, err := url.Parse("https://doi.org/" + f.opt.Doi)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -18,7 +18,7 @@ type headerLink struct {
|
||||
}
|
||||
|
||||
func parseLinkHeader(header string) (links []headerLink) {
|
||||
for link := range strings.SplitSeq(header, ",") {
|
||||
for _, link := range strings.Split(header, ",") {
|
||||
link = strings.TrimSpace(link)
|
||||
parsed := parseLink(link)
|
||||
if parsed != nil {
|
||||
@@ -30,7 +30,7 @@ func parseLinkHeader(header string) (links []headerLink) {
|
||||
|
||||
func parseLink(link string) (parsedLink *headerLink) {
|
||||
var parts []string
|
||||
for part := range strings.SplitSeq(link, ";") {
|
||||
for _, part := range strings.Split(link, ";") {
|
||||
parts = append(parts, strings.TrimSpace(part))
|
||||
}
|
||||
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
// Package api has type definitions for dosya.dev
|
||||
package api
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Response is the base response wrapper
|
||||
type Response struct {
|
||||
OK bool `json:"ok"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Error is an error response from the API.
|
||||
//
|
||||
// StatusCode and Status are filled in from the HTTP response. The API
|
||||
// sets ErrorCode on the errors a client is expected to act on rather
|
||||
// than just report, e.g. "concurrent_upload_limit".
|
||||
type Error struct {
|
||||
StatusCode int `json:"-"`
|
||||
Status string `json:"-"`
|
||||
OK bool `json:"ok"`
|
||||
Message string `json:"error"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
MaxConcurrentUploads int `json:"max_concurrent_uploads,omitempty"`
|
||||
}
|
||||
|
||||
// Error satisfies the error interface
|
||||
func (e *Error) Error() string {
|
||||
out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
|
||||
if e.Message != "" {
|
||||
out += ": " + e.Message
|
||||
}
|
||||
if e.ErrorCode != "" {
|
||||
out += " (" + e.ErrorCode + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileItem represents a file from the dosya API
|
||||
type FileItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
UpdatedAt float64 `json:"updated_at"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
|
||||
// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
|
||||
// are never re-read to hash the whole object - the same limitation S3 has).
|
||||
ContentHash string `json:"content_hash"`
|
||||
}
|
||||
|
||||
// FolderItem represents a folder from the dosya API
|
||||
type FolderItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
FileCount int `json:"file_count"`
|
||||
IsSynced int `json:"is_synced"`
|
||||
}
|
||||
|
||||
// ListResponse is the response from GET /api/files
|
||||
type ListResponse struct {
|
||||
Response
|
||||
Files []FileItem `json:"files"`
|
||||
Folders []FolderItem `json:"folders"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
// Pagination describes which page of files a ListResponse holds
|
||||
type Pagination struct {
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"per_page"`
|
||||
TotalFiles int `json:"total_files"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest is the request body for POST /api/folders
|
||||
type CreateFolderRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse is the response from POST /api/folders
|
||||
type CreateFolderResponse struct {
|
||||
Response
|
||||
Folder struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ParentID *string `json:"parent_id"`
|
||||
Workspace string `json:"workspace_id"`
|
||||
} `json:"folder"`
|
||||
}
|
||||
|
||||
// DeleteFolderResponse is the response from DELETE /api/folders/:id
|
||||
type DeleteFolderResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
Complete bool `json:"complete"`
|
||||
Remaining int `json:"remaining"`
|
||||
FilesAffected int `json:"files_affected"`
|
||||
FoldersRemoved int `json:"folders_removed"`
|
||||
}
|
||||
|
||||
// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
|
||||
type RenameFolderRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFolderRequest is the request body for PUT /api/folders/:id/move
|
||||
type MoveFolderRequest struct {
|
||||
ParentID *string `json:"parent_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// MoveFolderResponse is the response from PUT /api/folders/:id/move
|
||||
type MoveFolderResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// UploadInitRequest is the request body for POST /api/upload/init
|
||||
type UploadInitRequest struct {
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Region string `json:"region,omitempty"`
|
||||
FolderID *string `json:"folder_id"`
|
||||
FileID *string `json:"file_id,omitempty"`
|
||||
}
|
||||
|
||||
// UploadInitResponse is the response from POST /api/upload/init
|
||||
type UploadInitResponse struct {
|
||||
Response
|
||||
SessionID string `json:"session_id"`
|
||||
UploadURL string `json:"upload_url"`
|
||||
WorkspaceID string `json:"workspace_id"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Resumable *struct {
|
||||
PartSize int64 `json:"part_size"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
PartUploadURL string `json:"part_upload_url"`
|
||||
CompleteURL string `json:"complete_url"`
|
||||
StatusURL string `json:"status_url"`
|
||||
} `json:"resumable"`
|
||||
}
|
||||
|
||||
// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
|
||||
type UploadCompleteResponse struct {
|
||||
Response
|
||||
File struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MimeType string `json:"mime_type"`
|
||||
Extension string `json:"extension"`
|
||||
Region string `json:"region"`
|
||||
Version int `json:"version"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
|
||||
// "" for a multipart upload, which the server does not hash.
|
||||
ContentHash string `json:"content_hash"`
|
||||
} `json:"file"`
|
||||
}
|
||||
|
||||
// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
|
||||
type UploadPartResponse struct {
|
||||
Response
|
||||
PartNumber int `json:"part_number"`
|
||||
ETag string `json:"etag"`
|
||||
BytesUploaded int64 `json:"bytes_uploaded"`
|
||||
PartsCompleted int `json:"parts_completed"`
|
||||
TotalParts int `json:"total_parts"`
|
||||
}
|
||||
|
||||
// DeleteFileResponse is the response from DELETE /api/files/:id
|
||||
type DeleteFileResponse struct {
|
||||
Response
|
||||
Permanent bool `json:"permanent"`
|
||||
}
|
||||
|
||||
// RenameFileRequest is the request body for PUT /api/files/:id/rename
|
||||
type RenameFileRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// RenameFileResponse is the response from PUT /api/files/:id/rename
|
||||
type RenameFileResponse struct {
|
||||
Response
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// MoveFileRequest is the request body for PUT /api/files/:id/move
|
||||
type MoveFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest is the request body for POST /api/files/:id/copy
|
||||
type CopyFileRequest struct {
|
||||
FolderID *string `json:"folder_id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileResponse is the response from POST /api/files/:id/copy
|
||||
type CopyFileResponse struct {
|
||||
Response
|
||||
FileID string `json:"file_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// ShareLinkRequest is the request body for POST /api/files/:id/share
|
||||
type ShareLinkRequest struct {
|
||||
ExpiresInDays *int `json:"expires_in_days,omitempty"`
|
||||
}
|
||||
|
||||
// ShareLinkResponse is the response from POST /api/files/:id/share
|
||||
type ShareLinkResponse struct {
|
||||
Response
|
||||
Link struct {
|
||||
ID string `json:"id"`
|
||||
Token string `json:"token"`
|
||||
URL string `json:"url"`
|
||||
ExpiresAt *int64 `json:"expires_at"`
|
||||
CreatedAt float64 `json:"created_at"`
|
||||
} `json:"link"`
|
||||
}
|
||||
|
||||
// Workspace is one entry of the workspace list
|
||||
type Workspace struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// WorkspaceListResponse is the response from GET /api/workspaces
|
||||
type WorkspaceListResponse struct {
|
||||
Response
|
||||
Workspaces []Workspace `json:"workspaces"`
|
||||
}
|
||||
|
||||
// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
|
||||
type WorkspaceInfoResponse struct {
|
||||
Response
|
||||
Storage struct {
|
||||
Used int64 `json:"used"`
|
||||
Total int64 `json:"total"`
|
||||
Free int64 `json:"free"`
|
||||
} `json:"storage"`
|
||||
}
|
||||
@@ -1,772 +0,0 @@
|
||||
// Package dosya provides an interface to the dosya.dev cloud storage system.
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
rootID = "root"
|
||||
defaultBaseURL = "https://api.dosya.dev"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2
|
||||
attackConstant = 0
|
||||
)
|
||||
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "dosya",
|
||||
Description: "dosya.dev",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Help: "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
|
||||
Name: "api_key",
|
||||
Sensitive: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "workspace_id",
|
||||
Help: `Your workspace ID.
|
||||
|
||||
Leave this blank to use the one workspace the API key can reach, which
|
||||
is what a key restricted to a single workspace does. If the key can
|
||||
reach several, rclone will refuse and list their IDs for you to choose
|
||||
from.`,
|
||||
}, {
|
||||
Help: "API base URL.",
|
||||
Name: "api_url",
|
||||
Default: defaultBaseURL,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
Help: `Concurrency for multipart uploads.
|
||||
|
||||
This is the number of parts of the same file that are uploaded
|
||||
concurrently. The first part is always uploaded on its own, because
|
||||
the server creates the upload when it receives it.
|
||||
|
||||
Each part in flight is held in memory, so memory use grows with
|
||||
this value times the part size the server chooses.
|
||||
|
||||
If you are uploading large files over high-speed links and these
|
||||
uploads do not fully utilize your bandwidth, then increasing this
|
||||
may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeSingleQuote |
|
||||
encoder.EncodeDoubleQuote |
|
||||
encoder.EncodeLtGt |
|
||||
encoder.EncodeCtl |
|
||||
encoder.EncodeLeftSpace |
|
||||
encoder.EncodeRightSpace |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
APIKey string `config:"api_key"`
|
||||
WorkspaceID string `config:"workspace_id"`
|
||||
APIURL string `config:"api_url"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote dosya.dev filesystem
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
features *fs.Features
|
||||
rest *rest.Client
|
||||
pacer *fs.Pacer
|
||||
dirCache *dircache.DirCache
|
||||
}
|
||||
|
||||
// Object represents a dosya.dev file
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
file api.FileItem
|
||||
}
|
||||
|
||||
// findWorkspace fills in the workspace ID when the config leaves it blank.
|
||||
//
|
||||
// The ID is not shown anywhere in the dosya.dev web interface, so requiring
|
||||
// it meant asking people to find it in a developer console. A key restricted
|
||||
// to one workspace can only mean that one, and an account with a single
|
||||
// workspace has only one possible answer; anything else genuinely has to be
|
||||
// chosen, so list the choices in the error.
|
||||
func (f *Fs) findWorkspace(ctx context.Context) error {
|
||||
workspaces, err := f.listWorkspaces(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch len(workspaces) {
|
||||
case 0:
|
||||
return errors.New("no workspaces found for this API key")
|
||||
case 1:
|
||||
f.opt.WorkspaceID = workspaces[0].ID
|
||||
fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
|
||||
return nil
|
||||
}
|
||||
var choices strings.Builder
|
||||
for _, w := range workspaces {
|
||||
_, _ = fmt.Fprintf(&choices, "\n %s (%s)", w.ID, w.Name)
|
||||
}
|
||||
return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
|
||||
len(workspaces), choices.String())
|
||||
}
|
||||
|
||||
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
||||
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
|
||||
listing, err := f.listFilesAndFolders(ctx, pathID)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, folder := range listing.Folders {
|
||||
if f.opt.Enc.ToStandardName(folder.Name) == leaf {
|
||||
return folder.ID, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// CreateDir makes a directory with pathID as parent and name leaf
|
||||
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
|
||||
name := f.opt.Enc.FromStandardName(leaf)
|
||||
resp, err := f.createFolder(ctx, name, pathID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.Folder.ID, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *Fs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *Fs) Root() string {
|
||||
return f.root
|
||||
}
|
||||
|
||||
// String returns a description of the FS
|
||||
func (f *Fs) String() string {
|
||||
return fmt.Sprintf("dosya.dev root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Precision of the ModTimes in this Fs
|
||||
func (f *Fs) Precision() time.Duration {
|
||||
// Modification times are stored and returned to the second (the API
|
||||
// keeps them as Unix seconds, sent on upload via the source-mtime header).
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash types of the filesystem
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
// The server computes and stores the SHA-256 of a single-shot upload's
|
||||
// bytes; multipart uploads have none (see Object.Hash).
|
||||
return hash.NewHashSet(hash.SHA256)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// NewFs makes a new Fs object from the path
|
||||
func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(config, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
|
||||
}
|
||||
|
||||
// No DuplicateFiles: an upload that names no existing file but whose
|
||||
// (workspace, folder, name) is already taken becomes a new VERSION of
|
||||
// that file rather than a second row, so a name is unique within a
|
||||
// folder and the same upload twice returns the same file id.
|
||||
f.features = (&fs.Features{
|
||||
CaseInsensitive: false,
|
||||
CanHaveEmptyDirectories: true,
|
||||
ReadMimeType: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
client := fshttp.NewClient(ctx)
|
||||
f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
|
||||
f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
|
||||
|
||||
if f.opt.WorkspaceID == "" {
|
||||
err = f.findWorkspace(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
f.dirCache = dircache.New(root, rootID, f)
|
||||
|
||||
// Find the current root
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// Assume it is a file
|
||||
newRoot, remote := dircache.SplitPath(root)
|
||||
tempF := *f
|
||||
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
|
||||
tempF.root = newRoot
|
||||
// Make new Fs which is the parent
|
||||
err = tempF.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// No root so return old f
|
||||
return f, nil
|
||||
}
|
||||
_, err := tempF.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// File doesn't exist so return old f
|
||||
return f, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
f.features.Fill(ctx, &tempF)
|
||||
f.dirCache = tempF.dirCache
|
||||
f.root = tempF.root
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, folder := range listing.Folders {
|
||||
folderName := f.opt.Enc.ToStandardName(folder.Name)
|
||||
fullPath := getRemote(dir, folderName)
|
||||
modTime := time.Unix(int64(folder.CreatedAt), 0)
|
||||
d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
|
||||
entries = append(entries, d)
|
||||
f.dirCache.Put(fullPath, folder.ID)
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
fileName := f.opt.Enc.ToStandardName(file.Name)
|
||||
remote := getRemote(dir, fileName)
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}
|
||||
entries = append(entries, o)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
listing, err := f.listFilesAndFolders(ctx, directoryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, file := range listing.Files {
|
||||
if f.opt.Enc.ToStandardName(file.Name) == leaf {
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: file,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
// Put uploads a new file
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
existingObj, err := f.NewObject(ctx, src.Remote())
|
||||
switch err {
|
||||
case nil:
|
||||
return existingObj, existingObj.Update(ctx, in, src, options...)
|
||||
case fs.ErrorObjectNotFound:
|
||||
return f.put(ctx, in, src, options...)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// put uploads src as a new file
|
||||
func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
remote := src.Remote()
|
||||
|
||||
_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: fileFromUpload(resp, modTime),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fileFromUpload builds the FileItem for the object an upload just created.
|
||||
// The single-shot upload response omits updated_at, so the modification time
|
||||
// is taken from what was sent (modTime) rather than defaulting to the create
|
||||
// time - it is the time the server stored via the source-mtime header. The
|
||||
// SHA-256 comes straight from the response (empty for a multipart upload).
|
||||
func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
|
||||
updatedAt := resp.File.CreatedAt
|
||||
if !modTime.IsZero() {
|
||||
updatedAt = float64(modTime.Unix())
|
||||
}
|
||||
return api.FileItem{
|
||||
ID: resp.File.ID,
|
||||
Name: resp.File.Name,
|
||||
SizeBytes: resp.File.SizeBytes,
|
||||
MimeType: resp.File.MimeType,
|
||||
Extension: resp.File.Extension,
|
||||
Region: resp.File.Region,
|
||||
CreatedAt: resp.File.CreatedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
ContentHash: resp.File.ContentHash,
|
||||
}
|
||||
}
|
||||
|
||||
// Mkdir makes the directory (container, bucket)
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
return err
|
||||
}
|
||||
|
||||
// Rmdir removes the directory (container, bucket) if empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The server checks emptiness itself (?if_empty=1) and answers
|
||||
// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
|
||||
err = f.removeFolder(ctx, directoryID, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move src to this remote using server-side move operations
|
||||
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't move - not same remote type")
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
|
||||
_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find destination directory
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
needsMove := srcDirID != dstDirID
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
|
||||
// Moving and renaming in the same request (rather than a moveFile call
|
||||
// followed by a separate renameFile call) matters when both are needed:
|
||||
// a two-step move-then-rename can transiently place the file's CURRENT
|
||||
// name into a destination folder that already holds an unrelated file
|
||||
// with that name, even though the FINAL name is free - the server only
|
||||
// allows one active name per folder, so that intermediate state 409s.
|
||||
if needsMove || needsRename {
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Return new object. file.Name must reflect the destination leaf, not
|
||||
// the source's - a caller that reuses this object as the src of a
|
||||
// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
|
||||
// recomputes srcLeaf from file.Name; a stale name there makes that
|
||||
// second move see srcLeaf == dstLeaf and skip the rename entirely,
|
||||
// silently leaving the object under the wrong name.
|
||||
newFile := srcObj.file
|
||||
newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: newFile,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
|
||||
srcFs, ok := src.(*Fs)
|
||||
if !ok {
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
|
||||
srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srcLeaf := srcRemote
|
||||
if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
|
||||
srcLeaf = srcRemote[idx+1:]
|
||||
}
|
||||
needsRename := srcLeaf != dstLeaf
|
||||
newName := ""
|
||||
if needsRename {
|
||||
newName = f.opt.Enc.FromStandardName(dstLeaf)
|
||||
}
|
||||
|
||||
// Move folder to new parent only if parent changed. The new name goes
|
||||
// in the same request, as for files in Move: a move followed by a
|
||||
// separate rename can collide with an unrelated folder sharing the
|
||||
// CURRENT name in the destination even though the final name is free.
|
||||
if srcDirectoryID != dstDirectoryID {
|
||||
renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move directory: %w", err)
|
||||
}
|
||||
if renamed {
|
||||
needsRename = false
|
||||
}
|
||||
}
|
||||
|
||||
if needsRename {
|
||||
err = f.renameFolder(ctx, srcID, newName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side copy operations
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
|
||||
dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The copy API creates a new row named after the source file when
|
||||
// copying across folders, which collides with one already at the
|
||||
// destination. Copy under a temporary name, then delete the old
|
||||
// destination, then move the copy onto the name - so a copy that fails
|
||||
// leaves the destination as it was. Deleting it up front lost it
|
||||
// (prior art: backend/box).
|
||||
existingObj, err := f.NewObject(ctx, remote)
|
||||
switch err {
|
||||
case nil:
|
||||
tempRemote := remote + "-rclone-copy-" + random.String(8)
|
||||
fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
|
||||
tempObj, err := f.Copy(ctx, src, tempRemote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs.Debugf(tempRemote, "moving to real name %v", remote)
|
||||
if err := existingObj.Remove(ctx); err != nil {
|
||||
return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
|
||||
}
|
||||
return f.Move(ctx, tempObj, remote)
|
||||
case fs.ErrorObjectNotFound:
|
||||
// nothing at the destination, so copy straight onto the name
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dstName := f.opt.Enc.FromStandardName(dstLeaf)
|
||||
|
||||
resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
|
||||
// Servers that predate the name field ignore it and name the copy after
|
||||
// the source, or "Copy of <original>" in the same folder. Rename to the
|
||||
// requested destination name if it differs.
|
||||
if resp.Name != dstName {
|
||||
err = f.renameFile(ctx, resp.FileID, dstName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't rename copied file: %w", err)
|
||||
}
|
||||
resp.Name = dstName
|
||||
}
|
||||
|
||||
// A server-side copy is byte-identical to the source and the API
|
||||
// preserves the source's modification time and content hash onto the
|
||||
// copy, so the returned object must report the SOURCE's modtime and hash,
|
||||
// not the copy's wall-clock time. Reading "now" here made the object
|
||||
// rclone holds in memory disagree with the one read back from the remote
|
||||
// (its Fingerprint check compares the two exactly).
|
||||
return &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
file: api.FileItem{
|
||||
ID: resp.FileID,
|
||||
Name: resp.Name,
|
||||
SizeBytes: srcObj.file.SizeBytes,
|
||||
MimeType: srcObj.file.MimeType,
|
||||
Extension: srcObj.file.Extension,
|
||||
Region: srcObj.file.Region,
|
||||
CreatedAt: srcObj.file.CreatedAt,
|
||||
UpdatedAt: srcObj.file.UpdatedAt,
|
||||
ContentHash: srcObj.file.ContentHash,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Purge deletes all the files and the container
|
||||
func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
|
||||
err = f.removeFolder(ctx, directoryID, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.dirCache.FlushDir(dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
obj := o.(*Object)
|
||||
|
||||
resp, err := f.createShareLink(ctx, obj.file.ID, expire)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return resp.Link.URL, nil
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing as an
|
||||
// optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
info, err := f.getWorkspaceInfo(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage := &fs.Usage{
|
||||
Used: fs.NewUsageValue(info.Storage.Used),
|
||||
Total: fs.NewUsageValue(info.Storage.Total),
|
||||
Free: fs.NewUsageValue(info.Storage.Free),
|
||||
}
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
// Object methods
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// String returns a description of the Object
|
||||
func (o *Object) String() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// ModTime returns the modification date of the file
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
if o.file.UpdatedAt > 0 {
|
||||
return time.Unix(int64(o.file.UpdatedAt), 0)
|
||||
}
|
||||
return time.Unix(int64(o.file.CreatedAt), 0)
|
||||
}
|
||||
|
||||
// Size returns the size of the file
|
||||
func (o *Object) Size() int64 {
|
||||
return o.file.SizeBytes
|
||||
}
|
||||
|
||||
// Fs returns read only access to the Fs that this object is part of
|
||||
func (o *Object) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Hash returns the selected checksum of the file
|
||||
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.SHA256 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// May be "" - a multipart-uploaded object carries no server-side hash,
|
||||
// which rclone treats as "hash unavailable for this object" (as it does
|
||||
// for S3 multipart objects).
|
||||
return o.file.ContentHash, nil
|
||||
}
|
||||
|
||||
// Storable says whether this object can be stored
|
||||
func (o *Object) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// SetModTime sets the metadata on the object to set the modification date
|
||||
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
|
||||
return fs.ErrorCantSetModTime
|
||||
}
|
||||
|
||||
// Open opens the file for read
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
fs.FixRangeOption(options, o.file.SizeBytes)
|
||||
return o.fs.downloadFile(ctx, o.file.ID, options)
|
||||
}
|
||||
|
||||
// Update replaces the file content
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
// Find the directory ID for this file
|
||||
_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload as new version of existing file
|
||||
fileID := o.file.ID
|
||||
modTime := src.ModTime(ctx)
|
||||
resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update object metadata
|
||||
o.file = fileFromUpload(resp, modTime)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove removes this object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
return o.fs.deleteFile(ctx, o.file.ID)
|
||||
}
|
||||
|
||||
// MimeType of an Object if known, "" otherwise
|
||||
func (o *Object) MimeType(ctx context.Context) string {
|
||||
return o.file.MimeType
|
||||
}
|
||||
|
||||
// ID returns the ID of the Object if known, or "" if not
|
||||
func (o *Object) ID() string {
|
||||
return o.file.ID
|
||||
}
|
||||
|
||||
func getRemote(dir, fileName string) string {
|
||||
if dir == "" {
|
||||
return fileName
|
||||
}
|
||||
return dir + "/" + fileName
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.Copier = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ dircache.DirCacher = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.MimeTyper = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
)
|
||||
@@ -1,880 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// sourceMTimeHeader is the request header the upload endpoints read to set a
|
||||
// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
|
||||
const sourceMTimeHeader = "x-dosya-source-mtime"
|
||||
|
||||
// modTimeHeaders returns the ExtraHeaders that carry a source modification time
|
||||
// on an upload, or nil when there is nothing plausible to send. The server
|
||||
// clamps implausible values itself, but a zero time is never worth a header.
|
||||
func modTimeHeaders(modTime time.Time) map[string]string {
|
||||
if modTime.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
|
||||
}
|
||||
|
||||
// workspaceHintHeader names the workspace a by-id request is about. The API
|
||||
// maps an id to its workspace through an index the workspace fills about a
|
||||
// second after a write, so an id this client was handed moments ago can still
|
||||
// miss; the hint lets the server look for it in the named workspace instead.
|
||||
// It only locates: every route still checks membership against the workspace
|
||||
// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
|
||||
const workspaceHintHeader = "X-Dosya-Workspace"
|
||||
|
||||
// byIDHeaders returns the ExtraHeaders every /api/files/:id and
|
||||
// /api/folders/:id call carries, or nil when no workspace is configured.
|
||||
func (f *Fs) byIDHeaders() map[string]string {
|
||||
if f.opt.WorkspaceID == "" {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
|
||||
}
|
||||
|
||||
// retryErrorCodes is a slice of error codes that we will retry
|
||||
var retryErrorCodes = []int{
|
||||
429, // Too Many Requests
|
||||
500, // Internal Server Error
|
||||
502, // Bad Gateway
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
// deserve to be retried
|
||||
func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
|
||||
if retry {
|
||||
if d, ok := retryAfter(resp); ok {
|
||||
// The API answers 503 + Retry-After while a workspace is being
|
||||
// moved between storage homes, and Cloudflare sends it with 429.
|
||||
// The pacer's own backoff tops out at maxSleep, far short of that.
|
||||
fs.Debugf(nil, "Server asked to retry after %v", d)
|
||||
err = pacer.RetryAfterError(err, d)
|
||||
}
|
||||
}
|
||||
return retry, err
|
||||
}
|
||||
|
||||
// maxRetryAfter caps how long a single Retry-After is honoured for
|
||||
const maxRetryAfter = 5 * time.Minute
|
||||
|
||||
// retryAfter reads a 429 or 503 response's Retry-After header, which the API
|
||||
// sends as whole seconds
|
||||
func retryAfter(resp *http.Response) (time.Duration, bool) {
|
||||
if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
|
||||
return 0, false
|
||||
}
|
||||
seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
|
||||
if err != nil || seconds <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
|
||||
}
|
||||
|
||||
// errorHandler decodes a non-2xx response into an *api.Error so callers can
|
||||
// act on the machine-readable error_code rather than the message text.
|
||||
func errorHandler(resp *http.Response) error {
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading error out of body: %w", err)
|
||||
}
|
||||
apiErr := &api.Error{
|
||||
StatusCode: resp.StatusCode,
|
||||
Status: resp.Status,
|
||||
}
|
||||
if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
|
||||
// Not the API's JSON shape (e.g. an edge response that never
|
||||
// reached the API) - keep the body text so it isn't lost.
|
||||
apiErr.ErrorCode = ""
|
||||
apiErr.Message = strings.TrimSpace(string(body))
|
||||
}
|
||||
return apiErr
|
||||
}
|
||||
|
||||
// The workspace caps the number of in-flight uploads per user. The API
|
||||
// refuses an upload/init past that cap with a 400 carrying
|
||||
// error_code "concurrent_upload_limit", and documents it as an error to
|
||||
// wait on rather than give up on: slots free as uploads finish, and the
|
||||
// server marks sessions idle for 10 minutes as failed, so waiting up to
|
||||
// that long guarantees progress unless the workspace really is saturated.
|
||||
// A 400 is deliberately outside the pacer's retry codes, so the wait
|
||||
// lives in initUpload instead of shouldRetry.
|
||||
var (
|
||||
concurrentUploadRetryDelay = 5 * time.Second
|
||||
concurrentUploadWaitMax = 10 * time.Minute
|
||||
)
|
||||
|
||||
// abortUploadTimeout bounds the abort of a session the client is giving up on,
|
||||
// which runs outside the transfer's context so a cancelled transfer still
|
||||
// sends it
|
||||
const abortUploadTimeout = 10 * time.Second
|
||||
|
||||
// isConcurrentUploadLimit reports whether err is the API refusing an
|
||||
// upload because the workspace's concurrent upload limit is reached
|
||||
func isConcurrentUploadLimit(err error) bool {
|
||||
var apiErr *api.Error
|
||||
return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
|
||||
}
|
||||
|
||||
// listPageSize is the largest page GET /api/files will serve
|
||||
const listPageSize = 500
|
||||
|
||||
// listFilesAndFolders lists files and folders in a directory
|
||||
//
|
||||
// The API pages files (at most listPageSize per page) but returns every
|
||||
// folder on each page, so the files from all pages are gathered and the
|
||||
// folders are taken from the first page only. Pages are ordered by name,
|
||||
// which the API tie-breaks on id, so offsets are stable between requests.
|
||||
func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
|
||||
var listing *api.ListResponse
|
||||
for page := 1; ; page++ {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files",
|
||||
Parameters: map[string][]string{
|
||||
"workspace_id": {f.opt.WorkspaceID},
|
||||
"per_page": {strconv.Itoa(listPageSize)},
|
||||
"page": {strconv.Itoa(page)},
|
||||
"sort": {"name"},
|
||||
"dir": {"asc"},
|
||||
},
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
opts.Parameters["folder_id"] = []string{folderID}
|
||||
}
|
||||
|
||||
var result api.ListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list files: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if listing == nil {
|
||||
listing = &result
|
||||
} else {
|
||||
listing.Files = append(listing.Files, result.Files...)
|
||||
}
|
||||
if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
|
||||
return listing, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// createFolder creates a folder
|
||||
func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
|
||||
request := api.CreateFolderRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
Name: name,
|
||||
}
|
||||
if parentID != "" && parentID != rootID {
|
||||
request.ParentID = &parentID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/folders",
|
||||
}
|
||||
|
||||
var result api.CreateFolderResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// removeFolder deletes a folder
|
||||
//
|
||||
// ?permanent=1 trashes and purges a live folder in one request on today's
|
||||
// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
|
||||
// HTTP 409 "Folder is not empty" instead of purging a folder that still
|
||||
// holds a live file or a live child folder; that 409 is turned into
|
||||
// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
|
||||
// error, and it is never retried (409 is not one of retryErrorCodes). A big
|
||||
// subtree is purged in bounded passes (202, complete=false), so keep
|
||||
// calling until the purge is complete. An older server that ignores
|
||||
// permanent=1 only trashes on the first call (answering permanent=false),
|
||||
// so the loop's second pass sends the plain DELETE that purges it.
|
||||
func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
|
||||
parameters := map[string][]string{"permanent": {"1"}}
|
||||
if ifEmpty {
|
||||
parameters["if_empty"] = []string{"1"}
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/folders/" + folderID,
|
||||
Parameters: parameters,
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
for pass := 0; ; pass++ {
|
||||
var result api.DeleteFolderResponse
|
||||
var statusCode int
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
if resp != nil {
|
||||
statusCode = resp.StatusCode
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
if ifEmpty && statusCode == http.StatusConflict {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
if pass == 0 {
|
||||
return fmt.Errorf("couldn't remove folder: %w", err)
|
||||
}
|
||||
return fmt.Errorf("couldn't permanently remove folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
if result.Permanent && result.Complete {
|
||||
return nil
|
||||
}
|
||||
if pass >= maxFolderPurgePasses {
|
||||
return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
|
||||
const maxFolderPurgePasses = 1000
|
||||
|
||||
// renameFolder renames a folder
|
||||
func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
|
||||
request := api.RenameFolderRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFolder moves a folder to a new parent
|
||||
func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
|
||||
request := api.MoveFolderRequest{Name: newName}
|
||||
if newParentID != "" && newParentID != rootID {
|
||||
request.ParentID = &newParentID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/folders/" + folderID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.MoveFolderResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("couldn't move folder: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return false, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
// Servers that predate the name field ignore it and answer without a
|
||||
// name, leaving the caller to rename separately
|
||||
return newName != "" && result.Name == newName, nil
|
||||
}
|
||||
|
||||
// deleteFile deletes a file
|
||||
//
|
||||
// ?permanent=1 trashes and purges it in one request on today's API. An
|
||||
// older server that ignores the flag only soft-deletes it, answering
|
||||
// permanent=false, so a second DELETE on the now soft-deleted file purges
|
||||
// it from the database and R2, as it always has.
|
||||
func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/files/" + fileID,
|
||||
Parameters: map[string][]string{"permanent": {"1"}},
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.DeleteFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't delete file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
|
||||
// Answered into its own value so a field the second response omits
|
||||
// can't be read off the first.
|
||||
if !result.Permanent {
|
||||
var permanent api.DeleteFileResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't permanently delete file: %w", err)
|
||||
}
|
||||
if !permanent.OK {
|
||||
return fmt.Errorf("API error: %s", permanent.Error)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renameFile renames a file
|
||||
func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
|
||||
request := api.RenameFileRequest{Name: newName}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/rename",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.RenameFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't rename file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveFile moves a file to a new folder and, if newName is non-empty,
|
||||
// renames it in the same request
|
||||
func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
|
||||
request := api.MoveFileRequest{}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
if newName != "" {
|
||||
request.Name = &newName
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/files/" + fileID + "/move",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("couldn't move file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyFile copies a file to a folder
|
||||
func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
|
||||
request := api.CopyFileRequest{Name: name}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/copy",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.CopyFileResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// downloadFile downloads a file by its ID
|
||||
//
|
||||
// The download endpoint answers 302 with a presigned R2 URL in Location, so
|
||||
// this is two calls. Both go through the rest client, which is built on
|
||||
// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
|
||||
// --user-agent, --bind, --dump and the proxy settings apply to downloads as
|
||||
// they do to every other call, and errorHandler and shouldRetry decide the
|
||||
// errors and the retries uniformly.
|
||||
func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
|
||||
// NoRedirect stops the client following the 302 with the API's
|
||||
// Authorization header attached. A 302 is not a 2xx, so Call reports it
|
||||
// as an error (having read and closed the body); the Location header
|
||||
// is what actually matters, and finding it clears the error, as in
|
||||
// backend/linkbox.
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/files/" + fileID + "/download",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
NoRedirect: true,
|
||||
}
|
||||
var downloadURL string
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
if resp != nil {
|
||||
if location := resp.Header.Get("Location"); location != "" {
|
||||
downloadURL, err = location, nil
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get download URL: %w", err)
|
||||
}
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("no download URL in redirect response")
|
||||
}
|
||||
|
||||
// Fetch the object from the presigned URL. It carries its own
|
||||
// credentials and R2 refuses a request that also has an Authorization
|
||||
// header, so the client-level bearer token is removed by setting the
|
||||
// header empty: the rest client skips a header with an empty value.
|
||||
opts = rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: downloadURL,
|
||||
Options: options,
|
||||
ExtraHeaders: map[string]string{"Authorization": ""},
|
||||
}
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't download file: %w", err)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// uploadSmallFile uploads a file smaller than 10MB in a single request
|
||||
func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
Body: in,
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
// The body is a stream which can't be rewound, so a low level retry
|
||||
// would resend whatever is left of it - nothing - and the server
|
||||
// would store that as an empty file. Don't retry here; rclone's high
|
||||
// level retries redo the whole transfer instead.
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload file: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadPart uploads a single part of a multipart upload
|
||||
func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
|
||||
size := int64(len(part))
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Path: "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
|
||||
ContentLength: &size,
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
var result api.UploadPartResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
// A fresh reader per attempt so a retry resends the whole part
|
||||
opts.Body = bytes.NewReader(part)
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// completeUpload finalizes a multipart upload
|
||||
func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/" + sessionID + "/complete",
|
||||
ExtraHeaders: modTimeHeaders(modTime),
|
||||
}
|
||||
|
||||
var result api.UploadCompleteResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't complete upload: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// initUpload initializes a new upload session
|
||||
func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
|
||||
request := api.UploadInitRequest{
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
FileName: fileName,
|
||||
FileSize: fileSize,
|
||||
MimeType: mimeType,
|
||||
FileID: fileID,
|
||||
}
|
||||
if folderID != "" && folderID != rootID {
|
||||
request.FolderID = &folderID
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/upload/init",
|
||||
}
|
||||
|
||||
var result api.UploadInitResponse
|
||||
deadline := time.Now().Add(concurrentUploadWaitMax)
|
||||
for {
|
||||
result = api.UploadInitResponse{}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", err)
|
||||
}
|
||||
fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
|
||||
case <-time.After(concurrentUploadRetryDelay):
|
||||
}
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// uploadParts reads the source part by part and uploads the parts.
|
||||
//
|
||||
// Part 1 goes on its own: the server creates the R2 multipart upload (and a
|
||||
// new file's id and key) when it receives the first part, so parts sent
|
||||
// before that has finished would each start an upload of their own. The
|
||||
// rest go out opt.UploadConcurrency at a time. A part is only read from the
|
||||
// source once a buffer is free, so at most that many parts are held in
|
||||
// memory, and a short source is still found by the reader before the part
|
||||
// it could not fill is sent.
|
||||
func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
|
||||
readPart := func(buf []byte, partNum int) ([]byte, error) {
|
||||
currentPartSize := partSize
|
||||
if partNum == totalParts {
|
||||
currentPartSize = size - partSize*int64(totalParts-1)
|
||||
}
|
||||
part := buf[:currentPartSize]
|
||||
if _, err := io.ReadFull(in, part); err != nil {
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
|
||||
}
|
||||
return part, nil
|
||||
}
|
||||
|
||||
concurrency := max(f.opt.UploadConcurrency, 1)
|
||||
buffers := make(chan []byte, concurrency)
|
||||
buffers <- make([]byte, partSize)
|
||||
allocated := 1
|
||||
|
||||
first := <-buffers
|
||||
part, err := readPart(first, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
|
||||
return err
|
||||
}
|
||||
buffers <- first
|
||||
|
||||
// A failed read has to take the parts already in flight down with it,
|
||||
// or they upload to completion while the error waits on Wait.
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
g, gCtx := errgroup.WithContext(ctx)
|
||||
var readErr error
|
||||
for partNum := 2; partNum <= totalParts; partNum++ {
|
||||
// Take a free buffer, allocating one while under the limit.
|
||||
var buf []byte
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
default:
|
||||
if allocated < concurrency {
|
||||
buf = make([]byte, partSize)
|
||||
allocated++
|
||||
} else {
|
||||
select {
|
||||
case buf = <-buffers:
|
||||
case <-gCtx.Done():
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf == nil {
|
||||
break // a part failed; its error comes back from Wait
|
||||
}
|
||||
part, err := readPart(buf, partNum)
|
||||
if err != nil {
|
||||
readErr = err
|
||||
cancel()
|
||||
break
|
||||
}
|
||||
g.Go(func() error {
|
||||
defer func() { buffers <- buf }()
|
||||
_, err := f.uploadPart(gCtx, part, sessionID, partNum)
|
||||
return err
|
||||
})
|
||||
}
|
||||
err = g.Wait()
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// abortUpload retires an upload session the client is giving up on.
|
||||
//
|
||||
// Best effort: the server reclaims an abandoned session itself, so a failure
|
||||
// here - including an older server with no such route - costs the slot the
|
||||
// wait it would have cost anyway and is not worth failing an upload over.
|
||||
// Sent on a context of its own so a cancelled transfer still gives its slot
|
||||
// back rather than leaving it to the sweep.
|
||||
func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
|
||||
defer cancel()
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
Path: "/api/upload/" + sessionID,
|
||||
NoResponse: true,
|
||||
}
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err := f.rest.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadFile handles the full upload flow (small or multipart)
|
||||
func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
|
||||
// Both refusals live here so a new file and a new version of an existing
|
||||
// one answer the same way: upload/init needs a byte count up front, and
|
||||
// the server stores no zero-length object.
|
||||
if size < 0 {
|
||||
return nil, errors.New("can't upload files with unknown size")
|
||||
}
|
||||
if size == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
leaf := f.opt.Enc.FromStandardName(path.Base(remote))
|
||||
|
||||
initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A session abandoned here holds one of the workspace's upload slots
|
||||
// until the server's sweep retires it ten minutes later, and rclone's
|
||||
// high-level retry opens a fresh session rather than resuming this one -
|
||||
// so without this, a few failed attempts use up every slot.
|
||||
defer func() {
|
||||
if err != nil {
|
||||
f.abortUpload(ctx, initResp.SessionID)
|
||||
}
|
||||
}()
|
||||
|
||||
// Count what the source actually supplies, so a source which ends
|
||||
// early is reported as an error rather than stored as a truncated
|
||||
// file.
|
||||
counter := readers.NewCountingReader(in)
|
||||
|
||||
// Small file upload (single PUT, no resumable)
|
||||
if initResp.Resumable == nil {
|
||||
resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if got := int64(counter.BytesRead()); got != size {
|
||||
return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Multipart upload. Each part is read fully into memory before it is
|
||||
// sent, so a short source is caught before the part goes out and a
|
||||
// part which has to be retried is resent with all of its bytes.
|
||||
partSize := initResp.Resumable.PartSize
|
||||
totalParts := initResp.Resumable.TotalParts
|
||||
// Both bounds matter. partSize*(totalParts-1) >= size means a part would
|
||||
// have nothing to carry; size > partSize*totalParts means the last part
|
||||
// is bigger than partSize, and readPart would slice past its buffer and
|
||||
// panic.
|
||||
if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
|
||||
return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
|
||||
}
|
||||
if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f.completeUpload(ctx, initResp.SessionID, modTime)
|
||||
}
|
||||
|
||||
// createShareLink creates a public share link for a file
|
||||
func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
|
||||
request := api.ShareLinkRequest{}
|
||||
if expire < fs.DurationOff {
|
||||
days := int(time.Duration(expire).Hours() / 24)
|
||||
if days < 1 {
|
||||
days = 1
|
||||
}
|
||||
request.ExpiresInDays = &days
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/api/files/" + fileID + "/share",
|
||||
ExtraHeaders: f.byIDHeaders(),
|
||||
}
|
||||
|
||||
var result api.ShareLinkResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't create share link: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// getWorkspaceInfo fetches workspace storage info
|
||||
// listWorkspaces lists the workspaces this credential can reach. An API key
|
||||
// pinned to a workspace sees exactly one: its own.
|
||||
func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces",
|
||||
}
|
||||
|
||||
var result api.WorkspaceListResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't list workspaces: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return result.Workspaces, nil
|
||||
}
|
||||
|
||||
func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/api/workspaces/" + f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
var result api.WorkspaceInfoResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't get workspace info: %w", err)
|
||||
}
|
||||
if !result.OK {
|
||||
return nil, fmt.Errorf("API error: %s", result.Error)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestShouldRetryHonoursRetryAfter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
header string
|
||||
want time.Duration
|
||||
ok bool
|
||||
}{
|
||||
{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
|
||||
{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
|
||||
{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
|
||||
{"missing header", http.StatusServiceUnavailable, "", 0, false},
|
||||
{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
|
||||
{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
|
||||
if tc.header != "" {
|
||||
resp.Header.Set("Retry-After", tc.header)
|
||||
}
|
||||
retry, err := shouldRetry(ctx, resp, errorHandler(resp))
|
||||
assert.True(t, retry)
|
||||
d, ok := pacer.IsRetryAfter(err)
|
||||
assert.Equal(t, tc.ok, ok)
|
||||
if tc.ok {
|
||||
assert.Equal(t, tc.want, d)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
|
||||
resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
|
||||
resp.Header.Set("Retry-After", "30")
|
||||
retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
|
||||
assert.False(t, retry)
|
||||
_, ok := pacer.IsRetryAfter(err)
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestRemoveFolderPurgesPermanently(t *testing.T) {
|
||||
// a big subtree purges in bounded passes: 202 complete=false repeats
|
||||
// the same request until complete=true
|
||||
answers := []struct {
|
||||
status int
|
||||
body string
|
||||
}{
|
||||
{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
|
||||
{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
|
||||
}
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
require.Equal(t, "DELETE", r.Method)
|
||||
require.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "", r.URL.Query().Get("if_empty"))
|
||||
require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
|
||||
a := answers[calls]
|
||||
calls++
|
||||
return jsonResp(a.status, a.body), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
|
||||
assert.Equal(t, len(answers), calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", false)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "locked file")
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
|
||||
err := f.removeFolder(context.Background(), "fld_1", true)
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
|
||||
}
|
||||
|
||||
func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
|
||||
assert.Equal(t, 1, calls)
|
||||
}
|
||||
|
||||
func TestRmdirMakesOneRequest(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
require.NoError(t, f.Rmdir(context.Background(), "sub"))
|
||||
assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
|
||||
}
|
||||
|
||||
func TestRmdirMapsNonEmptyConflict(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
|
||||
}))
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
|
||||
f.dirCache.Put("sub", "fld_1")
|
||||
|
||||
err := f.Rmdir(context.Background(), "sub")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
|
||||
}
|
||||
|
||||
func TestDeleteFileSendsPermanentFlag(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
permanent bool
|
||||
wantCalls int
|
||||
}{
|
||||
{"today's API purges in one call", true, 1},
|
||||
{"an older server ignores the flag and still gets a second DELETE", false, 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
assert.Equal(t, "DELETE", r.Method)
|
||||
assert.Equal(t, "/api/files/fil_1", r.URL.Path)
|
||||
assert.Equal(t, "1", r.URL.Query().Get("permanent"))
|
||||
return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
|
||||
}))
|
||||
|
||||
require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
|
||||
assert.Equal(t, tc.wantCalls, calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// decodeBody reads a JSON request body into a map
|
||||
func decodeBody(t *testing.T, r *http.Request) map[string]any {
|
||||
b, err := io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
var m map[string]any
|
||||
require.NoError(t, json.Unmarshal(b, &m))
|
||||
return m
|
||||
}
|
||||
|
||||
func TestMoveFolderSendsName(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
newName string
|
||||
respBody string
|
||||
wantRenamed bool
|
||||
}{
|
||||
{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
|
||||
{"older server ignores name", "dst", `{"ok":true}`, false},
|
||||
{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_parent", body["parent_id"])
|
||||
if tc.newName == "" {
|
||||
assert.NotContains(t, body, "name")
|
||||
} else {
|
||||
assert.Equal(t, tc.newName, body["name"])
|
||||
}
|
||||
return jsonResp(http.StatusOK, tc.respBody), nil
|
||||
}))
|
||||
|
||||
renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantRenamed, renamed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyFileSendsName(t *testing.T) {
|
||||
f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
|
||||
body := decodeBody(t, r)
|
||||
assert.Equal(t, "fld_dst", body["folder_id"])
|
||||
assert.Equal(t, "final.txt", body["name"])
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
|
||||
}))
|
||||
|
||||
resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "fil_2", resp.FileID)
|
||||
assert.Equal(t, "final.txt", resp.Name)
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package dosya
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/dosya/api"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
|
||||
f := &Fs{}
|
||||
assert.Equal(t, time.Second, f.Precision())
|
||||
assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
|
||||
// Exactly SHA-256, nothing else.
|
||||
assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
|
||||
}
|
||||
|
||||
func TestObjectHashReturnsContentHash(t *testing.T) {
|
||||
o := &Object{file: api.FileItem{ContentHash: "abc123"}}
|
||||
|
||||
got, err := o.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "abc123", got)
|
||||
|
||||
// A multipart object with no server-side hash yields "" and no error,
|
||||
// exactly as an S3 multipart object does.
|
||||
empty := &Object{file: api.FileItem{ContentHash: ""}}
|
||||
got, err = empty.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", got)
|
||||
|
||||
// Any other algorithm is unsupported.
|
||||
_, err = o.Hash(context.Background(), hash.MD5)
|
||||
assert.ErrorIs(t, err, hash.ErrUnsupported)
|
||||
}
|
||||
|
||||
func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
|
||||
assert.True(t, mt.Equal(o.ModTime(context.Background())))
|
||||
}
|
||||
|
||||
func TestModTimeHeaders(t *testing.T) {
|
||||
assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
|
||||
h := modTimeHeaders(mt)
|
||||
require.NotNil(t, h)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
|
||||
}
|
||||
|
||||
func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
|
||||
resp := &api.UploadCompleteResponse{}
|
||||
resp.File.ID = "fil_1"
|
||||
resp.File.Name = "x.txt"
|
||||
resp.File.SizeBytes = 10
|
||||
resp.File.CreatedAt = 1_000
|
||||
resp.File.ContentHash = "deadbeef"
|
||||
|
||||
mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
|
||||
item := fileFromUpload(resp, mt)
|
||||
assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
|
||||
assert.Equal(t, "deadbeef", item.ContentHash)
|
||||
assert.Equal(t, float64(1_000), item.CreatedAt)
|
||||
|
||||
// A zero modtime falls back to the create time.
|
||||
item = fileFromUpload(resp, time.Time{})
|
||||
assert.Equal(t, float64(1_000), item.UpdatedAt)
|
||||
}
|
||||
|
||||
// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
|
||||
// fake transport and asserts the source-mtime header actually rides on the PUT.
|
||||
func TestUploadSendsSourceMTimeHeader(t *testing.T) {
|
||||
var putMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
|
||||
}
|
||||
switch {
|
||||
case r.Method == "POST" && r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
|
||||
case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
|
||||
putMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "h", resp.File.ContentHash)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
|
||||
// the completion request for a multipart upload (the server stores the modtime
|
||||
// there, not on the individual parts).
|
||||
func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
|
||||
var completeMTime string
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.ReadAll(r.Body)
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/api/upload/init":
|
||||
return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
|
||||
case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true}`), nil
|
||||
case r.URL.Path == "/api/upload/upl_1/complete":
|
||||
completeMTime = r.Header.Get(sourceMTimeHeader)
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
|
||||
mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
|
||||
_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
|
||||
}
|
||||
|
||||
// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
|
||||
// server-side Copy returns: it must report the SOURCE's modtime and hash (the
|
||||
// copy is byte-identical and the API preserves both), not the copy's wall
|
||||
// clock. Reading "now" here made rclone's in-memory object disagree with the
|
||||
// one read back from the remote and failed FsCopy's fingerprint check.
|
||||
func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
|
||||
rt := rtFunc(func(r *http.Request) (*http.Response, error) {
|
||||
switch {
|
||||
case r.Method == "GET" && r.URL.Path == "/api/files":
|
||||
// NewObject's existence probe on the destination: nothing there.
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
|
||||
case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
|
||||
return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
|
||||
}
|
||||
t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
|
||||
return nil, nil
|
||||
})
|
||||
f := newFakeFs(rt)
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
|
||||
mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
|
||||
ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
|
||||
ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
|
||||
CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
|
||||
}}
|
||||
|
||||
dst, err := f.Copy(context.Background(), src, "f-copy.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
|
||||
assert.Equal(t, int64(100), dst.Size())
|
||||
got, err := dst.Hash(context.Background(), hash.SHA256)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "srchash", got, "copy keeps the source hash")
|
||||
}
|
||||
Loaded 100 of 1346 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user