Compare commits
293
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13a1d23bb8 | ||
|
|
e11a066e00 | ||
|
|
f0dfe9280c | ||
|
|
b8d2ce8962 | ||
|
|
f60b09e7ab | ||
|
|
307f1edaf4 | ||
|
|
a7a3bee488 | ||
|
|
8e24a21ef2 | ||
|
|
239dae97db | ||
|
|
7056989972 | ||
|
|
056adc97d6 | ||
|
|
c2fd08192b | ||
|
|
9e0054983c | ||
|
|
7c336fdb6f | ||
|
|
5d07f521dc | ||
|
|
0d759d1280 | ||
|
|
93b1c50c14 | ||
|
|
d6819d9d46 | ||
|
|
65f2f59734 | ||
|
|
ac177f70cb | ||
|
|
9be4fc8c2b | ||
|
|
cf240b6c0f | ||
|
|
6159ea9cf5 | ||
|
|
be1fb99aee | ||
|
|
01095a539b | ||
|
|
6578f0ad08 | ||
|
|
f4ffa54952 | ||
|
|
abed695720 | ||
|
|
77892a8289 | ||
|
|
9f9569ad6f | ||
|
|
80cdc4dd06 | ||
|
|
216c9de2c4 | ||
|
|
7952910e40 | ||
|
|
7eed0b6825 | ||
|
|
c7da3ab77f | ||
|
|
879913da41 | ||
|
|
576e6145fc | ||
|
|
be73a72f93 | ||
|
|
36913cda94 | ||
|
|
699e4cfb23 | ||
|
|
bc5cad8792 | ||
|
|
9024962fc4 | ||
|
|
aa1f4ace64 | ||
|
|
dd1d750c55 | ||
|
|
5a2564c6e2 | ||
|
|
82cc80cc6f | ||
|
|
7d0a8bf850 | ||
|
|
fd8b28d36d | ||
|
|
c63ecace41 | ||
|
|
0c8c3d8fb9 | ||
|
|
5042f360f0 | ||
|
|
9601dbce87 | ||
|
|
e06f0b0595 | ||
|
|
b2866f0291 | ||
|
|
cf97f250df | ||
|
|
627b763d4b | ||
|
|
f14945f9c1 | ||
|
|
391661fdb4 | ||
|
|
faffd0a6f1 | ||
|
|
6cc3356f8e | ||
|
|
07e76419c9 | ||
|
|
60c4f35b56 | ||
|
|
15a9c0fd36 | ||
|
|
8b85ffbf03 | ||
|
|
26fb659fe4 | ||
|
|
7aa3d8a32f | ||
|
|
b7ebec865b | ||
|
|
a60d09c43d | ||
|
|
14a47937c0 | ||
|
|
64d6916161 | ||
|
|
ae778f1413 | ||
|
|
33859568d6 | ||
|
|
4b3aa5aea0 | ||
|
|
349487bb7f | ||
|
|
b70b2fff16 | ||
|
|
32307e9226 | ||
|
|
2bd6630c2e | ||
|
|
54c2078f25 | ||
|
|
4f284614a4 | ||
|
|
eef0b39a2c | ||
|
|
37f6336636 | ||
|
|
1049f88a1d | ||
|
|
327ca25a4d | ||
|
|
673e24a60f | ||
|
|
43db4c5dc7 | ||
|
|
88b484722a | ||
|
|
ed5bd327c0 | ||
|
|
341ce61a2a | ||
|
|
9abf9d38c0 | ||
|
|
f42aa5a586 | ||
|
|
40f2e847a3 | ||
|
|
846f193806 | ||
|
|
116d0f3708 | ||
|
|
140ec62683 | ||
|
|
8c746f36dd | ||
|
|
fac7ba4bd0 | ||
|
|
e3be333c29 | ||
|
|
67013ccf35 | ||
|
|
a1522468db | ||
|
|
58f30ab538 | ||
|
|
7539b2c5e1 | ||
|
|
6d39c63983 | ||
|
|
1bf73d862a | ||
|
|
e2a079d73f | ||
|
|
7d7f82854f | ||
|
|
b6113a505f | ||
|
|
2a754ef2fa | ||
|
|
85d13c4e34 | ||
|
|
49b4ca3412 | ||
|
|
315a148820 | ||
|
|
b873f838db | ||
|
|
1d6f86900b | ||
|
|
8a31921fb0 | ||
|
|
c64161822d | ||
|
|
d660960775 | ||
|
|
7615636093 | ||
|
|
cc9abcad06 | ||
|
|
5c4af237e4 | ||
|
|
90a57da9d5 | ||
|
|
824257583c | ||
|
|
cd857754c7 | ||
|
|
017d930dfc | ||
|
|
6ed68883fe | ||
|
|
2a5e0b28c9 | ||
|
|
264e75d892 | ||
|
|
4553c3de7b | ||
|
|
2e9e4a47a2 | ||
|
|
e44829a448 | ||
|
|
6b9f77459a | ||
|
|
4b2c39c585 | ||
|
|
dbf2499d85 | ||
|
|
760a134c95 | ||
|
|
63cfe260a2 | ||
|
|
3550275cd3 | ||
|
|
b728929f44 | ||
|
|
9ec918f137 | ||
|
|
3a9c7ceeb1 | ||
|
|
5502c0f8ae | ||
|
|
d707ae7cf4 | ||
|
|
9bef7f0dbf | ||
|
|
933bbf3ac8 | ||
|
|
ecc5972d6f | ||
|
|
07805796ab | ||
|
|
189e6dbf6a | ||
|
|
d47e289165 | ||
|
|
e51a0599a0 | ||
|
|
530a901de3 | ||
|
|
a64a8aad0e | ||
|
|
6529d2cd8f | ||
|
|
d9895fef9d | ||
|
|
8c7b7ac891 | ||
|
|
f814498561 | ||
|
|
5f4e4b1a20 | ||
|
|
28c187b9b4 | ||
|
|
e07afc4645 | ||
|
|
08932ab92a | ||
|
|
356ee57edb | ||
|
|
7c1660214d | ||
|
|
51b197c86f | ||
|
|
029ffd2761 | ||
|
|
f81cd7d279 | ||
|
|
1a0a4628d7 | ||
|
|
c10a4d465c | ||
|
|
3a6e07a613 | ||
|
|
c36f99d343 | ||
|
|
3e21a7261b | ||
|
|
fd439fab62 | ||
|
|
976aa6b416 | ||
|
|
b3a0383ca3 | ||
|
|
c13f129339 | ||
|
|
748d8c8957 | ||
|
|
4d379efcbb | ||
|
|
e5e6a4b5ae | ||
|
|
df18e8c55b | ||
|
|
f4e17d8b0b | ||
|
|
e5c69511bc | ||
|
|
175d4bc553 | ||
|
|
4851f1796c | ||
|
|
4ff8899b2c | ||
|
|
8f29a0b0a1 | ||
|
|
8b0e76e53b | ||
|
|
233fef5c4d | ||
|
|
b9586c3e03 | ||
|
|
0dc0ab1330 | ||
|
|
a6bbdb35a0 | ||
|
|
b33cb77b6c | ||
|
|
d51322bb5f | ||
|
|
e718ab6091 | ||
|
|
0a9e6e130f | ||
|
|
3358b9049c | ||
|
|
847734d421 | ||
|
|
f7b255d4ec | ||
|
|
24c752ed9e | ||
|
|
a99d155fd4 | ||
|
|
f72b32b470 | ||
|
|
9be7f99bf8 | ||
|
|
6858bf242e | ||
|
|
e8c6867e4c | ||
|
|
50fbd6b049 | ||
|
|
0783cab952 | ||
|
|
886ac7af1d | ||
|
|
3c40238f02 | ||
|
|
46ca0dd7fe | ||
|
|
2e968e7ce0 | ||
|
|
1886c552db | ||
|
|
38ab3dd5b1 | ||
|
|
1d02e1219a | ||
|
|
035d3f344c | ||
|
|
7d45aee70f | ||
|
|
f30789180d | ||
|
|
7cb05a84e9 | ||
|
|
6d4c625bfb | ||
|
|
4eccc40168 | ||
|
|
e451f9c999 | ||
|
|
321488441e | ||
|
|
bd99e05ff0 | ||
|
|
6440052fbd | ||
|
|
4afb59bc93 | ||
|
|
0343670375 | ||
|
|
5b2b372ba9 | ||
|
|
08c35ae741 | ||
|
|
ecea0cd6f9 | ||
|
|
80e6389a50 | ||
|
|
a3ccf4d8a0 | ||
|
|
31df39d356 | ||
|
|
03d3811f7f | ||
|
|
83b83f7768 | ||
|
|
71138082ea | ||
|
|
cf94824426 | ||
|
|
16971ab6b9 | ||
|
|
9f75af38e3 | ||
|
|
b5e4d39b05 | ||
|
|
4d19afdbbf | ||
|
|
2ebfedce85 | ||
|
|
1a4b85b6e7 | ||
|
|
5052b80298 | ||
|
|
fada870ff0 | ||
|
|
38f456c527 | ||
|
|
e6d82ac6ee | ||
|
|
4c74ded85a | ||
|
|
43848f5c42 | ||
|
|
fb895f69a1 | ||
|
|
b204090325 | ||
|
|
1821d86911 | ||
|
|
7ce67347fb | ||
|
|
0228bbff39 | ||
|
|
6890bd7738 | ||
|
|
bc5d1dfaf3 | ||
|
|
c33aeb705f | ||
|
|
12cf8e71df | ||
|
|
ec5ddb68a8 | ||
|
|
8335596207 | ||
|
|
4f56ab2341 | ||
|
|
8b5b7ecfd9 | ||
|
|
2aa2cfc70e | ||
|
|
7265b2331f | ||
|
|
0dd56ff2a3 | ||
|
|
2443cb284e | ||
|
|
0f3aa17fb6 | ||
|
|
8f74e7d331 | ||
|
|
ee92673e1b | ||
|
|
55655efabf | ||
|
|
700e6e11fd | ||
|
|
edb47076b5 | ||
|
|
e5fd97b8d2 | ||
|
|
bc57a31859 | ||
|
|
4adb48fbbc | ||
|
|
c41d0f7d3a | ||
|
|
d34ba258b0 | ||
|
|
05d54a95b8 | ||
|
|
f16b39165b | ||
|
|
86edb26fd5 | ||
|
|
203e1bdbf9 | ||
|
|
a522c056fe | ||
|
|
31adc7d89f | ||
|
|
c559ab7c58 | ||
|
|
80610ef774 | ||
|
|
a6c943a1ad | ||
|
|
53e0dbb5cb | ||
|
|
3a0000526b | ||
|
|
1fa6941e26 | ||
|
|
9bb7ad31e6 | ||
|
|
da8c6847ad | ||
|
|
d240d044c3 | ||
|
|
1056ace80f | ||
|
|
a06c1c0cb7 | ||
|
|
7672c3d586 | ||
|
|
f361cdf1cb | ||
|
|
26d3c71bab | ||
|
|
c76396f03c | ||
|
|
059ad47336 | ||
|
|
becc068d36 | ||
|
|
94deb6bd6f |
No files matched your search
+23
-21
@@ -29,12 +29,12 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.24']
|
||||
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '>=1.25.0-rc.1'
|
||||
go: '~1.26.0'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -45,14 +45,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '>=1.25.0-rc.1'
|
||||
go: '~1.26.0'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '>=1.25.0-rc.1'
|
||||
go: '~1.26.0'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -61,14 +61,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '>=1.25.0-rc.1'
|
||||
go: '~1.26.0'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '>=1.25.0-rc.1'
|
||||
go: '~1.26.0'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -78,14 +78,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '>=1.25.0-rc.1'
|
||||
go: '~1.26.0'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.24
|
||||
- job_name: go1.25
|
||||
os: ubuntu-latest
|
||||
go: '1.24'
|
||||
go: '~1.25.7'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -216,7 +216,7 @@ jobs:
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -224,12 +224,12 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '>=1.24.0-rc.1'
|
||||
go-version: '~1.26.0'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
- name: Cache
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
@@ -239,13 +239,13 @@ jobs:
|
||||
restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
|
||||
- name: Code quality test (Linux)
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: latest
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (Windows)
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
env:
|
||||
GOOS: "windows"
|
||||
with:
|
||||
@@ -253,7 +253,7 @@ jobs:
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (macOS)
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
env:
|
||||
GOOS: "darwin"
|
||||
with:
|
||||
@@ -261,7 +261,7 @@ jobs:
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (FreeBSD)
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
env:
|
||||
GOOS: "freebsd"
|
||||
with:
|
||||
@@ -269,7 +269,7 @@ jobs:
|
||||
skip-cache: true
|
||||
|
||||
- name: Code quality test (OpenBSD)
|
||||
uses: golangci/golangci-lint-action@v8
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
env:
|
||||
GOOS: "openbsd"
|
||||
with:
|
||||
@@ -291,7 +291,9 @@ jobs:
|
||||
README.md
|
||||
RELEASE.md
|
||||
CODE_OF_CONDUCT.md
|
||||
docs/content/{authors,bugs,changelog,docs,downloads,faq,filtering,gui,install,licence,overview,privacy}.md
|
||||
librclone\README.md
|
||||
backend\s3\README.md
|
||||
docs/content/{_index,authors,bugs,changelog,docs,downloads,faq,filtering,gui,install,licence,overview,privacy}.md
|
||||
|
||||
- name: Scan edits of autogenerated files
|
||||
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
|
||||
@@ -305,7 +307,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -313,7 +315,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '>=1.25.0-rc.1'
|
||||
go-version: '~1.26.0'
|
||||
|
||||
- name: Set global environment variables
|
||||
run: |
|
||||
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
df -h .
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
|
||||
- name: Load Go Build Cache for Docker
|
||||
id: go-cache
|
||||
uses: actions/cache@v4
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: |
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload Image Digest
|
||||
uses: actions/upload-artifact@v5
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: digests-${{ env.PLATFORM }}
|
||||
path: /tmp/digests/*
|
||||
@@ -198,7 +198,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Download Image Digests
|
||||
uses: actions/download-artifact@v6
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-*
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
sudo rm -rf /usr/share/dotnet || true
|
||||
df -h .
|
||||
- name: Checkout master
|
||||
uses: actions/checkout@v5
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Build and publish docker plugin
|
||||
|
||||
@@ -17,6 +17,14 @@ linters:
|
||||
#- prealloc # TODO
|
||||
- revive
|
||||
- unconvert
|
||||
exclusions:
|
||||
rules:
|
||||
- linters:
|
||||
- revive
|
||||
text: 'var-naming: avoid meaningless package names'
|
||||
- linters:
|
||||
- revive
|
||||
text: 'var-naming: avoid package names that conflict with Go standard library package names'
|
||||
# Configure checks. Mostly using defaults but with some commented exceptions.
|
||||
settings:
|
||||
govet:
|
||||
@@ -136,6 +144,7 @@ linters:
|
||||
- name: var-naming
|
||||
disabled: false
|
||||
|
||||
|
||||
formatters:
|
||||
enable:
|
||||
- goimports
|
||||
|
||||
@@ -41,3 +41,32 @@ single-title: # MD025
|
||||
# Markdown files we must use whatever works in the final HTML generated docs.
|
||||
# Suppress Markdownlint warning: Link fragments should be valid.
|
||||
link-fragments: false # MD051
|
||||
|
||||
# Restrict the languages and language identifiers to use for code blocks.
|
||||
# We only want those supported by both Hugo and GitHub. These are documented
|
||||
# here:
|
||||
# https://gohugo.io/content-management/syntax-highlighting/#languages
|
||||
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
|
||||
# In addition, we only want to allow identifiers (aliases) that correspond to
|
||||
# the same language in Hugo and GitHub, and preferrably also VSCode and other
|
||||
# commonly used tools, to avoid confusion. An example of this is that "shell"
|
||||
# by some are considered an identifier for shell scripts, i.e. an alias for
|
||||
# "sh", while others consider it an identifier for shell sessions, i.e. an
|
||||
# alias for "console". Although Hugo and GitHub in this case are consistent and
|
||||
# have choosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# avoids the confusion entirely.
|
||||
fenced-code-language: # MD040
|
||||
allowed_languages:
|
||||
- text
|
||||
- console
|
||||
- sh
|
||||
- bat
|
||||
- ini
|
||||
- json
|
||||
- yaml
|
||||
- go
|
||||
- python
|
||||
- c++
|
||||
- c#
|
||||
- java
|
||||
- powershell
|
||||
+72
-35
@@ -38,7 +38,7 @@ and [email](https://docs.github.com/en/github/setting-up-and-managing-your-githu
|
||||
Next open your terminal, change directory to your preferred folder and initialise
|
||||
your local rclone project:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git clone https://github.com/rclone/rclone.git
|
||||
cd rclone
|
||||
git remote rename origin upstream
|
||||
@@ -53,13 +53,13 @@ executed from the rclone folder created above.
|
||||
|
||||
Now [install Go](https://golang.org/doc/install) and verify your installation:
|
||||
|
||||
```sh
|
||||
```console
|
||||
go version
|
||||
```
|
||||
|
||||
Great, you can now compile and execute your own version of rclone:
|
||||
|
||||
```sh
|
||||
```console
|
||||
go build
|
||||
./rclone version
|
||||
```
|
||||
@@ -68,7 +68,7 @@ go build
|
||||
more accurate version number in the executable as well as enable you to specify
|
||||
more build options.) Finally make a branch to add your new feature
|
||||
|
||||
```sh
|
||||
```console
|
||||
git checkout -b my-new-feature
|
||||
```
|
||||
|
||||
@@ -80,7 +80,7 @@ and a quick view on the rclone [code organisation](#code-organisation).
|
||||
When ready - test the affected functionality and run the unit tests for the
|
||||
code you changed
|
||||
|
||||
```sh
|
||||
```console
|
||||
cd folder/with/changed/files
|
||||
go test -v
|
||||
```
|
||||
@@ -99,7 +99,7 @@ Make sure you
|
||||
|
||||
When you are done with that push your changes to GitHub:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git push -u origin my-new-feature
|
||||
```
|
||||
|
||||
@@ -119,7 +119,7 @@ or [squash your commits](#squashing-your-commits).
|
||||
|
||||
Follow the guideline for [commit messages](#commit-messages) and then:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git checkout my-new-feature # To switch to your branch
|
||||
git status # To see the new and changed files
|
||||
git add FILENAME # To select FILENAME for the commit
|
||||
@@ -130,7 +130,7 @@ git log # To verify the commit. Use q to quit the log
|
||||
|
||||
You can modify the message or changes in the latest commit using:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git commit --amend
|
||||
```
|
||||
|
||||
@@ -145,7 +145,7 @@ pushed to GitHub.
|
||||
|
||||
Your previously pushed commits are replaced by:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git push --force origin my-new-feature
|
||||
```
|
||||
|
||||
@@ -154,7 +154,7 @@ git push --force origin my-new-feature
|
||||
To base your changes on the latest version of the
|
||||
[rclone master](https://github.com/rclone/rclone/tree/master) (upstream):
|
||||
|
||||
```sh
|
||||
```console
|
||||
git checkout master
|
||||
git fetch upstream
|
||||
git merge --ff-only
|
||||
@@ -170,7 +170,7 @@ If you rebase commits that have been pushed to GitHub, then you will have to
|
||||
|
||||
To combine your commits into one commit:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git log # To count the commits to squash, e.g. the last 2
|
||||
git reset --soft HEAD~2 # To undo the 2 latest commits
|
||||
git status # To check everything is as expected
|
||||
@@ -178,13 +178,13 @@ git status # To check everything is as expected
|
||||
|
||||
If everything is fine, then make the new combined commit:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git commit # To commit the undone commits as one
|
||||
```
|
||||
|
||||
otherwise, you may roll back using:
|
||||
|
||||
```sh
|
||||
```console
|
||||
git reflog # To check that HEAD{1} is your previous state
|
||||
git reset --soft 'HEAD@{1}' # To roll back to your previous state
|
||||
```
|
||||
@@ -219,13 +219,13 @@ to check an error return).
|
||||
rclone's tests are run from the go testing framework, so at the top
|
||||
level you can run this to run all the tests.
|
||||
|
||||
```sh
|
||||
```console
|
||||
go test -v ./...
|
||||
```
|
||||
|
||||
You can also use `make`, if supported by your platform
|
||||
|
||||
```sh
|
||||
```console
|
||||
make quicktest
|
||||
```
|
||||
|
||||
@@ -246,7 +246,7 @@ need to make a remote called `TestDrive`.
|
||||
You can then run the unit tests in the drive directory. These tests
|
||||
are skipped if `TestDrive:` isn't defined.
|
||||
|
||||
```sh
|
||||
```console
|
||||
cd backend/drive
|
||||
go test -v
|
||||
```
|
||||
@@ -255,7 +255,7 @@ You can then run the integration tests which test all of rclone's
|
||||
operations. Normally these get run against the local file system,
|
||||
but they can be run against any of the remotes.
|
||||
|
||||
```sh
|
||||
```console
|
||||
cd fs/sync
|
||||
go test -v -remote TestDrive:
|
||||
go test -v -remote TestDrive: -fast-list
|
||||
@@ -268,9 +268,8 @@ If you want to use the integration test framework to run these tests
|
||||
altogether with an HTML report and test retries then from the
|
||||
project root:
|
||||
|
||||
```sh
|
||||
go install github.com/rclone/rclone/fstest/test_all
|
||||
test_all -backends drive
|
||||
```console
|
||||
go run ./fstest/test_all -backends drive
|
||||
```
|
||||
|
||||
### Full integration testing
|
||||
@@ -278,19 +277,19 @@ test_all -backends drive
|
||||
If you want to run all the integration tests against all the remotes,
|
||||
then change into the project root and run
|
||||
|
||||
```sh
|
||||
```console
|
||||
make check
|
||||
make test
|
||||
```
|
||||
|
||||
The commands may require some extra go packages which you can install with
|
||||
|
||||
```sh
|
||||
```console
|
||||
make build_dep
|
||||
```
|
||||
|
||||
The full integration tests are run daily on the integration test server. You can
|
||||
find the results at <https://pub.rclone.org/integration-tests/>
|
||||
find the results at <https://integration.rclone.org>
|
||||
|
||||
## Code Organisation
|
||||
|
||||
@@ -349,11 +348,13 @@ If you are adding a new feature then please update the documentation.
|
||||
|
||||
The documentation sources are generally in Markdown format, in conformance
|
||||
with the CommonMark specification and compatible with GitHub Flavored
|
||||
Markdown (GFM). The markdown format is checked as part of the lint operation
|
||||
that runs automatically on pull requests, to enforce standards and consistency.
|
||||
This is based on the [markdownlint](https://github.com/DavidAnson/markdownlint)
|
||||
tool, which can also be integrated into editors so you can perform the same
|
||||
checks while writing.
|
||||
Markdown (GFM). The markdown format and style is checked as part of the lint
|
||||
operation that runs automatically on pull requests, to enforce standards and
|
||||
consistency. This is based on the [markdownlint](https://github.com/DavidAnson/markdownlint)
|
||||
tool by David Anson, which can also be integrated into editors so you can
|
||||
perform the same checks while writing. It generally follows Ciro Santilli's
|
||||
[Markdown Style Guide](https://cirosantilli.com/markdown-style-guide), which
|
||||
is good source if you want to know more.
|
||||
|
||||
HTML pages, served as website <rclone.org>, are generated from the Markdown,
|
||||
using [Hugo](https://gohugo.io). Note that when generating the HTML pages,
|
||||
@@ -382,7 +383,7 @@ If you add a new general flag (not for a backend), then document it in
|
||||
alphabetical order.
|
||||
|
||||
If you add a new backend option/flag, then it should be documented in
|
||||
the source file in the `Help:` field.
|
||||
the source file in the `Help:` field:
|
||||
|
||||
- Start with the most important information about the option,
|
||||
as a single sentence on a single line.
|
||||
@@ -404,6 +405,30 @@ the source file in the `Help:` field.
|
||||
as an unordered list, therefore a single line break is enough to
|
||||
create a new list item. Also, for enumeration texts like name of
|
||||
countries, it looks better without an ending period/full stop character.
|
||||
- You can run `make backenddocs` to verify the resulting Markdown.
|
||||
- This will update the autogenerated sections of the backend docs Markdown
|
||||
files under `docs/content`.
|
||||
- It requires you to have [Python](https://www.python.org) installed.
|
||||
- The `backenddocs` make target runs the Python script `bin/make_backend_docs.py`,
|
||||
and you can also run this directly, optionally with the name of a backend
|
||||
as argument to only update the docs for a specific backend.
|
||||
- **Do not** commit the updated Markdown files. This operation is run as part of
|
||||
the release process. Since any manual changes in the autogenerated sections
|
||||
of the Markdown files will then be lost, we have a pull request check that
|
||||
reports error for any changes within the autogenerated sections. Should you
|
||||
have done manual changes outside of the autogenerated sections they must be
|
||||
committed, of course.
|
||||
- You can run `make serve` to verify the resulting website.
|
||||
- This will build the website and serve it locally, so you can open it in
|
||||
your web browser and verify that the end result looks OK. Check specifically
|
||||
any added links, also in light of the note above regarding different algorithms
|
||||
for generated header anchors.
|
||||
- It requires you to have the [Hugo](https://gohugo.io) tool available.
|
||||
- The `serve` make target depends on the `website` target, which runs the
|
||||
`hugo` command from the `docs` directory to build the website, and then
|
||||
it serves the website locally with an embedded web server using a command
|
||||
`hugo server --logLevel info -w --disableFastRender --ignoreCache`, so you
|
||||
can run similar Hugo commands directly as well.
|
||||
|
||||
When writing documentation for an entirely new backend,
|
||||
see [backend documentation](#backend-documentation).
|
||||
@@ -420,6 +445,11 @@ for small changes in the docs which makes it very easy. Just remember the
|
||||
caveat when linking to header anchors, noted above, which means that GitHub's
|
||||
Markdown preview may not be an entirely reliable verification of the results.
|
||||
|
||||
After your changes have been merged, you can verify them on
|
||||
[tip.rclone.org](https://tip.rclone.org). This site is updated daily with the
|
||||
current state of the master branch at 07:00 UTC. The changes will be on the main
|
||||
[rclone.org](https://rclone.org) site once they have been included in a release.
|
||||
|
||||
## Making a release
|
||||
|
||||
There are separate instructions for making a release in the RELEASE.md
|
||||
@@ -478,7 +508,7 @@ To add a dependency `github.com/ncw/new_dependency` see the
|
||||
instructions below. These will fetch the dependency and add it to
|
||||
`go.mod` and `go.sum`.
|
||||
|
||||
```sh
|
||||
```console
|
||||
go get github.com/ncw/new_dependency
|
||||
```
|
||||
|
||||
@@ -492,7 +522,7 @@ and `go.sum` in the same commit as your other changes.
|
||||
|
||||
If you need to update a dependency then run
|
||||
|
||||
```sh
|
||||
```console
|
||||
go get golang.org/x/crypto
|
||||
```
|
||||
|
||||
@@ -581,8 +611,7 @@ remote or an fs.
|
||||
- Add your backend to `fstest/test_all/config.yaml`
|
||||
- Once you've done that then you can use the integration test framework from
|
||||
the project root:
|
||||
- go install ./...
|
||||
- test_all -backends remote
|
||||
- `go run ./fstest/test_all -backends remote`
|
||||
|
||||
Or if you want to run the integration tests manually:
|
||||
|
||||
@@ -603,14 +632,22 @@ Add your backend to the docs - you'll need to pick an icon for it from
|
||||
alphabetical order of full name of remote (e.g. `drive` is ordered as
|
||||
`Google Drive`) but with the local file system last.
|
||||
|
||||
First add a data file about your backend in
|
||||
`docs/data/backends/remote.yaml` - this is used to build the overview
|
||||
tables and the tiering info.
|
||||
|
||||
- Create it with: `bin/manage_backends.py create docs/data/backends/remote.yaml`
|
||||
- Edit it to fill in the blanks. Look at the [tiers docs](https://rclone.org/tiers/).
|
||||
- Run this command to fill in the features: `bin/manage_backends.py features docs/data/backends/remote.yaml`
|
||||
|
||||
Next edit these files:
|
||||
|
||||
- `README.md` - main GitHub page
|
||||
- `docs/content/remote.md` - main docs page (note the backend options are
|
||||
automatically added to this file with `make backenddocs`)
|
||||
- make sure this has the `autogenerated options` comments in (see your
|
||||
reference backend docs)
|
||||
- update them in your backend with `bin/make_backend_docs.py remote`
|
||||
- `docs/content/overview.md` - overview docs - add an entry into the Features
|
||||
table and the Optional Features table.
|
||||
- `docs/content/docs.md` - list of remotes in config section
|
||||
- `docs/content/_index.md` - front page of rclone.org
|
||||
- `docs/layouts/chrome/navbar.html` - add it to the website navigation
|
||||
|
||||
Generated
+12277
-6370
File diff suppressed because it is too large.
Load diff
Generated
+7756
-2427
File diff suppressed because it is too large.
Load diff
@@ -216,6 +216,12 @@ beta:
|
||||
rclone -v copy build/ pub.rclone.org:/$(TAG)
|
||||
@echo Beta release ready at https://pub.rclone.org/$(TAG)/
|
||||
|
||||
privatebeta:
|
||||
go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) -include '^(darwin|windows|linux)/(arm64|amd64)$$' $(TAG)
|
||||
rclone -Pv copy build/ private-downloads:/beta/$(TAG)
|
||||
@echo Private beta release ready at private-downloads:/beta/$(TAG)/
|
||||
rclone link private-downloads:/beta/$(TAG)
|
||||
|
||||
log_since_last_release:
|
||||
git log $(LAST_TAG)..
|
||||
|
||||
|
||||
@@ -28,21 +28,26 @@ directories to and from different cloud storage providers.
|
||||
- Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
|
||||
- Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
|
||||
- ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
|
||||
- Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
|
||||
- Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
|
||||
- Box [:page_facing_up:](https://rclone.org/box/)
|
||||
- Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
|
||||
- China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
|
||||
- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
|
||||
- Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
|
||||
- Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
|
||||
- Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
|
||||
- Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
|
||||
- DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
- Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
- Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
|
||||
- Drime [:page_facing_up:](https://rclone.org/s3/#drime)
|
||||
- Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
- Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
|
||||
- Exaba [:page_facing_up:](https://rclone.org/s3/#exaba)
|
||||
- Fastly Object Storage [:page_facing_up:](https://rclone.org/s3/#fastly)
|
||||
- Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
|
||||
- FileLu [:page_facing_up:](https://rclone.org/filelu/)
|
||||
- Filen [:page_facing_up:](https://rclone.org/filen/)
|
||||
- Files.com [:page_facing_up:](https://rclone.org/filescom/)
|
||||
- FlashBlade [:page_facing_up:](https://rclone.org/s3/#pure-storage-flashblade)
|
||||
- FTP [:page_facing_up:](https://rclone.org/ftp/)
|
||||
@@ -59,6 +64,7 @@ directories to and from different cloud storage providers.
|
||||
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
|
||||
- ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
|
||||
- Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
|
||||
- Internxt [:page_facing_up:](https://rclone.org/internxt/)
|
||||
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
|
||||
@@ -109,9 +115,9 @@ directories to and from different cloud storage providers.
|
||||
- Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
|
||||
- Servercore Object Storage [:page_facing_up:](https://rclone.org/s3/#servercore)
|
||||
- SFTP [:page_facing_up:](https://rclone.org/sftp/)
|
||||
- Shade [:page_facing_up:](https://rclone.org/shade/)
|
||||
- SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
|
||||
- Spectra Logic [:page_facing_up:](https://rclone.org/s3/#spectralogic)
|
||||
- StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
|
||||
- Storj [:page_facing_up:](https://rclone.org/storj/)
|
||||
- SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
|
||||
- Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
|
||||
@@ -120,6 +126,7 @@ directories to and from different cloud storage providers.
|
||||
- Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
|
||||
- WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
- Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
|
||||
- Zadara Object Storage [:page_facing_up:](https://rclone.org/s3/#zadara)
|
||||
- Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
|
||||
- Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
|
||||
- The local filesystem [:page_facing_up:](https://rclone.org/local/)
|
||||
|
||||
+65
-11
@@ -21,6 +21,7 @@ This file describes how to make the various kinds of releases
|
||||
- make doc
|
||||
- git status - to check for new man pages - git add them
|
||||
- git commit -a -v -m "Version v1.XX.0"
|
||||
- make check
|
||||
- make retag
|
||||
- git push origin # without --follow-tags so it doesn't push the tag if it fails
|
||||
- git push --follow-tags origin
|
||||
@@ -60,7 +61,7 @@ If `make updatedirect` added a `toolchain` directive then remove it.
|
||||
We don't want to force a toolchain on our users. Linux packagers are
|
||||
often using a version of Go that is a few versions out of date.
|
||||
|
||||
```sh
|
||||
```console
|
||||
go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
|
||||
go get -d $(cat /tmp/potential-upgrades)
|
||||
go mod tidy -go=1.22 -compat=1.22
|
||||
@@ -70,7 +71,7 @@ If the `go mod tidy` fails use the output from it to remove the
|
||||
package which can't be upgraded from `/tmp/potential-upgrades` when
|
||||
done
|
||||
|
||||
```sh
|
||||
```console
|
||||
git co go.mod go.sum
|
||||
```
|
||||
|
||||
@@ -102,17 +103,70 @@ The above procedure will not upgrade major versions, so v2 to v3.
|
||||
However this tool can show which major versions might need to be
|
||||
upgraded:
|
||||
|
||||
```sh
|
||||
```console
|
||||
go run github.com/icholy/gomajor@latest list -major
|
||||
```
|
||||
|
||||
Expect API breakage when updating major versions.
|
||||
|
||||
## Updating Go
|
||||
|
||||
When a new Go stable is released update to it. We support the current
|
||||
stable Go and the previous release which is in line with the rest of
|
||||
the Go ecosystem.
|
||||
|
||||
These files will need editing:
|
||||
|
||||
- `.github/workflows/build.yml` - change current and previous Go versions
|
||||
- `docs/content/install.md` - change minimum Go version required
|
||||
- `fs/versioncheck.go` - update minimum Go version required
|
||||
- `go.mod` - update minimum Go version required
|
||||
|
||||
Check it builds
|
||||
|
||||
- `make GOTAGS=cmount`
|
||||
- `make compiletest`
|
||||
|
||||
Assuming `go1.XX` is current and `go1.YY` is previous version:
|
||||
|
||||
Use `git grep go1.YY` and `git grep go1.YY` to look for opportunities
|
||||
to remove build tags we no longer need.
|
||||
|
||||
Commit with message like this:
|
||||
|
||||
```text
|
||||
build: update to go1.YY and make go1.YY the minimum required version
|
||||
```
|
||||
|
||||
Send to CI and if it passes, merge.
|
||||
|
||||
### gofix
|
||||
|
||||
Updating the minimum required version of Go is a good opportunity to
|
||||
run the `go fix` command to modernize Go usage.
|
||||
|
||||
This needs to be run for all architectures.
|
||||
|
||||
```console
|
||||
GOOS=linux go fix -tags cmount ./...
|
||||
GOOS=freebsd go fix -tags cmount ./...
|
||||
GOOS=windows go fix -tags cmount ./...
|
||||
GOOS=darwin go fix -tags cmount ./...
|
||||
```
|
||||
|
||||
Examine the diff carefully.
|
||||
|
||||
Commit with message
|
||||
|
||||
```text
|
||||
build: modernize Go code with go fix for go1.YY
|
||||
```
|
||||
|
||||
## Tidy beta
|
||||
|
||||
At some point after the release run
|
||||
|
||||
```sh
|
||||
```console
|
||||
bin/tidy-beta v1.55
|
||||
```
|
||||
|
||||
@@ -159,7 +213,7 @@ which is a private repo containing artwork from sponsors.
|
||||
|
||||
Create an update website branch based off the last release
|
||||
|
||||
```sh
|
||||
```console
|
||||
git co -b update-website
|
||||
```
|
||||
|
||||
@@ -167,19 +221,19 @@ If the branch already exists, double check there are no commits that need saving
|
||||
|
||||
Now reset the branch to the last release
|
||||
|
||||
```sh
|
||||
```console
|
||||
git reset --hard v1.64.0
|
||||
```
|
||||
|
||||
Create the changes, check them in, test with `make serve` then
|
||||
|
||||
```sh
|
||||
```console
|
||||
make upload_test_website
|
||||
```
|
||||
|
||||
Check out <https://test.rclone.org> and when happy
|
||||
|
||||
```sh
|
||||
```console
|
||||
make upload_website
|
||||
```
|
||||
|
||||
@@ -189,14 +243,14 @@ Cherry pick any changes back to master and the stable branch if it is active.
|
||||
|
||||
To do a basic build of rclone's docker image to debug builds locally:
|
||||
|
||||
```sh
|
||||
```console
|
||||
docker buildx build --load -t rclone/rclone:testing --progress=plain .
|
||||
docker run --rm rclone/rclone:testing version
|
||||
```
|
||||
|
||||
To test the multipatform build
|
||||
|
||||
```sh
|
||||
```console
|
||||
docker buildx build -t rclone/rclone:testing --progress=plain --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 .
|
||||
```
|
||||
|
||||
@@ -204,6 +258,6 @@ To make a full build then set the tags correctly and add `--push`
|
||||
|
||||
Note that you can't only build one architecture - you need to build them all.
|
||||
|
||||
```sh
|
||||
```console
|
||||
docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
|
||||
```
|
||||
+4
-1
@@ -16,11 +16,13 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/compress"
|
||||
_ "github.com/rclone/rclone/backend/crypt"
|
||||
_ "github.com/rclone/rclone/backend/doi"
|
||||
_ "github.com/rclone/rclone/backend/drime"
|
||||
_ "github.com/rclone/rclone/backend/drive"
|
||||
_ "github.com/rclone/rclone/backend/dropbox"
|
||||
_ "github.com/rclone/rclone/backend/fichier"
|
||||
_ "github.com/rclone/rclone/backend/filefabric"
|
||||
_ "github.com/rclone/rclone/backend/filelu"
|
||||
_ "github.com/rclone/rclone/backend/filen"
|
||||
_ "github.com/rclone/rclone/backend/filescom"
|
||||
_ "github.com/rclone/rclone/backend/ftp"
|
||||
_ "github.com/rclone/rclone/backend/gofile"
|
||||
@@ -33,6 +35,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/iclouddrive"
|
||||
_ "github.com/rclone/rclone/backend/imagekit"
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
_ "github.com/rclone/rclone/backend/internxt"
|
||||
_ "github.com/rclone/rclone/backend/jottacloud"
|
||||
_ "github.com/rclone/rclone/backend/koofr"
|
||||
_ "github.com/rclone/rclone/backend/linkbox"
|
||||
@@ -55,6 +58,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
_ "github.com/rclone/rclone/backend/seafile"
|
||||
_ "github.com/rclone/rclone/backend/sftp"
|
||||
_ "github.com/rclone/rclone/backend/shade"
|
||||
_ "github.com/rclone/rclone/backend/sharefile"
|
||||
_ "github.com/rclone/rclone/backend/sia"
|
||||
_ "github.com/rclone/rclone/backend/smb"
|
||||
@@ -63,7 +67,6 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/swift"
|
||||
_ "github.com/rclone/rclone/backend/ulozto"
|
||||
_ "github.com/rclone/rclone/backend/union"
|
||||
_ "github.com/rclone/rclone/backend/uptobox"
|
||||
_ "github.com/rclone/rclone/backend/webdav"
|
||||
_ "github.com/rclone/rclone/backend/yandex"
|
||||
_ "github.com/rclone/rclone/backend/zoho"
|
||||
|
||||
@@ -0,0 +1,584 @@
|
||||
// Package auth supplies the authentication and client creation for the azure SDK
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
)
|
||||
|
||||
const (
|
||||
// Default storage account, key and blob endpoint for emulator support,
|
||||
// though it is a base64 key checked in here, it is publicly available secret.
|
||||
emulatorAccount = "devstoreaccount1"
|
||||
emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
||||
emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
|
||||
)
|
||||
|
||||
// ConfigOptions is the common authentication options for azure
|
||||
var ConfigOptions = []fs.Option{{
|
||||
Name: "account",
|
||||
Help: `Azure Storage Account Name.
|
||||
|
||||
Set this to the Azure Storage Account Name in use.
|
||||
|
||||
Leave blank to use SAS URL or Emulator, otherwise it needs to be set.
|
||||
|
||||
If this is blank and if env_auth is set it will be read from the
|
||||
environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible.
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "env_auth",
|
||||
Help: `Read credentials from runtime (environment variables, CLI or MSI).
|
||||
|
||||
See the [authentication docs](/azureblob#authentication) for full info.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: `Storage Account Shared Key.
|
||||
|
||||
Leave blank to use SAS URL or Emulator.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sas_url",
|
||||
Help: `SAS URL for container level access only.
|
||||
|
||||
Leave blank if using account/key or Emulator.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "connection_string",
|
||||
Help: `Storage Connection String.
|
||||
|
||||
Connection string for the storage. Leave blank if using other auth methods.
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "tenant",
|
||||
Help: `ID of the service principal's tenant. Also called its directory ID.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_id",
|
||||
Help: `The ID of the client in use.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_secret",
|
||||
Help: `One of the service principal's client secrets
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_certificate_path",
|
||||
Help: `Path to a PEM or PKCS12 certificate file including the private key.
|
||||
|
||||
Set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
}, {
|
||||
Name: "client_certificate_password",
|
||||
Help: `Password for the certificate file (optional).
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
|
||||
And the certificate has a password.
|
||||
`,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "client_send_certificate_chain",
|
||||
Help: `Send the certificate chain when using certificate auth.
|
||||
|
||||
Specifies whether an authentication request will include an x5c header
|
||||
to support subject name / issuer based authentication. When set to
|
||||
true, authentication requests include the x5c header.
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "username",
|
||||
Help: `User name (usually an email address)
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "password",
|
||||
Help: `The user's password
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
IsPassword: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "service_principal_file",
|
||||
Help: `Path to file containing credentials for use with a service principal.
|
||||
|
||||
Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
|
||||
|
||||
$ az ad sp create-for-rbac --name "<name>" \
|
||||
--role "Storage Blob Data Owner" \
|
||||
--scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
|
||||
> azure-principal.json
|
||||
|
||||
See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
|
||||
|
||||
It may be more convenient to put the credentials directly into the
|
||||
rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + `
|
||||
keys instead of setting ` + "`service_principal_file`" + `.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "disable_instance_discovery",
|
||||
Help: `Skip requesting Microsoft Entra instance metadata
|
||||
|
||||
This should be set true only by applications authenticating in
|
||||
disconnected clouds, or private clouds such as Azure Stack.
|
||||
|
||||
It determines whether rclone requests Microsoft Entra instance
|
||||
metadata from ` + "`https://login.microsoft.com/`" + ` before
|
||||
authenticating.
|
||||
|
||||
Setting this to true will skip this request, making you responsible
|
||||
for ensuring the configured authority is valid and trustworthy.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_msi",
|
||||
Help: `Use a managed service identity to authenticate (only works in Azure).
|
||||
|
||||
When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/)
|
||||
to authenticate to Azure Storage instead of a SAS token or account key.
|
||||
|
||||
If the VM(SS) on which this program is running has a system-assigned identity, it will
|
||||
be used by default. If the resource has no system-assigned but exactly one user-assigned identity,
|
||||
the user-assigned identity will be used by default. If the resource has multiple user-assigned
|
||||
identities, the identity to use must be explicitly specified using exactly one of the msi_object_id,
|
||||
msi_client_id, or msi_mi_res_id parameters.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "msi_object_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_client_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_mi_res_id",
|
||||
Help: "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "use_emulator",
|
||||
Help: "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_az",
|
||||
Help: `Use Azure CLI tool az for authentication
|
||||
|
||||
Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
as the sole means of authentication.
|
||||
|
||||
Setting this can be useful if you wish to use the az CLI on a host with
|
||||
a System Managed Identity that you do not want to use.
|
||||
|
||||
Don't set env_auth at the same time.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service.\n\nLeave blank normally.",
|
||||
Advanced: true,
|
||||
}}
|
||||
|
||||
// Options defines the common auth configuration for azure backends
|
||||
type Options struct {
|
||||
Account string `config:"account"`
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
Key string `config:"key"`
|
||||
SASURL string `config:"sas_url"`
|
||||
ConnectionString string `config:"connection_string"`
|
||||
Tenant string `config:"tenant"`
|
||||
ClientID string `config:"client_id"`
|
||||
ClientSecret string `config:"client_secret"`
|
||||
ClientCertificatePath string `config:"client_certificate_path"`
|
||||
ClientCertificatePassword string `config:"client_certificate_password"`
|
||||
ClientSendCertificateChain bool `config:"client_send_certificate_chain"`
|
||||
Username string `config:"username"`
|
||||
Password string `config:"password"`
|
||||
ServicePrincipalFile string `config:"service_principal_file"`
|
||||
DisableInstanceDiscovery bool `config:"disable_instance_discovery"`
|
||||
UseMSI bool `config:"use_msi"`
|
||||
MSIObjectID string `config:"msi_object_id"`
|
||||
MSIClientID string `config:"msi_client_id"`
|
||||
MSIResourceID string `config:"msi_mi_res_id"`
|
||||
UseEmulator bool `config:"use_emulator"`
|
||||
UseAZ bool `config:"use_az"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
}
|
||||
|
||||
type servicePrincipalCredentials struct {
|
||||
AppID string `json:"appId"`
|
||||
Password string `json:"password"`
|
||||
Tenant string `json:"tenant"`
|
||||
}
|
||||
|
||||
// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli.
|
||||
func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) {
|
||||
var spCredentials servicePrincipalCredentials
|
||||
if err := json.Unmarshal(credentialsData, &spCredentials); err != nil {
|
||||
return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
|
||||
}
|
||||
// TODO: support certificate credentials
|
||||
// Validate all fields present
|
||||
if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" {
|
||||
return nil, fmt.Errorf("missing fields in credentials file")
|
||||
}
|
||||
return &spCredentials, nil
|
||||
}
|
||||
|
||||
// Wrap the http.Transport to satisfy the Transporter interface
|
||||
type transporter struct {
|
||||
http.RoundTripper
|
||||
}
|
||||
|
||||
// Make a new transporter
|
||||
func newTransporter(ctx context.Context) transporter {
|
||||
return transporter{
|
||||
RoundTripper: fshttp.NewTransport(ctx),
|
||||
}
|
||||
}
|
||||
|
||||
// Do sends the HTTP request and returns the HTTP response or error.
|
||||
func (tr transporter) Do(req *http.Request) (*http.Response, error) {
|
||||
return tr.RoundTripper.RoundTrip(req)
|
||||
}
|
||||
|
||||
// NewClientOpts should be passed to configure NewClient
|
||||
type NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct {
|
||||
DefaultBaseURL string // Base URL, eg blob.core.windows.net
|
||||
Blob bool // set if this is blob storage
|
||||
RootContainer string // Container that rclone is looking at
|
||||
NewClient func(serviceURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error)
|
||||
NewClientFromConnectionString func(connectionString string, options *ClientOptions) (*Client, error)
|
||||
NewClientWithNoCredential func(serviceURL string, options *ClientOptions) (*Client, error)
|
||||
NewClientWithSharedKeyCredential func(serviceURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error)
|
||||
NewSharedKeyCredential func(accountName, accountKey string) (*SharedKeyCredential, error)
|
||||
SetClientOptions func(options *ClientOptions, policyClientOptions policy.ClientOptions)
|
||||
}
|
||||
|
||||
// NewClientResult is returned from NewClient
|
||||
type NewClientResult[Client any] struct {
|
||||
Client *Client // Client to access the Service
|
||||
Cred azcore.TokenCredential // how to generate tokens (may be nil)
|
||||
UsingSharedKeyCred bool // set if using shared key credentials
|
||||
Anonymous bool // true if anonymous authentication was used
|
||||
Container string // Container that SAS URL points to
|
||||
}
|
||||
|
||||
// NewClient creates a service client from the rclone options
|
||||
func NewClient[Client, ClientOptions, SharedKeyCredential any](ctx context.Context, conf NewClientOpts[Client, ClientOptions, SharedKeyCredential], opt *Options) (r NewClientResult[Client], err error) {
|
||||
var sharedKeyCred *SharedKeyCredential
|
||||
|
||||
// Client options specifying our own transport
|
||||
policyClientOptions := policy.ClientOptions{
|
||||
Transport: newTransporter(ctx),
|
||||
}
|
||||
// Can't do this with generics (yet)
|
||||
// clientOpt := service.ClientOptions{
|
||||
// ClientOptions: policyClientOptions,
|
||||
// }
|
||||
// So call back to user
|
||||
var clientOpt ClientOptions
|
||||
conf.SetClientOptions(&clientOpt, policyClientOptions)
|
||||
|
||||
// Here we auth by setting one of cred, sharedKeyCred, client or anonymous
|
||||
switch {
|
||||
case opt.EnvAuth:
|
||||
// Read account from environment if needed
|
||||
if opt.Account == "" {
|
||||
opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME")
|
||||
}
|
||||
// Read credentials from the environment
|
||||
options := azidentity.DefaultAzureCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
|
||||
}
|
||||
r.Cred, err = azidentity.NewDefaultAzureCredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create azure environment credential failed: %w", err)
|
||||
}
|
||||
case opt.UseEmulator:
|
||||
if opt.Account == "" {
|
||||
opt.Account = emulatorAccount
|
||||
}
|
||||
if opt.Key == "" {
|
||||
opt.Key = emulatorAccountKey
|
||||
}
|
||||
if opt.Endpoint == "" {
|
||||
opt.Endpoint = emulatorBlobEndpoint
|
||||
}
|
||||
if conf.NewSharedKeyCredential == nil {
|
||||
return r, errors.New("emulator use not supported")
|
||||
}
|
||||
sharedKeyCred, err = conf.NewSharedKeyCredential(opt.Account, opt.Key)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create new shared key credential for emulator failed: %w", err)
|
||||
}
|
||||
case opt.Account != "" && opt.Key != "":
|
||||
if conf.NewSharedKeyCredential == nil {
|
||||
return r, errors.New("shared key credentials not supported")
|
||||
}
|
||||
sharedKeyCred, err = conf.NewSharedKeyCredential(opt.Account, opt.Key)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create new shared key credential failed: %w", err)
|
||||
}
|
||||
case opt.SASURL != "":
|
||||
parts, err := sas.ParseURL(opt.SASURL)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to parse SAS URL: %w", err)
|
||||
}
|
||||
endpoint := opt.SASURL
|
||||
r.Container = parts.ContainerName
|
||||
// Check if we have container level SAS or account level SAS
|
||||
if conf.Blob && r.Container != "" {
|
||||
// Container level SAS
|
||||
if conf.RootContainer != "" && r.Container != conf.RootContainer {
|
||||
return r, fmt.Errorf("container name in SAS URL (%q) and container provided in command (%q) do not match", r.Container, conf.RootContainer)
|
||||
}
|
||||
// Rewrite the endpoint string to be without the container
|
||||
parts.ContainerName = ""
|
||||
endpoint = parts.String()
|
||||
}
|
||||
r.Client, err = conf.NewClientWithNoCredential(endpoint, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("unable to create SAS URL client: %w", err)
|
||||
}
|
||||
case opt.ConnectionString != "":
|
||||
r.Client, err = conf.NewClientFromConnectionString(opt.ConnectionString, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("unable to create connection string client: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "":
|
||||
// Service principal with client secret
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
r.Cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "":
|
||||
// Service principal with certificate
|
||||
//
|
||||
// Read the certificate
|
||||
data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath))
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error reading client certificate file: %w", err)
|
||||
}
|
||||
// NewClientCertificateCredential requires at least one *x509.Certificate, and a
|
||||
// crypto.PrivateKey.
|
||||
//
|
||||
// ParseCertificates returns these given certificate data in PEM or PKCS12 format.
|
||||
// It handles common scenarios but has limitations, for example it doesn't load PEM
|
||||
// encrypted private keys.
|
||||
var password []byte
|
||||
if opt.ClientCertificatePassword != "" {
|
||||
pw, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
password = []byte(pw)
|
||||
}
|
||||
certs, key, err := azidentity.ParseCertificates(data, password)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to parse client certificate file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientCertificateCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
SendCertificateChain: opt.ClientSendCertificateChain,
|
||||
}
|
||||
r.Cred, err = azidentity.NewClientCertificateCredential(
|
||||
opt.Tenant, opt.ClientID, certs, key, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
|
||||
// User with username and password
|
||||
//nolint:staticcheck // this is deprecated due to Azure policy
|
||||
options := azidentity.UsernamePasswordCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
password, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("user password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
r.Cred, err = azidentity.NewUsernamePasswordCredential(
|
||||
opt.Tenant, opt.ClientID, opt.Username, password, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("authenticate user with password failed: %w", err)
|
||||
}
|
||||
case opt.ServicePrincipalFile != "":
|
||||
// Loading service principal credentials from file.
|
||||
loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error opening service principal credentials file: %w", err)
|
||||
}
|
||||
parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error parsing service principal credentials file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
r.Cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.UseMSI:
|
||||
// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
|
||||
// Validate and ensure exactly one is set. (To do: better validation.)
|
||||
var b2i = map[bool]int{false: 0, true: 1}
|
||||
set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""]
|
||||
if set > 1 {
|
||||
return r, errors.New("more than one user-assigned identity ID is set")
|
||||
}
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
switch {
|
||||
case opt.MSIClientID != "":
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
case opt.MSIObjectID != "":
|
||||
// FIXME this doesn't appear to be in the new SDK?
|
||||
return r, fmt.Errorf("MSI object ID is currently unsupported")
|
||||
case opt.MSIResourceID != "":
|
||||
options.ID = azidentity.ResourceID(opt.MSIResourceID)
|
||||
}
|
||||
r.Cred, err = azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
r.Cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
case opt.UseAZ:
|
||||
var options = azidentity.AzureCLICredentialOptions{}
|
||||
r.Cred, err = azidentity.NewAzureCLICredential(&options)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
|
||||
}
|
||||
case opt.Account != "":
|
||||
// Anonymous access
|
||||
r.Anonymous = true
|
||||
default:
|
||||
return r, errors.New("no authentication method configured")
|
||||
}
|
||||
|
||||
// Make the client if not already created
|
||||
if r.Client == nil {
|
||||
// Work out what the endpoint is if it is still unset
|
||||
if opt.Endpoint == "" {
|
||||
if opt.Account == "" {
|
||||
return r, fmt.Errorf("account must be set: can't make service URL")
|
||||
}
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, conf.DefaultBaseURL))
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("failed to make azure storage URL from account: %w", err)
|
||||
}
|
||||
opt.Endpoint = u.String()
|
||||
}
|
||||
if sharedKeyCred != nil {
|
||||
// Shared key cred
|
||||
r.Client, err = conf.NewClientWithSharedKeyCredential(opt.Endpoint, sharedKeyCred, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create client with shared key failed: %w", err)
|
||||
}
|
||||
r.UsingSharedKeyCred = true
|
||||
} else if r.Cred != nil {
|
||||
// Azidentity cred
|
||||
r.Client, err = conf.NewClient(opt.Endpoint, r.Cred, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create client failed: %w", err)
|
||||
}
|
||||
} else if r.Anonymous {
|
||||
// Anonymous public access
|
||||
r.Client, err = conf.NewClientWithNoCredential(opt.Endpoint, &clientOpt)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("create public client failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.Client == nil {
|
||||
return r, fmt.Errorf("internal error: auth failed to make credentials or client")
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
+495
-564
File diff suppressed because it is too large.
Load diff
@@ -5,11 +5,16 @@ package azureblob
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
@@ -148,4 +153,417 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Features", f.testFeatures)
|
||||
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
||||
t.Run("Metadata", f.testMetadataPaths)
|
||||
}
|
||||
|
||||
// helper to read blob properties for an object
|
||||
func getProps(ctx context.Context, t *testing.T, o fs.Object) *blob.GetPropertiesResponse {
|
||||
ao := o.(*Object)
|
||||
props, err := ao.readMetaDataAlways(ctx)
|
||||
require.NoError(t, err)
|
||||
return props
|
||||
}
|
||||
|
||||
// helper to assert select headers and user metadata
|
||||
func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, want map[string]string, wantUserMeta map[string]string) {
|
||||
// Headers
|
||||
get := func(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return *p
|
||||
}
|
||||
if v, ok := want["content-type"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentType), "content-type")
|
||||
}
|
||||
if v, ok := want["cache-control"]; ok {
|
||||
assert.Equal(t, v, get(props.CacheControl), "cache-control")
|
||||
}
|
||||
if v, ok := want["content-disposition"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentDisposition), "content-disposition")
|
||||
}
|
||||
if v, ok := want["content-encoding"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentEncoding), "content-encoding")
|
||||
}
|
||||
if v, ok := want["content-language"]; ok {
|
||||
assert.Equal(t, v, get(props.ContentLanguage), "content-language")
|
||||
}
|
||||
// User metadata (case-insensitive keys from service)
|
||||
norm := make(map[string]*string, len(props.Metadata))
|
||||
for kk, vv := range props.Metadata {
|
||||
norm[strings.ToLower(kk)] = vv
|
||||
}
|
||||
for k, v := range wantUserMeta {
|
||||
pv, ok := norm[strings.ToLower(k)]
|
||||
if assert.True(t, ok, fmt.Sprintf("missing user metadata key %q", k)) {
|
||||
if pv == nil {
|
||||
assert.Equal(t, v, "", k)
|
||||
} else {
|
||||
assert.Equal(t, v, *pv, k)
|
||||
}
|
||||
} else {
|
||||
// Log available keys for diagnostics
|
||||
keys := make([]string, 0, len(props.Metadata))
|
||||
for kk := range props.Metadata {
|
||||
keys = append(keys, kk)
|
||||
}
|
||||
t.Logf("available user metadata keys: %v", keys)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// helper to read blob tags for an object
|
||||
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
|
||||
ao := o.(*Object)
|
||||
blb := ao.getBlobSVC()
|
||||
resp, err := blb.GetTags(ctx, nil)
|
||||
require.NoError(t, err)
|
||||
out := make(map[string]string)
|
||||
for _, tag := range resp.BlobTagSet {
|
||||
if tag.Key != nil {
|
||||
k := *tag.Key
|
||||
v := ""
|
||||
if tag.Value != nil {
|
||||
v = *tag.Value
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Test metadata across different write paths
|
||||
func (f *Fs) testMetadataPaths(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping in short mode")
|
||||
}
|
||||
|
||||
// Common expected metadata and headers
|
||||
baseMeta := fs.Metadata{
|
||||
"cache-control": "no-cache",
|
||||
"content-disposition": "inline",
|
||||
"content-language": "en-US",
|
||||
// Note: Don't set content-encoding here to avoid download decoding differences
|
||||
// We will set a custom user metadata key
|
||||
"potato": "royal",
|
||||
// and modtime
|
||||
"mtime": fstest.Time("2009-05-06T04:05:06.499999999Z").Format(time.RFC3339Nano),
|
||||
}
|
||||
|
||||
// Singlepart upload
|
||||
t.Run("PutSinglepart", func(t *testing.T) {
|
||||
// size less than chunk size
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("meta-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
// override content-type via metadata mapping
|
||||
meta := fs.Metadata{}
|
||||
meta.Merge(baseMeta)
|
||||
meta["content-type"] = "text/plain"
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
props := getProps(ctx, t, obj)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "text/plain",
|
||||
"cache-control": "no-cache",
|
||||
"content-disposition": "inline",
|
||||
"content-language": "en-US",
|
||||
}, map[string]string{
|
||||
"potato": "royal",
|
||||
})
|
||||
_ = http.StatusOK // keep import for parity but don't inspect RawResponse
|
||||
})
|
||||
|
||||
// Multipart upload
|
||||
t.Run("PutMultipart", func(t *testing.T) {
|
||||
// size greater than chunk size to force multipart
|
||||
contents := random.String(int(f.opt.ChunkSize + 1024))
|
||||
item := fstest.NewItem("meta-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{}
|
||||
meta.Merge(baseMeta)
|
||||
meta["content-type"] = "application/json"
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
props := getProps(ctx, t, obj)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "application/json",
|
||||
"cache-control": "no-cache",
|
||||
"content-disposition": "inline",
|
||||
"content-language": "en-US",
|
||||
}, map[string]string{
|
||||
"potato": "royal",
|
||||
})
|
||||
|
||||
// Tags: Singlepart upload
|
||||
t.Run("PutSinglepartTags", func(t *testing.T) {
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
"x-ms-tags": "env=dev,team=sync",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
tags := getTagsMap(ctx, t, obj)
|
||||
assert.Equal(t, "dev", tags["env"])
|
||||
assert.Equal(t, "sync", tags["team"])
|
||||
})
|
||||
|
||||
// Tags: Multipart upload
|
||||
t.Run("PutMultipartTags", func(t *testing.T) {
|
||||
contents := random.String(int(f.opt.ChunkSize + 2048))
|
||||
item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
meta := fs.Metadata{
|
||||
"x-ms-tags": "project=alpha,release=2025-08",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "application/octet-stream", meta)
|
||||
defer func() { _ = obj.Remove(ctx) }()
|
||||
|
||||
tags := getTagsMap(ctx, t, obj)
|
||||
assert.Equal(t, "alpha", tags["project"])
|
||||
assert.Equal(t, "2025-08", tags["release"])
|
||||
})
|
||||
})
|
||||
|
||||
// Singlepart copy with metadata-set mapping; omit content-type to exercise fallback
|
||||
t.Run("CopySinglepart", func(t *testing.T) {
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("meta-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
// set mapping via MetadataSet
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"cache-control": "private, max-age=60",
|
||||
"content-disposition": "attachment; filename=foo.txt",
|
||||
"content-language": "fr",
|
||||
// no content-type: should fallback to source
|
||||
"potato": "maris",
|
||||
}
|
||||
|
||||
// do copy
|
||||
dstName := "meta-copy-single-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
// content-type should fallback to source (text/plain)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "text/plain",
|
||||
"cache-control": "private, max-age=60",
|
||||
"content-disposition": "attachment; filename=foo.txt",
|
||||
"content-language": "fr",
|
||||
}, map[string]string{
|
||||
"potato": "maris",
|
||||
})
|
||||
// mtime should be populated on copy when --metadata is used
|
||||
// and should equal the source ModTime (RFC3339Nano)
|
||||
// Read user metadata (case-insensitive)
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
// parse and compare times ignoring formatting differences
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// CopySinglepart with only --metadata (no MetadataSet) must inject mtime and preserve src content-type
|
||||
t.Run("CopySinglepart_MetadataOnly", func(t *testing.T) {
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("meta-copy-single-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
|
||||
dstName := "meta-copy-single-only-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "text/plain",
|
||||
}, map[string]string{})
|
||||
// Assert mtime injected
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// Multipart copy with metadata-set mapping; omit content-type to exercise fallback
|
||||
t.Run("CopyMultipart", func(t *testing.T) {
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 1024))
|
||||
srcItem := fstest.NewItem("meta-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
// set mapping via MetadataSet
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"cache-control": "max-age=0, no-cache",
|
||||
// omit content-type to trigger fallback
|
||||
"content-language": "de",
|
||||
"potato": "desiree",
|
||||
}
|
||||
|
||||
dstName := "meta-copy-multi-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
// content-type should fallback to source (application/octet-stream)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "application/octet-stream",
|
||||
"cache-control": "max-age=0, no-cache",
|
||||
"content-language": "de",
|
||||
}, map[string]string{
|
||||
"potato": "desiree",
|
||||
})
|
||||
// mtime should be populated on copy when --metadata is used
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// CopyMultipart with only --metadata must inject mtime and preserve src content-type
|
||||
t.Run("CopyMultipart_MetadataOnly", func(t *testing.T) {
|
||||
contents := random.String(int(f.opt.CopyCutoff + 2048))
|
||||
srcItem := fstest.NewItem("meta-copy-multi-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
|
||||
dstName := "meta-copy-multi-only-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
props := getProps(ctx2, t, dst)
|
||||
assertHeadersAndMetadata(t, props, map[string]string{
|
||||
"content-type": "application/octet-stream",
|
||||
}, map[string]string{})
|
||||
m := props.Metadata
|
||||
var gotMtime string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "mtime") && v != nil {
|
||||
gotMtime = *v
|
||||
break
|
||||
}
|
||||
}
|
||||
if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
|
||||
parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
|
||||
}
|
||||
})
|
||||
|
||||
// Tags: Singlepart copy
|
||||
t.Run("CopySinglepartTags", func(t *testing.T) {
|
||||
// create small source
|
||||
contents := random.String(int(f.opt.ChunkSize / 2))
|
||||
srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
// set mapping via MetadataSet including tags
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"x-ms-tags": "copy=single,mode=test",
|
||||
}
|
||||
|
||||
dstName := "tags-copy-single-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
tags := getTagsMap(ctx2, t, dst)
|
||||
assert.Equal(t, "single", tags["copy"])
|
||||
assert.Equal(t, "test", tags["mode"])
|
||||
})
|
||||
|
||||
// Tags: Multipart copy
|
||||
t.Run("CopyMultipartTags", func(t *testing.T) {
|
||||
// create large source to force multipart
|
||||
contents := random.String(int(f.opt.CopyCutoff + 4096))
|
||||
srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
|
||||
defer func() { _ = srcObj.Remove(ctx) }()
|
||||
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
ci.MetadataSet = fs.Metadata{
|
||||
"x-ms-tags": "copy=multi,mode=test",
|
||||
}
|
||||
|
||||
dstName := "tags-copy-multi-dst.txt"
|
||||
dst, err := f.Copy(ctx2, srcObj, dstName)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = dst.Remove(ctx2) }()
|
||||
|
||||
tags := getTagsMap(ctx2, t, dst)
|
||||
assert.Equal(t, "multi", tags["copy"])
|
||||
assert.Equal(t, "test", tags["mode"])
|
||||
})
|
||||
|
||||
// Negative: invalid x-ms-tags must error
|
||||
t.Run("InvalidXMsTags", func(t *testing.T) {
|
||||
contents := random.String(32)
|
||||
item := fstest.NewItem("tags-invalid.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
// construct ObjectInfo with invalid x-ms-tags
|
||||
buf := strings.NewReader(contents)
|
||||
// Build obj info with metadata
|
||||
meta := fs.Metadata{
|
||||
"x-ms-tags": "badpair-without-equals",
|
||||
}
|
||||
// force metadata on
|
||||
ctx2, ci := fs.AddConfig(ctx)
|
||||
ci.Metadata = true
|
||||
obji := object.NewStaticObjectInfo(item.Path, item.ModTime, int64(len(contents)), true, nil, nil)
|
||||
obji = obji.WithMetadata(meta).WithMimeType("text/plain")
|
||||
_, err := f.Put(ctx2, buf, obji)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid tag")
|
||||
})
|
||||
}
|
||||
@@ -29,36 +29,28 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/fileerror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/service"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/share"
|
||||
"github.com/rclone/rclone/backend/azureblob/auth"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
)
|
||||
|
||||
@@ -73,199 +65,12 @@ func init() {
|
||||
Name: "azurefiles",
|
||||
Description: "Microsoft Azure Files",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: "account",
|
||||
Help: `Azure Storage Account Name.
|
||||
|
||||
Set this to the Azure Storage Account Name in use.
|
||||
|
||||
Leave blank to use SAS URL or connection string, otherwise it needs to be set.
|
||||
|
||||
If this is blank and if env_auth is set it will be read from the
|
||||
environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible.
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Options: slices.Concat(auth.ConfigOptions, []fs.Option{{
|
||||
Name: "share_name",
|
||||
Help: `Azure Files Share Name.
|
||||
|
||||
This is required and is the name of the share to access.
|
||||
`,
|
||||
}, {
|
||||
Name: "env_auth",
|
||||
Help: `Read credentials from runtime (environment variables, CLI or MSI).
|
||||
|
||||
See the [authentication docs](/azurefiles#authentication) for full info.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "key",
|
||||
Help: `Storage Account Shared Key.
|
||||
|
||||
Leave blank to use SAS URL or connection string.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sas_url",
|
||||
Help: `SAS URL.
|
||||
|
||||
Leave blank if using account/key or connection string.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "connection_string",
|
||||
Help: `Azure Files Connection String.`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "tenant",
|
||||
Help: `ID of the service principal's tenant. Also called its directory ID.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_id",
|
||||
Help: `The ID of the client in use.
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
- Service principal with certificate
|
||||
- User with username and password
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_secret",
|
||||
Help: `One of the service principal's client secrets
|
||||
|
||||
Set this if using
|
||||
- Service principal with client secret
|
||||
`,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "client_certificate_path",
|
||||
Help: `Path to a PEM or PKCS12 certificate file including the private key.
|
||||
|
||||
Set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
}, {
|
||||
Name: "client_certificate_password",
|
||||
Help: `Password for the certificate file (optional).
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
|
||||
And the certificate has a password.
|
||||
`,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "client_send_certificate_chain",
|
||||
Help: `Send the certificate chain when using certificate auth.
|
||||
|
||||
Specifies whether an authentication request will include an x5c header
|
||||
to support subject name / issuer based authentication. When set to
|
||||
true, authentication requests include the x5c header.
|
||||
|
||||
Optionally set this if using
|
||||
- Service principal with certificate
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "username",
|
||||
Help: `User name (usually an email address)
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "password",
|
||||
Help: `The user's password
|
||||
|
||||
Set this if using
|
||||
- User with username and password
|
||||
`,
|
||||
IsPassword: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "service_principal_file",
|
||||
Help: `Path to file containing credentials for use with a service principal.
|
||||
|
||||
Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
|
||||
|
||||
$ az ad sp create-for-rbac --name "<name>" \
|
||||
--role "Storage Files Data Owner" \
|
||||
--scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
|
||||
> azure-principal.json
|
||||
|
||||
See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to files data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
|
||||
|
||||
**NB** this section needs updating for Azure Files - pull requests appreciated!
|
||||
|
||||
It may be more convenient to put the credentials directly into the
|
||||
rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + `
|
||||
keys instead of setting ` + "`service_principal_file`" + `.
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_msi",
|
||||
Help: `Use a managed service identity to authenticate (only works in Azure).
|
||||
|
||||
When true, use a [managed service identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/)
|
||||
to authenticate to Azure Storage instead of a SAS token or account key.
|
||||
|
||||
If the VM(SS) on which this program is running has a system-assigned identity, it will
|
||||
be used by default. If the resource has no system-assigned but exactly one user-assigned identity,
|
||||
the user-assigned identity will be used by default. If the resource has multiple user-assigned
|
||||
identities, the identity to use must be explicitly specified using exactly one of the msi_object_id,
|
||||
msi_client_id, or msi_mi_res_id parameters.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "msi_object_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_client_id",
|
||||
Help: "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "msi_mi_res_id",
|
||||
Help: "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "disable_instance_discovery",
|
||||
Help: `Skip requesting Microsoft Entra instance metadata
|
||||
This should be set true only by applications authenticating in
|
||||
disconnected clouds, or private clouds such as Azure Stack.
|
||||
It determines whether rclone requests Microsoft Entra instance
|
||||
metadata from ` + "`https://login.microsoft.com/`" + ` before
|
||||
authenticating.
|
||||
Setting this to true will skip this request, making you responsible
|
||||
for ensuring the configured authority is valid and trustworthy.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_az",
|
||||
Help: `Use Azure CLI tool az for authentication
|
||||
Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
as the sole means of authentication.
|
||||
Setting this can be useful if you wish to use the az CLI on a host with
|
||||
a System Managed Identity that you do not want to use.
|
||||
Don't set env_auth at the same time.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service.\n\nLeave blank normally.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Upload chunk size.
|
||||
@@ -323,38 +128,18 @@ You will need this much free space in the share as the file will be this size te
|
||||
encoder.EncodeInvalidUtf8 |
|
||||
encoder.EncodeCtl | encoder.EncodeDel |
|
||||
encoder.EncodeDot | encoder.EncodeRightPeriod),
|
||||
}},
|
||||
}}),
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Account string `config:"account"`
|
||||
ShareName string `config:"share_name"`
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
Key string `config:"key"`
|
||||
SASURL string `config:"sas_url"`
|
||||
ConnectionString string `config:"connection_string"`
|
||||
Tenant string `config:"tenant"`
|
||||
ClientID string `config:"client_id"`
|
||||
ClientSecret string `config:"client_secret"`
|
||||
ClientCertificatePath string `config:"client_certificate_path"`
|
||||
ClientCertificatePassword string `config:"client_certificate_password"`
|
||||
ClientSendCertificateChain bool `config:"client_send_certificate_chain"`
|
||||
Username string `config:"username"`
|
||||
Password string `config:"password"`
|
||||
ServicePrincipalFile string `config:"service_principal_file"`
|
||||
DisableInstanceDiscovery bool `config:"disable_instance_discovery"`
|
||||
UseMSI bool `config:"use_msi"`
|
||||
MSIObjectID string `config:"msi_object_id"`
|
||||
MSIClientID string `config:"msi_client_id"`
|
||||
MSIResourceID string `config:"msi_mi_res_id"`
|
||||
UseAZ bool `config:"use_az"`
|
||||
Endpoint string `config:"endpoint"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
MaxStreamSize fs.SizeSuffix `config:"max_stream_size"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
auth.Options
|
||||
ShareName string `config:"share_name"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
MaxStreamSize fs.SizeSuffix `config:"max_stream_size"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a root directory inside a share. The root directory can be ""
|
||||
@@ -377,266 +162,29 @@ type Object struct {
|
||||
contentType string // content type if known
|
||||
}
|
||||
|
||||
// Wrap the http.Transport to satisfy the Transporter interface
|
||||
type transporter struct {
|
||||
http.RoundTripper
|
||||
}
|
||||
|
||||
// Make a new transporter
|
||||
func newTransporter(ctx context.Context) transporter {
|
||||
return transporter{
|
||||
RoundTripper: fshttp.NewTransport(ctx),
|
||||
}
|
||||
}
|
||||
|
||||
// Do sends the HTTP request and returns the HTTP response or error.
|
||||
func (tr transporter) Do(req *http.Request) (*http.Response, error) {
|
||||
return tr.RoundTripper.RoundTrip(req)
|
||||
}
|
||||
|
||||
type servicePrincipalCredentials struct {
|
||||
AppID string `json:"appId"`
|
||||
Password string `json:"password"`
|
||||
Tenant string `json:"tenant"`
|
||||
}
|
||||
|
||||
// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli.
|
||||
func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) {
|
||||
var spCredentials servicePrincipalCredentials
|
||||
if err := json.Unmarshal(credentialsData, &spCredentials); err != nil {
|
||||
return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
|
||||
}
|
||||
// TODO: support certificate credentials
|
||||
// Validate all fields present
|
||||
if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" {
|
||||
return nil, fmt.Errorf("missing fields in credentials file")
|
||||
}
|
||||
return &spCredentials, nil
|
||||
}
|
||||
|
||||
// Factored out from NewFs so that it can be tested with opt *Options and without m configmap.Mapper
|
||||
func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.Fs, error) {
|
||||
// Client options specifying our own transport
|
||||
policyClientOptions := policy.ClientOptions{
|
||||
Transport: newTransporter(ctx),
|
||||
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
|
||||
DefaultBaseURL: storageDefaultBaseURL,
|
||||
NewClient: service.NewClient,
|
||||
NewClientFromConnectionString: service.NewClientFromConnectionString,
|
||||
NewClientWithNoCredential: service.NewClientWithNoCredential,
|
||||
NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential,
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
options.ClientOptions = policyClientOptions
|
||||
},
|
||||
}
|
||||
backup := service.ShareTokenIntentBackup
|
||||
clientOpt := service.ClientOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
FileRequestIntent: &backup,
|
||||
res, err := auth.NewClient(ctx, conf, &opt.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// f.svc = res.Client
|
||||
// f.cred = res.Cred
|
||||
// f.sharedKeyCred = res.SharedKeyCred
|
||||
// f.anonymous = res.Anonymous
|
||||
|
||||
// Here we auth by setting one of cred, sharedKeyCred or f.client
|
||||
var (
|
||||
cred azcore.TokenCredential
|
||||
sharedKeyCred *service.SharedKeyCredential
|
||||
client *service.Client
|
||||
err error
|
||||
)
|
||||
switch {
|
||||
case opt.EnvAuth:
|
||||
// Read account from environment if needed
|
||||
if opt.Account == "" {
|
||||
opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME")
|
||||
}
|
||||
// Read credentials from the environment
|
||||
options := azidentity.DefaultAzureCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
DisableInstanceDiscovery: opt.DisableInstanceDiscovery,
|
||||
}
|
||||
cred, err = azidentity.NewDefaultAzureCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create azure environment credential failed: %w", err)
|
||||
}
|
||||
case opt.Account != "" && opt.Key != "":
|
||||
sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create new shared key credential failed: %w", err)
|
||||
}
|
||||
case opt.UseAZ:
|
||||
options := azidentity.AzureCLICredentialOptions{}
|
||||
cred, err = azidentity.NewAzureCLICredential(&options)
|
||||
fmt.Println(cred)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure CLI credentials: %w", err)
|
||||
}
|
||||
case opt.SASURL != "":
|
||||
client, err = service.NewClientWithNoCredential(opt.SASURL, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create SAS URL client: %w", err)
|
||||
}
|
||||
case opt.ConnectionString != "":
|
||||
client, err = service.NewClientFromConnectionString(opt.ConnectionString, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create connection string client: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "":
|
||||
// Service principal with client secret
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "":
|
||||
// Service principal with certificate
|
||||
//
|
||||
// Read the certificate
|
||||
data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading client certificate file: %w", err)
|
||||
}
|
||||
// NewClientCertificateCredential requires at least one *x509.Certificate, and a
|
||||
// crypto.PrivateKey.
|
||||
//
|
||||
// ParseCertificates returns these given certificate data in PEM or PKCS12 format.
|
||||
// It handles common scenarios but has limitations, for example it doesn't load PEM
|
||||
// encrypted private keys.
|
||||
var password []byte
|
||||
if opt.ClientCertificatePassword != "" {
|
||||
pw, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
password = []byte(pw)
|
||||
}
|
||||
certs, key, err := azidentity.ParseCertificates(data, password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse client certificate file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientCertificateCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
SendCertificateChain: opt.ClientSendCertificateChain,
|
||||
}
|
||||
cred, err = azidentity.NewClientCertificateCredential(
|
||||
opt.Tenant, opt.ClientID, certs, key, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
|
||||
// User with username and password
|
||||
//nolint:staticcheck // this is deprecated due to Azure policy
|
||||
options := azidentity.UsernamePasswordCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
password, err := obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("user password decode failed - did you obscure it?: %w", err)
|
||||
}
|
||||
cred, err = azidentity.NewUsernamePasswordCredential(
|
||||
opt.Tenant, opt.ClientID, opt.Username, password, &options,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("authenticate user with password failed: %w", err)
|
||||
}
|
||||
case opt.ServicePrincipalFile != "":
|
||||
// Loading service principal credentials from file.
|
||||
loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
|
||||
}
|
||||
parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing service principal credentials file: %w", err)
|
||||
}
|
||||
options := azidentity.ClientSecretCredentialOptions{
|
||||
ClientOptions: policyClientOptions,
|
||||
}
|
||||
cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating a client secret credential: %w", err)
|
||||
}
|
||||
case opt.UseMSI:
|
||||
// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
|
||||
// Validate and ensure exactly one is set. (To do: better validation.)
|
||||
b2i := map[bool]int{false: 0, true: 1}
|
||||
set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""]
|
||||
if set > 1 {
|
||||
return nil, errors.New("more than one user-assigned identity ID is set")
|
||||
}
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
switch {
|
||||
case opt.MSIClientID != "":
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
case opt.MSIObjectID != "":
|
||||
// FIXME this doesn't appear to be in the new SDK?
|
||||
return nil, fmt.Errorf("MSI object ID is currently unsupported")
|
||||
case opt.MSIResourceID != "":
|
||||
options.ID = azidentity.ResourceID(opt.MSIResourceID)
|
||||
}
|
||||
cred, err = azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, errors.New("no authentication method configured")
|
||||
}
|
||||
|
||||
// Make the client if not already created
|
||||
if client == nil {
|
||||
// Work out what the endpoint is if it is still unset
|
||||
if opt.Endpoint == "" {
|
||||
if opt.Account == "" {
|
||||
return nil, fmt.Errorf("account must be set: can't make service URL")
|
||||
}
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, storageDefaultBaseURL))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to make azure storage URL from account: %w", err)
|
||||
}
|
||||
opt.Endpoint = u.String()
|
||||
}
|
||||
if sharedKeyCred != nil {
|
||||
// Shared key cred
|
||||
client, err = service.NewClientWithSharedKeyCredential(opt.Endpoint, sharedKeyCred, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create client with shared key failed: %w", err)
|
||||
}
|
||||
} else if cred != nil {
|
||||
// Azidentity cred
|
||||
client, err = service.NewClient(opt.Endpoint, cred, &clientOpt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create client failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("internal error: auth failed to make credentials or client")
|
||||
}
|
||||
|
||||
shareClient := client.NewShareClient(opt.ShareName)
|
||||
shareClient := res.Client.NewShareClient(opt.ShareName)
|
||||
svc := shareClient.NewRootDirectoryClient()
|
||||
f := &Fs{
|
||||
shareClient: shareClient,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/azureblob/auth"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
@@ -28,22 +29,28 @@ func (f *Fs) InternalTestAuth(t *testing.T) {
|
||||
{
|
||||
name: "ConnectionString",
|
||||
options: &Options{
|
||||
ShareName: shareName,
|
||||
ConnectionString: "",
|
||||
ShareName: shareName,
|
||||
Options: auth.Options{
|
||||
ConnectionString: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "AccountAndKey",
|
||||
options: &Options{
|
||||
ShareName: shareName,
|
||||
Account: "",
|
||||
Key: "",
|
||||
Options: auth.Options{
|
||||
Account: "",
|
||||
Key: "",
|
||||
},
|
||||
}},
|
||||
{
|
||||
name: "SASUrl",
|
||||
options: &Options{
|
||||
ShareName: shareName,
|
||||
SASURL: "",
|
||||
Options: auth.Options{
|
||||
SASURL: "",
|
||||
},
|
||||
}},
|
||||
}
|
||||
|
||||
|
||||
+44
-22
@@ -48,6 +48,14 @@ type LifecycleRule struct {
|
||||
FileNamePrefix string `json:"fileNamePrefix"`
|
||||
}
|
||||
|
||||
// ServerSideEncryption is a configuration object for B2 Server-Side Encryption
|
||||
type ServerSideEncryption struct {
|
||||
Mode string `json:"mode"`
|
||||
Algorithm string `json:"algorithm"` // Encryption algorithm to use
|
||||
CustomerKey string `json:"customerKey"` // User provided Base64 encoded key that is used by the server to encrypt files
|
||||
CustomerKeyMd5 string `json:"customerKeyMd5"` // An MD5 hash of the decoded key
|
||||
}
|
||||
|
||||
// Timestamp is a UTC time when this file was uploaded. It is a base
|
||||
// 10 number of milliseconds since midnight, January 1, 1970 UTC. This
|
||||
// fits in a 64 bit integer such as the type "long" in the programming
|
||||
@@ -125,23 +133,32 @@ type File struct {
|
||||
Info map[string]string `json:"fileInfo"` // The custom information that was uploaded with the file. This is a JSON object, holding the name/value pairs that were uploaded with the file.
|
||||
}
|
||||
|
||||
// AuthorizeAccountResponse is as returned from the b2_authorize_account call
|
||||
type AuthorizeAccountResponse struct {
|
||||
// StorageAPI is as returned from the b2_authorize_account call
|
||||
type StorageAPI struct {
|
||||
AbsoluteMinimumPartSize int `json:"absoluteMinimumPartSize"` // The smallest possible size of a part of a large file.
|
||||
AccountID string `json:"accountId"` // The identifier for the account.
|
||||
Allowed struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
|
||||
BucketID string `json:"bucketId"` // When present, access is restricted to one bucket.
|
||||
BucketName string `json:"bucketName"` // When present, name of bucket - may be empty
|
||||
Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has.
|
||||
Buckets []struct { // When present, access is restricted to one or more buckets.
|
||||
ID string `json:"id"` // ID of bucket
|
||||
Name string `json:"name"` // When present, name of bucket - may be empty
|
||||
} `json:"buckets"`
|
||||
Capabilities []string `json:"capabilities"` // A list of strings, each one naming a capability the key has for every bucket.
|
||||
NamePrefix any `json:"namePrefix"` // When present, access is restricted to files whose names start with the prefix
|
||||
} `json:"allowed"`
|
||||
APIURL string `json:"apiUrl"` // The base URL to use for all API calls except for uploading and downloading files.
|
||||
AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
|
||||
DownloadURL string `json:"downloadUrl"` // The base URL to use for downloading files.
|
||||
MinimumPartSize int `json:"minimumPartSize"` // DEPRECATED: This field will always have the same value as recommendedPartSize. Use recommendedPartSize instead.
|
||||
RecommendedPartSize int `json:"recommendedPartSize"` // The recommended size for each part of a large file. We recommend using this part size for optimal upload performance.
|
||||
}
|
||||
|
||||
// AuthorizeAccountResponse is as returned from the b2_authorize_account call
|
||||
type AuthorizeAccountResponse struct {
|
||||
AccountID string `json:"accountId"` // The identifier for the account.
|
||||
AuthorizationToken string `json:"authorizationToken"` // An authorization token to use with all calls, other than b2_authorize_account, that need an Authorization header.
|
||||
APIs struct { // Supported APIs for this account / key. These are API-dependent JSON objects.
|
||||
Storage StorageAPI `json:"storageApi"`
|
||||
} `json:"apiInfo"`
|
||||
}
|
||||
|
||||
// ListBucketsRequest is parameters for b2_list_buckets call
|
||||
type ListBucketsRequest struct {
|
||||
AccountID string `json:"accountId"` // The identifier for the account.
|
||||
@@ -261,10 +278,11 @@ type GetFileInfoRequest struct {
|
||||
//
|
||||
// Example: { "src_last_modified_millis" : "1452802803026", "large_file_sha1" : "a3195dc1e7b46a2ff5da4b3c179175b75671e80d", "color": "blue" }
|
||||
type StartLargeFileRequest struct {
|
||||
BucketID string `json:"bucketId"` //The ID of the bucket that the file will go in.
|
||||
Name string `json:"fileName"` // The name of the file. See Files for requirements on file names.
|
||||
ContentType string `json:"contentType"` // The MIME type of the content of the file, which will be returned in the Content-Type header when downloading the file. Use the Content-Type b2/x-auto to automatically set the stored Content-Type post upload. In the case where a file extension is absent or the lookup fails, the Content-Type is set to application/octet-stream.
|
||||
Info map[string]string `json:"fileInfo"` // A JSON object holding the name/value pairs for the custom file info.
|
||||
BucketID string `json:"bucketId"` // The ID of the bucket that the file will go in.
|
||||
Name string `json:"fileName"` // The name of the file. See Files for requirements on file names.
|
||||
ContentType string `json:"contentType"` // The MIME type of the content of the file, which will be returned in the Content-Type header when downloading the file. Use the Content-Type b2/x-auto to automatically set the stored Content-Type post upload. In the case where a file extension is absent or the lookup fails, the Content-Type is set to application/octet-stream.
|
||||
Info map[string]string `json:"fileInfo"` // A JSON object holding the name/value pairs for the custom file info.
|
||||
ServerSideEncryption *ServerSideEncryption `json:"serverSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption
|
||||
}
|
||||
|
||||
// StartLargeFileResponse is the response to StartLargeFileRequest
|
||||
@@ -325,21 +343,25 @@ type CancelLargeFileResponse struct {
|
||||
|
||||
// CopyFileRequest is as passed to b2_copy_file
|
||||
type CopyFileRequest struct {
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
Name string `json:"fileName"` // The name of the new file being created.
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
MetadataDirective string `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE
|
||||
ContentType string `json:"contentType,omitempty"` // The MIME type of the content of the file (REPLACE only)
|
||||
Info map[string]string `json:"fileInfo,omitempty"` // This field stores the metadata that will be stored with the file. (REPLACE only)
|
||||
DestBucketID string `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
Name string `json:"fileName"` // The name of the new file being created.
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
MetadataDirective string `json:"metadataDirective,omitempty"` // The strategy for how to populate metadata for the new file: COPY or REPLACE
|
||||
ContentType string `json:"contentType,omitempty"` // The MIME type of the content of the file (REPLACE only)
|
||||
Info map[string]string `json:"fileInfo,omitempty"` // This field stores the metadata that will be stored with the file. (REPLACE only)
|
||||
DestBucketID string `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
|
||||
SourceServerSideEncryption *ServerSideEncryption `json:"sourceServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the source file
|
||||
DestinationServerSideEncryption *ServerSideEncryption `json:"destinationServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the destination file
|
||||
}
|
||||
|
||||
// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
|
||||
type CopyPartRequest struct {
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
LargeFileID string `json:"largeFileId"` // The ID of the large file the part will belong to, as returned by b2_start_large_file.
|
||||
PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1)
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
LargeFileID string `json:"largeFileId"` // The ID of the large file the part will belong to, as returned by b2_start_large_file.
|
||||
PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1)
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
SourceServerSideEncryption *ServerSideEncryption `json:"sourceServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the source file
|
||||
DestinationServerSideEncryption *ServerSideEncryption `json:"destinationServerSideEncryption,omitempty"` // A JSON object holding values related to Server-Side Encryption for the destination file
|
||||
}
|
||||
|
||||
// UpdateBucketRequest describes a request to modify a B2 bucket
|
||||
|
||||
+229
-73
@@ -8,7 +8,9 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -53,6 +55,9 @@ const (
|
||||
nameHeader = "X-Bz-File-Name"
|
||||
timestampHeader = "X-Bz-Upload-Timestamp"
|
||||
retryAfterHeader = "Retry-After"
|
||||
sseAlgorithmHeader = "X-Bz-Server-Side-Encryption-Customer-Algorithm"
|
||||
sseKeyHeader = "X-Bz-Server-Side-Encryption-Customer-Key"
|
||||
sseMd5Header = "X-Bz-Server-Side-Encryption-Customer-Key-Md5"
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 5 * time.Minute
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
@@ -67,7 +72,7 @@ const (
|
||||
|
||||
// Globals
|
||||
var (
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
errNotWithVersions = errors.New("can't modify files in --b2-versions mode")
|
||||
errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
|
||||
)
|
||||
|
||||
@@ -252,6 +257,51 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}, {
|
||||
Name: "sse_customer_algorithm",
|
||||
Help: "If using SSE-C, the server-side encryption algorithm used when storing this object in B2.",
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}, {
|
||||
Value: "AES256",
|
||||
Help: "Advanced Encryption Standard (256 bits key length)",
|
||||
}},
|
||||
}, {
|
||||
Name: "sse_customer_key",
|
||||
Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
|
||||
|
||||
Alternatively you can provide --sse-customer-key-base64.`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}},
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sse_customer_key_base64",
|
||||
Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
|
||||
|
||||
Alternatively you can provide --sse-customer-key.`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}},
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "sse_customer_key_md5",
|
||||
Help: `If using SSE-C you may provide the secret encryption key MD5 checksum (optional).
|
||||
|
||||
If you leave it blank, this is calculated automatically from the sse_customer_key provided.
|
||||
`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
}},
|
||||
Sensitive: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -274,6 +324,10 @@ type Options struct {
|
||||
DownloadAuthorizationDuration fs.Duration `config:"download_auth_duration"`
|
||||
Lifecycle int `config:"lifecycle"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
SSECustomerAlgorithm string `config:"sse_customer_algorithm"`
|
||||
SSECustomerKey string `config:"sse_customer_key"`
|
||||
SSECustomerKeyBase64 string `config:"sse_customer_key_base64"`
|
||||
SSECustomerKeyMD5 string `config:"sse_customer_key_md5"`
|
||||
}
|
||||
|
||||
// Fs represents a remote b2 server
|
||||
@@ -504,6 +558,24 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if opt.Endpoint == "" {
|
||||
opt.Endpoint = defaultEndpoint
|
||||
}
|
||||
if opt.SSECustomerKey != "" && opt.SSECustomerKeyBase64 != "" {
|
||||
return nil, errors.New("b2: can't use both sse_customer_key and sse_customer_key_base64 at the same time")
|
||||
} else if opt.SSECustomerKeyBase64 != "" {
|
||||
// Decode the Base64-encoded key and store it in the SSECustomerKey field
|
||||
decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKeyBase64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("b2: Could not decode sse_customer_key_base64: %w", err)
|
||||
}
|
||||
opt.SSECustomerKey = string(decoded)
|
||||
} else {
|
||||
// Encode the raw key as Base64
|
||||
opt.SSECustomerKeyBase64 = base64.StdEncoding.EncodeToString([]byte(opt.SSECustomerKey))
|
||||
}
|
||||
if opt.SSECustomerKey != "" && opt.SSECustomerKeyMD5 == "" {
|
||||
// Calculate CustomerKeyMd5 if not supplied
|
||||
md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))
|
||||
opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
|
||||
}
|
||||
ci := fs.GetConfig(ctx)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
@@ -535,17 +607,29 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to authorize account: %w", err)
|
||||
}
|
||||
// If this is a key limited to a single bucket, it must exist already
|
||||
if f.rootBucket != "" && f.info.Allowed.BucketID != "" {
|
||||
allowedBucket := f.opt.Enc.ToStandardName(f.info.Allowed.BucketName)
|
||||
if allowedBucket == "" {
|
||||
return nil, errors.New("bucket that application key is restricted to no longer exists")
|
||||
// If this is a key limited to one or more buckets, one of them must exist
|
||||
// and be ours.
|
||||
if f.rootBucket != "" && len(f.info.APIs.Storage.Allowed.Buckets) != 0 {
|
||||
buckets := f.info.APIs.Storage.Allowed.Buckets
|
||||
var rootFound = false
|
||||
var rootID string
|
||||
for _, b := range buckets {
|
||||
allowedBucket := f.opt.Enc.ToStandardName(b.Name)
|
||||
if allowedBucket == "" {
|
||||
fs.Debugf(f, "bucket %q that application key is restricted to no longer exists", b.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
if allowedBucket == f.rootBucket {
|
||||
rootFound = true
|
||||
rootID = b.ID
|
||||
}
|
||||
}
|
||||
if allowedBucket != f.rootBucket {
|
||||
return nil, fmt.Errorf("you must use bucket %q with this application key", allowedBucket)
|
||||
if !rootFound {
|
||||
return nil, fmt.Errorf("you must use bucket(s) %q with this application key", buckets)
|
||||
}
|
||||
f.cache.MarkOK(f.rootBucket)
|
||||
f.setBucketID(f.rootBucket, f.info.Allowed.BucketID)
|
||||
f.setBucketID(f.rootBucket, rootID)
|
||||
}
|
||||
if f.rootBucket != "" && f.rootDirectory != "" {
|
||||
// Check to see if the (bucket,directory) is actually an existing file
|
||||
@@ -571,7 +655,7 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
|
||||
defer f.authMu.Unlock()
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/b2api/v1/b2_authorize_account",
|
||||
Path: "/b2api/v4/b2_authorize_account",
|
||||
RootURL: f.opt.Endpoint,
|
||||
UserName: f.opt.Account,
|
||||
Password: f.opt.Key,
|
||||
@@ -584,13 +668,13 @@ func (f *Fs) authorizeAccount(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to authenticate: %w", err)
|
||||
}
|
||||
f.srv.SetRoot(f.info.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
|
||||
f.srv.SetRoot(f.info.APIs.Storage.APIURL+"/b2api/v1").SetHeader("Authorization", f.info.AuthorizationToken)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasPermission returns if the current AuthorizationToken has the selected permission
|
||||
func (f *Fs) hasPermission(permission string) bool {
|
||||
return slices.Contains(f.info.Allowed.Capabilities, permission)
|
||||
return slices.Contains(f.info.APIs.Storage.Allowed.Capabilities, permission)
|
||||
}
|
||||
|
||||
// getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
|
||||
@@ -995,44 +1079,83 @@ type listBucketFn func(*api.Bucket) error
|
||||
|
||||
// listBucketsToFn lists the buckets to the function supplied
|
||||
func (f *Fs) listBucketsToFn(ctx context.Context, bucketName string, fn listBucketFn) error {
|
||||
var account = api.ListBucketsRequest{
|
||||
AccountID: f.info.AccountID,
|
||||
BucketID: f.info.Allowed.BucketID,
|
||||
}
|
||||
if bucketName != "" && account.BucketID == "" {
|
||||
account.BucketName = f.opt.Enc.FromStandardName(bucketName)
|
||||
responses := make([]api.ListBucketsResponse, len(f.info.APIs.Storage.Allowed.Buckets))[:0]
|
||||
|
||||
call := func(id string) error {
|
||||
var account = api.ListBucketsRequest{
|
||||
AccountID: f.info.AccountID,
|
||||
BucketID: id,
|
||||
}
|
||||
if bucketName != "" && account.BucketID == "" {
|
||||
account.BucketName = f.opt.Enc.FromStandardName(bucketName)
|
||||
}
|
||||
|
||||
var response api.ListBucketsResponse
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_list_buckets",
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
responses = append(responses, response)
|
||||
return nil
|
||||
}
|
||||
|
||||
var response api.ListBucketsResponse
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_list_buckets",
|
||||
for i := range f.info.APIs.Storage.Allowed.Buckets {
|
||||
b := &f.info.APIs.Storage.Allowed.Buckets[i]
|
||||
// Empty names indicate a bucket that no longer exists, this is non-fatal
|
||||
// for multi-bucket API keys.
|
||||
if b.Name == "" {
|
||||
continue
|
||||
}
|
||||
// When requesting a specific bucket skip over non-matching names
|
||||
if bucketName != "" && b.Name != bucketName {
|
||||
continue
|
||||
}
|
||||
|
||||
err := call(b.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
if len(f.info.APIs.Storage.Allowed.Buckets) == 0 {
|
||||
err := call("")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
f.bucketIDMutex.Lock()
|
||||
f.bucketTypeMutex.Lock()
|
||||
f._bucketID = make(map[string]string, 1)
|
||||
f._bucketType = make(map[string]string, 1)
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
|
||||
f.cache.MarkOK(bucket.Name)
|
||||
f._bucketID[bucket.Name] = bucket.ID
|
||||
f._bucketType[bucket.Name] = bucket.Type
|
||||
|
||||
for ri := range responses {
|
||||
response := &responses[ri]
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
|
||||
f.cache.MarkOK(bucket.Name)
|
||||
f._bucketID[bucket.Name] = bucket.ID
|
||||
f._bucketType[bucket.Name] = bucket.Type
|
||||
}
|
||||
}
|
||||
f.bucketTypeMutex.Unlock()
|
||||
f.bucketIDMutex.Unlock()
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
err = fn(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
for ri := range responses {
|
||||
response := &responses[ri]
|
||||
for i := range response.Buckets {
|
||||
bucket := &response.Buckets[i]
|
||||
err := fn(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1435,6 +1558,16 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
|
||||
Name: f.opt.Enc.FromStandardPath(dstPath),
|
||||
DestBucketID: destBucketID,
|
||||
}
|
||||
if f.opt.SSECustomerKey != "" && f.opt.SSECustomerKeyMD5 != "" {
|
||||
serverSideEncryptionConfig := api.ServerSideEncryption{
|
||||
Mode: "SSE-C",
|
||||
Algorithm: f.opt.SSECustomerAlgorithm,
|
||||
CustomerKey: f.opt.SSECustomerKeyBase64,
|
||||
CustomerKeyMd5: f.opt.SSECustomerKeyMD5,
|
||||
}
|
||||
request.SourceServerSideEncryption = &serverSideEncryptionConfig
|
||||
request.DestinationServerSideEncryption = &serverSideEncryptionConfig
|
||||
}
|
||||
if newInfo == nil {
|
||||
request.MetadataDirective = "COPY"
|
||||
} else {
|
||||
@@ -1524,7 +1657,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
bucket, bucketPath := f.split(remote)
|
||||
var RootURL string
|
||||
if f.opt.DownloadURL == "" {
|
||||
RootURL = f.info.DownloadURL
|
||||
RootURL = f.info.APIs.Storage.DownloadURL
|
||||
} else {
|
||||
RootURL = f.opt.DownloadURL
|
||||
}
|
||||
@@ -1866,15 +1999,16 @@ var _ io.ReadCloser = &openFile{}
|
||||
|
||||
func (o *Object) getOrHead(ctx context.Context, method string, options []fs.OpenOption) (resp *http.Response, info *api.File, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: method,
|
||||
Options: options,
|
||||
NoResponse: method == "HEAD",
|
||||
Method: method,
|
||||
Options: options,
|
||||
NoResponse: method == "HEAD",
|
||||
ExtraHeaders: map[string]string{},
|
||||
}
|
||||
|
||||
// Use downloadUrl from backblaze if downloadUrl is not set
|
||||
// otherwise use the custom downloadUrl
|
||||
if o.fs.opt.DownloadURL == "" {
|
||||
opts.RootURL = o.fs.info.DownloadURL
|
||||
opts.RootURL = o.fs.info.APIs.Storage.DownloadURL
|
||||
} else {
|
||||
opts.RootURL = o.fs.opt.DownloadURL
|
||||
}
|
||||
@@ -1886,6 +2020,11 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
|
||||
bucket, bucketPath := o.split()
|
||||
opts.Path += "/file/" + urlEncode(o.fs.opt.Enc.FromStandardName(bucket)) + "/" + urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath))
|
||||
}
|
||||
if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
opts.ExtraHeaders[sseAlgorithmHeader] = o.fs.opt.SSECustomerAlgorithm
|
||||
opts.ExtraHeaders[sseKeyHeader] = o.fs.opt.SSECustomerKeyBase64
|
||||
opts.ExtraHeaders[sseMd5Header] = o.fs.opt.SSECustomerKeyMD5
|
||||
}
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
@@ -2150,6 +2289,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
},
|
||||
ContentLength: &size,
|
||||
}
|
||||
if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
opts.ExtraHeaders[sseAlgorithmHeader] = o.fs.opt.SSECustomerAlgorithm
|
||||
opts.ExtraHeaders[sseKeyHeader] = o.fs.opt.SSECustomerKeyBase64
|
||||
opts.ExtraHeaders[sseMd5Header] = o.fs.opt.SSECustomerKeyMD5
|
||||
}
|
||||
var response api.FileInfo
|
||||
// Don't retry, return a retry error instead
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
@@ -2241,7 +2385,10 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
bucket, bucketPath := o.split()
|
||||
if o.fs.opt.Versions {
|
||||
return errNotWithVersions
|
||||
t, path := api.RemoveVersion(bucketPath)
|
||||
if !t.IsZero() {
|
||||
return o.fs.deleteByID(ctx, o.id, path)
|
||||
}
|
||||
}
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
return errNotWithVersionAt
|
||||
@@ -2264,32 +2411,36 @@ func (o *Object) ID() string {
|
||||
|
||||
var lifecycleHelp = fs.CommandHelp{
|
||||
Name: "lifecycle",
|
||||
Short: "Read or set the lifecycle for a bucket",
|
||||
Short: "Read or set the lifecycle for a bucket.",
|
||||
Long: `This command can be used to read or set the lifecycle for a bucket.
|
||||
|
||||
Usage Examples:
|
||||
|
||||
To show the current lifecycle rules:
|
||||
|
||||
rclone backend lifecycle b2:bucket
|
||||
` + "```console" + `
|
||||
rclone backend lifecycle b2:bucket
|
||||
` + "```" + `
|
||||
|
||||
This will dump something like this showing the lifecycle rules.
|
||||
|
||||
[
|
||||
{
|
||||
"daysFromHidingToDeleting": 1,
|
||||
"daysFromUploadingToHiding": null,
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": null,
|
||||
"fileNamePrefix": ""
|
||||
}
|
||||
]
|
||||
` + "```json" + `
|
||||
[
|
||||
{
|
||||
"daysFromHidingToDeleting": 1,
|
||||
"daysFromUploadingToHiding": null,
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": null,
|
||||
"fileNamePrefix": ""
|
||||
}
|
||||
]
|
||||
` + "```" + `
|
||||
|
||||
If there are no lifecycle rules (the default) then it will just return [].
|
||||
If there are no lifecycle rules (the default) then it will just return ` + "`[]`" + `.
|
||||
|
||||
To reset the current lifecycle rules:
|
||||
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=30
|
||||
rclone backend lifecycle b2:bucket -o daysFromUploadingToHiding=5 -o daysFromHidingToDeleting=1
|
||||
` + "```console" + `
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=30
|
||||
rclone backend lifecycle b2:bucket -o daysFromUploadingToHiding=5 -o daysFromHidingToDeleting=1
|
||||
` + "```" + `
|
||||
|
||||
This will run and then print the new lifecycle rules as above.
|
||||
|
||||
@@ -2301,14 +2452,17 @@ the daysFromHidingToDeleting to 1 day. You can enable hard_delete in
|
||||
the config also which will mean deletions won't cause versions but
|
||||
overwrites will still cause versions to be made.
|
||||
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=1
|
||||
` + "```console" + `
|
||||
rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=1
|
||||
` + "```" + `
|
||||
|
||||
See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
|
||||
`,
|
||||
See: <https://www.backblaze.com/docs/cloud-storage-lifecycle-rules>`,
|
||||
Opts: map[string]string{
|
||||
"daysFromHidingToDeleting": "After a file has been hidden for this many days it is deleted. 0 is off.",
|
||||
"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": "Cancels any unfinished large file versions after this many days",
|
||||
"daysFromHidingToDeleting": `After a file has been hidden for this many days
|
||||
it is deleted. 0 is off.`,
|
||||
"daysFromUploadingToHiding": `This many days after uploading a file is hidden.`,
|
||||
"daysFromStartingToCancelingUnfinishedLargeFiles": `Cancels any unfinished
|
||||
large file versions after this many days.`,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2391,13 +2545,14 @@ max-age, which defaults to 24 hours.
|
||||
Note that you can use --interactive/-i or --dry-run with this command to see what
|
||||
it would do.
|
||||
|
||||
rclone backend cleanup b2:bucket/path/to/object
|
||||
rclone backend cleanup -o max-age=7w b2:bucket/path/to/object
|
||||
` + "```console" + `
|
||||
rclone backend cleanup b2:bucket/path/to/object
|
||||
rclone backend cleanup -o max-age=7w b2:bucket/path/to/object
|
||||
` + "```" + `
|
||||
|
||||
Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
|
||||
`,
|
||||
Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.`,
|
||||
Opts: map[string]string{
|
||||
"max-age": "Max age of upload to delete",
|
||||
"max-age": "Max age of upload to delete.",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2420,8 +2575,9 @@ var cleanupHiddenHelp = fs.CommandHelp{
|
||||
Note that you can use --interactive/-i or --dry-run with this command to see what
|
||||
it would do.
|
||||
|
||||
rclone backend cleanup-hidden b2:bucket/path/to/dir
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend cleanup-hidden b2:bucket/path/to/dir
|
||||
` + "```",
|
||||
}
|
||||
|
||||
func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out any, err error) {
|
||||
|
||||
+33
-1
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/transferaccounter"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -144,6 +145,14 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
|
||||
request.ContentType = newInfo.ContentType
|
||||
request.Info = newInfo.Info
|
||||
}
|
||||
if o.fs.opt.SSECustomerKey != "" && o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
request.ServerSideEncryption = &api.ServerSideEncryption{
|
||||
Mode: "SSE-C",
|
||||
Algorithm: o.fs.opt.SSECustomerAlgorithm,
|
||||
CustomerKey: o.fs.opt.SSECustomerKeyBase64,
|
||||
CustomerKeyMd5: o.fs.opt.SSECustomerKeyMD5,
|
||||
}
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_start_large_file",
|
||||
@@ -295,6 +304,12 @@ func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader i
|
||||
ContentLength: &sizeWithHash,
|
||||
}
|
||||
|
||||
if up.o.fs.opt.SSECustomerKey != "" && up.o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
opts.ExtraHeaders[sseAlgorithmHeader] = up.o.fs.opt.SSECustomerAlgorithm
|
||||
opts.ExtraHeaders[sseKeyHeader] = up.o.fs.opt.SSECustomerKeyBase64
|
||||
opts.ExtraHeaders[sseMd5Header] = up.o.fs.opt.SSECustomerKeyMD5
|
||||
}
|
||||
|
||||
var response api.UploadPartResponse
|
||||
|
||||
resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
@@ -334,6 +349,17 @@ func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64)
|
||||
PartNumber: int64(part + 1),
|
||||
Range: fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
|
||||
}
|
||||
|
||||
if up.o.fs.opt.SSECustomerKey != "" && up.o.fs.opt.SSECustomerKeyMD5 != "" {
|
||||
serverSideEncryptionConfig := api.ServerSideEncryption{
|
||||
Mode: "SSE-C",
|
||||
Algorithm: up.o.fs.opt.SSECustomerAlgorithm,
|
||||
CustomerKey: up.o.fs.opt.SSECustomerKeyBase64,
|
||||
CustomerKeyMd5: up.o.fs.opt.SSECustomerKeyMD5,
|
||||
}
|
||||
request.SourceServerSideEncryption = &serverSideEncryptionConfig
|
||||
request.DestinationServerSideEncryption = &serverSideEncryptionConfig
|
||||
}
|
||||
var response api.UploadPartResponse
|
||||
resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
retry, err := up.f.shouldRetry(ctx, resp, err)
|
||||
@@ -474,10 +500,12 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
|
||||
defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
|
||||
fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
|
||||
var (
|
||||
account = transferaccounter.Get(ctx)
|
||||
g, gCtx = errgroup.WithContext(ctx)
|
||||
remaining = up.size
|
||||
)
|
||||
g.SetLimit(up.f.opt.UploadConcurrency)
|
||||
account.Start()
|
||||
for part := range up.parts {
|
||||
// Fail fast, in case an errgroup managed function returns an error
|
||||
// gCtx is cancelled. There is no point in copying all the other parts.
|
||||
@@ -489,7 +517,11 @@ func (up *largeUpload) Copy(ctx context.Context) (err error) {
|
||||
|
||||
part := part // for the closure
|
||||
g.Go(func() (err error) {
|
||||
return up.copyChunk(gCtx, part, reqSize)
|
||||
err = up.copyChunk(gCtx, part, reqSize)
|
||||
if err == nil {
|
||||
account.Add(reqSize)
|
||||
}
|
||||
return err
|
||||
})
|
||||
remaining -= reqSize
|
||||
}
|
||||
|
||||
+32
-17
@@ -87,13 +87,11 @@ func init() {
|
||||
Description: "Box",
|
||||
NewFs: NewFs,
|
||||
Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
jsonFile, ok := m.Get("box_config_file")
|
||||
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
|
||||
boxAccessToken, boxAccessTokenOk := m.Get("access_token")
|
||||
var err error
|
||||
// If using box config.json, use JWT auth
|
||||
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
|
||||
err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
|
||||
if usesJWTAuth(m) {
|
||||
err = refreshJWTToken(ctx, name, m)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to configure token with jwt authentication: %w", err)
|
||||
}
|
||||
@@ -114,6 +112,11 @@ func init() {
|
||||
}, {
|
||||
Name: "box_config_file",
|
||||
Help: "Box App config.json location\n\nLeave blank normally." + env.ShellExpandHelp,
|
||||
}, {
|
||||
Name: "config_credentials",
|
||||
Help: "Box App config.json contents.\n\nLeave blank normally.",
|
||||
Hide: fs.OptionHideBoth,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: "access_token",
|
||||
Help: "Box App Primary Access Token\n\nLeave blank normally.",
|
||||
@@ -184,9 +187,17 @@ See: https://developer.box.com/guides/authentication/jwt/as-user/
|
||||
})
|
||||
}
|
||||
|
||||
func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
|
||||
jsonFile = env.ShellExpand(jsonFile)
|
||||
boxConfig, err := getBoxConfig(jsonFile)
|
||||
func usesJWTAuth(m configmap.Mapper) bool {
|
||||
jsonFile, okFile := m.Get("box_config_file")
|
||||
jsonFileCredentials, okCredentials := m.Get("config_credentials")
|
||||
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
|
||||
return (okFile || okCredentials) && boxSubTypeOk && (jsonFile != "" || jsonFileCredentials != "") && boxSubType != ""
|
||||
}
|
||||
|
||||
func refreshJWTToken(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
boxSubType, _ := m.Get("box_sub_type")
|
||||
|
||||
boxConfig, err := getBoxConfig(m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get box config: %w", err)
|
||||
}
|
||||
@@ -205,12 +216,19 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
|
||||
return err
|
||||
}
|
||||
|
||||
func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
|
||||
file, err := os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("box: failed to read Box config: %w", err)
|
||||
func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
|
||||
configFileCredentials, _ := m.Get("config_credentials")
|
||||
configFileBytes := []byte(configFileCredentials)
|
||||
|
||||
if configFileCredentials == "" {
|
||||
configFile, _ := m.Get("box_config_file")
|
||||
configFileBytes, err = os.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("box: failed to read Box config: %w", err)
|
||||
}
|
||||
}
|
||||
err = json.Unmarshal(file, &boxConfig)
|
||||
|
||||
err = json.Unmarshal(configFileBytes, &boxConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("box: failed to parse Box config: %w", err)
|
||||
}
|
||||
@@ -485,15 +503,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
f.srv.SetHeader("as-user", f.opt.Impersonate)
|
||||
}
|
||||
|
||||
jsonFile, ok := m.Get("box_config_file")
|
||||
boxSubType, boxSubTypeOk := m.Get("box_sub_type")
|
||||
|
||||
if ts != nil {
|
||||
// If using box config.json and JWT, renewing should just refresh the token and
|
||||
// should do so whether there are uploads pending or not.
|
||||
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
|
||||
if usesJWTAuth(m) {
|
||||
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
|
||||
err := refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
|
||||
err := refreshJWTToken(ctx, name, m)
|
||||
return err
|
||||
})
|
||||
f.tokenRenewer.Start()
|
||||
|
||||
+130
-188
@@ -2,10 +2,8 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
@@ -46,6 +44,7 @@ const (
|
||||
minCompressionRatio = 1.1
|
||||
|
||||
gzFileExt = ".gz"
|
||||
zstdFileExt = ".zst"
|
||||
metaFileExt = ".json"
|
||||
uncompressedFileExt = ".bin"
|
||||
)
|
||||
@@ -54,6 +53,7 @@ const (
|
||||
const (
|
||||
Uncompressed = 0
|
||||
Gzip = 2
|
||||
Zstd = 4
|
||||
)
|
||||
|
||||
var nameRegexp = regexp.MustCompile(`^(.+?)\.([A-Za-z0-9-_]{11})$`)
|
||||
@@ -66,6 +66,10 @@ func init() {
|
||||
Value: "gzip",
|
||||
Help: "Standard gzip compression with fastest parameters.",
|
||||
},
|
||||
{
|
||||
Value: "zstd",
|
||||
Help: "Zstandard compression — fast modern algorithm offering adjustable speed-to-compression tradeoffs.",
|
||||
},
|
||||
}
|
||||
|
||||
// Register our remote
|
||||
@@ -87,17 +91,23 @@ func init() {
|
||||
Examples: compressionModeOptions,
|
||||
}, {
|
||||
Name: "level",
|
||||
Help: `GZIP compression level (-2 to 9).
|
||||
|
||||
Generally -1 (default, equivalent to 5) is recommended.
|
||||
Levels 1 to 9 increase compression at the cost of speed. Going past 6
|
||||
generally offers very little return.
|
||||
|
||||
Level -2 uses Huffman encoding only. Only use if you know what you
|
||||
are doing.
|
||||
Level 0 turns off compression.`,
|
||||
Default: sgzip.DefaultCompression,
|
||||
Advanced: true,
|
||||
Help: `GZIP (levels -2 to 9):
|
||||
- -2 — Huffman encoding only. Only use if you know what you're doing.
|
||||
- -1 (default) — recommended; equivalent to level 5.
|
||||
- 0 — turns off compression.
|
||||
- 1–9 — increase compression at the cost of speed. Going past 6 generally offers very little return.
|
||||
|
||||
ZSTD (levels 0 to 4):
|
||||
- 0 — turns off compression entirely.
|
||||
- 1 — fastest compression with the lowest ratio.
|
||||
- 2 (default) — good balance of speed and compression.
|
||||
- 3 — better compression, but uses about 2–3x more CPU than the default.
|
||||
- 4 — best possible compression ratio (highest CPU cost).
|
||||
|
||||
Notes:
|
||||
- Choose GZIP for wide compatibility; ZSTD for better speed/ratio tradeoffs.
|
||||
- Negative gzip levels: -2 = Huffman-only, -1 = default (≈ level 5).`,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "ram_cache_limit",
|
||||
Help: `Some remotes don't allow the upload of files with unknown size.
|
||||
@@ -112,6 +122,47 @@ this limit will be cached on disk.`,
|
||||
})
|
||||
}
|
||||
|
||||
// compressionModeHandler defines the interface for handling different compression modes
|
||||
type compressionModeHandler interface {
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
processFileNameGetFileExtension(compressionMode int) string
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error)
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
isCompressible(r io.Reader, compressionMode int) (bool, error)
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error)
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error)
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error)
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Remote string `config:"remote"`
|
||||
@@ -125,12 +176,13 @@ type Options struct {
|
||||
// Fs represents a wrapped fs.Fs
|
||||
type Fs struct {
|
||||
fs.Fs
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
mode int // compression mode id
|
||||
features *fs.Features // optional features
|
||||
wrapper fs.Fs
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
mode int // compression mode id
|
||||
features *fs.Features // optional features
|
||||
modeHandler compressionModeHandler // compression mode handler
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
@@ -167,13 +219,28 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
|
||||
return nil, fmt.Errorf("failed to make remote %s:%q to wrap: %w", wName, remotePath, err)
|
||||
}
|
||||
|
||||
compressionMode := compressionModeFromName(opt.CompressionMode)
|
||||
var modeHandler compressionModeHandler
|
||||
|
||||
switch compressionMode {
|
||||
case Gzip:
|
||||
modeHandler = &gzipModeHandler{}
|
||||
case Zstd:
|
||||
modeHandler = &zstdModeHandler{}
|
||||
case Uncompressed:
|
||||
modeHandler = &uncompressedModeHandler{}
|
||||
default:
|
||||
modeHandler = &unknownModeHandler{}
|
||||
}
|
||||
|
||||
// Create the wrapping fs
|
||||
f := &Fs{
|
||||
Fs: wrappedFs,
|
||||
name: name,
|
||||
root: rpath,
|
||||
opt: *opt,
|
||||
mode: compressionModeFromName(opt.CompressionMode),
|
||||
Fs: wrappedFs,
|
||||
name: name,
|
||||
root: rpath,
|
||||
opt: *opt,
|
||||
mode: compressionMode,
|
||||
modeHandler: modeHandler,
|
||||
}
|
||||
// Correct root if definitely pointing to a file
|
||||
if err == fs.ErrorIsFile {
|
||||
@@ -215,10 +282,13 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
|
||||
return f, err
|
||||
}
|
||||
|
||||
// compressionModeFromName converts a compression mode name to its int representation.
|
||||
func compressionModeFromName(name string) int {
|
||||
switch name {
|
||||
case "gzip":
|
||||
return Gzip
|
||||
case "zstd":
|
||||
return Zstd
|
||||
default:
|
||||
return Uncompressed
|
||||
}
|
||||
@@ -242,7 +312,7 @@ func base64ToInt64(str string) (int64, error) {
|
||||
|
||||
// Processes a file name for a compressed file. Returns the original file name, the extension, and the size of the original file.
|
||||
// Returns -2 for the original size if the file is uncompressed.
|
||||
func processFileName(compressedFileName string) (origFileName string, extension string, origSize int64, err error) {
|
||||
func processFileName(compressedFileName string, modeHandler compressionModeHandler) (origFileName string, extension string, origSize int64, err error) {
|
||||
// Separate the filename and size from the extension
|
||||
extensionPos := strings.LastIndex(compressedFileName, ".")
|
||||
if extensionPos == -1 {
|
||||
@@ -261,7 +331,8 @@ func processFileName(compressedFileName string) (origFileName string, extension
|
||||
if err != nil {
|
||||
return "", "", 0, errors.New("could not decode size")
|
||||
}
|
||||
return match[1], gzFileExt, size, nil
|
||||
ext := modeHandler.processFileNameGetFileExtension(compressionModeFromName(compressedFileName[extensionPos+1:]))
|
||||
return match[1], ext, size, nil
|
||||
}
|
||||
|
||||
// Generates the file name for a metadata file
|
||||
@@ -286,11 +357,15 @@ func unwrapMetadataFile(filename string) (string, bool) {
|
||||
|
||||
// makeDataName generates the file name for a data file with specified compression mode
|
||||
func makeDataName(remote string, size int64, mode int) (newRemote string) {
|
||||
if mode != Uncompressed {
|
||||
switch mode {
|
||||
case Gzip:
|
||||
newRemote = remote + "." + int64ToBase64(size) + gzFileExt
|
||||
} else {
|
||||
case Zstd:
|
||||
newRemote = remote + "." + int64ToBase64(size) + zstdFileExt
|
||||
default:
|
||||
newRemote = remote + uncompressedFileExt
|
||||
}
|
||||
|
||||
return newRemote
|
||||
}
|
||||
|
||||
@@ -304,7 +379,7 @@ func (f *Fs) dataName(remote string, size int64, compressed bool) (name string)
|
||||
|
||||
// addData parses an object and adds it to the DirEntries
|
||||
func (f *Fs) addData(entries *fs.DirEntries, o fs.Object) {
|
||||
origFileName, _, size, err := processFileName(o.Remote())
|
||||
origFileName, _, size, err := processFileName(o.Remote(), f.modeHandler)
|
||||
if err != nil {
|
||||
fs.Errorf(o, "Error on parsing file name: %v", err)
|
||||
return
|
||||
@@ -427,8 +502,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error decoding metadata: %w", err)
|
||||
}
|
||||
size, err := f.modeHandler.newObjectGetOriginalSize(meta)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading metadata: %w", err)
|
||||
}
|
||||
// Create our Object
|
||||
o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
|
||||
o, err := f.Fs.NewObject(ctx, makeDataName(remote, size, meta.Mode))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -437,7 +516,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
|
||||
// checkCompressAndType checks if an object is compressible and determines it's mime type
|
||||
// returns a multireader with the bytes that were read to determine mime type
|
||||
func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool, mimeType string, err error) {
|
||||
func checkCompressAndType(in io.Reader, compressionMode int, modeHandler compressionModeHandler) (newReader io.Reader, compressible bool, mimeType string, err error) {
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
buf := make([]byte, heuristicBytes)
|
||||
n, err := in.Read(buf)
|
||||
@@ -446,7 +525,7 @@ func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool,
|
||||
return nil, false, "", err
|
||||
}
|
||||
mime := mimetype.Detect(buf)
|
||||
compressible, err = isCompressible(bytes.NewReader(buf))
|
||||
compressible, err = modeHandler.isCompressible(bytes.NewReader(buf), compressionMode)
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
}
|
||||
@@ -454,26 +533,6 @@ func checkCompressAndType(in io.Reader) (newReader io.Reader, compressible bool,
|
||||
return wrap(in), compressible, mime.String(), nil
|
||||
}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func isCompressible(r io.Reader) (bool, error) {
|
||||
var b bytes.Buffer
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err := io.Copy(w, r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
// verifyObjectHash verifies the Objects hash
|
||||
func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.MultiHasher, ht hash.Type) error {
|
||||
srcHash := hasher.Sums()[ht]
|
||||
@@ -494,9 +553,9 @@ func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.Mul
|
||||
|
||||
type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
|
||||
|
||||
type compressionResult struct {
|
||||
type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
|
||||
err error
|
||||
meta sgzip.GzipMetadata
|
||||
meta T
|
||||
}
|
||||
|
||||
// replicating some of operations.Rcat functionality because we want to support remotes without streaming
|
||||
@@ -537,106 +596,18 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
|
||||
return nil, fmt.Errorf("failed to write temporary local file: %w", err)
|
||||
}
|
||||
if _, err = tempFile.Seek(0, 0); err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("failed to seek temporary local file: %w", err)
|
||||
}
|
||||
finfo, err := tempFile.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("failed to stat temporary local file: %w", err)
|
||||
}
|
||||
return f.Fs.Put(ctx, tempFile, object.NewStaticObjectInfo(dstFileName, modTime, finfo.Size(), false, nil, f.Fs))
|
||||
}
|
||||
|
||||
// Put a compressed version of a file. Returns a wrappable object and metadata.
|
||||
func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, mimeType string) (fs.Object, *ObjectMetadata, error) {
|
||||
// Unwrap reader accounting
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
|
||||
// Add the metadata hasher
|
||||
metaHasher := md5.New()
|
||||
in = io.TeeReader(in, metaHasher)
|
||||
|
||||
// Compress the file
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
results := make(chan compressionResult)
|
||||
go func() {
|
||||
gz, err := sgzip.NewWriterLevel(pipeWriter, f.opt.CompressionLevel)
|
||||
if err != nil {
|
||||
results <- compressionResult{err: err, meta: sgzip.GzipMetadata{}}
|
||||
return
|
||||
}
|
||||
_, err = io.Copy(gz, in)
|
||||
gzErr := gz.Close()
|
||||
if gzErr != nil {
|
||||
fs.Errorf(nil, "Failed to close compress: %v", gzErr)
|
||||
if err == nil {
|
||||
err = gzErr
|
||||
}
|
||||
}
|
||||
closeErr := pipeWriter.Close()
|
||||
if closeErr != nil {
|
||||
fs.Errorf(nil, "Failed to close pipe: %v", closeErr)
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}
|
||||
results <- compressionResult{err: err, meta: gz.MetaData()}
|
||||
}()
|
||||
wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize)) // Probably no longer needed as sgzip has it's own buffering
|
||||
|
||||
// Find a hash the destination supports to compute a hash of
|
||||
// the compressed data.
|
||||
ht := f.Fs.Hashes().GetOne()
|
||||
var hasher *hash.MultiHasher
|
||||
var err error
|
||||
if ht != hash.None {
|
||||
// unwrap the accounting again
|
||||
wrappedIn, wrap = accounting.UnWrap(wrappedIn)
|
||||
hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// add the hasher and re-wrap the accounting
|
||||
wrappedIn = io.TeeReader(wrappedIn, hasher)
|
||||
wrappedIn = wrap(wrappedIn)
|
||||
}
|
||||
|
||||
// Transfer the data
|
||||
o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
|
||||
//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
|
||||
if err != nil {
|
||||
if o != nil {
|
||||
removeErr := o.Remove(ctx)
|
||||
if removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially transferred object: %v", err)
|
||||
}
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
// Check whether we got an error during compression
|
||||
result := <-results
|
||||
err = result.err
|
||||
if err != nil {
|
||||
if o != nil {
|
||||
removeErr := o.Remove(ctx)
|
||||
if removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially compressed object: %v", err)
|
||||
}
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Generate metadata
|
||||
meta := newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)
|
||||
|
||||
// Check the hashes of the compressed data if we were comparing them
|
||||
if ht != hash.None && hasher != nil {
|
||||
err = f.verifyObjectHash(ctx, o, hasher, ht)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return o, meta, nil
|
||||
return f.modeHandler.putCompress(ctx, f, in, src, options, mimeType)
|
||||
}
|
||||
|
||||
// Put an uncompressed version of a file. Returns a wrappable object and metadata.
|
||||
@@ -680,7 +651,8 @@ func (f *Fs) putUncompress(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return o, newMetadata(o.Size(), Uncompressed, sgzip.GzipMetadata{}, hex.EncodeToString(sum), mimeType), nil
|
||||
|
||||
return f.modeHandler.putUncompressGetNewMetadata(o, Uncompressed, hex.EncodeToString(sum), mimeType, sum)
|
||||
}
|
||||
|
||||
// This function will write a metadata struct to a metadata Object for an src. Returns a wrappable metadata object.
|
||||
@@ -751,7 +723,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
o, err := f.NewObject(ctx, src.Remote())
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// Get our file compressibility
|
||||
in, compressible, mimeType, err := checkCompressAndType(in)
|
||||
in, compressible, mimeType, err := checkCompressAndType(in, f.mode, f.modeHandler)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -771,7 +743,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
}
|
||||
found := err == nil
|
||||
|
||||
in, compressible, mimeType, err := checkCompressAndType(in)
|
||||
in, compressible, mimeType, err := checkCompressAndType(in, f.mode, f.modeHandler)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1090,11 +1062,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, duration fs.Duration
|
||||
|
||||
// ObjectMetadata describes the metadata for an Object.
|
||||
type ObjectMetadata struct {
|
||||
Mode int // Compression mode of the file.
|
||||
Size int64 // Size of the object.
|
||||
MD5 string // MD5 hash of the file.
|
||||
MimeType string // Mime type of the file
|
||||
CompressionMetadata sgzip.GzipMetadata
|
||||
Mode int // Compression mode of the file.
|
||||
Size int64 // Size of the object.
|
||||
MD5 string // MD5 hash of the file.
|
||||
MimeType string // Mime type of the file
|
||||
CompressionMetadataGzip *sgzip.GzipMetadata // Metadata for Gzip compression
|
||||
CompressionMetadataZstd *SzstdMetadata // Metadata for Zstd compression
|
||||
}
|
||||
|
||||
// Object with external metadata
|
||||
@@ -1107,17 +1080,6 @@ type Object struct {
|
||||
meta *ObjectMetadata // Metadata struct for this object (nil if not loaded)
|
||||
}
|
||||
|
||||
// This function generates a metadata object
|
||||
func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mimeType string) *ObjectMetadata {
|
||||
meta := new(ObjectMetadata)
|
||||
meta.Size = size
|
||||
meta.Mode = mode
|
||||
meta.CompressionMetadata = cmeta
|
||||
meta.MD5 = md5
|
||||
meta.MimeType = mimeType
|
||||
return meta
|
||||
}
|
||||
|
||||
// This function will read the metadata from a metadata object.
|
||||
func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {
|
||||
// Open our meradata object
|
||||
@@ -1165,7 +1127,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return o.mo, o.mo.Update(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
in, compressible, mimeType, err := checkCompressAndType(in)
|
||||
in, compressible, mimeType, err := checkCompressAndType(in, o.meta.Mode, o.f.modeHandler)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1278,7 +1240,7 @@ func (o *Object) String() string {
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *Object) Remote() string {
|
||||
origFileName, _, _, err := processFileName(o.Object.Remote())
|
||||
origFileName, _, _, err := processFileName(o.Object.Remote(), o.f.modeHandler)
|
||||
if err != nil {
|
||||
fs.Errorf(o.f, "Could not get remote path for: %s", o.Object.Remote())
|
||||
return o.Object.Remote()
|
||||
@@ -1381,7 +1343,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
return o.Object.Open(ctx, options...)
|
||||
}
|
||||
// Get offset and limit from OpenOptions, pass the rest to the underlying remote
|
||||
var openOptions = []fs.OpenOption{&fs.SeekOption{Offset: 0}}
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
@@ -1389,31 +1350,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
|
||||
offset = x.Offset
|
||||
case *fs.RangeOption:
|
||||
offset, limit = x.Decode(o.Size())
|
||||
default:
|
||||
openOptions = append(openOptions, option)
|
||||
}
|
||||
}
|
||||
// Get a chunkedreader for the wrapped object
|
||||
chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize, chunkStreams)
|
||||
// Get file handle
|
||||
var file io.Reader
|
||||
if offset != 0 {
|
||||
file, err = sgzip.NewReaderAt(chunkedReader, &o.meta.CompressionMetadata, offset)
|
||||
} else {
|
||||
file, err = sgzip.NewReader(chunkedReader)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fileReader io.Reader
|
||||
if limit != -1 {
|
||||
fileReader = io.LimitReader(file, limit)
|
||||
} else {
|
||||
fileReader = file
|
||||
}
|
||||
// Return a ReadCloser
|
||||
return ReadCloserWrapper{Reader: fileReader, Closer: chunkedReader}, nil
|
||||
var retCloser io.Closer = chunkedReader
|
||||
return o.f.modeHandler.openGetReadCloser(ctx, o, offset, limit, chunkedReader, retCloser, options...)
|
||||
}
|
||||
|
||||
// ObjectInfo describes a wrapped fs.ObjectInfo for being the source
|
||||
|
||||
@@ -48,7 +48,27 @@ func TestRemoteGzip(t *testing.T) {
|
||||
opt.ExtraConfig = []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "compress"},
|
||||
{Name: name, Key: "remote", Value: tempdir},
|
||||
{Name: name, Key: "compression_mode", Value: "gzip"},
|
||||
{Name: name, Key: "mode", Value: "gzip"},
|
||||
{Name: name, Key: "level", Value: "-1"},
|
||||
}
|
||||
opt.QuickTestOK = true
|
||||
fstests.Run(t, &opt)
|
||||
}
|
||||
|
||||
// TestRemoteZstd tests ZSTD compression
|
||||
func TestRemoteZstd(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
tempdir := filepath.Join(os.TempDir(), "rclone-compress-test-zstd")
|
||||
name := "TestCompressZstd"
|
||||
opt := defaultOpt
|
||||
opt.RemoteName = name + ":"
|
||||
opt.ExtraConfig = []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "compress"},
|
||||
{Name: name, Key: "remote", Value: tempdir},
|
||||
{Name: name, Key: "mode", Value: "zstd"},
|
||||
{Name: name, Key: "level", Value: "2"},
|
||||
}
|
||||
opt.QuickTestOK = true
|
||||
fstests.Run(t, &opt)
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/buengese/sgzip"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
// gzipModeHandler implements compressionModeHandler for gzip
|
||||
type gzipModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err = io.Copy(w, r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (g *gzipModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
if meta.CompressionMetadataGzip == nil {
|
||||
return 0, errors.New("missing gzip metadata")
|
||||
}
|
||||
return meta.CompressionMetadataGzip.Size, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (g *gzipModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
var file io.Reader
|
||||
|
||||
if offset != 0 {
|
||||
file, err = sgzip.NewReaderAt(cr, o.meta.CompressionMetadataGzip, offset)
|
||||
} else {
|
||||
file, err = sgzip.NewReader(cr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fileReader io.Reader
|
||||
if limit != -1 {
|
||||
fileReader = io.LimitReader(file, limit)
|
||||
} else {
|
||||
fileReader = file
|
||||
}
|
||||
// Return a ReadCloser
|
||||
return ReadCloserWrapper{Reader: fileReader, Closer: closer}, nil
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (g *gzipModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
if compressionMode == Gzip {
|
||||
return gzFileExt
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (g *gzipModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
// Unwrap reader accounting
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
|
||||
// Add the metadata hasher
|
||||
metaHasher := md5.New()
|
||||
in = io.TeeReader(in, metaHasher)
|
||||
|
||||
// Compress the file
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
|
||||
resultsGzip := make(chan compressionResult[sgzip.GzipMetadata])
|
||||
go func() {
|
||||
gz, err := sgzip.NewWriterLevel(pipeWriter, f.opt.CompressionLevel)
|
||||
if err != nil {
|
||||
resultsGzip <- compressionResult[sgzip.GzipMetadata]{err: err, meta: sgzip.GzipMetadata{}}
|
||||
close(resultsGzip)
|
||||
return
|
||||
}
|
||||
_, err = io.Copy(gz, in)
|
||||
gzErr := gz.Close()
|
||||
if gzErr != nil && err == nil {
|
||||
err = gzErr
|
||||
}
|
||||
closeErr := pipeWriter.Close()
|
||||
if closeErr != nil && err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
resultsGzip <- compressionResult[sgzip.GzipMetadata]{err: err, meta: gz.MetaData()}
|
||||
close(resultsGzip)
|
||||
}()
|
||||
|
||||
wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize)) // Probably no longer needed as sgzip has it's own buffering
|
||||
|
||||
// Find a hash the destination supports to compute a hash of
|
||||
// the compressed data.
|
||||
ht := f.Fs.Hashes().GetOne()
|
||||
var hasher *hash.MultiHasher
|
||||
var err error
|
||||
if ht != hash.None {
|
||||
// unwrap the accounting again
|
||||
wrappedIn, wrap = accounting.UnWrap(wrappedIn)
|
||||
hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// add the hasher and re-wrap the accounting
|
||||
wrappedIn = io.TeeReader(wrappedIn, hasher)
|
||||
wrappedIn = wrap(wrappedIn)
|
||||
}
|
||||
|
||||
// Transfer the data
|
||||
o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
|
||||
if err != nil {
|
||||
if o != nil {
|
||||
if removeErr := o.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
|
||||
}
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
// Check whether we got an error during compression
|
||||
result := <-resultsGzip
|
||||
if result.err != nil {
|
||||
if o != nil {
|
||||
if removeErr := o.Remove(ctx); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove partially compressed object: %v", removeErr)
|
||||
}
|
||||
}
|
||||
return nil, nil, result.err
|
||||
}
|
||||
|
||||
// Generate metadata
|
||||
meta := g.newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)
|
||||
|
||||
// Check the hashes of the compressed data if we were comparing them
|
||||
if ht != hash.None && hasher != nil {
|
||||
err = f.verifyObjectHash(ctx, o, hasher, ht)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return o, meta, nil
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (g *gzipModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return o, g.newMetadata(o.Size(), mode, sgzip.GzipMetadata{}, hex.EncodeToString(sum), mimeType), nil
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (g *gzipModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
meta, ok := cmeta.(sgzip.GzipMetadata)
|
||||
if !ok {
|
||||
panic("invalid cmeta type: expected sgzip.GzipMetadata")
|
||||
}
|
||||
|
||||
objMeta := new(ObjectMetadata)
|
||||
objMeta.Size = size
|
||||
objMeta.Mode = mode
|
||||
objMeta.CompressionMetadataGzip = &meta
|
||||
objMeta.CompressionMetadataZstd = nil
|
||||
objMeta.MD5 = md5
|
||||
objMeta.MimeType = mimeType
|
||||
|
||||
return objMeta
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
szstd "github.com/a1ex3/zstd-seekable-format-go/pkg"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
const szstdChunkSize int = 1 << 20 // 1 MiB chunk size
|
||||
|
||||
// SzstdMetadata holds metadata for szstd compressed files.
|
||||
type SzstdMetadata struct {
|
||||
BlockSize int // BlockSize is the size of the blocks in the zstd file
|
||||
Size int64 // Size is the uncompressed size of the file
|
||||
BlockData []uint32 // BlockData is the block data for the zstd file, used for seeking
|
||||
}
|
||||
|
||||
// SzstdWriter is a writer that compresses data in szstd format.
|
||||
type SzstdWriter struct {
|
||||
enc *zstd.Encoder
|
||||
w szstd.ConcurrentWriter
|
||||
metadata SzstdMetadata
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewWriterSzstd creates a new szstd writer with the specified options.
|
||||
// It initializes the szstd writer with a zstd encoder and returns a pointer to the SzstdWriter.
|
||||
// The writer can be used to write data in chunks, and it will automatically handle block sizes and metadata.
|
||||
func NewWriterSzstd(w io.Writer, opts ...zstd.EOption) (*SzstdWriter, error) {
|
||||
encoder, err := zstd.NewWriter(nil, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sw, err := szstd.NewWriter(w, encoder)
|
||||
if err != nil {
|
||||
if err := encoder.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &SzstdWriter{
|
||||
enc: encoder,
|
||||
w: sw,
|
||||
metadata: SzstdMetadata{
|
||||
BlockSize: szstdChunkSize,
|
||||
Size: 0,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Write writes data to the szstd writer in chunks of szstdChunkSize.
|
||||
// It handles the block size and metadata updates automatically.
|
||||
func (w *SzstdWriter) Write(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if w.metadata.BlockData == nil {
|
||||
numBlocks := (len(p) + w.metadata.BlockSize - 1) / w.metadata.BlockSize
|
||||
w.metadata.BlockData = make([]uint32, 1, numBlocks+1)
|
||||
w.metadata.BlockData[0] = 0
|
||||
}
|
||||
|
||||
start := 0
|
||||
total := len(p)
|
||||
|
||||
var writerFunc szstd.FrameSource = func() ([]byte, error) {
|
||||
if start >= total {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
end := min(start+w.metadata.BlockSize, total)
|
||||
chunk := p[start:end]
|
||||
size := end - start
|
||||
|
||||
w.mu.Lock()
|
||||
w.metadata.Size += int64(size)
|
||||
w.mu.Unlock()
|
||||
|
||||
start = end
|
||||
return chunk, nil
|
||||
}
|
||||
|
||||
// write sizes of compressed blocks in the callback
|
||||
err := w.w.WriteMany(context.Background(), writerFunc,
|
||||
szstd.WithWriteCallback(func(size uint32) {
|
||||
w.mu.Lock()
|
||||
lastOffset := w.metadata.BlockData[len(w.metadata.BlockData)-1]
|
||||
w.metadata.BlockData = append(w.metadata.BlockData, lastOffset+size)
|
||||
w.mu.Unlock()
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// Close closes the SzstdWriter and its underlying encoder.
|
||||
func (w *SzstdWriter) Close() error {
|
||||
if err := w.w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.enc.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetMetadata returns the metadata of the szstd writer.
|
||||
func (w *SzstdWriter) GetMetadata() SzstdMetadata {
|
||||
return w.metadata
|
||||
}
|
||||
|
||||
// SzstdReaderAt is a reader that allows random access in szstd compressed data.
|
||||
type SzstdReaderAt struct {
|
||||
r szstd.Reader
|
||||
decoder *zstd.Decoder
|
||||
metadata *SzstdMetadata
|
||||
pos int64
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewReaderAtSzstd creates a new SzstdReaderAt at the specified io.ReadSeeker.
|
||||
func NewReaderAtSzstd(rs io.ReadSeeker, meta *SzstdMetadata, offset int64, opts ...zstd.DOption) (*SzstdReaderAt, error) {
|
||||
decoder, err := zstd.NewReader(nil, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, err := szstd.NewReader(rs, decoder)
|
||||
if err != nil {
|
||||
decoder.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sr := &SzstdReaderAt{
|
||||
r: r,
|
||||
decoder: decoder,
|
||||
metadata: meta,
|
||||
pos: 0,
|
||||
}
|
||||
|
||||
// Set initial position to the provided offset
|
||||
if _, err := sr.Seek(offset, io.SeekStart); err != nil {
|
||||
if err := sr.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return sr, nil
|
||||
}
|
||||
|
||||
// Seek sets the offset for the next Read.
|
||||
func (s *SzstdReaderAt) Seek(offset int64, whence int) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
pos, err := s.r.Seek(offset, whence)
|
||||
if err == nil {
|
||||
s.pos = pos
|
||||
}
|
||||
return pos, err
|
||||
}
|
||||
|
||||
func (s *SzstdReaderAt) Read(p []byte) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
n, err := s.r.Read(p)
|
||||
if err == nil {
|
||||
s.pos += int64(n)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadAt reads data at the specified offset.
|
||||
func (s *SzstdReaderAt) ReadAt(p []byte, off int64) (int, error) {
|
||||
if off < 0 {
|
||||
return 0, errors.New("invalid offset")
|
||||
}
|
||||
if off >= s.metadata.Size {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
endOff := min(off+int64(len(p)), s.metadata.Size)
|
||||
|
||||
// Find all blocks covered by the range
|
||||
type blockInfo struct {
|
||||
index int // Block index
|
||||
offsetInBlock int64 // Offset within the block for starting reading
|
||||
bytesToRead int64 // How many bytes to read from this block
|
||||
}
|
||||
|
||||
var blocks []blockInfo
|
||||
uncompressedOffset := int64(0)
|
||||
currentOff := off
|
||||
|
||||
for i := 0; i < len(s.metadata.BlockData)-1; i++ {
|
||||
blockUncompressedEnd := min(uncompressedOffset+int64(s.metadata.BlockSize), s.metadata.Size)
|
||||
|
||||
if currentOff < blockUncompressedEnd && endOff > uncompressedOffset {
|
||||
offsetInBlock := max(0, currentOff-uncompressedOffset)
|
||||
bytesToRead := min(blockUncompressedEnd-uncompressedOffset-offsetInBlock, endOff-currentOff)
|
||||
|
||||
blocks = append(blocks, blockInfo{
|
||||
index: i,
|
||||
offsetInBlock: offsetInBlock,
|
||||
bytesToRead: bytesToRead,
|
||||
})
|
||||
|
||||
currentOff += bytesToRead
|
||||
if currentOff >= endOff {
|
||||
break
|
||||
}
|
||||
}
|
||||
uncompressedOffset = blockUncompressedEnd
|
||||
}
|
||||
|
||||
if len(blocks) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// Parallel block decoding
|
||||
type decodeResult struct {
|
||||
index int
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
|
||||
resultCh := make(chan decodeResult, len(blocks))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, runtime.NumCPU())
|
||||
|
||||
for _, block := range blocks {
|
||||
wg.Add(1)
|
||||
go func(block blockInfo) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
|
||||
startOffset := int64(s.metadata.BlockData[block.index])
|
||||
endOffset := int64(s.metadata.BlockData[block.index+1])
|
||||
compressedSize := endOffset - startOffset
|
||||
|
||||
compressed := make([]byte, compressedSize)
|
||||
_, err := s.r.ReadAt(compressed, startOffset)
|
||||
if err != nil && err != io.EOF {
|
||||
resultCh <- decodeResult{index: block.index, err: err}
|
||||
return
|
||||
}
|
||||
|
||||
decoded, err := s.decoder.DecodeAll(compressed, nil)
|
||||
if err != nil {
|
||||
resultCh <- decodeResult{index: block.index, err: err}
|
||||
return
|
||||
}
|
||||
|
||||
resultCh <- decodeResult{index: block.index, data: decoded, err: nil}
|
||||
}(block)
|
||||
}
|
||||
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(resultCh)
|
||||
}()
|
||||
|
||||
// Collect results in block index order
|
||||
totalRead := 0
|
||||
results := make(map[int]decodeResult)
|
||||
expected := len(blocks)
|
||||
minIndex := blocks[0].index
|
||||
|
||||
for res := range resultCh {
|
||||
results[res.index] = res
|
||||
for {
|
||||
if result, ok := results[minIndex]; ok {
|
||||
if result.err != nil {
|
||||
return 0, result.err
|
||||
}
|
||||
// find the corresponding blockInfo
|
||||
var blk blockInfo
|
||||
for _, b := range blocks {
|
||||
if b.index == result.index {
|
||||
blk = b
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
start := blk.offsetInBlock
|
||||
end := start + blk.bytesToRead
|
||||
copy(p[totalRead:totalRead+int(blk.bytesToRead)], result.data[start:end])
|
||||
totalRead += int(blk.bytesToRead)
|
||||
minIndex++
|
||||
if minIndex-blocks[0].index >= len(blocks) {
|
||||
break
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(results) == expected && minIndex-blocks[0].index >= len(blocks) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return totalRead, nil
|
||||
}
|
||||
|
||||
// Close closes the SzstdReaderAt and underlying decoder.
|
||||
func (s *SzstdReaderAt) Close() error {
|
||||
if err := s.r.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.decoder.Close()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
)
|
||||
|
||||
// uncompressedModeHandler implements compressionModeHandler for uncompressed files
|
||||
type uncompressedModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (u *uncompressedModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (u *uncompressedModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (u *uncompressedModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
return o.Object.Open(ctx, options...)
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (u *uncompressedModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (u *uncompressedModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unsupported compression mode %d", f.mode)
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (u *uncompressedModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unsupported compression mode %d", Uncompressed)
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (u *uncompressedModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
)
|
||||
|
||||
// unknownModeHandler implements compressionModeHandler for unknown compression types
|
||||
type unknownModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (unk *unknownModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
return false, fmt.Errorf("unknown compression mode %d", compressionMode)
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (unk *unknownModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (unk *unknownModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
return nil, fmt.Errorf("unknown compression mode %d", o.meta.Mode)
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (unk *unknownModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (unk *unknownModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unknown compression mode %d", f.mode)
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (unk *unknownModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return nil, nil, fmt.Errorf("unknown compression mode")
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (unk *unknownModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package compress
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunkedreader"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
)
|
||||
|
||||
// zstdModeHandler implements compressionModeHandler for zstd
|
||||
type zstdModeHandler struct{}
|
||||
|
||||
// isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
|
||||
// the configured threshold
|
||||
func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
|
||||
var b bytes.Buffer
|
||||
var n int64
|
||||
w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, err = io.Copy(w, r)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ratio := float64(n) / float64(b.Len())
|
||||
return ratio > minCompressionRatio, nil
|
||||
}
|
||||
|
||||
// newObjectGetOriginalSize returns the original file size from the metadata
|
||||
func (z *zstdModeHandler) newObjectGetOriginalSize(meta *ObjectMetadata) (int64, error) {
|
||||
if meta.CompressionMetadataZstd == nil {
|
||||
return 0, errors.New("missing zstd metadata")
|
||||
}
|
||||
return meta.CompressionMetadataZstd.Size, nil
|
||||
}
|
||||
|
||||
// openGetReadCloser opens a compressed object and returns a ReadCloser in the Open method
|
||||
func (z *zstdModeHandler) openGetReadCloser(
|
||||
ctx context.Context,
|
||||
o *Object,
|
||||
offset int64,
|
||||
limit int64,
|
||||
cr chunkedreader.ChunkedReader,
|
||||
closer io.Closer,
|
||||
options ...fs.OpenOption,
|
||||
) (rc io.ReadCloser, err error) {
|
||||
var file io.Reader
|
||||
|
||||
if offset != 0 {
|
||||
file, err = NewReaderAtSzstd(cr, o.meta.CompressionMetadataZstd, offset)
|
||||
} else {
|
||||
file, err = zstd.NewReader(cr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fileReader io.Reader
|
||||
if limit != -1 {
|
||||
fileReader = io.LimitReader(file, limit)
|
||||
} else {
|
||||
fileReader = file
|
||||
}
|
||||
// Return a ReadCloser
|
||||
return ReadCloserWrapper{Reader: fileReader, Closer: closer}, nil
|
||||
}
|
||||
|
||||
// processFileNameGetFileExtension returns the file extension for the given compression mode
|
||||
func (z *zstdModeHandler) processFileNameGetFileExtension(compressionMode int) string {
|
||||
if compressionMode == Zstd {
|
||||
return zstdFileExt
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// putCompress compresses the input data and uploads it to the remote, returning the new object and its metadata
|
||||
func (z *zstdModeHandler) putCompress(
|
||||
ctx context.Context,
|
||||
f *Fs,
|
||||
in io.Reader,
|
||||
src fs.ObjectInfo,
|
||||
options []fs.OpenOption,
|
||||
mimeType string,
|
||||
) (fs.Object, *ObjectMetadata, error) {
|
||||
// Unwrap reader accounting
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
|
||||
// Add the metadata hasher
|
||||
metaHasher := md5.New()
|
||||
in = io.TeeReader(in, metaHasher)
|
||||
|
||||
// Compress the file
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
|
||||
resultsZstd := make(chan compressionResult[SzstdMetadata])
|
||||
go func() {
|
||||
writer, err := NewWriterSzstd(pipeWriter, zstd.WithEncoderLevel(zstd.EncoderLevel(f.opt.CompressionLevel)))
|
||||
if err != nil {
|
||||
resultsZstd <- compressionResult[SzstdMetadata]{err: err}
|
||||
close(resultsZstd)
|
||||
return
|
||||
}
|
||||
_, err = io.Copy(writer, in)
|
||||
if wErr := writer.Close(); wErr != nil && err == nil {
|
||||
err = wErr
|
||||
}
|
||||
if cErr := pipeWriter.Close(); cErr != nil && err == nil {
|
||||
err = cErr
|
||||
}
|
||||
|
||||
resultsZstd <- compressionResult[SzstdMetadata]{err: err, meta: writer.GetMetadata()}
|
||||
close(resultsZstd)
|
||||
}()
|
||||
|
||||
wrappedIn := wrap(bufio.NewReaderSize(pipeReader, bufferSize))
|
||||
|
||||
ht := f.Fs.Hashes().GetOne()
|
||||
var hasher *hash.MultiHasher
|
||||
var err error
|
||||
if ht != hash.None {
|
||||
wrappedIn, wrap = accounting.UnWrap(wrappedIn)
|
||||
hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
wrappedIn = io.TeeReader(wrappedIn, hasher)
|
||||
wrappedIn = wrap(wrappedIn)
|
||||
}
|
||||
|
||||
o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
result := <-resultsZstd
|
||||
if result.err != nil {
|
||||
if o != nil {
|
||||
_ = o.Remove(ctx)
|
||||
}
|
||||
return nil, nil, result.err
|
||||
}
|
||||
|
||||
// Build metadata using uncompressed size for filename
|
||||
meta := z.newMetadata(result.meta.Size, f.mode, result.meta, hex.EncodeToString(metaHasher.Sum(nil)), mimeType)
|
||||
if ht != hash.None && hasher != nil {
|
||||
err = f.verifyObjectHash(ctx, o, hasher, ht)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
return o, meta, nil
|
||||
}
|
||||
|
||||
// putUncompressGetNewMetadata returns metadata in the putUncompress method for a specific compression algorithm
|
||||
func (z *zstdModeHandler) putUncompressGetNewMetadata(o fs.Object, mode int, md5 string, mimeType string, sum []byte) (fs.Object, *ObjectMetadata, error) {
|
||||
return o, z.newMetadata(o.Size(), mode, SzstdMetadata{}, hex.EncodeToString(sum), mimeType), nil
|
||||
}
|
||||
|
||||
// This function generates a metadata object for sgzip.GzipMetadata or SzstdMetadata.
|
||||
// Warning: This function panics if cmeta is not of the expected type.
|
||||
func (z *zstdModeHandler) newMetadata(size int64, mode int, cmeta any, md5 string, mimeType string) *ObjectMetadata {
|
||||
meta, ok := cmeta.(SzstdMetadata)
|
||||
if !ok {
|
||||
panic("invalid cmeta type: expected SzstdMetadata")
|
||||
}
|
||||
|
||||
objMeta := new(ObjectMetadata)
|
||||
objMeta.Size = size
|
||||
objMeta.Mode = mode
|
||||
objMeta.CompressionMetadataGzip = nil
|
||||
objMeta.CompressionMetadataZstd = &meta
|
||||
objMeta.MD5 = md5
|
||||
objMeta.MimeType = mimeType
|
||||
|
||||
return objMeta
|
||||
}
|
||||
@@ -403,14 +403,14 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
if ciphertext == "" {
|
||||
return "", nil
|
||||
}
|
||||
pos := strings.Index(ciphertext, ".")
|
||||
if pos == -1 {
|
||||
before, after, ok := strings.Cut(ciphertext, ".")
|
||||
if !ok {
|
||||
return "", ErrorNotAnEncryptedFile
|
||||
} // No .
|
||||
num := ciphertext[:pos]
|
||||
num := before
|
||||
if num == "!" {
|
||||
// No rotation; probably original was not valid unicode
|
||||
return ciphertext[pos+1:], nil
|
||||
return after, nil
|
||||
}
|
||||
dir, err := strconv.Atoi(num)
|
||||
if err != nil {
|
||||
@@ -425,7 +425,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
|
||||
var result bytes.Buffer
|
||||
|
||||
inQuote := false
|
||||
for _, runeValue := range ciphertext[pos+1:] {
|
||||
for _, runeValue := range after {
|
||||
switch {
|
||||
case inQuote:
|
||||
_, _ = result.WriteRune(runeValue)
|
||||
|
||||
+12
-10
@@ -923,28 +923,30 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
|
||||
var commandHelp = []fs.CommandHelp{
|
||||
{
|
||||
Name: "encode",
|
||||
Short: "Encode the given filename(s)",
|
||||
Short: "Encode the given filename(s).",
|
||||
Long: `This encodes the filenames given as arguments returning a list of
|
||||
strings of the encoded results.
|
||||
|
||||
Usage Example:
|
||||
Usage examples:
|
||||
|
||||
rclone backend encode crypt: file1 [file2...]
|
||||
rclone rc backend/command command=encode fs=crypt: file1 [file2...]
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend encode crypt: file1 [file2...]
|
||||
rclone rc backend/command command=encode fs=crypt: file1 [file2...]
|
||||
` + "```",
|
||||
},
|
||||
{
|
||||
Name: "decode",
|
||||
Short: "Decode the given filename(s)",
|
||||
Short: "Decode the given filename(s).",
|
||||
Long: `This decodes the filenames given as arguments returning a list of
|
||||
strings of the decoded results. It will return an error if any of the
|
||||
inputs are invalid.
|
||||
|
||||
Usage Example:
|
||||
Usage examples:
|
||||
|
||||
rclone backend decode crypt: encryptedfile1 [encryptedfile2...]
|
||||
rclone rc backend/command command=decode fs=crypt: encryptedfile1 [encryptedfile2...]
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend decode crypt: encryptedfile1 [encryptedfile2...]
|
||||
rclone rc backend/command command=decode fs=crypt: encryptedfile1 [encryptedfile2...]
|
||||
` + "```",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+14
-10
@@ -77,7 +77,7 @@ The DOI provider can be set when rclone does not automatically recognize a suppo
|
||||
Name: "doi_resolver_api_url",
|
||||
Help: `The URL of the DOI resolver API to use.
|
||||
|
||||
The DOI resolver can be set for testing or for cases when the the canonical DOI resolver API cannot be used.
|
||||
The DOI resolver can be set for testing or for cases when the canonical DOI resolver API cannot be used.
|
||||
|
||||
Defaults to "https://doi.org/api".`,
|
||||
Required: false,
|
||||
@@ -563,21 +563,26 @@ var commandHelp = []fs.CommandHelp{{
|
||||
Short: "Show metadata about the DOI.",
|
||||
Long: `This command returns a JSON object with some information about the DOI.
|
||||
|
||||
rclone backend medatadata doi:
|
||||
Usage example:
|
||||
|
||||
It returns a JSON object representing metadata about the DOI.
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend metadata doi:
|
||||
` + "```" + `
|
||||
|
||||
It returns a JSON object representing metadata about the DOI.`,
|
||||
}, {
|
||||
Name: "set",
|
||||
Short: "Set command for updating the config parameters.",
|
||||
Long: `This set command can be used to update the config parameters
|
||||
for a running doi backend.
|
||||
|
||||
Usage Examples:
|
||||
Usage examples:
|
||||
|
||||
rclone backend set doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: -o doi=NEW_DOI
|
||||
` + "```console" + `
|
||||
rclone backend set doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=doi: -o doi=NEW_DOI
|
||||
` + "```" + `
|
||||
|
||||
The option keys are named as they are in the config file.
|
||||
|
||||
@@ -585,8 +590,7 @@ This rebuilds the connection to the doi backend when it is called with
|
||||
the new parameters. Only new parameters need be passed as the values
|
||||
will default to those currently in use.
|
||||
|
||||
It doesn't return anything.
|
||||
`,
|
||||
It doesn't return anything.`,
|
||||
}}
|
||||
|
||||
// Command the backend to run a named command
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
// Package api has type definitions for drime
|
||||
//
|
||||
// Converted from the API docs with help from https://mholt.github.io/json-to-go/
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Types of things in Item
|
||||
const (
|
||||
ItemTypeFolder = "folder"
|
||||
)
|
||||
|
||||
// User information
|
||||
type User struct {
|
||||
Email string `json:"email"`
|
||||
ID json.Number `json:"id"`
|
||||
Avatar string `json:"avatar"`
|
||||
ModelType string `json:"model_type"`
|
||||
OwnsEntry bool `json:"owns_entry"`
|
||||
EntryPermissions []any `json:"entry_permissions"`
|
||||
DisplayName string `json:"display_name"`
|
||||
}
|
||||
|
||||
// Permissions for a file
|
||||
type Permissions struct {
|
||||
FilesUpdate bool `json:"files.update"`
|
||||
FilesCreate bool `json:"files.create"`
|
||||
FilesDownload bool `json:"files.download"`
|
||||
FilesDelete bool `json:"files.delete"`
|
||||
}
|
||||
|
||||
// Item describes a folder or a file as returned by /drive/file-entries
|
||||
type Item struct {
|
||||
ID json.Number `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description any `json:"description"`
|
||||
FileName string `json:"file_name"`
|
||||
Mime string `json:"mime"`
|
||||
Color any `json:"color"`
|
||||
Backup bool `json:"backup"`
|
||||
Tracked int `json:"tracked"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
UserID json.Number `json:"user_id"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt any `json:"deleted_at"`
|
||||
IsDeleted int `json:"is_deleted"`
|
||||
Path string `json:"path"`
|
||||
DiskPrefix any `json:"disk_prefix"`
|
||||
Type string `json:"type"`
|
||||
Extension any `json:"extension"`
|
||||
FileHash any `json:"file_hash"`
|
||||
Public bool `json:"public"`
|
||||
Thumbnail bool `json:"thumbnail"`
|
||||
MuxStatus any `json:"mux_status"`
|
||||
ThumbnailURL any `json:"thumbnail_url"`
|
||||
WorkspaceID int `json:"workspace_id"`
|
||||
IsEncrypted int `json:"is_encrypted"`
|
||||
Iv any `json:"iv"`
|
||||
VaultID any `json:"vault_id"`
|
||||
OwnerID int `json:"owner_id"`
|
||||
Hash string `json:"hash"`
|
||||
URL string `json:"url"`
|
||||
Users []User `json:"users"`
|
||||
Tags []any `json:"tags"`
|
||||
Permissions Permissions `json:"permissions"`
|
||||
}
|
||||
|
||||
// Listing response
|
||||
type Listing struct {
|
||||
CurrentPage int `json:"current_page"`
|
||||
Data []Item `json:"data"`
|
||||
From int `json:"from"`
|
||||
LastPage int `json:"last_page"`
|
||||
NextPage int `json:"next_page"`
|
||||
PerPage int `json:"per_page"`
|
||||
PrevPage int `json:"prev_page"`
|
||||
To int `json:"to"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// UploadResponse for a file
|
||||
type UploadResponse struct {
|
||||
Status string `json:"status"`
|
||||
FileEntry Item `json:"fileEntry"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest for a folder
|
||||
type CreateFolderRequest struct {
|
||||
Name string `json:"name"`
|
||||
ParentID json.Number `json:"parentId,omitempty"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse for a folder
|
||||
type CreateFolderResponse struct {
|
||||
Status string `json:"status"`
|
||||
Folder Item `json:"folder"`
|
||||
}
|
||||
|
||||
// Error is returned from drime when things go wrong
|
||||
type Error struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// Error returns a string for the error and satisfies the error interface
|
||||
func (e Error) Error() string {
|
||||
out := fmt.Sprintf("Error %q", e.Message)
|
||||
return out
|
||||
}
|
||||
|
||||
// Check Error satisfies the error interface
|
||||
var _ error = (*Error)(nil)
|
||||
|
||||
// DeleteRequest is the input to DELETE /file-entries
|
||||
type DeleteRequest struct {
|
||||
EntryIDs []string `json:"entryIds"`
|
||||
DeleteForever bool `json:"deleteForever"`
|
||||
}
|
||||
|
||||
// DeleteResponse is the input to DELETE /file-entries
|
||||
type DeleteResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Errors map[string]string `json:"errors"`
|
||||
}
|
||||
|
||||
// UpdateItemRequest describes the updates to be done to an item for PUT /file-entries/{id}/
|
||||
type UpdateItemRequest struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
// UpdateItemResponse is returned by PUT /file-entries/{id}/
|
||||
type UpdateItemResponse struct {
|
||||
Status string `json:"status"`
|
||||
FileEntry Item `json:"fileEntry"`
|
||||
}
|
||||
|
||||
// MoveRequest is the input to /file-entries/move
|
||||
type MoveRequest struct {
|
||||
EntryIDs []string `json:"entryIds"`
|
||||
DestinationID string `json:"destinationId"`
|
||||
}
|
||||
|
||||
// MoveResponse is returned by POST /file-entries/move
|
||||
type MoveResponse struct {
|
||||
Status string `json:"status"`
|
||||
Entries []Item `json:"entries"`
|
||||
}
|
||||
|
||||
// CopyRequest is the input to /file-entries/duplicate
|
||||
type CopyRequest struct {
|
||||
EntryIDs []string `json:"entryIds"`
|
||||
DestinationID string `json:"destinationId"`
|
||||
}
|
||||
|
||||
// CopyResponse is returned by POST /file-entries/duplicate
|
||||
type CopyResponse struct {
|
||||
Status string `json:"status"`
|
||||
Entries []Item `json:"entries"`
|
||||
}
|
||||
|
||||
// MultiPartCreateRequest is the input of POST /s3/multipart/create
|
||||
type MultiPartCreateRequest struct {
|
||||
Filename string `json:"filename"`
|
||||
Mime string `json:"mime"`
|
||||
Size int64 `json:"size"`
|
||||
Extension string `json:"extension"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
|
||||
// MultiPartCreateResponse is returned by POST /s3/multipart/create
|
||||
type MultiPartCreateResponse struct {
|
||||
UploadID string `json:"uploadId"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// CompletedPart Type for completed parts when making a multipart upload.
|
||||
type CompletedPart struct {
|
||||
ETag string `json:"ETag"`
|
||||
PartNumber int32 `json:"PartNumber"`
|
||||
}
|
||||
|
||||
// MultiPartGetURLsRequest is the input of POST /s3/multipart/batch-sign-part-urls
|
||||
type MultiPartGetURLsRequest struct {
|
||||
UploadID string `json:"uploadId"`
|
||||
Key string `json:"key"`
|
||||
PartNumbers []int `json:"partNumbers"`
|
||||
}
|
||||
|
||||
// MultiPartGetURLsResponse is the result of POST /s3/multipart/batch-sign-part-urls
|
||||
type MultiPartGetURLsResponse struct {
|
||||
URLs []struct {
|
||||
URL string `json:"url"`
|
||||
PartNumber int32 `json:"partNumber"`
|
||||
} `json:"urls"`
|
||||
}
|
||||
|
||||
// MultiPartCompleteRequest is the input to POST /s3/multipart/complete
|
||||
type MultiPartCompleteRequest struct {
|
||||
UploadID string `json:"uploadId"`
|
||||
Key string `json:"key"`
|
||||
Parts []CompletedPart `json:"parts"`
|
||||
}
|
||||
|
||||
// MultiPartCompleteResponse is the result of POST /s3/multipart/complete
|
||||
type MultiPartCompleteResponse struct {
|
||||
Location string `json:"location"`
|
||||
}
|
||||
|
||||
// MultiPartEntriesRequest is the input to POST /s3/entries
|
||||
type MultiPartEntriesRequest struct {
|
||||
ClientMime string `json:"clientMime"`
|
||||
ClientName string `json:"clientName"`
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
ClientExtension string `json:"clientExtension"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
|
||||
// MultiPartEntriesResponse is the result of POST /s3/entries
|
||||
type MultiPartEntriesResponse struct {
|
||||
FileEntry Item `json:"fileEntry"`
|
||||
}
|
||||
|
||||
// MultiPartAbort is the input of POST /s3/multipart/abort
|
||||
type MultiPartAbort struct {
|
||||
UploadID string `json:"uploadId"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// SpaceUsageResponse is returned by GET /user/space-usage
|
||||
type SpaceUsageResponse struct {
|
||||
Used int64 `json:"used"`
|
||||
Available int64 `json:"available"`
|
||||
Status string `json:"status"`
|
||||
SEO any `json:"seo"`
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,33 @@
|
||||
// Drime filesystem interface
|
||||
package drime
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestDrime:",
|
||||
NilObject: (*Object)(nil),
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: minChunkSize,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
return f.setUploadChunkSize(cs)
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
return f.setUploadCutoff(cs)
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
+160
-115
@@ -733,6 +733,17 @@ two accounts.
|
||||
Advanced: true,
|
||||
Default: rwOff,
|
||||
Examples: rwExamples,
|
||||
}, {
|
||||
Name: "metadata_enforce_expansive_access",
|
||||
Help: `Whether the request should enforce expansive access rules.
|
||||
|
||||
From Feb 2026 this flag will be set by default so this flag can be used for
|
||||
testing before then.
|
||||
|
||||
See: https://developers.google.com/workspace/drive/api/guides/limited-expansive-access
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -812,6 +823,7 @@ type Options struct {
|
||||
MetadataOwner rwChoice `config:"metadata_owner"`
|
||||
MetadataPermissions rwChoice `config:"metadata_permissions"`
|
||||
MetadataLabels rwChoice `config:"metadata_labels"`
|
||||
EnforceExpansiveAccess bool `config:"metadata_enforce_expansive_access"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
EnvAuth bool `config:"env_auth"`
|
||||
}
|
||||
@@ -3092,6 +3104,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
_, err = f.svc.Permissions.Create(id, permission).
|
||||
Fields("").
|
||||
SupportsAllDrives(true).
|
||||
EnforceExpansiveAccess(f.opt.EnforceExpansiveAccess).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -3396,9 +3409,11 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
|
||||
return nil, fmt.Errorf("failed to find source dir: %w", err)
|
||||
}
|
||||
isDir = true
|
||||
} else {
|
||||
} else if do, ok := srcObj.(fs.IDer); ok {
|
||||
// source was a file
|
||||
srcID = srcObj.(*Object).id
|
||||
srcID = do.ID()
|
||||
} else {
|
||||
return nil, fmt.Errorf("unknown source object: %T", srcObj)
|
||||
}
|
||||
srcID = actualID(srcID) // link to underlying object not to shortcut
|
||||
|
||||
@@ -3664,41 +3679,47 @@ func (f *Fs) rescue(ctx context.Context, dirID string, delete bool) (err error)
|
||||
|
||||
var commandHelp = []fs.CommandHelp{{
|
||||
Name: "get",
|
||||
Short: "Get command for fetching the drive config parameters",
|
||||
Long: `This is a get command which will be used to fetch the various drive config parameters
|
||||
Short: "Get command for fetching the drive config parameters.",
|
||||
Long: `This is a get command which will be used to fetch the various drive config
|
||||
parameters.
|
||||
|
||||
Usage Examples:
|
||||
Usage examples:
|
||||
|
||||
rclone backend get drive: [-o service_account_file] [-o chunk_size]
|
||||
rclone rc backend/command command=get fs=drive: [-o service_account_file] [-o chunk_size]
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend get drive: [-o service_account_file] [-o chunk_size]
|
||||
rclone rc backend/command command=get fs=drive: [-o service_account_file] [-o chunk_size]
|
||||
` + "```",
|
||||
Opts: map[string]string{
|
||||
"chunk_size": "show the current upload chunk size",
|
||||
"service_account_file": "show the current service account file",
|
||||
"chunk_size": "Show the current upload chunk size.",
|
||||
"service_account_file": "Show the current service account file.",
|
||||
},
|
||||
}, {
|
||||
Name: "set",
|
||||
Short: "Set command for updating the drive config parameters",
|
||||
Long: `This is a set command which will be used to update the various drive config parameters
|
||||
Short: "Set command for updating the drive config parameters.",
|
||||
Long: `This is a set command which will be used to update the various drive config
|
||||
parameters.
|
||||
|
||||
Usage Examples:
|
||||
Usage examples:
|
||||
|
||||
rclone backend set drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
|
||||
rclone rc backend/command command=set fs=drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend set drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
|
||||
rclone rc backend/command command=set fs=drive: [-o service_account_file=sa.json] [-o chunk_size=67108864]
|
||||
` + "```",
|
||||
Opts: map[string]string{
|
||||
"chunk_size": "update the current upload chunk size",
|
||||
"service_account_file": "update the current service account file",
|
||||
"chunk_size": "Update the current upload chunk size.",
|
||||
"service_account_file": "Update the current service account file.",
|
||||
},
|
||||
}, {
|
||||
Name: "shortcut",
|
||||
Short: "Create shortcuts from files or directories",
|
||||
Short: "Create shortcuts from files or directories.",
|
||||
Long: `This command creates shortcuts from files or directories.
|
||||
|
||||
Usage:
|
||||
Usage examples:
|
||||
|
||||
rclone backend shortcut drive: source_item destination_shortcut
|
||||
rclone backend shortcut drive: source_item -o target=drive2: destination_shortcut
|
||||
` + "```console" + `
|
||||
rclone backend shortcut drive: source_item destination_shortcut
|
||||
rclone backend shortcut drive: source_item -o target=drive2: destination_shortcut
|
||||
` + "```" + `
|
||||
|
||||
In the first example this creates a shortcut from the "source_item"
|
||||
which can be a file or a directory to the "destination_shortcut". The
|
||||
@@ -3708,90 +3729,100 @@ from "drive:"
|
||||
In the second example this creates a shortcut from the "source_item"
|
||||
relative to "drive:" to the "destination_shortcut" relative to
|
||||
"drive2:". This may fail with a permission error if the user
|
||||
authenticated with "drive2:" can't read files from "drive:".
|
||||
`,
|
||||
authenticated with "drive2:" can't read files from "drive:".`,
|
||||
Opts: map[string]string{
|
||||
"target": "optional target remote for the shortcut destination",
|
||||
"target": "Optional target remote for the shortcut destination.",
|
||||
},
|
||||
}, {
|
||||
Name: "drives",
|
||||
Short: "List the Shared Drives available to this account",
|
||||
Short: "List the Shared Drives available to this account.",
|
||||
Long: `This command lists the Shared Drives (Team Drives) available to this
|
||||
account.
|
||||
|
||||
Usage:
|
||||
Usage example:
|
||||
|
||||
rclone backend [-o config] drives drive:
|
||||
` + "```console" + `
|
||||
rclone backend [-o config] drives drive:
|
||||
` + "```" + `
|
||||
|
||||
This will return a JSON list of objects like this
|
||||
This will return a JSON list of objects like this:
|
||||
|
||||
[
|
||||
{
|
||||
"id": "0ABCDEF-01234567890",
|
||||
"kind": "drive#teamDrive",
|
||||
"name": "My Drive"
|
||||
},
|
||||
{
|
||||
"id": "0ABCDEFabcdefghijkl",
|
||||
"kind": "drive#teamDrive",
|
||||
"name": "Test Drive"
|
||||
}
|
||||
]
|
||||
` + "```json" + `
|
||||
[
|
||||
{
|
||||
"id": "0ABCDEF-01234567890",
|
||||
"kind": "drive#teamDrive",
|
||||
"name": "My Drive"
|
||||
},
|
||||
{
|
||||
"id": "0ABCDEFabcdefghijkl",
|
||||
"kind": "drive#teamDrive",
|
||||
"name": "Test Drive"
|
||||
}
|
||||
]
|
||||
` + "```" + `
|
||||
|
||||
With the -o config parameter it will output the list in a format
|
||||
suitable for adding to a config file to make aliases for all the
|
||||
drives found and a combined drive.
|
||||
|
||||
[My Drive]
|
||||
type = alias
|
||||
remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=:
|
||||
` + "```ini" + `
|
||||
[My Drive]
|
||||
type = alias
|
||||
remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=:
|
||||
|
||||
[Test Drive]
|
||||
type = alias
|
||||
remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
|
||||
[Test Drive]
|
||||
type = alias
|
||||
remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
|
||||
|
||||
[AllDrives]
|
||||
type = combine
|
||||
upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
|
||||
[AllDrives]
|
||||
type = combine
|
||||
upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
|
||||
` + "```" + `
|
||||
|
||||
Adding this to the rclone config file will cause those team drives to
|
||||
be accessible with the aliases shown. Any illegal characters will be
|
||||
substituted with "_" and duplicate names will have numbers suffixed.
|
||||
It will also add a remote called AllDrives which shows all the shared
|
||||
drives combined into one directory tree.
|
||||
`,
|
||||
drives combined into one directory tree.`,
|
||||
}, {
|
||||
Name: "untrash",
|
||||
Short: "Untrash files and directories",
|
||||
Short: "Untrash files and directories.",
|
||||
Long: `This command untrashes all the files and directories in the directory
|
||||
passed in recursively.
|
||||
|
||||
Usage:
|
||||
Usage example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend untrash drive:directory
|
||||
rclone backend --interactive untrash drive:directory subdir
|
||||
` + "```" + `
|
||||
|
||||
This takes an optional directory to trash which make this easier to
|
||||
use via the API.
|
||||
|
||||
rclone backend untrash drive:directory
|
||||
rclone backend --interactive untrash drive:directory subdir
|
||||
|
||||
Use the --interactive/-i or --dry-run flag to see what would be restored before restoring it.
|
||||
Use the --interactive/-i or --dry-run flag to see what would be restored before
|
||||
restoring it.
|
||||
|
||||
Result:
|
||||
|
||||
{
|
||||
"Untrashed": 17,
|
||||
"Errors": 0
|
||||
}
|
||||
`,
|
||||
` + "```json" + `
|
||||
{
|
||||
"Untrashed": 17,
|
||||
"Errors": 0
|
||||
}
|
||||
` + "```",
|
||||
}, {
|
||||
Name: "copyid",
|
||||
Short: "Copy files by ID",
|
||||
Long: `This command copies files by ID
|
||||
Short: "Copy files by ID.",
|
||||
Long: `This command copies files by ID.
|
||||
|
||||
Usage:
|
||||
Usage examples:
|
||||
|
||||
rclone backend copyid drive: ID path
|
||||
rclone backend copyid drive: ID1 path1 ID2 path2
|
||||
` + "```console" + `
|
||||
rclone backend copyid drive: ID path
|
||||
rclone backend copyid drive: ID1 path1 ID2 path2
|
||||
` + "```" + `
|
||||
|
||||
It copies the drive file with ID given to the path (an rclone path which
|
||||
will be passed internally to rclone copyto). The ID and path pairs can be
|
||||
@@ -3804,17 +3835,19 @@ component will be used as the file name.
|
||||
If the destination is a drive backend then server-side copying will be
|
||||
attempted if possible.
|
||||
|
||||
Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
|
||||
`,
|
||||
Use the --interactive/-i or --dry-run flag to see what would be copied before
|
||||
copying.`,
|
||||
}, {
|
||||
Name: "moveid",
|
||||
Short: "Move files by ID",
|
||||
Long: `This command moves files by ID
|
||||
Short: "Move files by ID.",
|
||||
Long: `This command moves files by ID.
|
||||
|
||||
Usage:
|
||||
Usage examples:
|
||||
|
||||
rclone backend moveid drive: ID path
|
||||
rclone backend moveid drive: ID1 path1 ID2 path2
|
||||
` + "```console" + `
|
||||
rclone backend moveid drive: ID path
|
||||
rclone backend moveid drive: ID1 path1 ID2 path2
|
||||
` + "```" + `
|
||||
|
||||
It moves the drive file with ID given to the path (an rclone path which
|
||||
will be passed internally to rclone moveto).
|
||||
@@ -3826,58 +3859,65 @@ component will be used as the file name.
|
||||
If the destination is a drive backend then server-side moving will be
|
||||
attempted if possible.
|
||||
|
||||
Use the --interactive/-i or --dry-run flag to see what would be moved beforehand.
|
||||
`,
|
||||
Use the --interactive/-i or --dry-run flag to see what would be moved beforehand.`,
|
||||
}, {
|
||||
Name: "exportformats",
|
||||
Short: "Dump the export formats for debug purposes",
|
||||
Short: "Dump the export formats for debug purposes.",
|
||||
}, {
|
||||
Name: "importformats",
|
||||
Short: "Dump the import formats for debug purposes",
|
||||
Short: "Dump the import formats for debug purposes.",
|
||||
}, {
|
||||
Name: "query",
|
||||
Short: "List files using Google Drive query language",
|
||||
Long: `This command lists files based on a query
|
||||
Short: "List files using Google Drive query language.",
|
||||
Long: `This command lists files based on a query.
|
||||
|
||||
Usage:
|
||||
Usage example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend query drive: query
|
||||
` + "```" + `
|
||||
|
||||
rclone backend query drive: query
|
||||
|
||||
The query syntax is documented at [Google Drive Search query terms and
|
||||
operators](https://developers.google.com/drive/api/guides/ref-search-terms).
|
||||
|
||||
For example:
|
||||
|
||||
rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
|
||||
` + "```console" + `
|
||||
rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
|
||||
` + "```" + `
|
||||
|
||||
If the query contains literal ' or \ characters, these need to be escaped with
|
||||
\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a
|
||||
file named "foo ' \.txt":
|
||||
|
||||
rclone backend query drive: "name = 'foo \' \\\.txt'"
|
||||
` + "```console" + `
|
||||
rclone backend query drive: "name = 'foo \' \\\.txt'"
|
||||
` + "```" + `
|
||||
|
||||
The result is a JSON array of matches, for example:
|
||||
|
||||
[
|
||||
{
|
||||
"createdTime": "2017-06-29T19:58:28.537Z",
|
||||
"id": "0AxBe_CDEF4zkGHI4d0FjYko2QkD",
|
||||
"md5Checksum": "68518d16be0c6fbfab918be61d658032",
|
||||
"mimeType": "text/plain",
|
||||
"modifiedTime": "2024-02-02T10:40:02.874Z",
|
||||
"name": "foo ' \\.txt",
|
||||
"parents": [
|
||||
"0BxAe_BCDE4zkFGZpcWJGek0xbzC"
|
||||
],
|
||||
"resourceKey": "0-ABCDEFGHIXJQpIGqBJq3MC",
|
||||
"sha1Checksum": "8f284fa768bfb4e45d076a579ab3905ab6bfa893",
|
||||
"size": "311",
|
||||
"webViewLink": "https://drive.google.com/file/d/0AxBe_CDEF4zkGHI4d0FjYko2QkD/view?usp=drivesdk\u0026resourcekey=0-ABCDEFGHIXJQpIGqBJq3MC"
|
||||
}
|
||||
]`,
|
||||
` + "```json" + `
|
||||
[
|
||||
{
|
||||
"createdTime": "2017-06-29T19:58:28.537Z",
|
||||
"id": "0AxBe_CDEF4zkGHI4d0FjYko2QkD",
|
||||
"md5Checksum": "68518d16be0c6fbfab918be61d658032",
|
||||
"mimeType": "text/plain",
|
||||
"modifiedTime": "2024-02-02T10:40:02.874Z",
|
||||
"name": "foo ' \\.txt",
|
||||
"parents": [
|
||||
"0BxAe_BCDE4zkFGZpcWJGek0xbzC"
|
||||
],
|
||||
"resourceKey": "0-ABCDEFGHIXJQpIGqBJq3MC",
|
||||
"sha1Checksum": "8f284fa768bfb4e45d076a579ab3905ab6bfa893",
|
||||
"size": "311",
|
||||
"webViewLink": "https://drive.google.com/file/d/0AxBe_CDEF4zkGHI4d0FjYko2QkD/view?usp=drivesdk\u0026resourcekey=0-ABCDEFGHIXJQpIGqBJq3MC"
|
||||
}
|
||||
]
|
||||
` + "```console",
|
||||
}, {
|
||||
Name: "rescue",
|
||||
Short: "Rescue or delete any orphaned files",
|
||||
Short: "Rescue or delete any orphaned files.",
|
||||
Long: `This command rescues or deletes any orphaned files or directories.
|
||||
|
||||
Sometimes files can get orphaned in Google Drive. This means that they
|
||||
@@ -3886,26 +3926,31 @@ are no longer in any folder in Google Drive.
|
||||
This command finds those files and either rescues them to a directory
|
||||
you specify or deletes them.
|
||||
|
||||
Usage:
|
||||
|
||||
This can be used in 3 ways.
|
||||
|
||||
First, list all orphaned files
|
||||
First, list all orphaned files:
|
||||
|
||||
rclone backend rescue drive:
|
||||
` + "```console" + `
|
||||
rclone backend rescue drive:
|
||||
` + "```" + `
|
||||
|
||||
Second rescue all orphaned files to the directory indicated
|
||||
Second rescue all orphaned files to the directory indicated:
|
||||
|
||||
rclone backend rescue drive: "relative/path/to/rescue/directory"
|
||||
` + "```console" + `
|
||||
rclone backend rescue drive: "relative/path/to/rescue/directory"
|
||||
` + "```" + `
|
||||
|
||||
e.g. To rescue all orphans to a directory called "Orphans" in the top level
|
||||
E.g. to rescue all orphans to a directory called "Orphans" in the top level:
|
||||
|
||||
rclone backend rescue drive: Orphans
|
||||
` + "```console" + `
|
||||
rclone backend rescue drive: Orphans
|
||||
` + "```" + `
|
||||
|
||||
Third delete all orphaned files to the trash
|
||||
Third delete all orphaned files to the trash:
|
||||
|
||||
rclone backend rescue drive: -o delete
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend rescue drive: -o delete
|
||||
` + "```",
|
||||
}}
|
||||
|
||||
// Command the backend to run a named command
|
||||
|
||||
@@ -643,6 +643,44 @@ func (f *Fs) InternalTestAgeQuery(t *testing.T) {
|
||||
assert.Contains(t, subFs.lastQuery, timeQuery)
|
||||
}
|
||||
|
||||
// TestIntegration/FsMkdir/FsPutFiles/Internal/SingleQuoteFolder
|
||||
func (f *Fs) InternalTestSingleQuoteFolder(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Test various folder names containing single quotes
|
||||
for _, name := range []string{
|
||||
"'",
|
||||
"''",
|
||||
"'a'",
|
||||
"it's a test",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := "singleQuoteTest/" + name
|
||||
err := f.Mkdir(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
err := f.Rmdir(ctx, dir)
|
||||
assert.NoError(t, err)
|
||||
}()
|
||||
|
||||
entries, err := f.List(ctx, "singleQuoteTest")
|
||||
require.NoError(t, err)
|
||||
|
||||
found := false
|
||||
for _, entry := range entries {
|
||||
if entry.Remote() == dir {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "directory %q not found in listing", name)
|
||||
})
|
||||
}
|
||||
|
||||
err := f.Rmdir(ctx, "singleQuoteTest")
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
// These tests all depend on each other so run them as nested tests
|
||||
t.Run("DocumentImport", func(t *testing.T) {
|
||||
@@ -662,6 +700,7 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("CopyOrMoveID", f.InternalTestCopyOrMoveID)
|
||||
t.Run("Query", f.InternalTestQuery)
|
||||
t.Run("AgeQuery", f.InternalTestAgeQuery)
|
||||
t.Run("SingleQuoteFolder", f.InternalTestSingleQuoteFolder)
|
||||
t.Run("ShouldRetry", f.InternalTestShouldRetry)
|
||||
}
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions [
|
||||
_, err := f.svc.Permissions.Create(info.Id, perm).
|
||||
SupportsAllDrives(true).
|
||||
SendNotificationEmail(false).
|
||||
EnforceExpansiveAccess(f.opt.EnforceExpansiveAccess).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
@@ -483,6 +484,7 @@ func (f *Fs) setOwner(ctx context.Context, info *drive.File, owner string) (err
|
||||
SupportsAllDrives(true).
|
||||
TransferOwnership(true).
|
||||
// SendNotificationEmail(false). - required apparently!
|
||||
EnforceExpansiveAccess(f.opt.EnforceExpansiveAccess).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
@@ -1330,6 +1330,16 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
var result *files.RelocationResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
result, err = f.srv.MoveV2(&arg)
|
||||
switch e := err.(type) {
|
||||
case files.MoveV2APIError:
|
||||
// There seems to be a bit of eventual consistency here which causes this to
|
||||
// fail on just created objects
|
||||
// See: https://github.com/rclone/rclone/issues/8881
|
||||
if e.EndpointError != nil && e.EndpointError.FromLookup != nil && e.EndpointError.FromLookup.Tag == files.LookupErrorNotFound {
|
||||
fs.Debugf(srcObj, "Retrying move on %v error", err)
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -3,6 +3,19 @@ package api
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// MultipartInitResponse represents the response from multipart/init.
|
||||
type MultipartInitResponse struct {
|
||||
Status int `json:"status"`
|
||||
Msg string `json:"msg"`
|
||||
Result struct {
|
||||
UploadID string `json:"upload_id"`
|
||||
SessID string `json:"sess_id"`
|
||||
Server string `json:"server"`
|
||||
FolderID int64 `json:"folder_id"`
|
||||
ObjectPath string `json:"object_path"`
|
||||
} `json:"result"`
|
||||
}
|
||||
|
||||
// CreateFolderResponse represents the response for creating a folder.
|
||||
type CreateFolderResponse struct {
|
||||
Status int `json:"status"`
|
||||
|
||||
+24
-19
@@ -21,6 +21,11 @@ import (
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultUploadCutoff = fs.SizeSuffix(500 * 1024 * 1024)
|
||||
defaultChunkSize = fs.SizeSuffix(64 * 1024 * 1024)
|
||||
)
|
||||
|
||||
// Register the backend with Rclone
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
@@ -33,6 +38,17 @@ func init() {
|
||||
Required: true,
|
||||
Sensitive: true,
|
||||
},
|
||||
{
|
||||
Name: "upload_cutoff",
|
||||
Help: "Cutoff for switching to chunked upload. Any files larger than this will be uploaded in chunks of chunk_size.",
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "Chunk size to use for uploading. Used for multipart uploads.",
|
||||
Default: defaultChunkSize,
|
||||
Advanced: true,
|
||||
},
|
||||
{
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -72,8 +88,10 @@ func init() {
|
||||
|
||||
// Options defines the configuration for the FileLu backend
|
||||
type Options struct {
|
||||
Key string `config:"key"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
Key string `config:"key"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
}
|
||||
|
||||
// Fs represents the FileLu file system
|
||||
@@ -189,7 +207,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
return f.deleteFolder(ctx, fullPath)
|
||||
}
|
||||
|
||||
// List returns a list of files and folders
|
||||
// List returns a list of files and folders for the given directory
|
||||
func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
// Compose full path for API call
|
||||
@@ -250,23 +267,11 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
|
||||
// Put uploads a file directly to the destination folder in the FileLu storage system.
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
if src.Size() == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: src.Remote(),
|
||||
}
|
||||
|
||||
err := f.uploadFile(ctx, in, src.Remote())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
newObject := &Object{
|
||||
fs: f,
|
||||
remote: src.Remote(),
|
||||
size: src.Size(),
|
||||
modTime: src.ModTime(ctx),
|
||||
}
|
||||
fs.Infof(f, "Put: Successfully uploaded new file %q", src.Remote())
|
||||
return newObject, nil
|
||||
return o, o.Update(ctx, in, src, options...)
|
||||
}
|
||||
|
||||
// Move moves the file to the specified location
|
||||
|
||||
+127
-153
@@ -1,9 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -16,40 +14,82 @@ import (
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
// multipartInit starts a new multipart upload and returns server details.
|
||||
func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/multipart/init",
|
||||
Parameters: url.Values{
|
||||
"key": {f.opt.Key},
|
||||
"filename": {filename},
|
||||
"folder_path": {folderPath},
|
||||
},
|
||||
}
|
||||
|
||||
var result api.MultipartInitResponse
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return fserrors.ShouldRetry(err), err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if result.Status != 200 {
|
||||
return nil, fmt.Errorf("multipart init error: %s", result.Msg)
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// completeMultipart finalizes the multipart upload on the file server.
|
||||
func (f *Fs) completeMultipart(ctx context.Context, server string, uploadID string, sessID string, objectPath string) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", server, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("X-RC-Upload-Id", uploadID)
|
||||
req.Header.Set("X-Sess-ID", sessID)
|
||||
req.Header.Set("X-Object-Path", objectPath)
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != 202 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("completeMultipart failed %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createFolder creates a folder at the specified path.
|
||||
func (f *Fs) createFolder(ctx context.Context, dirPath string) (*api.CreateFolderResponse, error) {
|
||||
encodedDir := f.fromStandardPath(dirPath)
|
||||
apiURL := fmt.Sprintf("%s/folder/create?folder_path=%s&key=%s",
|
||||
f.endpoint,
|
||||
url.QueryEscape(encodedDir),
|
||||
url.QueryEscape(f.opt.Key), // assuming f.opt.Key is the correct field
|
||||
)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/folder/create",
|
||||
Parameters: url.Values{
|
||||
"folder_path": {encodedDir},
|
||||
"key": {f.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
var resp *http.Response
|
||||
result := api.CreateFolderResponse{}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var innerErr error
|
||||
resp, innerErr = f.client.Do(req)
|
||||
return fserrors.ShouldRetry(innerErr), innerErr
|
||||
var result api.CreateFolderResponse
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return fserrors.ShouldRetry(err), err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err = json.NewDecoder(resp.Body).Decode(&result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error decoding response: %w", err)
|
||||
}
|
||||
if result.Status != 200 {
|
||||
return nil, fmt.Errorf("error: %s", result.Msg)
|
||||
}
|
||||
@@ -61,44 +101,29 @@ func (f *Fs) createFolder(ctx context.Context, dirPath string) (*api.CreateFolde
|
||||
// getFolderList List both files and folders in a directory.
|
||||
func (f *Fs) getFolderList(ctx context.Context, path string) (*api.FolderListResponse, error) {
|
||||
encodedDir := f.fromStandardPath(path)
|
||||
apiURL := fmt.Sprintf("%s/folder/list?folder_path=%s&key=%s",
|
||||
f.endpoint,
|
||||
url.QueryEscape(encodedDir),
|
||||
url.QueryEscape(f.opt.Key),
|
||||
)
|
||||
|
||||
var body []byte
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/folder/list",
|
||||
Parameters: url.Values{
|
||||
"folder_path": {encodedDir},
|
||||
"key": {f.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
var response api.FolderListResponse
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &response)
|
||||
if err != nil {
|
||||
return shouldRetry(err), fmt.Errorf("failed to list directory: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
body, err = io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error reading response body: %w", err)
|
||||
}
|
||||
|
||||
return shouldRetryHTTP(resp.StatusCode), nil
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var response api.FolderListResponse
|
||||
if err := json.NewDecoder(bytes.NewReader(body)).Decode(&response); err != nil {
|
||||
return nil, fmt.Errorf("error decoding response: %w", err)
|
||||
}
|
||||
if response.Status != 200 {
|
||||
if strings.Contains(response.Msg, "Folder not found") {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
@@ -115,42 +140,28 @@ func (f *Fs) getFolderList(ctx context.Context, path string) (*api.FolderListRes
|
||||
}
|
||||
|
||||
return &response, nil
|
||||
|
||||
}
|
||||
|
||||
// deleteFolder deletes a folder at the specified path.
|
||||
func (f *Fs) deleteFolder(ctx context.Context, fullPath string) error {
|
||||
fullPath = f.fromStandardPath(fullPath)
|
||||
deleteURL := fmt.Sprintf("%s/folder/delete?folder_path=%s&key=%s",
|
||||
f.endpoint,
|
||||
url.QueryEscape(fullPath),
|
||||
url.QueryEscape(f.opt.Key),
|
||||
)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/folder/delete",
|
||||
Parameters: url.Values{
|
||||
"folder_path": {fullPath},
|
||||
"key": {f.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
delResp := api.DeleteFolderResponse{}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", deleteURL, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := f.client.Do(req)
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &delResp)
|
||||
if err != nil {
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &delResp); err != nil {
|
||||
return false, fmt.Errorf("error decoding delete response: %w", err)
|
||||
}
|
||||
if delResp.Status != 200 {
|
||||
return false, fmt.Errorf("delete error: %s", delResp.Msg)
|
||||
}
|
||||
@@ -167,38 +178,27 @@ func (f *Fs) deleteFolder(ctx context.Context, fullPath string) error {
|
||||
// getDirectLink of file from FileLu to download.
|
||||
func (f *Fs) getDirectLink(ctx context.Context, filePath string) (string, int64, error) {
|
||||
filePath = f.fromStandardPath(filePath)
|
||||
apiURL := fmt.Sprintf("%s/file/direct_link?file_path=%s&key=%s",
|
||||
f.endpoint,
|
||||
url.QueryEscape(filePath),
|
||||
url.QueryEscape(f.opt.Key),
|
||||
)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/file/direct_link",
|
||||
Parameters: url.Values{
|
||||
"file_path": {filePath},
|
||||
"key": {f.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
result := api.FileDirectLinkResponse{}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
if err != nil {
|
||||
return shouldRetry(err), fmt.Errorf("failed to fetch direct link: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return false, fmt.Errorf("error decoding response: %w", err)
|
||||
}
|
||||
|
||||
if result.Status != 200 {
|
||||
return false, fmt.Errorf("API error: %s", result.Msg)
|
||||
}
|
||||
|
||||
return shouldRetryHTTP(resp.StatusCode), nil
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
@@ -210,39 +210,31 @@ func (f *Fs) getDirectLink(ctx context.Context, filePath string) (string, int64,
|
||||
// deleteFile deletes a file based on filePath
|
||||
func (f *Fs) deleteFile(ctx context.Context, filePath string) error {
|
||||
filePath = f.fromStandardPath(filePath)
|
||||
apiURL := fmt.Sprintf("%s/file/remove?file_path=%s&key=%s",
|
||||
f.endpoint,
|
||||
url.QueryEscape(filePath),
|
||||
url.QueryEscape(f.opt.Key),
|
||||
)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/file/remove",
|
||||
Parameters: url.Values{
|
||||
"file_path": {filePath},
|
||||
"key": {f.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
result := api.DeleteFileResponse{}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return shouldRetry(err), fmt.Errorf("failed to fetch direct link: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return false, fmt.Errorf("error decoding response: %w", err)
|
||||
return shouldRetry(err), fmt.Errorf("failed to delete file: %w", err)
|
||||
}
|
||||
|
||||
if result.Status != 200 {
|
||||
return false, fmt.Errorf("API error: %s", result.Msg)
|
||||
}
|
||||
|
||||
return shouldRetryHTTP(resp.StatusCode), nil
|
||||
return false, nil
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -275,46 +267,28 @@ func (f *Fs) getAccountInfo(ctx context.Context) (*api.AccountInfoResponse, erro
|
||||
|
||||
// getFileInfo retrieves file information based on file code
|
||||
func (f *Fs) getFileInfo(ctx context.Context, fileCode string) (*api.FileInfoResponse, error) {
|
||||
u, _ := url.Parse(f.endpoint + "/file/info2")
|
||||
q := u.Query()
|
||||
q.Set("file_code", fileCode) // raw path — Go handles escaping properly here
|
||||
q.Set("key", f.opt.Key)
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
apiURL := f.endpoint + "/file/info2?" + u.RawQuery
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/file/info2",
|
||||
Parameters: url.Values{
|
||||
"file_code": {fileCode},
|
||||
"key": {f.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
result := api.FileInfoResponse{}
|
||||
|
||||
var body []byte
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
if err != nil {
|
||||
return shouldRetry(err), fmt.Errorf("failed to fetch file info: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
body, err = io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("error reading response body: %w", err)
|
||||
}
|
||||
|
||||
return shouldRetryHTTP(resp.StatusCode), nil
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := api.FileInfoResponse{}
|
||||
|
||||
if err := json.NewDecoder(bytes.NewReader(body)).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("error decoding response: %w", err)
|
||||
}
|
||||
|
||||
if result.Status != 200 || len(result.Result) == 0 {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -13,8 +14,108 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
// multipartUpload uploads a file in fixed-size chunks using the multipart API.
|
||||
func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) error {
|
||||
dir := path.Dir(remote)
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
|
||||
if dir != "" {
|
||||
_ = f.Mkdir(ctx, dir)
|
||||
}
|
||||
|
||||
folder := strings.Trim(dir, "/")
|
||||
if folder != "" {
|
||||
folder = "/" + folder
|
||||
}
|
||||
|
||||
file := path.Base(remote)
|
||||
|
||||
initResp, err := f.multipartInit(ctx, folder, file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multipart init failed: %w", err)
|
||||
}
|
||||
|
||||
uploadID := initResp.Result.UploadID
|
||||
sessID := initResp.Result.SessID
|
||||
server := initResp.Result.Server
|
||||
objectPath := initResp.Result.ObjectPath
|
||||
|
||||
chunkSize := int(f.opt.ChunkSize)
|
||||
buf := make([]byte, 0, chunkSize)
|
||||
tmp := make([]byte, 1024*1024)
|
||||
partNo := 1
|
||||
|
||||
for {
|
||||
n, errRead := in.Read(tmp)
|
||||
if n > 0 {
|
||||
buf = append(buf, tmp[:n]...)
|
||||
|
||||
// If buffer reached chunkSize, upload a full part
|
||||
if len(buf) >= chunkSize {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
partNo++
|
||||
buf = buf[:0]
|
||||
}
|
||||
}
|
||||
|
||||
if errRead == io.EOF {
|
||||
break
|
||||
}
|
||||
if errRead != nil {
|
||||
return fmt.Errorf("read failed: %w", errRead)
|
||||
}
|
||||
}
|
||||
|
||||
if len(buf) > 0 {
|
||||
err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return fmt.Errorf("upload part %d failed: %w", partNo, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("complete multipart failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadPart sends a single multipart chunk to the upload server.
|
||||
func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
|
||||
url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("X-RC-Upload-Id", uploadID)
|
||||
req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
|
||||
req.Header.Set("X-Sess-ID", sessID)
|
||||
req.Header.Set("X-Object-Path", objectPath)
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return fmt.Errorf("uploadPart failed: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFile uploads a file to FileLu
|
||||
func (f *Fs) uploadFile(ctx context.Context, fileContent io.Reader, fileFullPath string) error {
|
||||
directory := path.Dir(fileFullPath)
|
||||
@@ -68,9 +169,15 @@ func (f *Fs) uploadFile(ctx context.Context, fileContent io.Reader, fileFullPath
|
||||
return nil
|
||||
}
|
||||
|
||||
// getUploadServer gets the upload server URL with proper key authentication
|
||||
func (f *Fs) getUploadServer(ctx context.Context) (string, string, error) {
|
||||
apiURL := fmt.Sprintf("%s/upload/server?key=%s", f.endpoint, url.QueryEscape(f.opt.Key))
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/upload/server",
|
||||
Parameters: url.Values{
|
||||
"key": {f.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Status int `json:"status"`
|
||||
@@ -80,36 +187,21 @@ func (f *Fs) getUploadServer(ctx context.Context) (string, string, error) {
|
||||
}
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := f.client.Do(req)
|
||||
_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
if err != nil {
|
||||
return shouldRetry(err), fmt.Errorf("failed to get upload server: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return false, fmt.Errorf("error decoding response: %w", err)
|
||||
}
|
||||
|
||||
if result.Status != 200 {
|
||||
return false, fmt.Errorf("API error: %s", result.Msg)
|
||||
}
|
||||
|
||||
return shouldRetryHTTP(resp.StatusCode), nil
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
if result.Status != 200 {
|
||||
return "", "", fmt.Errorf("API error: %s", result.Msg)
|
||||
}
|
||||
|
||||
return result.Result, result.SessID, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package filelu
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -16,6 +14,7 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
// Object describes a FileLu object
|
||||
@@ -88,6 +87,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
}
|
||||
}
|
||||
|
||||
// Wrap the response body to handle offset and count
|
||||
var reader io.ReadCloser
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", directLink, nil)
|
||||
@@ -109,22 +109,25 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
return false, fmt.Errorf("failed to download file: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Wrap the response body to handle offset and count
|
||||
currentContents, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to read response body: %w", err)
|
||||
if offset > 0 {
|
||||
_, err = io.CopyN(io.Discard, resp.Body, offset)
|
||||
if err != nil {
|
||||
_ = resp.Body.Close()
|
||||
return false, fmt.Errorf("failed to skip offset: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if offset > 0 {
|
||||
if offset > int64(len(currentContents)) {
|
||||
return false, fmt.Errorf("offset %d exceeds file size %d", offset, len(currentContents))
|
||||
if count > 0 {
|
||||
reader = struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{
|
||||
Reader: io.LimitReader(resp.Body, count),
|
||||
Closer: resp.Body,
|
||||
}
|
||||
currentContents = currentContents[offset:]
|
||||
} else {
|
||||
reader = resp.Body
|
||||
}
|
||||
if count > 0 && count < int64(len(currentContents)) {
|
||||
currentContents = currentContents[:count]
|
||||
}
|
||||
reader = io.NopCloser(bytes.NewReader(currentContents))
|
||||
|
||||
return false, nil
|
||||
})
|
||||
@@ -137,15 +140,23 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
|
||||
// Update updates the object with new data
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
if src.Size() <= 0 {
|
||||
return fs.ErrorCantUploadEmptyFiles
|
||||
size := src.Size()
|
||||
|
||||
if size <= int64(o.fs.opt.UploadCutoff) {
|
||||
err := o.fs.uploadFile(ctx, in, o.remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
fullPath := path.Join(o.fs.root, o.remote)
|
||||
err := o.fs.multipartUpload(ctx, in, fullPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
err := o.fs.uploadFile(ctx, in, o.remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file: %w", err)
|
||||
}
|
||||
o.size = src.Size()
|
||||
o.size = size
|
||||
o.modTime = src.ModTime(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -183,8 +194,14 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
return "", fmt.Errorf("no valid file code found in the remote path")
|
||||
}
|
||||
|
||||
apiURL := fmt.Sprintf("%s/file/info?file_code=%s&key=%s",
|
||||
o.fs.endpoint, url.QueryEscape(fileCode), url.QueryEscape(o.fs.opt.Key))
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/file/info",
|
||||
Parameters: url.Values{
|
||||
"file_code": {fileCode},
|
||||
"key": {o.fs.opt.Key},
|
||||
},
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Status int `json:"status"`
|
||||
@@ -193,29 +210,18 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
Hash string `json:"hash"`
|
||||
} `json:"result"`
|
||||
}
|
||||
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := o.fs.client.Do(req)
|
||||
_, err := o.fs.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
if err != nil {
|
||||
return shouldRetry(err), err
|
||||
}
|
||||
defer func() {
|
||||
if err := resp.Body.Close(); err != nil {
|
||||
fs.Logf(nil, "Failed to close response body: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return shouldRetryHTTP(resp.StatusCode), nil
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if result.Status != 200 || len(result.Result) == 0 {
|
||||
return "", fmt.Errorf("error: unable to fetch hash: %s", result.Msg)
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,14 @@
|
||||
package filen
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestFilen:",
|
||||
NilObject: (*Object)(nil),
|
||||
})
|
||||
}
|
||||
+9
-3
@@ -204,6 +204,12 @@ Example:
|
||||
Help: `URL for HTTP CONNECT proxy
|
||||
|
||||
Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
|
||||
|
||||
Supports the format http://user:pass@host:port, http://host:port, http://host.
|
||||
|
||||
Example:
|
||||
|
||||
http://myUser:myPass@proxyhostname.example.com:8000
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -892,7 +898,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
|
||||
resultchan := make(chan []*ftp.Entry, 1)
|
||||
errchan := make(chan error, 1)
|
||||
go func() {
|
||||
go func(c *ftp.ServerConn) {
|
||||
result, err := c.List(f.dirFromStandardPath(path.Join(f.root, dir)))
|
||||
f.putFtpConnection(&c, err)
|
||||
if err != nil {
|
||||
@@ -900,7 +906,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return
|
||||
}
|
||||
resultchan <- result
|
||||
}()
|
||||
}(c)
|
||||
|
||||
// Wait for List for up to Timeout seconds
|
||||
timer := time.NewTimer(f.ci.TimeoutOrInfinite())
|
||||
@@ -1292,7 +1298,7 @@ func (f *ftpReadCloser) Close() error {
|
||||
// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
|
||||
if errX := textprotoError(err); errX != nil {
|
||||
switch errX.Code {
|
||||
case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
|
||||
case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend, ftp.StatusRequestedFileActionOK:
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,9 +346,26 @@ can't check the size and hash but the file contents will be decompressed.
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for the service.\n\nLeave blank normally.",
|
||||
Name: "endpoint",
|
||||
Help: `Custom endpoint for the storage API. Leave blank to use the provider default.
|
||||
|
||||
When using a custom endpoint that includes a subpath (e.g. example.org/custom/endpoint),
|
||||
the subpath will be ignored during upload operations due to a limitation in the
|
||||
underlying Google API Go client library.
|
||||
Download and listing operations will work correctly with the full endpoint path.
|
||||
If you require subpath support for uploads, avoid using subpaths in your custom
|
||||
endpoint configuration.`,
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "storage.example.org",
|
||||
Help: "Specify a custom endpoint",
|
||||
}, {
|
||||
Value: "storage.example.org:4443",
|
||||
Help: "Specifying a custom endpoint with port",
|
||||
}, {
|
||||
Value: "storage.example.org:4443/gcs/api",
|
||||
Help: "Specifying a subpath, see the note, uploads won't use the custom path!",
|
||||
}},
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
|
||||
@@ -116,8 +116,8 @@ type Date struct {
|
||||
type DateFilter struct {
|
||||
Dates []Date `json:"dates,omitempty"`
|
||||
Ranges []struct {
|
||||
StartDate Date `json:"startDate,omitempty"`
|
||||
EndDate Date `json:"endDate,omitempty"`
|
||||
StartDate Date `json:"startDate"`
|
||||
EndDate Date `json:"endDate"`
|
||||
} `json:"ranges,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
+25
-16
@@ -43,33 +43,42 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
|
||||
var commandHelp = []fs.CommandHelp{{
|
||||
Name: "drop",
|
||||
Short: "Drop cache",
|
||||
Short: "Drop cache.",
|
||||
Long: `Completely drop checksum cache.
|
||||
Usage Example:
|
||||
rclone backend drop hasher:
|
||||
`,
|
||||
|
||||
Usage example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend drop hasher:
|
||||
` + "```",
|
||||
}, {
|
||||
Name: "dump",
|
||||
Short: "Dump the database",
|
||||
Long: "Dump cache records covered by the current remote",
|
||||
Short: "Dump the database.",
|
||||
Long: "Dump cache records covered by the current remote.",
|
||||
}, {
|
||||
Name: "fulldump",
|
||||
Short: "Full dump of the database",
|
||||
Long: "Dump all cache records in the database",
|
||||
Short: "Full dump of the database.",
|
||||
Long: "Dump all cache records in the database.",
|
||||
}, {
|
||||
Name: "import",
|
||||
Short: "Import a SUM file",
|
||||
Short: "Import a SUM file.",
|
||||
Long: `Amend hash cache from a SUM file and bind checksums to files by size/time.
|
||||
Usage Example:
|
||||
rclone backend import hasher:subdir md5 /path/to/sum.md5
|
||||
`,
|
||||
|
||||
Usage example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend import hasher:subdir md5 /path/to/sum.md5
|
||||
` + "```",
|
||||
}, {
|
||||
Name: "stickyimport",
|
||||
Short: "Perform fast import of a SUM file",
|
||||
Short: "Perform fast import of a SUM file.",
|
||||
Long: `Fill hash cache from a SUM file without verifying file fingerprints.
|
||||
Usage Example:
|
||||
rclone backend stickyimport hasher:subdir md5 remote:path/to/sum.md5
|
||||
`,
|
||||
|
||||
Usage example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend stickyimport hasher:subdir md5 remote:path/to/sum.md5
|
||||
` + "```",
|
||||
}}
|
||||
|
||||
func (f *Fs) dbDump(ctx context.Context, full bool, root string) error {
|
||||
|
||||
+130
-10
@@ -11,6 +11,7 @@ import (
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -37,6 +38,10 @@ func init() {
|
||||
Description: "HTTP",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
System: systemMetadataInfo,
|
||||
Help: `HTTP metadata keys are case insensitive and are always returned in lower case.`,
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: "url",
|
||||
Help: "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
|
||||
@@ -98,6 +103,40 @@ sizes of any files, and some files that don't exist may be in the listing.`,
|
||||
fs.Register(fsi)
|
||||
}
|
||||
|
||||
// system metadata keys which this backend owns
|
||||
var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||
"cache-control": {
|
||||
Help: "Cache-Control header",
|
||||
Type: "string",
|
||||
Example: "no-cache",
|
||||
},
|
||||
"content-disposition": {
|
||||
Help: "Content-Disposition header",
|
||||
Type: "string",
|
||||
Example: "inline",
|
||||
},
|
||||
"content-disposition-filename": {
|
||||
Help: "Filename retrieved from Content-Disposition header",
|
||||
Type: "string",
|
||||
Example: "file.txt",
|
||||
},
|
||||
"content-encoding": {
|
||||
Help: "Content-Encoding header",
|
||||
Type: "string",
|
||||
Example: "gzip",
|
||||
},
|
||||
"content-language": {
|
||||
Help: "Content-Language header",
|
||||
Type: "string",
|
||||
Example: "en-US",
|
||||
},
|
||||
"content-type": {
|
||||
Help: "Content-Type header",
|
||||
Type: "string",
|
||||
Example: "text/plain",
|
||||
},
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Endpoint string `config:"url"`
|
||||
@@ -126,6 +165,13 @@ type Object struct {
|
||||
size int64
|
||||
modTime time.Time
|
||||
contentType string
|
||||
|
||||
// Metadata as pointers to strings as they often won't be present
|
||||
contentDisposition *string // Content-Disposition: header
|
||||
contentDispositionFilename *string // Filename retrieved from Content-Disposition: header
|
||||
cacheControl *string // Cache-Control: header
|
||||
contentEncoding *string // Content-Encoding: header
|
||||
contentLanguage *string // Content-Language: header
|
||||
}
|
||||
|
||||
// statusError returns an error if the res contained an error
|
||||
@@ -277,6 +323,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ci: ci,
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
ReadMetadata: true,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
@@ -429,6 +476,29 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// parseFilename extracts the filename from a Content-Disposition header
|
||||
func parseFilename(contentDisposition string) (string, error) {
|
||||
// Normalize the contentDisposition to canonical MIME format
|
||||
mediaType, params, err := mime.ParseMediaType(contentDisposition)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse contentDisposition: %v", err)
|
||||
}
|
||||
|
||||
// Check if the contentDisposition is an attachment
|
||||
if strings.ToLower(mediaType) != "attachment" {
|
||||
return "", fmt.Errorf("not an attachment: %s", mediaType)
|
||||
}
|
||||
|
||||
// Extract the filename from the parameters
|
||||
filename, ok := params["filename"]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("filename not found in contentDisposition")
|
||||
}
|
||||
|
||||
// Decode filename if it contains special encoding
|
||||
return textproto.TrimString(filename), nil
|
||||
}
|
||||
|
||||
// Adds the configured headers to the request if any
|
||||
func addHeaders(req *http.Request, opt *Options) {
|
||||
for i := 0; i < len(opt.Headers); i += 2 {
|
||||
@@ -513,9 +583,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
entriesMu.Unlock()
|
||||
}
|
||||
for range checkers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
wg.Go(func() {
|
||||
for remote := range in {
|
||||
file := &Object{
|
||||
fs: f,
|
||||
@@ -531,7 +599,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
fs.Debugf(remote, "skipping because of error: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
for _, name := range names {
|
||||
isDir := name[len(name)-1] == '/'
|
||||
@@ -577,6 +645,9 @@ func (o *Object) String() string {
|
||||
|
||||
// Remote the name of the remote HTTP file, relative to the fs root
|
||||
func (o *Object) Remote() string {
|
||||
if o.contentDispositionFilename != nil {
|
||||
return *o.contentDispositionFilename
|
||||
}
|
||||
return o.remote
|
||||
}
|
||||
|
||||
@@ -634,6 +705,29 @@ func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
|
||||
o.modTime = t
|
||||
o.contentType = res.Header.Get("Content-Type")
|
||||
o.size = rest.ParseSizeFromHeaders(res.Header)
|
||||
contentDisposition := res.Header.Get("Content-Disposition")
|
||||
if contentDisposition != "" {
|
||||
o.contentDisposition = &contentDisposition
|
||||
}
|
||||
if o.contentDisposition != nil {
|
||||
var filename string
|
||||
filename, err = parseFilename(*o.contentDisposition)
|
||||
if err == nil && filename != "" {
|
||||
o.contentDispositionFilename = &filename
|
||||
}
|
||||
}
|
||||
cacheControl := res.Header.Get("Cache-Control")
|
||||
if cacheControl != "" {
|
||||
o.cacheControl = &cacheControl
|
||||
}
|
||||
contentEncoding := res.Header.Get("Content-Encoding")
|
||||
if contentEncoding != "" {
|
||||
o.contentEncoding = &contentEncoding
|
||||
}
|
||||
contentLanguage := res.Header.Get("Content-Language")
|
||||
if contentLanguage != "" {
|
||||
o.contentLanguage = &contentLanguage
|
||||
}
|
||||
|
||||
// If NoSlash is set then check ContentType to see if it is a directory
|
||||
if o.fs.opt.NoSlash {
|
||||
@@ -722,11 +816,13 @@ var commandHelp = []fs.CommandHelp{{
|
||||
Long: `This set command can be used to update the config parameters
|
||||
for a running http backend.
|
||||
|
||||
Usage Examples:
|
||||
Usage examples:
|
||||
|
||||
rclone backend set remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=remote: -o url=https://example.com
|
||||
` + "```console" + `
|
||||
rclone backend set remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
|
||||
rclone rc backend/command command=set fs=remote: -o url=https://example.com
|
||||
` + "```" + `
|
||||
|
||||
The option keys are named as they are in the config file.
|
||||
|
||||
@@ -734,8 +830,7 @@ This rebuilds the connection to the http backend when it is called with
|
||||
the new parameters. Only new parameters need be passed as the values
|
||||
will default to those currently in use.
|
||||
|
||||
It doesn't return anything.
|
||||
`,
|
||||
It doesn't return anything.`,
|
||||
}}
|
||||
|
||||
// Command the backend to run a named command
|
||||
@@ -771,6 +866,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
}
|
||||
}
|
||||
|
||||
// Metadata returns metadata for an object
|
||||
//
|
||||
// It should return nil if there is no Metadata
|
||||
func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
|
||||
metadata = make(fs.Metadata, 6)
|
||||
if o.contentType != "" {
|
||||
metadata["content-type"] = o.contentType
|
||||
}
|
||||
|
||||
// Set system metadata
|
||||
setMetadata := func(k string, v *string) {
|
||||
if v == nil || *v == "" {
|
||||
return
|
||||
}
|
||||
metadata[k] = *v
|
||||
}
|
||||
setMetadata("content-disposition", o.contentDisposition)
|
||||
setMetadata("content-disposition-filename", o.contentDispositionFilename)
|
||||
setMetadata("cache-control", o.cacheControl)
|
||||
setMetadata("content-language", o.contentLanguage)
|
||||
setMetadata("content-encoding", o.contentEncoding)
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
@@ -778,4 +897,5 @@ var (
|
||||
_ fs.Object = &Object{}
|
||||
_ fs.MimeTyper = &Object{}
|
||||
_ fs.Commander = &Fs{}
|
||||
_ fs.Metadataer = &Object{}
|
||||
)
|
||||
@@ -60,6 +60,17 @@ func prepareServer(t *testing.T) configmap.Simple {
|
||||
what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path)
|
||||
assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0])
|
||||
assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2])
|
||||
|
||||
// Set the content disposition header for the fifth file
|
||||
// later we will check if it is set using the metadata method
|
||||
if r.URL.Path == "/five.txt.gz" {
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=\"five.txt.gz\"")
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Content-Language", "en-US")
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
}
|
||||
|
||||
fileServer.ServeHTTP(w, r)
|
||||
})
|
||||
|
||||
@@ -102,27 +113,33 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
|
||||
|
||||
sort.Sort(entries)
|
||||
|
||||
require.Equal(t, 4, len(entries))
|
||||
require.Equal(t, 5, len(entries))
|
||||
|
||||
e := entries[0]
|
||||
assert.Equal(t, "four", e.Remote())
|
||||
assert.Equal(t, "five.txt.gz", e.Remote())
|
||||
assert.Equal(t, int64(-1), e.Size())
|
||||
_, ok := e.(fs.Directory)
|
||||
_, ok := e.(fs.Object)
|
||||
assert.True(t, ok)
|
||||
|
||||
e = entries[1]
|
||||
assert.Equal(t, "four", e.Remote())
|
||||
assert.Equal(t, int64(-1), e.Size())
|
||||
_, ok = e.(fs.Directory)
|
||||
assert.True(t, ok)
|
||||
|
||||
e = entries[2]
|
||||
assert.Equal(t, "one%.txt", e.Remote())
|
||||
assert.Equal(t, int64(5+lineEndSize), e.Size())
|
||||
_, ok = e.(*Object)
|
||||
assert.True(t, ok)
|
||||
|
||||
e = entries[2]
|
||||
e = entries[3]
|
||||
assert.Equal(t, "three", e.Remote())
|
||||
assert.Equal(t, int64(-1), e.Size())
|
||||
_, ok = e.(fs.Directory)
|
||||
assert.True(t, ok)
|
||||
|
||||
e = entries[3]
|
||||
e = entries[4]
|
||||
assert.Equal(t, "two.html", e.Remote())
|
||||
if noSlash {
|
||||
assert.Equal(t, int64(-1), e.Size())
|
||||
@@ -218,6 +235,23 @@ func TestNewObjectWithLeadingSlash(t *testing.T) {
|
||||
assert.Equal(t, fs.ErrorObjectNotFound, err)
|
||||
}
|
||||
|
||||
func TestNewObjectWithMetadata(t *testing.T) {
|
||||
f := prepare(t)
|
||||
o, err := f.NewObject(context.Background(), "/five.txt.gz")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "five.txt.gz", o.Remote())
|
||||
ho, ok := o.(*Object)
|
||||
assert.True(t, ok)
|
||||
metadata, err := ho.Metadata(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "text/plain; charset=utf-8", metadata["content-type"])
|
||||
assert.Equal(t, "attachment; filename=\"five.txt.gz\"", metadata["content-disposition"])
|
||||
assert.Equal(t, "five.txt.gz", metadata["content-disposition-filename"])
|
||||
assert.Equal(t, "no-cache", metadata["cache-control"])
|
||||
assert.Equal(t, "en-US", metadata["content-language"])
|
||||
assert.Equal(t, "gzip", metadata["content-encoding"])
|
||||
}
|
||||
|
||||
func TestOpen(t *testing.T) {
|
||||
m := prepareServer(t)
|
||||
|
||||
|
||||
Binary file not shown.
@@ -599,7 +599,7 @@ type UpdateFileInfo struct {
|
||||
Signature string `json:"signature,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
WrappingKey string `json:"wrapping_key,omitempty"`
|
||||
} `json:"data,omitempty"`
|
||||
} `json:"data"`
|
||||
DocumentID string `json:"document_id"`
|
||||
FileFlags FileFlags `json:"file_flags"`
|
||||
Mtime int64 `json:"mtime"`
|
||||
@@ -849,10 +849,10 @@ type DriveItem struct {
|
||||
NumberOfItems int64 `json:"numberOfItems"`
|
||||
Status string `json:"status"`
|
||||
Extension string `json:"extension,omitempty"`
|
||||
DateModified time.Time `json:"dateModified,omitempty"`
|
||||
DateChanged time.Time `json:"dateChanged,omitempty"`
|
||||
DateModified time.Time `json:"dateModified"`
|
||||
DateChanged time.Time `json:"dateChanged"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
LastOpenTime time.Time `json:"lastOpenTime,omitempty"`
|
||||
LastOpenTime time.Time `json:"lastOpenTime"`
|
||||
Urls struct {
|
||||
URLDownload string `json:"url_download"`
|
||||
} `json:"urls"`
|
||||
|
||||
@@ -72,7 +72,7 @@ func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadPara
|
||||
|
||||
response := &UploadResult{}
|
||||
|
||||
formReader, contentType, _, err := rest.MultipartUpload(ctx, file, formParams, "file", param.FileName)
|
||||
formReader, contentType, _, err := rest.MultipartUpload(ctx, file, formParams, "file", param.FileName, "application/octet-stream")
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
// Package internxt provides authentication handling
|
||||
package internxt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
internxtauth "github.com/internxt/rclone-adapter/auth"
|
||||
internxtconfig "github.com/internxt/rclone-adapter/config"
|
||||
sdkerrors "github.com/internxt/rclone-adapter/errors"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
type userInfo struct {
|
||||
RootFolderID string
|
||||
Bucket string
|
||||
BridgeUser string
|
||||
UserID string
|
||||
}
|
||||
|
||||
type userInfoConfig struct {
|
||||
Token string
|
||||
}
|
||||
|
||||
// getUserInfo fetches user metadata from the refresh endpoint
|
||||
func getUserInfo(ctx context.Context, cfg *userInfoConfig) (*userInfo, error) {
|
||||
// Call the refresh endpoint to get all user metadata
|
||||
refreshCfg := internxtconfig.NewDefaultToken(cfg.Token)
|
||||
resp, err := internxtauth.RefreshToken(ctx, refreshCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch user info: %w", err)
|
||||
}
|
||||
|
||||
if resp.User.Bucket == "" {
|
||||
return nil, errors.New("API response missing user.bucket")
|
||||
}
|
||||
if resp.User.RootFolderID == "" {
|
||||
return nil, errors.New("API response missing user.rootFolderId")
|
||||
}
|
||||
if resp.User.BridgeUser == "" {
|
||||
return nil, errors.New("API response missing user.bridgeUser")
|
||||
}
|
||||
if resp.User.UserID == "" {
|
||||
return nil, errors.New("API response missing user.userId")
|
||||
}
|
||||
|
||||
info := &userInfo{
|
||||
RootFolderID: resp.User.RootFolderID,
|
||||
Bucket: resp.User.Bucket,
|
||||
BridgeUser: resp.User.BridgeUser,
|
||||
UserID: resp.User.UserID,
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "User info: rootFolderId=%s, bucket=%s",
|
||||
info.RootFolderID, info.Bucket)
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// parseJWTExpiry extracts the expiry time from a JWT token string
|
||||
func parseJWTExpiry(tokenString string) (time.Time, error) {
|
||||
parser := jwt.NewParser(jwt.WithoutClaimsValidation())
|
||||
token, _, err := parser.ParseUnverified(tokenString, jwt.MapClaims{})
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("failed to parse token: %w", err)
|
||||
}
|
||||
|
||||
claims, ok := token.Claims.(jwt.MapClaims)
|
||||
if !ok {
|
||||
return time.Time{}, errors.New("invalid token claims")
|
||||
}
|
||||
|
||||
exp, ok := claims["exp"].(float64)
|
||||
if !ok {
|
||||
return time.Time{}, errors.New("token missing expiration")
|
||||
}
|
||||
|
||||
return time.Unix(int64(exp), 0), nil
|
||||
}
|
||||
|
||||
// jwtToOAuth2Token converts a JWT string to an oauth2.Token with expiry
|
||||
func jwtToOAuth2Token(jwtString string) (*oauth2.Token, error) {
|
||||
expiry, err := parseJWTExpiry(jwtString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &oauth2.Token{
|
||||
AccessToken: jwtString,
|
||||
TokenType: "Bearer",
|
||||
Expiry: expiry,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// computeBasicAuthHeader creates the BasicAuthHeader for bucket operations
|
||||
func computeBasicAuthHeader(bridgeUser, userID string) string {
|
||||
sum := sha256.Sum256([]byte(userID))
|
||||
hexPass := hex.EncodeToString(sum[:])
|
||||
creds := fmt.Sprintf("%s:%s", bridgeUser, hexPass)
|
||||
return "Basic " + base64.StdEncoding.EncodeToString([]byte(creds))
|
||||
}
|
||||
|
||||
// refreshJWTToken refreshes the token using Internxt's refresh endpoint
|
||||
func refreshJWTToken(ctx context.Context, name string, m configmap.Mapper) error {
|
||||
currentToken, err := oauthutil.GetToken(name, m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get current token: %w", err)
|
||||
}
|
||||
|
||||
cfg := internxtconfig.NewDefaultToken(currentToken.AccessToken)
|
||||
resp, err := internxtauth.RefreshToken(ctx, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("refresh request failed: %w", err)
|
||||
}
|
||||
|
||||
if resp.NewToken == "" {
|
||||
return errors.New("refresh response missing newToken")
|
||||
}
|
||||
|
||||
// Convert JWT to oauth2.Token format
|
||||
token, err := jwtToOAuth2Token(resp.NewToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse refreshed token: %w", err)
|
||||
}
|
||||
|
||||
err = oauthutil.PutToken(name, m, token, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save token: %w", err)
|
||||
}
|
||||
|
||||
if resp.User.Bucket != "" {
|
||||
m.Set("bucket", resp.User.Bucket)
|
||||
}
|
||||
|
||||
fs.Debugf(name, "Token refreshed successfully, new expiry: %v", token.Expiry)
|
||||
return nil
|
||||
}
|
||||
|
||||
// reLogin performs a full re-login using stored email+password credentials.
|
||||
// Returns the AccessResponse on success, or an error if 2FA is required or login fails.
|
||||
func (f *Fs) reLogin(ctx context.Context) (*internxtauth.AccessResponse, error) {
|
||||
password, err := obscure.Reveal(f.opt.Pass)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't decrypt password: %w", err)
|
||||
}
|
||||
|
||||
cfg := internxtconfig.NewDefaultToken("")
|
||||
cfg.HTTPClient = fshttp.NewClient(ctx)
|
||||
|
||||
loginResp, err := internxtauth.Login(ctx, cfg, f.opt.Email)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("re-login check failed: %w", err)
|
||||
}
|
||||
|
||||
if loginResp.TFA {
|
||||
return nil, errors.New("account requires 2FA - please run: rclone config reconnect " + f.name + ":")
|
||||
}
|
||||
|
||||
resp, err := internxtauth.DoLogin(ctx, cfg, f.opt.Email, password, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("re-login failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// refreshOrReLogin tries to refresh the JWT token first; if that fails with 401,
|
||||
// it falls back to a full re-login using stored credentials.
|
||||
func (f *Fs) refreshOrReLogin(ctx context.Context) error {
|
||||
refreshErr := refreshJWTToken(ctx, f.name, f.m)
|
||||
if refreshErr == nil {
|
||||
newToken, err := oauthutil.GetToken(f.name, f.m)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get refreshed token: %w", err)
|
||||
}
|
||||
f.cfg.Token = newToken.AccessToken
|
||||
f.cfg.BasicAuthHeader = computeBasicAuthHeader(f.bridgeUser, f.userID)
|
||||
fs.Debugf(f, "Token refresh succeeded")
|
||||
return nil
|
||||
}
|
||||
|
||||
var httpErr *sdkerrors.HTTPError
|
||||
if !errors.As(refreshErr, &httpErr) || httpErr.StatusCode() != 401 {
|
||||
if fserrors.ShouldRetry(refreshErr) {
|
||||
return refreshErr
|
||||
}
|
||||
return refreshErr
|
||||
}
|
||||
|
||||
fs.Debugf(f, "Token refresh returned 401, attempting re-login with stored credentials")
|
||||
|
||||
resp, err := f.reLogin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("re-login fallback failed: %w", err)
|
||||
}
|
||||
|
||||
oauthToken, err := jwtToOAuth2Token(resp.NewToken)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse re-login token: %w", err)
|
||||
}
|
||||
err = oauthutil.PutToken(f.name, f.m, oauthToken, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save re-login token: %w", err)
|
||||
}
|
||||
|
||||
f.cfg.Token = oauthToken.AccessToken
|
||||
f.bridgeUser = resp.User.BridgeUser
|
||||
f.userID = resp.User.UserID
|
||||
f.cfg.BasicAuthHeader = computeBasicAuthHeader(f.bridgeUser, f.userID)
|
||||
f.cfg.Bucket = resp.User.Bucket
|
||||
f.cfg.RootFolderID = resp.User.RootFolderID
|
||||
|
||||
fs.Debugf(f, "Re-login succeeded, new token expiry: %v", oauthToken.Expiry)
|
||||
return nil
|
||||
}
|
||||
|
||||
// reAuthorize is called after getting 401 from the server.
|
||||
// It serializes re-auth attempts and uses a circuit-breaker to avoid infinite loops.
|
||||
func (f *Fs) reAuthorize(ctx context.Context) error {
|
||||
f.authMu.Lock()
|
||||
defer f.authMu.Unlock()
|
||||
|
||||
if f.authFailed {
|
||||
return errors.New("re-authorization permanently failed")
|
||||
}
|
||||
|
||||
err := f.refreshOrReLogin(ctx)
|
||||
if err != nil {
|
||||
f.authFailed = true
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,14 @@
|
||||
package internxt_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestInternxt:",
|
||||
})
|
||||
}
|
||||
@@ -1070,12 +1070,11 @@ func (f *Fs) Hashes() hash.Set {
|
||||
var commandHelp = []fs.CommandHelp{
|
||||
{
|
||||
Name: "noop",
|
||||
Short: "A null operation for testing backend commands",
|
||||
Long: `This is a test command which has some options
|
||||
you can try to change the output.`,
|
||||
Short: "A null operation for testing backend commands.",
|
||||
Long: `This is a test command which has some options you can try to change the output.`,
|
||||
Opts: map[string]string{
|
||||
"echo": "echo the input arguments",
|
||||
"error": "return an error based on option value",
|
||||
"echo": "Echo the input arguments.",
|
||||
"error": "Return an error based on option value.",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -33,12 +33,10 @@ func TestRemove(t *testing.T) {
|
||||
assert.True(t, exists())
|
||||
// close the file in the background
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
wg.Go(func() {
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
require.NoError(t, fd.Close())
|
||||
}()
|
||||
})
|
||||
// delete the open file
|
||||
err = remove(name)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
func remove(name string) (err error) {
|
||||
const maxTries = 10
|
||||
var sleepTime = 1 * time.Millisecond
|
||||
for i := 0; i < maxTries; i++ {
|
||||
for i := range maxTries {
|
||||
err = os.Remove(name)
|
||||
if err == nil {
|
||||
break
|
||||
|
||||
@@ -112,7 +112,7 @@ type ListItem struct {
|
||||
Count struct {
|
||||
Folders int `json:"folders"`
|
||||
Files int `json:"files"`
|
||||
} `json:"count,omitempty"`
|
||||
} `json:"count"`
|
||||
Kind string `json:"kind"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
@@ -154,7 +154,7 @@ type FolderInfoResponse struct {
|
||||
Type string `json:"type"`
|
||||
Home string `json:"home"`
|
||||
List []ListItem `json:"list"`
|
||||
} `json:"body,omitempty"`
|
||||
} `json:"body"`
|
||||
Time int64 `json:"time"`
|
||||
Status int `json:"status"`
|
||||
Email string `json:"email"`
|
||||
|
||||
+67
-38
@@ -17,11 +17,10 @@ Improvements:
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -47,6 +46,9 @@ const (
|
||||
maxSleep = 2 * time.Second
|
||||
eventWaitTime = 500 * time.Millisecond
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
|
||||
sessionIDConfigKey = "session_id"
|
||||
masterKeyConfigKey = "master_key"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -70,6 +72,24 @@ func init() {
|
||||
Help: "Password.",
|
||||
Required: true,
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "2fa",
|
||||
Help: `The 2FA code of your MEGA account if the account is set up with one`,
|
||||
Required: false,
|
||||
}, {
|
||||
Name: sessionIDConfigKey,
|
||||
Help: "Session (internal use only)",
|
||||
Required: false,
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: masterKeyConfigKey,
|
||||
Help: "Master key (internal use only)",
|
||||
Required: false,
|
||||
Advanced: true,
|
||||
Sensitive: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: "debug",
|
||||
Help: `Output more debug from Mega.
|
||||
@@ -113,6 +133,9 @@ Enabling it will increase CPU usage and add network overhead.`,
|
||||
type Options struct {
|
||||
User string `config:"user"`
|
||||
Pass string `config:"pass"`
|
||||
TwoFA string `config:"2fa"`
|
||||
SessionID string `config:"session_id"`
|
||||
MasterKey string `config:"master_key"`
|
||||
Debug bool `config:"debug"`
|
||||
HardDelete bool `config:"hard_delete"`
|
||||
UseHTTPS bool `config:"use_https"`
|
||||
@@ -209,6 +232,19 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
ci := fs.GetConfig(ctx)
|
||||
|
||||
// Create Fs
|
||||
root = parsePath(root)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
DuplicateFiles: true,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
// cache *mega.Mega on username so we can reuse and share
|
||||
// them between remotes. They are expensive to make as they
|
||||
// contain all the objects and sharing the objects makes the
|
||||
@@ -218,25 +254,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
defer megaCacheMu.Unlock()
|
||||
srv := megaCache[opt.User]
|
||||
if srv == nil {
|
||||
// srv = mega.New().SetClient(fshttp.NewClient(ctx))
|
||||
|
||||
// Workaround for Mega's use of insecure cipher suites which are no longer supported by default since Go 1.22.
|
||||
// Relevant issues:
|
||||
// https://github.com/rclone/rclone/issues/8565
|
||||
// https://github.com/meganz/webclient/issues/103
|
||||
clt := fshttp.NewClient(ctx)
|
||||
clt.Transport = fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
|
||||
var ids []uint16
|
||||
// Read default ciphers
|
||||
for _, cs := range tls.CipherSuites() {
|
||||
ids = append(ids, cs.ID)
|
||||
}
|
||||
// Insecure but Mega uses TLS_RSA_WITH_AES_128_GCM_SHA256 for storage endpoints
|
||||
// (e.g. https://gfs302n114.userstorage.mega.co.nz) as of June 18, 2025.
|
||||
t.TLSClientConfig.CipherSuites = append(ids, tls.TLS_RSA_WITH_AES_128_GCM_SHA256)
|
||||
})
|
||||
srv = mega.New().SetClient(clt)
|
||||
|
||||
srv = mega.New().SetClient(fshttp.NewClient(ctx))
|
||||
srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
|
||||
srv.SetHTTPS(opt.UseHTTPS)
|
||||
srv.SetLogger(func(format string, v ...any) {
|
||||
@@ -248,25 +266,29 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
})
|
||||
}
|
||||
|
||||
err := srv.Login(opt.User, opt.Pass)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't login: %w", err)
|
||||
if opt.SessionID == "" {
|
||||
fs.Debugf(f, "Using username and password to initialize the Mega API")
|
||||
err := srv.MultiFactorLogin(opt.User, opt.Pass, opt.TwoFA)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't login: %w", err)
|
||||
}
|
||||
megaCache[opt.User] = srv
|
||||
m.Set(sessionIDConfigKey, srv.GetSessionID())
|
||||
encodedMasterKey := base64.StdEncoding.EncodeToString(srv.GetMasterKey())
|
||||
m.Set(masterKeyConfigKey, encodedMasterKey)
|
||||
} else {
|
||||
fs.Debugf(f, "Using previously stored session ID and master key to initialize the Mega API")
|
||||
decodedMasterKey, err := base64.StdEncoding.DecodeString(opt.MasterKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't decode master key: %w", err)
|
||||
}
|
||||
err = srv.LoginWithKeys(opt.SessionID, decodedMasterKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't login with session ID and master key: %w", err)
|
||||
}
|
||||
}
|
||||
megaCache[opt.User] = srv
|
||||
}
|
||||
|
||||
root = parsePath(root)
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: srv,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
DuplicateFiles: true,
|
||||
CanHaveEmptyDirectories: true,
|
||||
}).Fill(ctx, f)
|
||||
f.srv = srv
|
||||
|
||||
// Find the root node and check if it is a file or not
|
||||
_, err = f.findRoot(ctx, false)
|
||||
@@ -319,6 +341,8 @@ func (f *Fs) findDir(rootNode *mega.Node, dir string) (node *mega.Node, err erro
|
||||
node, err = f.findNode(rootNode, dir)
|
||||
if err == mega.ENOENT {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
} else if err == nil && node == nil {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
} else if err == nil && node.GetType() == mega.FILE {
|
||||
return nil, fs.ErrorIsFile
|
||||
}
|
||||
@@ -330,6 +354,8 @@ func (f *Fs) findObject(rootNode *mega.Node, file string) (node *mega.Node, err
|
||||
node, err = f.findNode(rootNode, file)
|
||||
if err == mega.ENOENT {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
} else if err == nil && node == nil {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
} else if err == nil && node.GetType() != mega.FILE {
|
||||
return nil, fs.ErrorIsDir // all other node types are directories
|
||||
}
|
||||
@@ -424,6 +450,9 @@ func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {
|
||||
|
||||
// Check for preexisting root
|
||||
absRoot := f.srv.FS.GetRoot()
|
||||
if absRoot == nil {
|
||||
return nil, errors.New("couldn't find root directory: filesystem not initialized (is the account authenticated?)")
|
||||
}
|
||||
node, err := f.findDir(absRoot, f.root)
|
||||
//log.Printf("findRoot findDir %p %v", node, err)
|
||||
if err == nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
@@ -24,7 +25,8 @@ import (
|
||||
var (
|
||||
hashType = hash.MD5
|
||||
// the object storage is persistent
|
||||
buckets = newBucketsInfo()
|
||||
buckets = newBucketsInfo()
|
||||
errWriteOnly = errors.New("can't read when using --memory-discard")
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -33,12 +35,32 @@ func init() {
|
||||
Name: "memory",
|
||||
Description: "In memory object storage system.",
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{},
|
||||
Options: []fs.Option{{
|
||||
Name: "discard",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
Help: `If set all writes will be discarded and reads will return an error
|
||||
|
||||
If set then when files are uploaded the contents not be saved. The
|
||||
files will appear to have been uploaded but will give an error on
|
||||
read. Files will have their MD5 sum calculated on upload which takes
|
||||
very little CPU time and allows the transfers to be checked.
|
||||
|
||||
This can be useful for testing performance.
|
||||
|
||||
Probably most easily used by using the connection string syntax:
|
||||
|
||||
:memory,discard:bucket
|
||||
|
||||
`,
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct{}
|
||||
type Options struct {
|
||||
Discard bool `config:"discard"`
|
||||
}
|
||||
|
||||
// Fs represents a remote memory server
|
||||
type Fs struct {
|
||||
@@ -164,6 +186,7 @@ type objectData struct {
|
||||
hash string
|
||||
mimeType string
|
||||
data []byte
|
||||
size int64
|
||||
}
|
||||
|
||||
// Object describes a memory object
|
||||
@@ -558,7 +581,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hashType {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
if o.od.hash == "" {
|
||||
if o.od.hash == "" && !o.fs.opt.Discard {
|
||||
sum := md5.Sum(o.od.data)
|
||||
o.od.hash = hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -567,7 +590,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
|
||||
// Size returns the size of an object in bytes
|
||||
func (o *Object) Size() int64 {
|
||||
return int64(len(o.od.data))
|
||||
return o.od.size
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
@@ -593,6 +616,9 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// Open an object for read
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
if o.fs.opt.Discard {
|
||||
return nil, errWriteOnly
|
||||
}
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options {
|
||||
switch x := option.(type) {
|
||||
@@ -624,13 +650,24 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// The new object may have been created if an error is returned
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
bucket, bucketPath := o.split()
|
||||
data, err := io.ReadAll(in)
|
||||
var data []byte
|
||||
var size int64
|
||||
var hash string
|
||||
if o.fs.opt.Discard {
|
||||
h := md5.New()
|
||||
size, err = io.Copy(h, in)
|
||||
hash = hex.EncodeToString(h.Sum(nil))
|
||||
} else {
|
||||
data, err = io.ReadAll(in)
|
||||
size = int64(len(data))
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update memory object: %w", err)
|
||||
}
|
||||
o.od = &objectData{
|
||||
data: data,
|
||||
hash: "",
|
||||
size: size,
|
||||
hash: hash,
|
||||
modTime: src.ModTime(ctx),
|
||||
mimeType: fs.MimeType(ctx, src),
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ Please choose the 'y' option to set your own password then enter your secret.`,
|
||||
|
||||
var commandHelp = []fs.CommandHelp{{
|
||||
Name: "du",
|
||||
Short: "Return disk usage information for a specified directory",
|
||||
Short: "Return disk usage information for a specified directory.",
|
||||
Long: `The usage information returned, includes the targeted directory as well as all
|
||||
files stored in any sub-directories that may exist.`,
|
||||
}, {
|
||||
@@ -96,7 +96,12 @@ files stored in any sub-directories that may exist.`,
|
||||
Long: `The desired path location (including applicable sub-directories) ending in
|
||||
the object that will be the target of the symlink (for example, /links/mylink).
|
||||
Include the file extension for the object, if applicable.
|
||||
` + "`rclone backend symlink <src> <path>`",
|
||||
|
||||
Usage example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend symlink <src> <path>
|
||||
` + "```",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -50,12 +50,12 @@ type Identity struct {
|
||||
// to represent a set of identities associated with various events for
|
||||
// an item, such as created by or last modified by.
|
||||
type IdentitySet struct {
|
||||
User Identity `json:"user,omitempty"`
|
||||
Application Identity `json:"application,omitempty"`
|
||||
Device Identity `json:"device,omitempty"`
|
||||
Group Identity `json:"group,omitempty"`
|
||||
SiteGroup Identity `json:"siteGroup,omitempty"` // The SharePoint group associated with this action. Optional.
|
||||
SiteUser Identity `json:"siteUser,omitempty"` // The SharePoint user associated with this action. Optional.
|
||||
User Identity `json:"user"`
|
||||
Application Identity `json:"application"`
|
||||
Device Identity `json:"device"`
|
||||
Group Identity `json:"group"`
|
||||
SiteGroup Identity `json:"siteGroup"` // The SharePoint group associated with this action. Optional.
|
||||
SiteUser Identity `json:"siteUser"` // The SharePoint user associated with this action. Optional.
|
||||
}
|
||||
|
||||
// Quota groups storage space quota-related information on OneDrive into a single structure.
|
||||
@@ -133,7 +133,7 @@ type RemoteItemFacet struct {
|
||||
|
||||
// FolderFacet groups folder-related data on OneDrive into a single structure
|
||||
type FolderFacet struct {
|
||||
ChildCount int64 `json:"childCount"` // Number of children contained immediately within this container.
|
||||
ChildCount int64 `json:"childCount,omitempty"` // Number of children contained immediately within this container.
|
||||
}
|
||||
|
||||
// HashesType groups different types of hashes into a single structure, for an item on OneDrive.
|
||||
@@ -155,8 +155,8 @@ type FileFacet struct {
|
||||
// facet can be used to specify the last modified date or created date
|
||||
// of the item as it was on the local device.
|
||||
type FileSystemInfoFacet struct {
|
||||
CreatedDateTime Timestamp `json:"createdDateTime,omitempty"` // The UTC date and time the file was created on a client.
|
||||
LastModifiedDateTime Timestamp `json:"lastModifiedDateTime,omitempty"` // The UTC date and time the file was last modified on a client.
|
||||
CreatedDateTime Timestamp `json:"createdDateTime"` // The UTC date and time the file was created on a client.
|
||||
LastModifiedDateTime Timestamp `json:"lastModifiedDateTime"` // The UTC date and time the file was last modified on a client.
|
||||
}
|
||||
|
||||
// DeletedFacet indicates that the item on OneDrive has been
|
||||
@@ -175,10 +175,10 @@ type PackageFacet struct {
|
||||
// SharedType indicates a DriveItem has been shared with others. The resource includes information about how the item is shared.
|
||||
// If a Driveitem has a non-null shared facet, the item has been shared.
|
||||
type SharedType struct {
|
||||
Owner IdentitySet `json:"owner,omitempty"` // The identity of the owner of the shared item. Read-only.
|
||||
Scope string `json:"scope,omitempty"` // Indicates the scope of how the item is shared: anonymous, organization, or users. Read-only.
|
||||
SharedBy IdentitySet `json:"sharedBy,omitempty"` // The identity of the user who shared the item. Read-only.
|
||||
SharedDateTime Timestamp `json:"sharedDateTime,omitempty"` // The UTC date and time when the item was shared. Read-only.
|
||||
Owner IdentitySet `json:"owner"` // The identity of the owner of the shared item. Read-only.
|
||||
Scope string `json:"scope,omitempty"` // Indicates the scope of how the item is shared: anonymous, organization, or users. Read-only.
|
||||
SharedBy IdentitySet `json:"sharedBy"` // The identity of the user who shared the item. Read-only.
|
||||
SharedDateTime Timestamp `json:"sharedDateTime"` // The UTC date and time when the item was shared. Read-only.
|
||||
}
|
||||
|
||||
// SharingInvitationType groups invitation-related data items into a single structure.
|
||||
|
||||
@@ -60,7 +60,7 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
|
||||
ReadOnly: true,
|
||||
},
|
||||
"description": {
|
||||
Help: "A short description of the file. Max 1024 characters. Only supported for OneDrive Personal.",
|
||||
Help: "A short description of the file. Max 1024 characters. No longer supported by Microsoft.",
|
||||
Type: "string",
|
||||
Example: "Contract for signing",
|
||||
},
|
||||
@@ -259,12 +259,8 @@ func (m *Metadata) Set(ctx context.Context, metadata fs.Metadata) (numSet int, e
|
||||
m.btime = t
|
||||
numSet++
|
||||
case "description":
|
||||
if m.fs.driveType != driveTypePersonal {
|
||||
fs.Debugf(m.remote, "metadata description is only supported for OneDrive Personal -- skipping: %s", v)
|
||||
continue
|
||||
}
|
||||
m.description = v
|
||||
numSet++
|
||||
fs.Debugf(m.remote, "metadata description is no longer supported -- skipping: %s", v)
|
||||
continue
|
||||
case "permissions":
|
||||
if !m.fs.opt.MetadataPermissions.IsSet(rwWrite) {
|
||||
continue
|
||||
@@ -292,9 +288,6 @@ func (m *Metadata) toAPIMetadata() api.Metadata {
|
||||
update := api.Metadata{
|
||||
FileSystemInfo: &api.FileSystemInfoFacet{},
|
||||
}
|
||||
if m.description != "" && m.fs.driveType == driveTypePersonal {
|
||||
update.Description = m.description
|
||||
}
|
||||
if !m.mtime.IsZero() {
|
||||
update.FileSystemInfo.LastModifiedDateTime = api.Timestamp(m.mtime)
|
||||
}
|
||||
@@ -603,12 +596,10 @@ func (m *Metadata) addPermission(ctx context.Context, p *api.PermissionsType) (n
|
||||
|
||||
req := &api.AddPermissionsRequest{
|
||||
Recipients: fillRecipients(p, m.fs.driveType),
|
||||
RequireSignIn: m.fs.driveType != driveTypePersonal, // personal and business have conflicting requirements
|
||||
RequireSignIn: true,
|
||||
Roles: p.Roles,
|
||||
}
|
||||
if m.fs.driveType != driveTypePersonal {
|
||||
req.RetainInheritedPermissions = false // not supported for personal
|
||||
}
|
||||
req.RetainInheritedPermissions = false
|
||||
|
||||
if p.Link != nil && p.Link.Scope == api.AnonymousScope {
|
||||
link, err := m.fs.PublicLink(ctx, m.remote, fs.DurationOff, false)
|
||||
@@ -816,15 +807,13 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, meta, err = f.createDir(ctx, parentID, dir, leaf, metadata)
|
||||
_, meta, err = f.createDir(ctx, parentID, dir, leaf, metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if f.driveType != driveTypePersonal {
|
||||
// for some reason, OneDrive Business needs this extra step to set modtime, while Personal does not. Seems like a bug...
|
||||
fs.Debugf(dir, "setting time %v", meta.mtime)
|
||||
info, err = meta.Write(ctx, false)
|
||||
}
|
||||
// for some reason, OneDrive Business and Personal needs this extra step to set modtime. Seems like a bug...
|
||||
fs.Debugf(dir, "setting time %v", meta.mtime)
|
||||
info, err = meta.Write(ctx, false)
|
||||
} else if err == nil {
|
||||
// Directory exists and needs updating
|
||||
info, meta, err = f.updateDir(ctx, dirID, dir, metadata)
|
||||
|
||||
@@ -403,7 +403,7 @@ This is why this flag is not set as the default.
|
||||
|
||||
As a rule of thumb if nearly all of your data is under rclone's root
|
||||
directory (the |root/directory| in |onedrive:root/directory|) then
|
||||
using this flag will be be a big performance win. If your data is
|
||||
using this flag will be a big performance win. If your data is
|
||||
mostly not under the root then using this flag will be a big
|
||||
performance loss.
|
||||
|
||||
@@ -2554,7 +2554,7 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
|
||||
}
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.srv.Call(ctx, &opts)
|
||||
resp, err = o.fs.unAuth.Call(ctx, &opts)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
return
|
||||
|
||||
@@ -136,11 +136,6 @@ func (f *Fs) TestReadPermissions(t *testing.T, r *fstest.Run) {
|
||||
_, expectedMeta := f.putWithMeta(ctx, t, &file1, []*api.PermissionsType{}) // return var intentionally switched here
|
||||
permissions := defaultPermissions(f.driveType)
|
||||
_, actualMeta := f.putWithMeta(ctx, t, &file1, permissions)
|
||||
if f.driveType == driveTypePersonal {
|
||||
perms, ok := actualMeta["permissions"]
|
||||
assert.False(t, ok, fmt.Sprintf("permissions metadata key was unexpectedly found: %v", perms))
|
||||
return
|
||||
}
|
||||
assert.JSONEq(t, expectedMeta["permissions"], actualMeta["permissions"])
|
||||
}
|
||||
|
||||
@@ -159,7 +154,7 @@ func (f *Fs) TestReadMetadata(t *testing.T, r *fstest.Run) {
|
||||
if slices.Contains(optionals, k) {
|
||||
continue
|
||||
}
|
||||
if k == "description" && f.driveType != driveTypePersonal {
|
||||
if k == "description" {
|
||||
continue // not supported
|
||||
}
|
||||
gotV, ok := actualMeta[k]
|
||||
@@ -196,7 +191,7 @@ func (f *Fs) TestDirectoryMetadata(t *testing.T, r *fstest.Run) {
|
||||
if slices.Contains(optionals, k) {
|
||||
continue
|
||||
}
|
||||
if k == "description" && f.driveType != driveTypePersonal {
|
||||
if k == "description" {
|
||||
continue // not supported
|
||||
}
|
||||
gotV, ok := actualMeta[k]
|
||||
@@ -417,9 +412,7 @@ func (f *Fs) compareMeta(t *testing.T, expectedMeta, actualMeta fs.Metadata, ign
|
||||
compareTimeStrings(t, k, v, gotV, time.Second)
|
||||
continue
|
||||
case "description":
|
||||
if f.driveType != driveTypePersonal {
|
||||
continue // not supported
|
||||
}
|
||||
continue // not supported
|
||||
}
|
||||
assert.True(t, ok, fmt.Sprintf("expected metadata key is missing: %v", k))
|
||||
assert.Equal(t, v, gotV, actualMeta)
|
||||
|
||||
@@ -30,20 +30,25 @@ const (
|
||||
|
||||
var commandHelp = []fs.CommandHelp{{
|
||||
Name: operationRename,
|
||||
Short: "change the name of an object",
|
||||
Short: "change the name of an object.",
|
||||
Long: `This command can be used to rename a object.
|
||||
|
||||
Usage Examples:
|
||||
Usage example:
|
||||
|
||||
rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
|
||||
` + "```",
|
||||
Opts: nil,
|
||||
}, {
|
||||
Name: operationListMultiPart,
|
||||
Short: "List the unfinished multipart uploads",
|
||||
Short: "List the unfinished multipart uploads.",
|
||||
Long: `This command lists the unfinished multipart uploads in JSON format.
|
||||
|
||||
rclone backend list-multipart-uploads oos:bucket/path/to/object
|
||||
Usage example:
|
||||
|
||||
` + "```console" + `
|
||||
rclone backend list-multipart-uploads oos:bucket/path/to/object
|
||||
` + "```" + `
|
||||
|
||||
It returns a dictionary of buckets with values as lists of unfinished
|
||||
multipart uploads.
|
||||
@@ -51,70 +56,82 @@ multipart uploads.
|
||||
You can call it with no bucket in which case it lists all bucket, with
|
||||
a bucket or with a bucket and path.
|
||||
|
||||
{
|
||||
"test-bucket": [
|
||||
{
|
||||
"namespace": "test-namespace",
|
||||
"bucket": "test-bucket",
|
||||
"object": "600m.bin",
|
||||
"uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8",
|
||||
"timeCreated": "2022-07-29T06:21:16.595Z",
|
||||
"storageTier": "Standard"
|
||||
}
|
||||
]
|
||||
`,
|
||||
` + "```json" + `
|
||||
{
|
||||
"test-bucket": [
|
||||
{
|
||||
"namespace": "test-namespace",
|
||||
"bucket": "test-bucket",
|
||||
"object": "600m.bin",
|
||||
"uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8",
|
||||
"timeCreated": "2022-07-29T06:21:16.595Z",
|
||||
"storageTier": "Standard"
|
||||
}
|
||||
]
|
||||
}`,
|
||||
}, {
|
||||
Name: operationCleanup,
|
||||
Short: "Remove unfinished multipart uploads.",
|
||||
Long: `This command removes unfinished multipart uploads of age greater than
|
||||
max-age which defaults to 24 hours.
|
||||
|
||||
Note that you can use --interactive/-i or --dry-run with this command to see what
|
||||
it would do.
|
||||
Note that you can use --interactive/-i or --dry-run with this command to see
|
||||
what it would do.
|
||||
|
||||
rclone backend cleanup oos:bucket/path/to/object
|
||||
rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
|
||||
Usage examples:
|
||||
|
||||
Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
|
||||
`,
|
||||
` + "```console" + `
|
||||
rclone backend cleanup oos:bucket/path/to/object
|
||||
rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
|
||||
` + "```" + `
|
||||
|
||||
Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.`,
|
||||
Opts: map[string]string{
|
||||
"max-age": "Max age of upload to delete",
|
||||
"max-age": "Max age of upload to delete.",
|
||||
},
|
||||
}, {
|
||||
Name: operationRestore,
|
||||
Short: "Restore objects from Archive to Standard storage",
|
||||
Long: `This command can be used to restore one or more objects from Archive to Standard storage.
|
||||
Short: "Restore objects from Archive to Standard storage.",
|
||||
Long: `This command can be used to restore one or more objects from Archive to
|
||||
Standard storage.
|
||||
|
||||
Usage Examples:
|
||||
Usage examples:
|
||||
|
||||
rclone backend restore oos:bucket/path/to/directory -o hours=HOURS
|
||||
rclone backend restore oos:bucket -o hours=HOURS
|
||||
` + "```console" + `
|
||||
rclone backend restore oos:bucket/path/to/directory -o hours=HOURS
|
||||
rclone backend restore oos:bucket -o hours=HOURS
|
||||
` + "```" + `
|
||||
|
||||
This flag also obeys the filters. Test first with --interactive/-i or --dry-run flags
|
||||
|
||||
rclone --interactive backend restore --include "*.txt" oos:bucket/path -o hours=72
|
||||
` + "```console" + `
|
||||
rclone --interactive backend restore --include "*.txt" oos:bucket/path -o hours=72
|
||||
` + "```" + `
|
||||
|
||||
All the objects shown will be marked for restore, then
|
||||
All the objects shown will be marked for restore, then:
|
||||
|
||||
rclone backend restore --include "*.txt" oos:bucket/path -o hours=72
|
||||
` + "```console" + `
|
||||
rclone backend restore --include "*.txt" oos:bucket/path -o hours=72
|
||||
` + "```" + `
|
||||
|
||||
It returns a list of status dictionaries with Object Name and Status
|
||||
keys. The Status will be "RESTORED"" if it was successful or an error message
|
||||
if not.
|
||||
It returns a list of status dictionaries with Object Name and Status keys.
|
||||
The Status will be "RESTORED"" if it was successful or an error message if not.
|
||||
|
||||
[
|
||||
{
|
||||
"Object": "test.txt"
|
||||
"Status": "RESTORED",
|
||||
},
|
||||
{
|
||||
"Object": "test/file4.txt"
|
||||
"Status": "RESTORED",
|
||||
}
|
||||
]
|
||||
`,
|
||||
` + "```json" + `
|
||||
[
|
||||
{
|
||||
"Object": "test.txt"
|
||||
"Status": "RESTORED",
|
||||
},
|
||||
{
|
||||
"Object": "test/file4.txt"
|
||||
"Status": "RESTORED",
|
||||
}
|
||||
]
|
||||
` + "```",
|
||||
Opts: map[string]string{
|
||||
"hours": "The number of hours for which this object will be restored. Default is 24 hrs.",
|
||||
"hours": `The number of hours for which this object will be restored.
|
||||
Default is 24 hrs.`,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -60,9 +60,6 @@ type StateChangeConf struct {
|
||||
func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (any, error) {
|
||||
// fs.Debugf(entityType, "Waiting for state to become: %s", conf.Target)
|
||||
|
||||
notfoundTick := 0
|
||||
targetOccurrence := 0
|
||||
|
||||
// Set a default for times to check for not found
|
||||
if conf.NotFoundChecks == 0 {
|
||||
conf.NotFoundChecks = 20
|
||||
@@ -84,9 +81,11 @@ func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType
|
||||
// cancellation channel for the refresh loop
|
||||
cancelCh := make(chan struct{})
|
||||
|
||||
result := Result{}
|
||||
|
||||
go func() {
|
||||
notfoundTick := 0
|
||||
targetOccurrence := 0
|
||||
result := Result{}
|
||||
|
||||
defer close(resCh)
|
||||
|
||||
select {
|
||||
|
||||
@@ -222,3 +222,11 @@ type UserInfo struct {
|
||||
} `json:"steps"`
|
||||
} `json:"journey"`
|
||||
}
|
||||
|
||||
// DiffResult is the response from /diff
|
||||
type DiffResult struct {
|
||||
Result int `json:"result"`
|
||||
DiffID int64 `json:"diffid"`
|
||||
Entries []map[string]any `json:"entries"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
+134
-1
@@ -171,6 +171,7 @@ type Fs struct {
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
pacer *fs.Pacer // pacer for API calls
|
||||
tokenRenewer *oauthutil.Renew // renew the token on expiry
|
||||
lastDiffID int64 // change tracking state for diff long-polling
|
||||
}
|
||||
|
||||
// Object describes a pcloud object
|
||||
@@ -1033,6 +1034,137 @@ func (f *Fs) Shutdown(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChangeNotify implements fs.Features.ChangeNotify
|
||||
func (f *Fs) ChangeNotify(ctx context.Context, notify func(string, fs.EntryType), ch <-chan time.Duration) {
|
||||
// Start long-poll loop in background
|
||||
go f.changeNotifyLoop(ctx, notify, ch)
|
||||
}
|
||||
|
||||
// changeNotifyLoop contains the blocking long-poll logic.
|
||||
func (f *Fs) changeNotifyLoop(ctx context.Context, notify func(string, fs.EntryType), ch <-chan time.Duration) {
|
||||
// Standard polling interval
|
||||
interval := 30 * time.Second
|
||||
|
||||
// Start with diffID = 0 to get the current state
|
||||
var diffID int64
|
||||
|
||||
// Helper to process changes from the diff API
|
||||
handleChanges := func(entries []map[string]any) {
|
||||
notifiedPaths := make(map[string]bool)
|
||||
|
||||
for _, entry := range entries {
|
||||
meta, ok := entry["metadata"].(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Robust extraction of ParentFolderID
|
||||
var pid int64
|
||||
if val, ok := meta["parentfolderid"]; ok {
|
||||
switch v := val.(type) {
|
||||
case float64:
|
||||
pid = int64(v)
|
||||
case int64:
|
||||
pid = v
|
||||
case int:
|
||||
pid = int64(v)
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the path using dirCache.GetInv
|
||||
// pCloud uses "d" prefix for directory IDs in cache, but API returns numbers
|
||||
dirID := fmt.Sprintf("d%d", pid)
|
||||
parentPath, ok := f.dirCache.GetInv(dirID)
|
||||
|
||||
if !ok {
|
||||
// Parent not in cache, so we can ignore this change as it is outside
|
||||
// of what the mount has seen or cares about.
|
||||
continue
|
||||
}
|
||||
|
||||
name, _ := meta["name"].(string)
|
||||
fullPath := path.Join(parentPath, name)
|
||||
|
||||
// Determine EntryType (File or Directory)
|
||||
entryType := fs.EntryObject
|
||||
if isFolder, ok := meta["isfolder"].(bool); ok && isFolder {
|
||||
entryType = fs.EntryDirectory
|
||||
}
|
||||
|
||||
// Deduplicate notifications for this batch
|
||||
if !notifiedPaths[fullPath] {
|
||||
fs.Debugf(f, "ChangeNotify: detected change in %q (type: %v)", fullPath, entryType)
|
||||
notify(fullPath, entryType)
|
||||
notifiedPaths[fullPath] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
// Check context and channel
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case newInterval, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
interval = newInterval
|
||||
default:
|
||||
}
|
||||
|
||||
// Setup /diff Request
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/diff",
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
|
||||
if diffID != 0 {
|
||||
opts.Parameters.Set("diffid", strconv.FormatInt(diffID, 10))
|
||||
opts.Parameters.Set("block", "1")
|
||||
} else {
|
||||
opts.Parameters.Set("last", "0")
|
||||
}
|
||||
|
||||
// Perform Long-Poll
|
||||
// Timeout set to 90s (server usually blocks for 60s max)
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
||||
var result api.DiffResult
|
||||
|
||||
_, err := f.srv.CallJSON(reqCtx, &opts, nil, &result)
|
||||
cancel()
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return
|
||||
}
|
||||
// Ignore timeout errors as they are normal for long-polling
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
fs.Infof(f, "ChangeNotify: polling error: %v. Waiting %v.", err, interval)
|
||||
time.Sleep(interval)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// If result is not 0, reset DiffID to resync
|
||||
if result.Result != 0 {
|
||||
diffID = 0
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
if result.DiffID != 0 {
|
||||
diffID = result.DiffID
|
||||
f.lastDiffID = diffID
|
||||
}
|
||||
|
||||
if len(result.Entries) > 0 {
|
||||
handleChanges(result.Entries)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
// EU region supports SHA1 and SHA256 (but rclone doesn't
|
||||
@@ -1327,7 +1459,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// opts.Body=0), so upload it as a multipart form POST with
|
||||
// Content-Length set.
|
||||
if size == 0 {
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, opts.Parameters, "content", leaf)
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, opts.Parameters, "content", leaf, opts.ContentType)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to make multipart upload for 0 length file: %w", err)
|
||||
}
|
||||
@@ -1401,6 +1533,7 @@ var (
|
||||
_ fs.ListPer = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.Shutdowner = (*Fs)(nil)
|
||||
_ fs.ChangeNotifier = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
)
|
||||
+18
-18
@@ -75,8 +75,8 @@ type ErrorDetails struct {
|
||||
Type string `json:"@type,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Metadata struct{} `json:"metadata,omitempty"` // TODO: undiscovered yet
|
||||
Locale string `json:"locale,omitempty"` // e.g. "en"
|
||||
Metadata struct{} `json:"metadata"` // TODO: undiscovered yet
|
||||
Locale string `json:"locale,omitempty"` // e.g. "en"
|
||||
Message string `json:"message,omitempty"`
|
||||
StackEntries []any `json:"stack_entries,omitempty"` // TODO: undiscovered yet
|
||||
Detail string `json:"detail,omitempty"`
|
||||
@@ -189,8 +189,8 @@ type File struct {
|
||||
Apps []*FileApp `json:"apps,omitempty"`
|
||||
Audit *FileAudit `json:"audit,omitempty"`
|
||||
Collection string `json:"collection,omitempty"` // TODO
|
||||
CreatedTime Time `json:"created_time,omitempty"`
|
||||
DeleteTime Time `json:"delete_time,omitempty"`
|
||||
CreatedTime Time `json:"created_time"`
|
||||
DeleteTime Time `json:"delete_time"`
|
||||
FileCategory string `json:"file_category,omitempty"` // "AUDIO", "VIDEO"
|
||||
FileExtension string `json:"file_extension,omitempty"`
|
||||
FolderType string `json:"folder_type,omitempty"`
|
||||
@@ -202,7 +202,7 @@ type File struct {
|
||||
Md5Checksum string `json:"md5_checksum,omitempty"`
|
||||
Medias []*Media `json:"medias,omitempty"`
|
||||
MimeType string `json:"mime_type,omitempty"`
|
||||
ModifiedTime Time `json:"modified_time,omitempty"` // updated when renamed or moved
|
||||
ModifiedTime Time `json:"modified_time"` // updated when renamed or moved
|
||||
Name string `json:"name,omitempty"`
|
||||
OriginalFileIndex int `json:"original_file_index,omitempty"` // TODO
|
||||
OriginalURL string `json:"original_url,omitempty"`
|
||||
@@ -221,7 +221,7 @@ type File struct {
|
||||
ThumbnailLink string `json:"thumbnail_link,omitempty"`
|
||||
Trashed bool `json:"trashed,omitempty"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
UserModifiedTime Time `json:"user_modified_time,omitempty"`
|
||||
UserModifiedTime Time `json:"user_modified_time"`
|
||||
WebContentLink string `json:"web_content_link,omitempty"`
|
||||
Writable bool `json:"writable,omitempty"`
|
||||
}
|
||||
@@ -252,7 +252,7 @@ type Media struct {
|
||||
AudioCodec string `json:"audio_codec,omitempty"` // "pcm_bluray", "aac"
|
||||
VideoType string `json:"video_type,omitempty"` // "mpegts"
|
||||
HdrType string `json:"hdr_type,omitempty"`
|
||||
} `json:"video,omitempty"`
|
||||
} `json:"video"`
|
||||
Link *Link `json:"link,omitempty"`
|
||||
NeedMoreQuota bool `json:"need_more_quota,omitempty"`
|
||||
VipTypes []any `json:"vip_types,omitempty"` // TODO maybe list of something?
|
||||
@@ -290,11 +290,11 @@ type FileApp struct {
|
||||
NeedMoreQuota bool `json:"need_more_quota,omitempty"`
|
||||
IconLink string `json:"icon_link,omitempty"`
|
||||
IsDefault bool `json:"is_default,omitempty"`
|
||||
Params struct{} `json:"params,omitempty"` // TODO
|
||||
Params struct{} `json:"params"` // TODO
|
||||
CategoryIDs []any `json:"category_ids,omitempty"`
|
||||
AdSceneType int `json:"ad_scene_type,omitempty"`
|
||||
Space string `json:"space,omitempty"`
|
||||
Links struct{} `json:"links,omitempty"` // TODO
|
||||
Links struct{} `json:"links"` // TODO
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -320,8 +320,8 @@ type Task struct {
|
||||
FileName string `json:"file_name,omitempty"`
|
||||
FileSize string `json:"file_size,omitempty"`
|
||||
Message string `json:"message,omitempty"` // e.g. "Saving"
|
||||
CreatedTime Time `json:"created_time,omitempty"`
|
||||
UpdatedTime Time `json:"updated_time,omitempty"`
|
||||
CreatedTime Time `json:"created_time"`
|
||||
UpdatedTime Time `json:"updated_time"`
|
||||
ThirdTaskID string `json:"third_task_id,omitempty"` // TODO
|
||||
Phase string `json:"phase,omitempty"` // e.g. "PHASE_TYPE_RUNNING"
|
||||
Progress int `json:"progress,omitempty"`
|
||||
@@ -368,7 +368,7 @@ type ResumableParams struct {
|
||||
AccessKeySecret string `json:"access_key_secret,omitempty"`
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Expiration Time `json:"expiration,omitempty"`
|
||||
Expiration Time `json:"expiration"`
|
||||
Key string `json:"key,omitempty"`
|
||||
SecurityToken string `json:"security_token,omitempty"`
|
||||
}
|
||||
@@ -409,7 +409,7 @@ type About struct {
|
||||
Kind string `json:"kind,omitempty"` // "drive#about"
|
||||
Quota *Quota `json:"quota,omitempty"`
|
||||
ExpiresAt string `json:"expires_at,omitempty"`
|
||||
Quotas struct{} `json:"quotas,omitempty"` // maybe []*Quota?
|
||||
Quotas struct{} `json:"quotas"` // maybe []*Quota?
|
||||
}
|
||||
|
||||
// Quota informs drive quota
|
||||
@@ -445,8 +445,8 @@ type User struct {
|
||||
PhoneNumber string `json:"phone_number,omitempty"`
|
||||
Password string `json:"password,omitempty"` // "SET" if configured
|
||||
Status string `json:"status,omitempty"` // "ACTIVE"
|
||||
CreatedAt Time `json:"created_at,omitempty"`
|
||||
PasswordUpdatedAt Time `json:"password_updated_at,omitempty"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
PasswordUpdatedAt Time `json:"password_updated_at"`
|
||||
}
|
||||
|
||||
// UserProvider details third-party authentication
|
||||
@@ -464,11 +464,11 @@ type VIP struct {
|
||||
Message string `json:"message,omitempty"`
|
||||
RedirectURI string `json:"redirect_uri,omitempty"`
|
||||
Data struct {
|
||||
Expire Time `json:"expire,omitempty"`
|
||||
Expire Time `json:"expire"`
|
||||
Status string `json:"status,omitempty"` // "invalid" or "ok"
|
||||
Type string `json:"type,omitempty"` // "novip" or "platinum"
|
||||
UserID string `json:"user_id,omitempty"` // same as User.Sub
|
||||
} `json:"data,omitempty"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// DecompressResult is a response to RequestDecompress
|
||||
@@ -538,7 +538,7 @@ type CaptchaToken struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
ExpiresIn int64 `json:"expires_in"` // currently 300s
|
||||
// API doesn't provide Expiry field and thus it should be populated from ExpiresIn on retrieval
|
||||
Expiry time.Time `json:"expiry,omitempty"`
|
||||
Expiry time.Time `json:"expiry"`
|
||||
URL string `json:"url,omitempty"` // a link for users to solve captcha
|
||||
}
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestLinkValid(t *testing.T) {
|
||||
Expire: Time(time.Now().Add(time.Hour)),
|
||||
},
|
||||
expected: true,
|
||||
desc: "should fallback to Expire field when URL expire parameter is unparseable",
|
||||
desc: "should fallback to Expire field when URL expire parameter is unparsable",
|
||||
},
|
||||
{
|
||||
name: "invalid when both URL expire and Expire field are expired",
|
||||
|
||||
+37
-22
@@ -1384,7 +1384,7 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
|
||||
for i := range iVal.NumField() {
|
||||
params.Set(iTyp.Field(i).Tag.Get("json"), iVal.Field(i).String())
|
||||
}
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, params, "file", name)
|
||||
formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, params, "file", name, "application/octet-stream")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to make multipart upload: %w", err)
|
||||
}
|
||||
@@ -1608,7 +1608,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// add offline download task for url
|
||||
func (f *Fs) addURL(ctx context.Context, url, path string) (*api.Task, error) {
|
||||
func (f *Fs) addURL(ctx context.Context, url, name, path string) (*api.Task, error) {
|
||||
req := api.RequestNewTask{
|
||||
Kind: api.KindOfFile,
|
||||
UploadType: "UPLOAD_TYPE_URL",
|
||||
@@ -1617,6 +1617,9 @@ func (f *Fs) addURL(ctx context.Context, url, path string) (*api.Task, error) {
|
||||
},
|
||||
FolderType: "DOWNLOAD",
|
||||
}
|
||||
if name != "" {
|
||||
req.Name = f.opt.Enc.FromStandardName(name)
|
||||
}
|
||||
if parentID, err := f.dirCache.FindDir(ctx, path, false); err == nil {
|
||||
req.ParentID = parentIDForRequest(parentID)
|
||||
req.FolderType = ""
|
||||
@@ -1678,39 +1681,47 @@ func (f *Fs) decompressDir(ctx context.Context, filename, id, password string, s
|
||||
|
||||
var commandHelp = []fs.CommandHelp{{
|
||||
Name: "addurl",
|
||||
Short: "Add offline download task for url",
|
||||
Short: "Add offline download task for url.",
|
||||
Long: `This command adds offline download task for url.
|
||||
|
||||
Usage:
|
||||
Usage examples:
|
||||
|
||||
rclone backend addurl pikpak:dirpath url
|
||||
` + "```console" + `
|
||||
rclone backend addurl pikpak:dirpath url
|
||||
rclone backend addurl pikpak:dirpath url -o name=custom_filename.zip
|
||||
` + "```" + `
|
||||
|
||||
Downloads will be stored in 'dirpath'. If 'dirpath' is invalid,
|
||||
download will fallback to default 'My Pack' folder.
|
||||
`,
|
||||
Downloads will be stored in 'dirpath'. If 'dirpath' is invalid,
|
||||
download will fallback to default 'My Pack' folder.`,
|
||||
Opts: map[string]string{
|
||||
"name": "Custom filename for the downloaded file.",
|
||||
},
|
||||
}, {
|
||||
Name: "decompress",
|
||||
Short: "Request decompress of a file/files in a folder",
|
||||
Short: "Request decompress of a file/files in a folder.",
|
||||
Long: `This command requests decompress of file/files in a folder.
|
||||
|
||||
Usage:
|
||||
Usage examples:
|
||||
|
||||
rclone backend decompress pikpak:dirpath {filename} -o password=password
|
||||
rclone backend decompress pikpak:dirpath {filename} -o delete-src-file
|
||||
` + "```console" + `
|
||||
rclone backend decompress pikpak:dirpath {filename} -o password=password
|
||||
rclone backend decompress pikpak:dirpath {filename} -o delete-src-file
|
||||
` + "```" + `
|
||||
|
||||
An optional argument 'filename' can be specified for a file located in
|
||||
'pikpak:dirpath'. You may want to pass '-o password=password' for a
|
||||
password-protected files. Also, pass '-o delete-src-file' to delete
|
||||
An optional argument 'filename' can be specified for a file located in
|
||||
'pikpak:dirpath'. You may want to pass '-o password=password' for a
|
||||
password-protected files. Also, pass '-o delete-src-file' to delete
|
||||
source files after decompression finished.
|
||||
|
||||
Result:
|
||||
|
||||
{
|
||||
"Decompressed": 17,
|
||||
"SourceDeleted": 0,
|
||||
"Errors": 0
|
||||
}
|
||||
`,
|
||||
` + "```json" + `
|
||||
{
|
||||
"Decompressed": 17,
|
||||
"SourceDeleted": 0,
|
||||
"Errors": 0
|
||||
}
|
||||
` + "```",
|
||||
}}
|
||||
|
||||
// Command the backend to run a named command
|
||||
@@ -1728,7 +1739,11 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
if len(arg) != 1 {
|
||||
return nil, errors.New("need exactly 1 argument")
|
||||
}
|
||||
return f.addURL(ctx, arg[0], "")
|
||||
filename := ""
|
||||
if name, ok := opt["name"]; ok {
|
||||
filename = name
|
||||
}
|
||||
return f.addURL(ctx, arg[0], filename, "")
|
||||
case "decompress":
|
||||
filename := ""
|
||||
if len(arg) > 0 {
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
protonDriveAPI "github.com/henrybear327/Proton-API-Bridge"
|
||||
"github.com/henrybear327/go-proton-api"
|
||||
protonDriveAPI "github.com/rclone/Proton-API-Bridge"
|
||||
"github.com/rclone/go-proton-api"
|
||||
|
||||
"github.com/pquerna/otp/totp"
|
||||
|
||||
|
||||
+171
-147
@@ -1,4 +1,4 @@
|
||||
## Adding a new s3 provider
|
||||
# Adding a new s3 provider
|
||||
|
||||
It is quite easy to add a new S3 provider to rclone.
|
||||
|
||||
@@ -12,179 +12,203 @@ All tags can be found in `backend/s3/providers.go` Provider Struct.
|
||||
Looking through a few of the yaml files as examples should make things
|
||||
clear. `AWS.yaml` as the most config. pasting.
|
||||
|
||||
### YAML
|
||||
## YAML
|
||||
|
||||
In `backend/s3/provider/YourProvider.yaml`
|
||||
|
||||
- name
|
||||
- description
|
||||
- More like the full name often "YourProvider + Object Storage"
|
||||
- More like the full name often "YourProvider + Object Storage"
|
||||
- [Region]
|
||||
- Any regions your provider supports or the defaults (use `region: {}` for this)
|
||||
- Example from AWS.yaml:
|
||||
```yaml
|
||||
region:
|
||||
us-east-1: |-
|
||||
The default endpoint - a good choice if you are unsure.
|
||||
US Region, Northern Virginia, or Pacific Northwest.
|
||||
Leave location constraint empty.
|
||||
```
|
||||
- The defaults (as seen in Rclone.yaml):
|
||||
```yaml
|
||||
region:
|
||||
"": |-
|
||||
Use this if unsure.
|
||||
Will use v4 signatures and an empty region.
|
||||
other-v2-signature: |-
|
||||
Use this only if v4 signatures don't work.
|
||||
E.g. pre Jewel/v10 CEPH.
|
||||
```
|
||||
- Any regions your provider supports or the defaults (use `region: {}` for this)
|
||||
- Example from AWS.yaml:
|
||||
|
||||
```yaml
|
||||
region:
|
||||
us-east-1: |-
|
||||
The default endpoint - a good choice if you are unsure.
|
||||
US Region, Northern Virginia, or Pacific Northwest.
|
||||
Leave location constraint empty.
|
||||
```
|
||||
|
||||
- The defaults (as seen in Rclone.yaml):
|
||||
|
||||
```yaml
|
||||
region:
|
||||
"": |-
|
||||
Use this if unsure.
|
||||
Will use v4 signatures and an empty region.
|
||||
other-v2-signature: |-
|
||||
Use this only if v4 signatures don't work.
|
||||
E.g. pre Jewel/v10 CEPH.
|
||||
```
|
||||
|
||||
- [Endpoint]
|
||||
- Any endpoints your provider supports
|
||||
- Example from Mega.yaml
|
||||
```yaml
|
||||
endpoint:
|
||||
s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam)
|
||||
```
|
||||
- Any endpoints your provider supports
|
||||
|
||||
- Example from Mega.yaml
|
||||
|
||||
```yaml
|
||||
endpoint:
|
||||
s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam)
|
||||
```
|
||||
|
||||
- [Location Constraint]
|
||||
- The Location Constraint of your remote, often same as region.
|
||||
- Example from AWS.yaml
|
||||
```yaml
|
||||
location_constraint:
|
||||
"": Empty for US Region, Northern Virginia, or Pacific Northwest
|
||||
us-east-2: US East (Ohio) Region
|
||||
```
|
||||
- The Location Constraint of your remote, often same as region.
|
||||
- Example from AWS.yaml
|
||||
|
||||
```yaml
|
||||
location_constraint:
|
||||
"": Empty for US Region, Northern Virginia, or Pacific Northwest
|
||||
us-east-2: US East (Ohio) Region
|
||||
```
|
||||
|
||||
- [ACL]
|
||||
- Identical across *most* providers. Select the default with `acl: {}`
|
||||
- Example from AWS.yaml
|
||||
```yaml
|
||||
acl:
|
||||
private: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
No one else has access rights (default).
|
||||
public-read: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
The AllUsers group gets READ access.
|
||||
public-read-write: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
The AllUsers group gets READ and WRITE access.
|
||||
Granting this on a bucket is generally not recommended.
|
||||
authenticated-read: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
The AuthenticatedUsers group gets READ access.
|
||||
bucket-owner-read: |-
|
||||
Object owner gets FULL_CONTROL.
|
||||
Bucket owner gets READ access.
|
||||
If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
bucket-owner-full-control: |-
|
||||
Both the object owner and the bucket owner get FULL_CONTROL over the object.
|
||||
If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
```
|
||||
- Identical across *most* providers. Select the default with `acl: {}`
|
||||
- Example from AWS.yaml
|
||||
|
||||
```yaml
|
||||
acl:
|
||||
private: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
No one else has access rights (default).
|
||||
public-read: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
The AllUsers group gets READ access.
|
||||
public-read-write: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
The AllUsers group gets READ and WRITE access.
|
||||
Granting this on a bucket is generally not recommended.
|
||||
authenticated-read: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
The AuthenticatedUsers group gets READ access.
|
||||
bucket-owner-read: |-
|
||||
Object owner gets FULL_CONTROL.
|
||||
Bucket owner gets READ access.
|
||||
If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
bucket-owner-full-control: |-
|
||||
Both the object owner and the bucket owner get FULL_CONTROL over the object.
|
||||
If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
```
|
||||
|
||||
- [Storage Class]
|
||||
- Identical across *most* providers.
|
||||
- Defaults from AWS.yaml
|
||||
```yaml
|
||||
storage_class:
|
||||
"": Default
|
||||
STANDARD: Standard storage class
|
||||
REDUCED_REDUNDANCY: Reduced redundancy storage class
|
||||
STANDARD_IA: Standard Infrequent Access storage class
|
||||
ONEZONE_IA: One Zone Infrequent Access storage class
|
||||
GLACIER: Glacier Flexible Retrieval storage class
|
||||
DEEP_ARCHIVE: Glacier Deep Archive storage class
|
||||
INTELLIGENT_TIERING: Intelligent-Tiering storage class
|
||||
GLACIER_IR: Glacier Instant Retrieval storage class
|
||||
```
|
||||
- Identical across *most* providers.
|
||||
- Defaults from AWS.yaml
|
||||
|
||||
```yaml
|
||||
storage_class:
|
||||
"": Default
|
||||
STANDARD: Standard storage class
|
||||
REDUCED_REDUNDANCY: Reduced redundancy storage class
|
||||
STANDARD_IA: Standard Infrequent Access storage class
|
||||
ONEZONE_IA: One Zone Infrequent Access storage class
|
||||
GLACIER: Glacier Flexible Retrieval storage class
|
||||
DEEP_ARCHIVE: Glacier Deep Archive storage class
|
||||
INTELLIGENT_TIERING: Intelligent-Tiering storage class
|
||||
GLACIER_IR: Glacier Instant Retrieval storage class
|
||||
```
|
||||
|
||||
- [Server Side Encryption]
|
||||
- Not common, identical across *most* providers.
|
||||
- Defaults from AWS.yaml
|
||||
```yaml
|
||||
server_side_encryption:
|
||||
"": None
|
||||
AES256: AES256
|
||||
aws:kms: aws:kms
|
||||
```
|
||||
- Not common, identical across *most* providers.
|
||||
- Defaults from AWS.yaml
|
||||
|
||||
```yaml
|
||||
server_side_encryption:
|
||||
"": None
|
||||
AES256: AES256
|
||||
aws:kms: aws:kms
|
||||
```
|
||||
|
||||
- [Advanced Options]
|
||||
- All advanced options are Boolean - if true the configurator asks about that value, if not it doesn't:
|
||||
```go
|
||||
BucketACL bool `yaml:"bucket_acl,omitempty"`
|
||||
DirectoryBucket bool `yaml:"directory_bucket,omitempty"`
|
||||
LeavePartsOnError bool `yaml:"leave_parts_on_error,omitempty"`
|
||||
RequesterPays bool `yaml:"requester_pays,omitempty"`
|
||||
SSECustomerAlgorithm bool `yaml:"sse_customer_algorithm,omitempty"`
|
||||
SSECustomerKey bool `yaml:"sse_customer_key,omitempty"`
|
||||
SSECustomerKeyBase64 bool `yaml:"sse_customer_key_base64,omitempty"`
|
||||
SSECustomerKeyMd5 bool `yaml:"sse_customer_key_md5,omitempty"`
|
||||
SSEKmsKeyID bool `yaml:"sse_kms_key_id,omitempty"`
|
||||
STSEndpoint bool `yaml:"sts_endpoint,omitempty"`
|
||||
UseAccelerateEndpoint bool `yaml:"use_accelerate_endpoint,omitempty"`
|
||||
```
|
||||
- Example from AWS.yaml:
|
||||
```yaml
|
||||
bucket_acl: true
|
||||
directory_bucket: true
|
||||
leave_parts_on_error: true
|
||||
requester_pays: true
|
||||
sse_customer_algorithm: true
|
||||
sse_customer_key: true
|
||||
sse_customer_key_base64: true
|
||||
sse_customer_key_md5: true
|
||||
sse_kms_key_id: true
|
||||
sts_endpoint: true
|
||||
use_accelerate_endpoint: true
|
||||
```
|
||||
- All advanced options are Boolean - if true the configurator asks about that
|
||||
value, if not it doesn't:
|
||||
|
||||
```go
|
||||
BucketACL bool `yaml:"bucket_acl,omitempty"`
|
||||
DirectoryBucket bool `yaml:"directory_bucket,omitempty"`
|
||||
LeavePartsOnError bool `yaml:"leave_parts_on_error,omitempty"`
|
||||
RequesterPays bool `yaml:"requester_pays,omitempty"`
|
||||
SSECustomerAlgorithm bool `yaml:"sse_customer_algorithm,omitempty"`
|
||||
SSECustomerKey bool `yaml:"sse_customer_key,omitempty"`
|
||||
SSECustomerKeyBase64 bool `yaml:"sse_customer_key_base64,omitempty"`
|
||||
SSECustomerKeyMd5 bool `yaml:"sse_customer_key_md5,omitempty"`
|
||||
SSEKmsKeyID bool `yaml:"sse_kms_key_id,omitempty"`
|
||||
STSEndpoint bool `yaml:"sts_endpoint,omitempty"`
|
||||
UseAccelerateEndpoint bool `yaml:"use_accelerate_endpoint,omitempty"`
|
||||
```
|
||||
|
||||
- Example from AWS.yaml:
|
||||
|
||||
```yaml
|
||||
bucket_acl: true
|
||||
directory_bucket: true
|
||||
leave_parts_on_error: true
|
||||
requester_pays: true
|
||||
sse_customer_algorithm: true
|
||||
sse_customer_key: true
|
||||
sse_customer_key_base64: true
|
||||
sse_customer_key_md5: true
|
||||
sse_kms_key_id: true
|
||||
sts_endpoint: true
|
||||
use_accelerate_endpoint: true
|
||||
```
|
||||
|
||||
- Quirks
|
||||
- Quirks are discovered through documentation and running the tests as seen below.
|
||||
- Most quirks are *bool as to have 3 values, `true`, `false` and `dont care`.
|
||||
```go
|
||||
type Quirks struct {
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`
|
||||
MightGzip *bool `yaml:"might_gzip,omitempty"`
|
||||
UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
|
||||
UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
|
||||
UseXID *bool `yaml:"use_x_id,omitempty"`
|
||||
SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
|
||||
CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
|
||||
MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
|
||||
MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
|
||||
}
|
||||
- Quirks are discovered through documentation and running the tests as seen below.
|
||||
- Most quirks are *bool as to have 3 values, `true`, `false` and `dont care`.
|
||||
|
||||
```go
|
||||
type Quirks struct {
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`
|
||||
MightGzip *bool `yaml:"might_gzip,omitempty"`
|
||||
UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
|
||||
UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
|
||||
UseXID *bool `yaml:"use_x_id,omitempty"`
|
||||
SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
|
||||
EtagIsNotMD5 *bool `yaml:"etag_is_not_md5,omitempty"`
|
||||
CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
|
||||
MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
|
||||
MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
|
||||
}
|
||||
```
|
||||
- Example from AWS.yaml
|
||||
|
||||
- Example from AWS.yaml
|
||||
|
||||
```yaml
|
||||
quirks:
|
||||
might_gzip: false # Never auto gzips objects
|
||||
use_unsigned_payload: false # AWS has trailer support
|
||||
quirks:
|
||||
might_gzip: false # Never auto gzips objects
|
||||
use_unsigned_payload: false # AWS has trailer support
|
||||
```
|
||||
|
||||
Note that if you omit a section, eg `region` then the user won't be
|
||||
asked that question, and if you add an empty section e.g. `region: {}`
|
||||
then the defaults from the `Other.yaml` will be used.
|
||||
|
||||
### DOCS
|
||||
## DOCS
|
||||
|
||||
- `docs/content/s3.md`
|
||||
- Add the provider at the top of the page.
|
||||
- Add a section about the provider linked from there.
|
||||
- Make sure this is in alphabetical order in the `Providers` section.
|
||||
- Add a transcript of a trial `rclone config` session
|
||||
- Edit the transcript to remove things which might change in subsequent versions
|
||||
- **Do not** alter or add to the autogenerated parts of `s3.md`
|
||||
- Rule of thumb: don't edit anything not mentioned above.
|
||||
- **Do not** run `make backenddocs` or `bin/make_backend_docs.py s3`
|
||||
- This will make autogenerated changes!
|
||||
- Add the provider at the top of the page.
|
||||
- Add a section about the provider linked from there.
|
||||
- Make sure this is in alphabetical order in the `Providers` section.
|
||||
- Add a transcript of a trial `rclone config` session
|
||||
- Edit the transcript to remove things which might change in subsequent versions
|
||||
- **Do not** alter or add to the autogenerated parts of `s3.md`
|
||||
- Rule of thumb: don't edit anything not mentioned above.
|
||||
- **Do not** run `make backenddocs` or `bin/make_backend_docs.py s3`
|
||||
- This will make autogenerated changes!
|
||||
- `README.md` - this is the home page in github
|
||||
- Add the provider and a link to the section you wrote in `docs/contents/s3.md`
|
||||
- Add the provider and a link to the section you wrote in `docs/contents/s3.md`
|
||||
- `docs/content/_index.md` - this is the home page of rclone.org
|
||||
- Add the provider and a link to the section you wrote in `docs/contents/s3.md`
|
||||
- Add the provider and a link to the section you wrote in `docs/contents/s3.md`
|
||||
- Once you've written the docs, run `make serve` and check they look OK
|
||||
in the web browser and the links (internal and external) all work.
|
||||
in the web browser and the links (internal and external) all work.
|
||||
|
||||
### TESTS
|
||||
## TESTS
|
||||
|
||||
Once you've written the code, test `rclone config` works to your
|
||||
satisfaction and looks correct, and check the integration tests work
|
||||
|
||||
@@ -137,3 +137,4 @@ use_accelerate_endpoint: true
|
||||
quirks:
|
||||
might_gzip: false # Never auto gzips objects
|
||||
use_unsigned_payload: false # AWS has trailer support which means it adds checksums in the trailer without seeking
|
||||
use_data_integrity_protections: true
|
||||
@@ -0,0 +1,15 @@
|
||||
name: BizflyCloud
|
||||
description: Bizfly Cloud Simple Storage
|
||||
region:
|
||||
hn: Ha Noi
|
||||
hcm: Ho Chi Minh
|
||||
endpoint:
|
||||
hn.ss.bfcplatform.vn: Hanoi endpoint
|
||||
hcm.ss.bfcplatform.vn: Ho Chi Minh endpoint
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
force_path_style: true
|
||||
list_url_encode: false
|
||||
use_multipart_etag: false
|
||||
use_already_exists: false
|
||||
@@ -0,0 +1,26 @@
|
||||
name: Fastly
|
||||
description: Fastly Object Storage
|
||||
region:
|
||||
au-east-1: AU East 1
|
||||
eu-central: EU Central
|
||||
eu-south-1: EU South 1
|
||||
jp-central-1: JP Central 1
|
||||
uk-east-1: UK East 1
|
||||
us-central-1: US Central 1
|
||||
us-east: US East
|
||||
us-west: US West
|
||||
endpoint:
|
||||
au-east-1.object.fastlystorage.app: AU East 1
|
||||
eu-central.object.fastlystorage.app: EU Central
|
||||
eu-south-1.object.fastlystorage.app: EU South 1
|
||||
jp-central-1.object.fastlystorage.app: JP Central 1
|
||||
uk-east-1.object.fastlystorage.app: UK East 1
|
||||
us-central-1.object.fastlystorage.app: US Central 1
|
||||
us-east.object.fastlystorage.app: US East
|
||||
us-west.object.fastlystorage.app: US West
|
||||
quirks:
|
||||
force_path_style: true
|
||||
use_already_exists: false
|
||||
use_multipart_etag: false
|
||||
use_multipart_uploads: false
|
||||
etag_is_not_md5: true
|
||||
@@ -15,7 +15,7 @@ endpoint:
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
list_version: 1
|
||||
list_version: 2
|
||||
force_path_style: true
|
||||
list_url_encode: false
|
||||
use_multipart_etag: false
|
||||
@@ -3,11 +3,17 @@ description: IONOS Cloud
|
||||
region:
|
||||
de: Frankfurt, Germany
|
||||
eu-central-2: Berlin, Germany
|
||||
eu-central-3: Berlin, Germany
|
||||
eu-central-4: Frankfurt, Germany
|
||||
eu-south-2: Logrono, Spain
|
||||
us-central-1: Lenexa, USA
|
||||
endpoint:
|
||||
s3-eu-central-1.ionoscloud.com: Frankfurt, Germany
|
||||
s3-eu-central-2.ionoscloud.com: Berlin, Germany
|
||||
s3-eu-south-2.ionoscloud.com: Logrono, Spain
|
||||
s3.eu-central-1.ionoscloud.com: Frankfurt, Germany
|
||||
s3.eu-central-2.ionoscloud.com: Berlin, Germany
|
||||
s3.eu-central-3.ionoscloud.com: Berlin, Germany
|
||||
s3.eu-central-4.ionoscloud.com: Frankfurt, Germany
|
||||
s3.eu-south-2.ionoscloud.com: Logrono, Spain
|
||||
s3.us-central-1.ionoscloud.com: Lenexa, USA
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
|
||||
@@ -1,26 +1,26 @@
|
||||
name: Linode
|
||||
description: Linode Object Storage
|
||||
endpoint:
|
||||
nl-ams-1.linodeobjects.com: Amsterdam (Netherlands), nl-ams-1
|
||||
us-southeast-1.linodeobjects.com: Atlanta, GA (USA), us-southeast-1
|
||||
in-maa-1.linodeobjects.com: Chennai (India), in-maa-1
|
||||
us-ord-1.linodeobjects.com: Chicago, IL (USA), us-ord-1
|
||||
eu-central-1.linodeobjects.com: Frankfurt (Germany), eu-central-1
|
||||
id-cgk-1.linodeobjects.com: Jakarta (Indonesia), id-cgk-1
|
||||
gb-lon-1.linodeobjects.com: London 2 (Great Britain), gb-lon-1
|
||||
us-lax-1.linodeobjects.com: Los Angeles, CA (USA), us-lax-1
|
||||
es-mad-1.linodeobjects.com: Madrid (Spain), es-mad-1
|
||||
au-mel-1.linodeobjects.com: Melbourne (Australia), au-mel-1
|
||||
us-mia-1.linodeobjects.com: Miami, FL (USA), us-mia-1
|
||||
it-mil-1.linodeobjects.com: Milan (Italy), it-mil-1
|
||||
us-east-1.linodeobjects.com: Newark, NJ (USA), us-east-1
|
||||
jp-osa-1.linodeobjects.com: Osaka (Japan), jp-osa-1
|
||||
fr-par-1.linodeobjects.com: Paris (France), fr-par-1
|
||||
br-gru-1.linodeobjects.com: São Paulo (Brazil), br-gru-1
|
||||
us-sea-1.linodeobjects.com: Seattle, WA (USA), us-sea-1
|
||||
ap-south-1.linodeobjects.com: Singapore, ap-south-1
|
||||
sg-sin-1.linodeobjects.com: Singapore 2, sg-sin-1
|
||||
se-sto-1.linodeobjects.com: Stockholm (Sweden), se-sto-1
|
||||
us-iad-1.linodeobjects.com: Washington, DC, (USA), us-iad-1
|
||||
nl-ams-1.linodeobjects.com: Amsterdam, NL (nl-ams-1)
|
||||
us-southeast-1.linodeobjects.com: Atlanta, GA, US (us-southeast-1)
|
||||
in-maa-1.linodeobjects.com: Chennai, IN (in-maa-1)
|
||||
us-ord-1.linodeobjects.com: Chicago, IL, US (us-ord-1)
|
||||
eu-central-1.linodeobjects.com: Frankfurt, DE (eu-central-1)
|
||||
id-cgk-1.linodeobjects.com: Jakarta, ID (id-cgk-1)
|
||||
gb-lon-1.linodeobjects.com: London 2, UK (gb-lon-1)
|
||||
us-lax-1.linodeobjects.com: Los Angeles, CA, US (us-lax-1)
|
||||
es-mad-1.linodeobjects.com: Madrid, ES (es-mad-1)
|
||||
us-mia-1.linodeobjects.com: Miami, FL, US (us-mia-1)
|
||||
it-mil-1.linodeobjects.com: Milan, IT (it-mil-1)
|
||||
us-east-1.linodeobjects.com: Newark, NJ, US (us-east-1)
|
||||
jp-osa-1.linodeobjects.com: Osaka, JP (jp-osa-1)
|
||||
fr-par-1.linodeobjects.com: Paris, FR (fr-par-1)
|
||||
br-gru-1.linodeobjects.com: Sao Paulo, BR (br-gru-1)
|
||||
us-sea-1.linodeobjects.com: Seattle, WA, US (us-sea-1)
|
||||
ap-south-1.linodeobjects.com: Singapore, SG (ap-south-1)
|
||||
sg-sin-1.linodeobjects.com: Singapore 2, SG (sg-sin-1)
|
||||
se-sto-1.linodeobjects.com: Stockholm, SE (se-sto-1)
|
||||
jp-tyo-1.linodeobjects.com: Tokyo 3, JP (jp-tyo-1)
|
||||
us-iad-10.linodeobjects.com: Washington, DC, US (us-iad-10)
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
@@ -18,11 +18,11 @@ storage_class:
|
||||
GLACIER: |-
|
||||
Archived storage.
|
||||
Prices are lower, but it needs to be restored first to be accessed.
|
||||
Available in FR-PAR and NL-AMS regions.
|
||||
Available in the FR-PAR region only.
|
||||
ONEZONE_IA: |-
|
||||
One Zone - Infrequent Access.
|
||||
A good choice for storing secondary backup copies or easily re-creatable data.
|
||||
Available in the FR-PAR region only.
|
||||
Available in all regions.
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
max_upload_parts: 1000
|
||||
@@ -2,7 +2,17 @@ name: Selectel
|
||||
description: Selectel Object Storage
|
||||
region:
|
||||
ru-1: St. Petersburg
|
||||
ru-3: St. Petersburg
|
||||
ru-7: Moscow
|
||||
gis-1: Moscow
|
||||
kz-1: Kazakhstan
|
||||
uz-2: Uzbekistan
|
||||
endpoint:
|
||||
s3.ru-1.storage.selcloud.ru: Saint Petersburg
|
||||
s3.ru-1.storage.selcloud.ru: St. Petersburg
|
||||
s3.ru-3.storage.selcloud.ru: St. Petersburg
|
||||
s3.ru-7.storage.selcloud.ru: Moscow
|
||||
s3.gis-1.storage.selcloud.ru: Moscow
|
||||
s3.kz-1.storage.selcloud.ru: Kazakhstan
|
||||
s3.uz-2.storage.selcloud.ru: Uzbekistan
|
||||
quirks:
|
||||
list_url_encode: false
|
||||
@@ -1,14 +0,0 @@
|
||||
name: StackPath
|
||||
description: StackPath Object Storage
|
||||
region: {}
|
||||
endpoint:
|
||||
s3.us-east-2.stackpathstorage.com: US East Endpoint
|
||||
s3.us-west-1.stackpathstorage.com: US West Endpoint
|
||||
s3.eu-central-1.stackpathstorage.com: EU Endpoint
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
list_version: 1
|
||||
force_path_style: true
|
||||
list_url_encode: false
|
||||
use_already_exists: false
|
||||
@@ -0,0 +1,9 @@
|
||||
name: Zadara
|
||||
description: Zadara Object Storage
|
||||
region:
|
||||
us-east-1: |-
|
||||
The default region.
|
||||
Leave location constraint empty.
|
||||
endpoint: {}
|
||||
quirks:
|
||||
force_path_style: true
|
||||
+16
-14
@@ -20,20 +20,22 @@ var NewYamlMap = orderedmap.New[string, string]
|
||||
|
||||
// Quirks defines all the S3 provider quirks
|
||||
type Quirks struct {
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`
|
||||
MightGzip *bool `yaml:"might_gzip,omitempty"`
|
||||
UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
|
||||
UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
|
||||
UseXID *bool `yaml:"use_x_id,omitempty"`
|
||||
SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
|
||||
CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
|
||||
MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
|
||||
MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
UseAcceptEncodingGzip *bool `yaml:"use_accept_encoding_gzip,omitempty"`
|
||||
UseDataIntegrityProtections *bool `yaml:"use_data_integrity_protections,omitempty"`
|
||||
MightGzip *bool `yaml:"might_gzip,omitempty"`
|
||||
UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
|
||||
UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
|
||||
UseXID *bool `yaml:"use_x_id,omitempty"`
|
||||
SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
|
||||
EtagIsNotMD5 *bool `yaml:"etag_is_not_md5,omitempty"`
|
||||
CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
|
||||
MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
|
||||
MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
|
||||
}
|
||||
|
||||
// Provider defines the configurable data in each provider.yaml
|
||||
|
||||
+692
-184
File diff suppressed because it is too large.
Load diff
@@ -498,10 +498,236 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
|
||||
// Purge gets tested later
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestObjectLock(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a temporary bucket with Object Lock enabled to test on.
|
||||
// This exercises our BucketObjectLockEnabled option and isolates
|
||||
// the test from the main test bucket.
|
||||
lockBucket := f.rootBucket + "-object-lock-" + random.String(8)
|
||||
lockBucket = strings.ToLower(lockBucket)
|
||||
|
||||
// Try to create bucket with Object Lock enabled
|
||||
objectLockEnabled := true
|
||||
req := s3.CreateBucketInput{
|
||||
Bucket: &lockBucket,
|
||||
ACL: types.BucketCannedACL(f.opt.BucketACL),
|
||||
ObjectLockEnabledForBucket: &objectLockEnabled,
|
||||
}
|
||||
if f.opt.LocationConstraint != "" {
|
||||
req.CreateBucketConfiguration = &types.CreateBucketConfiguration{
|
||||
LocationConstraint: types.BucketLocationConstraint(f.opt.LocationConstraint),
|
||||
}
|
||||
}
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.CreateBucket(ctx, &req)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
t.Skipf("Object Lock not supported by this provider: CreateBucket with Object Lock failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify Object Lock is actually enabled on the new bucket.
|
||||
// Some S3-compatible servers (e.g. rclone serve s3) accept the
|
||||
// ObjectLockEnabledForBucket flag but don't actually implement Object Lock.
|
||||
var lockCfg *s3.GetObjectLockConfigurationOutput
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
lockCfg, err = f.c.GetObjectLockConfiguration(ctx, &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: &lockBucket,
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil || lockCfg.ObjectLockConfiguration == nil ||
|
||||
lockCfg.ObjectLockConfiguration.ObjectLockEnabled != types.ObjectLockEnabledEnabled {
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
t.Skipf("Object Lock not functional on this provider (GetObjectLockConfiguration: %v)", err)
|
||||
}
|
||||
|
||||
// Switch f to use the Object Lock bucket for this test
|
||||
oldBucket := f.rootBucket
|
||||
oldRoot := f.root
|
||||
oldRootDir := f.rootDirectory
|
||||
f.rootBucket = lockBucket
|
||||
f.root = lockBucket
|
||||
f.rootDirectory = ""
|
||||
defer func() {
|
||||
f.rootBucket = oldBucket
|
||||
f.root = oldRoot
|
||||
f.rootDirectory = oldRootDir
|
||||
}()
|
||||
|
||||
// Helper to remove an object with Object Lock protection
|
||||
removeLocked := func(t *testing.T, obj fs.Object) {
|
||||
t.Helper()
|
||||
o := obj.(*Object)
|
||||
// Remove legal hold if present
|
||||
_ = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
|
||||
// Enable bypass governance retention for deletion
|
||||
o.fs.opt.BypassGovernanceRetention = true
|
||||
err := obj.Remove(ctx)
|
||||
o.fs.opt.BypassGovernanceRetention = false
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Clean up the temporary bucket after all sub-tests
|
||||
defer func() {
|
||||
// List and remove all object versions
|
||||
var objectVersions []types.ObjectIdentifier
|
||||
listReq := &s3.ListObjectVersionsInput{Bucket: &lockBucket}
|
||||
for {
|
||||
var resp *s3.ListObjectVersionsOutput
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.c.ListObjectVersions(ctx, listReq)
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Failed to list object versions for cleanup: %v", err)
|
||||
break
|
||||
}
|
||||
for _, v := range resp.Versions {
|
||||
objectVersions = append(objectVersions, types.ObjectIdentifier{
|
||||
Key: v.Key,
|
||||
VersionId: v.VersionId,
|
||||
})
|
||||
}
|
||||
for _, m := range resp.DeleteMarkers {
|
||||
objectVersions = append(objectVersions, types.ObjectIdentifier{
|
||||
Key: m.Key,
|
||||
VersionId: m.VersionId,
|
||||
})
|
||||
}
|
||||
if !aws.ToBool(resp.IsTruncated) {
|
||||
break
|
||||
}
|
||||
listReq.KeyMarker = resp.NextKeyMarker
|
||||
listReq.VersionIdMarker = resp.NextVersionIdMarker
|
||||
}
|
||||
if len(objectVersions) > 0 {
|
||||
bypass := true
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.DeleteObjects(ctx, &s3.DeleteObjectsInput{
|
||||
Bucket: &lockBucket,
|
||||
BypassGovernanceRetention: &bypass,
|
||||
Delete: &types.Delete{
|
||||
Objects: objectVersions,
|
||||
Quiet: aws.Bool(true),
|
||||
},
|
||||
})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: &lockBucket})
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
}()
|
||||
|
||||
retainUntilDate := time.Now().UTC().Add(24 * time.Hour).Truncate(time.Second)
|
||||
|
||||
t.Run("Retention", func(t *testing.T) {
|
||||
// Set Object Lock options for this test
|
||||
f.opt.ObjectLockMode = "GOVERNANCE"
|
||||
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
|
||||
defer func() {
|
||||
f.opt.ObjectLockMode = ""
|
||||
f.opt.ObjectLockRetainUntilDate = ""
|
||||
}()
|
||||
|
||||
// Upload an object with Object Lock retention
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-retention", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
// Read back metadata and verify Object Lock settings
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
|
||||
gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
|
||||
require.NoError(t, err)
|
||||
assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
|
||||
})
|
||||
|
||||
t.Run("LegalHold", func(t *testing.T) {
|
||||
// Set Object Lock legal hold option
|
||||
f.opt.ObjectLockLegalHoldStatus = "ON"
|
||||
defer func() {
|
||||
f.opt.ObjectLockLegalHoldStatus = ""
|
||||
}()
|
||||
|
||||
// Upload an object with legal hold
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-legal-hold", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
// Verify legal hold is ON
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
|
||||
|
||||
// Set legal hold to OFF
|
||||
err = o.setObjectLegalHold(ctx, types.ObjectLockLegalHoldStatusOff)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Clear cached metadata and re-read
|
||||
o.meta = nil
|
||||
gotMetadata, err = o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "OFF", gotMetadata["object-lock-legal-hold-status"])
|
||||
})
|
||||
|
||||
t.Run("SetAfterUpload", func(t *testing.T) {
|
||||
// Test the post-upload API path (PutObjectRetention + PutObjectLegalHold)
|
||||
f.opt.ObjectLockSetAfterUpload = true
|
||||
f.opt.ObjectLockMode = "GOVERNANCE"
|
||||
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
|
||||
f.opt.ObjectLockLegalHoldStatus = "ON"
|
||||
defer func() {
|
||||
f.opt.ObjectLockSetAfterUpload = false
|
||||
f.opt.ObjectLockMode = ""
|
||||
f.opt.ObjectLockRetainUntilDate = ""
|
||||
f.opt.ObjectLockLegalHoldStatus = ""
|
||||
}()
|
||||
|
||||
// Upload an object - lock applied AFTER upload via separate API calls
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-after-upload", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
// Verify all Object Lock settings were applied
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
|
||||
gotRetainDate, err := time.Parse(time.RFC3339, gotMetadata["object-lock-retain-until-date"])
|
||||
require.NoError(t, err)
|
||||
assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
|
||||
assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Metadata", f.InternalTestMetadata)
|
||||
t.Run("NoHead", f.InternalTestNoHead)
|
||||
t.Run("Versions", f.InternalTestVersions)
|
||||
t.Run("ObjectLock", f.InternalTestObjectLock)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -92,3 +93,87 @@ var (
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
_ fstests.SetCopyCutoffer = (*Fs)(nil)
|
||||
)
|
||||
|
||||
func TestParseRetainUntilDate(t *testing.T) {
|
||||
now := time.Now()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
checkFunc func(t *testing.T, result time.Time)
|
||||
}{
|
||||
{
|
||||
name: "RFC3339 date",
|
||||
input: "2030-01-15T10:30:00Z",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected, _ := time.Parse(time.RFC3339, "2030-01-15T10:30:00Z")
|
||||
assert.Equal(t, expected, result)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "RFC3339 date with timezone",
|
||||
input: "2030-06-15T10:30:00+02:00",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected, _ := time.Parse(time.RFC3339, "2030-06-15T10:30:00+02:00")
|
||||
assert.Equal(t, expected, result)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "duration days",
|
||||
input: "365d",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected := now.Add(365 * 24 * time.Hour)
|
||||
diff := result.Sub(expected)
|
||||
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~365 days from now")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "duration hours",
|
||||
input: "24h",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected := now.Add(24 * time.Hour)
|
||||
diff := result.Sub(expected)
|
||||
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~24 hours from now")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "duration minutes",
|
||||
input: "30m",
|
||||
wantErr: false,
|
||||
checkFunc: func(t *testing.T, result time.Time) {
|
||||
expected := now.Add(30 * time.Minute)
|
||||
diff := result.Sub(expected)
|
||||
assert.Less(t, diff.Abs(), 2*time.Second, "result should be ~30 minutes from now")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid input",
|
||||
input: "not-a-date",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := parseRetainUntilDate(tt.input)
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
if tt.checkFunc != nil {
|
||||
tt.checkFunc(t, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,7 @@ func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV
|
||||
// setFrom_typesObject_typesObjectVersion copies matching elements from a to b
|
||||
func setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVersion) {
|
||||
a.ChecksumAlgorithm = b.ChecksumAlgorithm
|
||||
a.ChecksumType = b.ChecksumType
|
||||
a.ETag = b.ETag
|
||||
a.Key = b.Key
|
||||
a.LastModified = b.LastModified
|
||||
@@ -82,6 +83,7 @@ func setFrom_typesObject_typesObjectVersion(a *types.Object, b *types.ObjectVers
|
||||
func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipartUploadInput, b *s3.HeadObjectOutput) {
|
||||
a.BucketKeyEnabled = b.BucketKeyEnabled
|
||||
a.CacheControl = b.CacheControl
|
||||
a.ChecksumType = b.ChecksumType
|
||||
a.ContentDisposition = b.ContentDisposition
|
||||
a.ContentEncoding = b.ContentEncoding
|
||||
a.ContentLanguage = b.ContentLanguage
|
||||
@@ -160,12 +162,15 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
|
||||
a.CacheControl = b.CacheControl
|
||||
a.ChecksumCRC32 = b.ChecksumCRC32
|
||||
a.ChecksumCRC32C = b.ChecksumCRC32C
|
||||
a.ChecksumCRC64NVME = b.ChecksumCRC64NVME
|
||||
a.ChecksumSHA1 = b.ChecksumSHA1
|
||||
a.ChecksumSHA256 = b.ChecksumSHA256
|
||||
a.ChecksumType = b.ChecksumType
|
||||
a.ContentDisposition = b.ContentDisposition
|
||||
a.ContentEncoding = b.ContentEncoding
|
||||
a.ContentLanguage = b.ContentLanguage
|
||||
a.ContentLength = b.ContentLength
|
||||
a.ContentRange = b.ContentRange
|
||||
a.ContentType = b.ContentType
|
||||
a.DeleteMarker = b.DeleteMarker
|
||||
a.ETag = b.ETag
|
||||
@@ -187,6 +192,7 @@ func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.
|
||||
a.SSEKMSKeyId = b.SSEKMSKeyId
|
||||
a.ServerSideEncryption = b.ServerSideEncryption
|
||||
a.StorageClass = b.StorageClass
|
||||
a.TagCount = b.TagCount
|
||||
a.VersionId = b.VersionId
|
||||
a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
|
||||
a.ResultMetadata = b.ResultMetadata
|
||||
@@ -232,6 +238,7 @@ func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.P
|
||||
a.CacheControl = b.CacheControl
|
||||
a.ChecksumCRC32 = b.ChecksumCRC32
|
||||
a.ChecksumCRC32C = b.ChecksumCRC32C
|
||||
a.ChecksumCRC64NVME = b.ChecksumCRC64NVME
|
||||
a.ChecksumSHA1 = b.ChecksumSHA1
|
||||
a.ChecksumSHA256 = b.ChecksumSHA256
|
||||
a.ContentDisposition = b.ContentDisposition
|
||||
@@ -270,6 +277,8 @@ func setFrom_s3CopyObjectInput_s3PutObjectInput(a *s3.CopyObjectInput, b *s3.Put
|
||||
a.GrantRead = b.GrantRead
|
||||
a.GrantReadACP = b.GrantReadACP
|
||||
a.GrantWriteACP = b.GrantWriteACP
|
||||
a.IfMatch = b.IfMatch
|
||||
a.IfNoneMatch = b.IfNoneMatch
|
||||
a.Metadata = b.Metadata
|
||||
a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
|
||||
a.ObjectLockMode = b.ObjectLockMode
|
||||
|
||||
@@ -688,7 +688,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
|
||||
"need_idx_progress": {"true"},
|
||||
"replace": {"1"},
|
||||
}
|
||||
formReader, contentType, _, err := rest.MultipartUpload(ctx, in, parameters, "file", f.opt.Enc.FromStandardName(filename))
|
||||
formReader, contentType, _, err := rest.MultipartUpload(ctx, in, parameters, "file", f.opt.Enc.FromStandardName(filename), "application/octet-stream")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to make multipart upload: %w", err)
|
||||
}
|
||||
|
||||
+19
-10
@@ -519,6 +519,12 @@ Example:
|
||||
Help: `URL for HTTP CONNECT proxy
|
||||
|
||||
Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
|
||||
|
||||
Supports the format http://user:pass@host:port, http://host:port, http://host.
|
||||
|
||||
Example:
|
||||
|
||||
http://myUser:myPass@proxyhostname.example.com:8000
|
||||
`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -919,15 +925,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
opt.Port = "22"
|
||||
}
|
||||
|
||||
// get proxy URL if set
|
||||
if opt.HTTPProxy != "" {
|
||||
proxyURL, err := url.Parse(opt.HTTPProxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse HTTP Proxy URL: %w", err)
|
||||
}
|
||||
f.proxyURL = proxyURL
|
||||
}
|
||||
|
||||
// Set up sshConfig here from opt
|
||||
// **NB** everything else should be setup in NewFsWithConnection
|
||||
sshConfig := &ssh.ClientConfig{
|
||||
User: opt.User,
|
||||
Auth: []ssh.AuthMethod{},
|
||||
@@ -1175,11 +1174,21 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
||||
f.mkdirLock = newStringLock()
|
||||
f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
|
||||
f.savedpswd = ""
|
||||
|
||||
// set the pool drainer timer going
|
||||
if f.opt.IdleTimeout > 0 {
|
||||
f.drain = time.AfterFunc(time.Duration(f.opt.IdleTimeout), func() { _ = f.drainPool(ctx) })
|
||||
}
|
||||
|
||||
// get proxy URL if set
|
||||
if opt.HTTPProxy != "" {
|
||||
proxyURL, err := url.Parse(opt.HTTPProxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse HTTP Proxy URL: %w", err)
|
||||
}
|
||||
f.proxyURL = proxyURL
|
||||
}
|
||||
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
SlowHash: true,
|
||||
@@ -1249,7 +1258,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
|
||||
fs.Debugf(f, "Failed to resolve path using RealPath: %v", err)
|
||||
cwd, err := c.sftpClient.Getwd()
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to to read current directory - using relative paths: %v", err)
|
||||
fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
|
||||
} else {
|
||||
f.absRoot = path.Join(cwd, f.root)
|
||||
fs.Debugf(f, "Relative path joined with current directory to get absolute path %q", f.absRoot)
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
// Package api has type definitions for shade
|
||||
package api
|
||||
|
||||
// ListDirResponse -------------------------------------------------
|
||||
// Format from shade api
|
||||
type ListDirResponse struct {
|
||||
Type string `json:"type"` // "file" or "tree"
|
||||
Path string `json:"path"` // Full path including root
|
||||
Ino int `json:"ino"` // inode number
|
||||
Mtime int64 `json:"mtime"` // Modified time in milliseconds
|
||||
Ctime int64 `json:"ctime"` // Created time in milliseconds
|
||||
Size int64 `json:"size"` // Size in bytes
|
||||
Hash string `json:"hash"` // MD5 hash
|
||||
Draft bool `json:"draft"` // Whether this is a draft file
|
||||
}
|
||||
|
||||
// PartURL Type for multipart upload/download
|
||||
type PartURL struct {
|
||||
URL string `json:"url"`
|
||||
Headers map[string]string `json:"headers,omitempty"`
|
||||
}
|
||||
|
||||
// CompletedPart Type for completed parts when making a multipart upload.
|
||||
type CompletedPart struct {
|
||||
ETag string
|
||||
PartNumber int32
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,21 @@
|
||||
package shade_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/shade"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
name := "TestShade"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
NilObject: (*shade.Object)(nil),
|
||||
SkipInvalidUTF8: true,
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "eventually_consistent_delay", Value: "7"},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
//multipart upload for shade
|
||||
|
||||
package shade
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/backend/shade/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
type shadeChunkWriter struct {
|
||||
initToken string
|
||||
chunkSize int64
|
||||
size int64
|
||||
f *Fs
|
||||
o *Object
|
||||
completedParts []api.CompletedPart
|
||||
completedPartsMu sync.Mutex
|
||||
}
|
||||
|
||||
// uploadMultipart handles multipart upload for larger files
|
||||
func (o *Object) uploadMultipart(ctx context.Context, src fs.ObjectInfo, in io.Reader, options ...fs.OpenOption) error {
|
||||
|
||||
chunkWriter, err := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
|
||||
Open: o.fs,
|
||||
OpenOptions: options,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var shadeWriter = chunkWriter.(*shadeChunkWriter)
|
||||
o.size = shadeWriter.size
|
||||
return nil
|
||||
}
|
||||
|
||||
// OpenChunkWriter returns the chunk size and a ChunkWriter
|
||||
//
|
||||
// Pass in the remote and the src object
|
||||
// You can also use options to hint at the desired chunk size
|
||||
func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
|
||||
// Temporary Object under construction
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
|
||||
uploadParts := f.opt.MaxUploadParts
|
||||
if uploadParts < 1 {
|
||||
uploadParts = 1
|
||||
} else if uploadParts > maxUploadParts {
|
||||
uploadParts = maxUploadParts
|
||||
}
|
||||
size := src.Size()
|
||||
fs.FixRangeOption(options, size)
|
||||
|
||||
// calculate size of parts
|
||||
chunkSize := f.opt.ChunkSize
|
||||
|
||||
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
|
||||
// buffers here (default 64 MB). With a maximum number of parts (10,000) this will be a file of
|
||||
// 640 GB.
|
||||
if size == -1 {
|
||||
warnStreamUpload.Do(func() {
|
||||
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
|
||||
chunkSize, fs.SizeSuffix(int64(chunkSize)*int64(uploadParts)))
|
||||
})
|
||||
} else {
|
||||
chunkSize = chunksize.Calculator(src, size, uploadParts, chunkSize)
|
||||
}
|
||||
|
||||
token, err := o.fs.refreshJWTToken(ctx)
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to get token: %w", err)
|
||||
}
|
||||
|
||||
err = f.ensureParentDirectories(ctx, remote)
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to ensure parent directories: %w", err)
|
||||
}
|
||||
|
||||
fullPath := remote
|
||||
if f.root != "" {
|
||||
fullPath = path.Join(f.root, remote)
|
||||
}
|
||||
|
||||
// Initiate multipart upload
|
||||
type initRequest struct {
|
||||
Path string `json:"path"`
|
||||
PartSize int64 `json:"partSize"`
|
||||
}
|
||||
reqBody := initRequest{
|
||||
Path: fullPath,
|
||||
PartSize: int64(chunkSize),
|
||||
}
|
||||
|
||||
var initResp struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: fmt.Sprintf("/%s/upload/multipart", o.fs.drive),
|
||||
RootURL: o.fs.endpoint,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Authorization": "Bearer " + token,
|
||||
},
|
||||
Options: options,
|
||||
}
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
res, err := o.fs.srv.CallJSON(ctx, &opts, reqBody, &initResp)
|
||||
if err != nil {
|
||||
return res != nil && res.StatusCode == http.StatusTooManyRequests, err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to initiate multipart upload: %w", err)
|
||||
}
|
||||
|
||||
chunkWriter := &shadeChunkWriter{
|
||||
initToken: initResp.Token,
|
||||
chunkSize: int64(chunkSize),
|
||||
size: size,
|
||||
f: f,
|
||||
o: o,
|
||||
}
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(chunkSize),
|
||||
Concurrency: f.opt.Concurrency,
|
||||
LeavePartsOnError: false,
|
||||
}
|
||||
return info, chunkWriter, err
|
||||
}
|
||||
|
||||
// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
|
||||
func (s *shadeChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
|
||||
|
||||
token, err := s.f.refreshJWTToken(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Read chunk
|
||||
var chunk bytes.Buffer
|
||||
n, err := io.Copy(&chunk, reader)
|
||||
|
||||
if n == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to read chunk: %w", err)
|
||||
}
|
||||
// Get presigned URL for this part
|
||||
var partURL api.PartURL
|
||||
|
||||
partOpts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: fmt.Sprintf("/%s/upload/multipart/part/%d?token=%s", s.f.drive, chunkNumber+1, url.QueryEscape(s.initToken)),
|
||||
RootURL: s.f.endpoint,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Authorization": "Bearer " + token,
|
||||
},
|
||||
}
|
||||
|
||||
err = s.f.pacer.Call(func() (bool, error) {
|
||||
res, err := s.f.srv.CallJSON(ctx, &partOpts, nil, &partURL)
|
||||
if err != nil {
|
||||
return res != nil && res.StatusCode == http.StatusTooManyRequests, err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get part URL: %w", err)
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: partURL.URL,
|
||||
Body: &chunk,
|
||||
ContentType: "",
|
||||
ContentLength: &n,
|
||||
}
|
||||
|
||||
// Add headers
|
||||
var uploadRes *http.Response
|
||||
if len(partURL.Headers) > 0 {
|
||||
opts.ExtraHeaders = make(map[string]string)
|
||||
maps.Copy(opts.ExtraHeaders, partURL.Headers)
|
||||
}
|
||||
|
||||
err = s.f.pacer.Call(func() (bool, error) {
|
||||
uploadRes, err = s.f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
return uploadRes != nil && uploadRes.StatusCode == http.StatusTooManyRequests, err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to upload part %d: %w", chunk, err)
|
||||
}
|
||||
|
||||
if uploadRes.StatusCode != http.StatusOK && uploadRes.StatusCode != http.StatusCreated {
|
||||
body, _ := io.ReadAll(uploadRes.Body)
|
||||
fs.CheckClose(uploadRes.Body, &err)
|
||||
return 0, fmt.Errorf("part upload failed with status %d: %s", uploadRes.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// Get ETag from response
|
||||
etag := uploadRes.Header.Get("ETag")
|
||||
fs.CheckClose(uploadRes.Body, &err)
|
||||
|
||||
s.completedPartsMu.Lock()
|
||||
defer s.completedPartsMu.Unlock()
|
||||
s.completedParts = append(s.completedParts, api.CompletedPart{
|
||||
PartNumber: int32(chunkNumber + 1),
|
||||
ETag: etag,
|
||||
})
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Close complete chunked writer finalising the file.
|
||||
func (s *shadeChunkWriter) Close(ctx context.Context) error {
|
||||
|
||||
// Complete multipart upload
|
||||
sort.Slice(s.completedParts, func(i, j int) bool {
|
||||
return s.completedParts[i].PartNumber < s.completedParts[j].PartNumber
|
||||
})
|
||||
|
||||
type completeRequest struct {
|
||||
Parts []api.CompletedPart `json:"parts"`
|
||||
}
|
||||
var completeBody completeRequest
|
||||
|
||||
if s.completedParts == nil {
|
||||
completeBody = completeRequest{Parts: []api.CompletedPart{}}
|
||||
} else {
|
||||
completeBody = completeRequest{Parts: s.completedParts}
|
||||
}
|
||||
|
||||
token, err := s.f.refreshJWTToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
completeOpts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: fmt.Sprintf("/%s/upload/multipart/complete?token=%s", s.f.drive, url.QueryEscape(s.initToken)),
|
||||
RootURL: s.f.endpoint,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Authorization": "Bearer " + token,
|
||||
},
|
||||
}
|
||||
|
||||
var response http.Response
|
||||
|
||||
err = s.f.pacer.Call(func() (bool, error) {
|
||||
res, err := s.f.srv.CallJSON(ctx, &completeOpts, completeBody, &response)
|
||||
|
||||
if err != nil && res == nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if res.StatusCode == http.StatusTooManyRequests {
|
||||
return true, err // Retry on 429
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
return false, fmt.Errorf("complete multipart failed with status %d: %s", res.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to complete multipart upload: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Abort chunk write
|
||||
//
|
||||
// You can and should call Abort without calling Close.
|
||||
func (s *shadeChunkWriter) Abort(ctx context.Context) error {
|
||||
token, err := s.f.refreshJWTToken(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: fmt.Sprintf("/%s/upload/abort/multipart?token=%s", s.f.drive, url.QueryEscape(s.initToken)),
|
||||
RootURL: s.f.endpoint,
|
||||
ExtraHeaders: map[string]string{
|
||||
"Authorization": "Bearer " + token,
|
||||
},
|
||||
}
|
||||
|
||||
err = s.f.pacer.Call(func() (bool, error) {
|
||||
res, err := s.f.srv.Call(ctx, &opts)
|
||||
if err != nil {
|
||||
fs.Debugf(s.f, "Failed to abort multipart upload: %v", err)
|
||||
return false, nil // Don't retry abort
|
||||
}
|
||||
if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated {
|
||||
fs.Debugf(s.f, "Abort returned status %d", res.StatusCode)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to abort multipart upload: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loaded 100 of 589 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user