Compare commits
103
Commits
master
...
v1.74-stable
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5bc93a2a7a | ||
|
|
d9b28b9e7d | ||
|
|
874a804f52 | ||
|
|
e58f09739a | ||
|
|
7e78929575 | ||
|
|
015fd0eba1 | ||
|
|
1a74673244 | ||
|
|
c89b766cf4 | ||
|
|
9ebca99cd2 | ||
|
|
0f4a8fad64 | ||
|
|
888a80e64b | ||
|
|
b440d18897 | ||
|
|
c9ff593a7f | ||
|
|
39487c5ebf | ||
|
|
eb09655949 | ||
|
|
c2d711e714 | ||
|
|
2a6591d1cd | ||
|
|
b9fd40d182 | ||
|
|
e4093ec952 | ||
|
|
2ebb9a6c72 | ||
|
|
76a96a91b4 | ||
|
|
c98ab6ae97 | ||
|
|
e9dcfa12c2 | ||
|
|
a8e9793c02 | ||
|
|
81cfc9ab1b | ||
|
|
efa0e81074 | ||
|
|
5d94fa3184 | ||
|
|
ad7f846048 | ||
|
|
1a55ea17bf | ||
|
|
f0b43b59b4 | ||
|
|
af2853f7c9 | ||
|
|
bc6d5894f2 | ||
|
|
82c899e39c | ||
|
|
215dd7a06a | ||
|
|
5e190ba2a6 | ||
|
|
a19391014f | ||
|
|
dfdcda829e | ||
|
|
e0746b3b8d | ||
|
|
3d560453a1 | ||
|
|
792a8ac91f | ||
|
|
3f80fe0b07 | ||
|
|
660b630979 | ||
|
|
85a12f22c8 | ||
|
|
0e5e79ed50 | ||
|
|
3b00bbf730 | ||
|
|
3654e9cfb7 | ||
|
|
40cd3c99e6 | ||
|
|
6653752779 | ||
|
|
6e67723dc5 | ||
|
|
37e4117f48 | ||
|
|
9222ed2c5a | ||
|
|
48da1774f4 | ||
|
|
2a7b0d0d3b | ||
|
|
fc4349dc16 | ||
|
|
dff3a0731e | ||
|
|
5f5e2ef9cf | ||
|
|
d4397faa94 | ||
|
|
7324ee850a | ||
|
|
3c54a37488 | ||
|
|
a2f8b493ed | ||
|
|
641702fc7f | ||
|
|
f25d9e2bf3 | ||
|
|
ec28206d8a | ||
|
|
ddbc886395 | ||
|
|
c8ceb209fc | ||
|
|
5b4eb0eefb | ||
|
|
223e473e6e | ||
|
|
bac27547d5 | ||
|
|
b22fe9811c | ||
|
|
ce64f4edb8 | ||
|
|
f3986e2bb7 | ||
|
|
f0961f4a8d | ||
|
|
139ccc7ffd | ||
|
|
b3ddec50c6 | ||
|
|
3cd224e03f | ||
|
|
52af9caa14 | ||
|
|
b2cb7f89c1 | ||
|
|
2292ba0cd4 | ||
|
|
07c87327ae | ||
|
|
6cc0d1a0b0 | ||
|
|
7a5e01b015 | ||
|
|
bf111df84a | ||
|
|
0b73e2b858 | ||
|
|
e0f210cdc4 | ||
|
|
1c06bb9272 | ||
|
|
b2e1aa65dc | ||
|
|
3a414d8262 | ||
|
|
24054305ce | ||
|
|
076fb2bc53 | ||
|
|
e1fd3725bc | ||
|
|
5780a30c30 | ||
|
|
17c2db7289 | ||
|
|
44c56ed481 | ||
|
|
ebca82f7db | ||
|
|
e9d280b793 | ||
|
|
4f93bd482f | ||
|
|
b6c68882e5 | ||
|
|
23dc9d0515 | ||
|
|
be7c0f31a4 | ||
|
|
3ac9b75a83 | ||
|
|
5e9cb39084 | ||
|
|
87d0b59a51 | ||
|
|
76596b6727 |
No files matched your search
+121
-10
@@ -37,7 +37,7 @@ jobs:
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.5'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -48,14 +48,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.5'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.5'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -64,14 +64,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.5'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.5'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -81,14 +81,14 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.5'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.25
|
||||
os: ubuntu-latest
|
||||
go: '~1.25.7'
|
||||
go: '~1.25.12'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
@@ -97,16 +97,58 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ matrix.go }}
|
||||
check-latest: true
|
||||
# Caching is handled explicitly below. setup-go's built-in cache
|
||||
# keys only on go.sum, so all matrix jobs collide on one key and
|
||||
# only the first to finish saves - which loses the cross-compile
|
||||
# build cache.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
# The module cache holds downloaded source and is identical across
|
||||
# jobs (it depends only on go.sum, not on OS, arch or Go version), so
|
||||
# it is shared via a single key to avoid storing the same ~600 MiB in
|
||||
# every job's cache and overflowing the repo cache limit.
|
||||
- name: Module cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
# The build cache holds compiled artifacts which are specific to the
|
||||
# OS, arch and Go version, so it is kept per job.
|
||||
- name: Build cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-${{ matrix.job_name }}-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
|
||||
- name: Set environment variables
|
||||
run: |
|
||||
@@ -205,6 +247,15 @@ jobs:
|
||||
# Deploy binaries if enabled in config && not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
lint:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -227,15 +278,23 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.26.0'
|
||||
go-version: '~1.26.5'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ~/go/pkg/mod
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/go/pkg/mod
|
||||
~/.cache/go-build
|
||||
~/.cache/golangci-lint
|
||||
key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
@@ -302,6 +361,15 @@ jobs:
|
||||
run: bin/check_autogenerated_edits.py 'origin/${{ github.base_ref }}'
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
|
||||
android:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 30
|
||||
@@ -309,6 +377,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
run: |
|
||||
echo "year-week=$(date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
@@ -316,9 +390,37 @@ jobs:
|
||||
|
||||
# Upgrade together with NDK version
|
||||
- name: Set up Go
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.26.0'
|
||||
go-version: '~1.26.5'
|
||||
# Caching is handled explicitly below to share the module cache
|
||||
# with the other jobs - see the build job for the rationale.
|
||||
cache: false
|
||||
|
||||
- name: Get Go paths
|
||||
id: go-paths
|
||||
run: |
|
||||
echo "build-cache=$(go env GOCACHE)" >> $GITHUB_OUTPUT
|
||||
echo "mod-cache=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Module cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.mod-cache }}
|
||||
key: go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-mod-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-mod-
|
||||
|
||||
- name: Build cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ${{ steps.go-paths.outputs.build-cache }}
|
||||
key: go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
|
||||
restore-keys: |
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
|
||||
go-build-android-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-
|
||||
|
||||
- name: Set global environment variables
|
||||
run: |
|
||||
@@ -394,3 +496,12 @@ jobs:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# Upload artifacts if not a PR && not a fork
|
||||
if: env.RCLONE_CONFIG_PASS != '' && github.head_ref == '' && github.repository == 'rclone/rclone'
|
||||
|
||||
- name: Trim module cache before saving
|
||||
if: always()
|
||||
run: |
|
||||
# The downloaded module .zip archives are not needed to build from
|
||||
# the already-extracted module cache, so drop them before the cache
|
||||
# is saved - this roughly halves the module cache size.
|
||||
modcache=$(go env GOMODCACHE | tr '\\' '/')
|
||||
find "$modcache/cache/download" -name '*.zip' -type f -delete 2>/dev/null || true
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
default: true
|
||||
|
||||
# Use specific styles, to be consistent accross all documents.
|
||||
# Use specific styles, to be consistent across all documents.
|
||||
# Default is to accept any as long as it is consistent within the same document.
|
||||
heading-style: # MD003
|
||||
style: atx
|
||||
@@ -48,12 +48,12 @@ link-fragments: false # MD051
|
||||
# https://gohugo.io/content-management/syntax-highlighting/#languages
|
||||
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
|
||||
# In addition, we only want to allow identifiers (aliases) that correspond to
|
||||
# the same language in Hugo and GitHub, and preferrably also VSCode and other
|
||||
# the same language in Hugo and GitHub, and preferably also VSCode and other
|
||||
# commonly used tools, to avoid confusion. An example of this is that "shell"
|
||||
# by some are considered an identifier for shell scripts, i.e. an alias for
|
||||
# "sh", while others consider it an identifier for shell sessions, i.e. an
|
||||
# alias for "console". Although Hugo and GitHub in this case are consistent and
|
||||
# have choosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# have chosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# avoids the confusion entirely.
|
||||
fenced-code-language: # MD040
|
||||
allowed_languages:
|
||||
|
||||
Generated
+687
-43
File diff suppressed because it is too large.
Load diff
@@ -1,6 +1,6 @@
|
||||
% rclone(1) User Manual
|
||||
% Nick Craig-Wood
|
||||
% May 01, 2026
|
||||
% Jul 08, 2026
|
||||
|
||||
# NAME
|
||||
|
||||
@@ -5416,12 +5416,12 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,command=e
|
||||
|
||||
```console
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20260501
|
||||
// Output: stories/The Quick Brown Fox!-20260708
|
||||
```
|
||||
|
||||
```console
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2026-05-01 0208PM
|
||||
// Output: stories/The Quick Brown Fox!-2026-07-08 0504PM
|
||||
```
|
||||
|
||||
```console
|
||||
@@ -5631,8 +5631,9 @@ Copy files from source to dest, skipping identical files.
|
||||
If source:path is a file or directory then it copies it to a file or
|
||||
directory named dest:path.
|
||||
|
||||
This can be used to upload single files to other than their current
|
||||
name. If the source is a directory then it acts exactly like the
|
||||
This can be used to copy a single file to a destination with a name
|
||||
different from its source - for example, uploading and renaming in one
|
||||
step. If the source is a directory then it acts exactly like the
|
||||
[copy](https://rclone.org/commands/rclone_copy/) command.
|
||||
|
||||
So
|
||||
@@ -14066,6 +14067,17 @@ Versioning is not currently supported.
|
||||
Metadata will only be saved in memory other than the rclone `mtime`
|
||||
metadata which will be set as the modification time of the file.
|
||||
|
||||
## Object names
|
||||
|
||||
`serve s3` stores objects as files in the backend, so object keys are
|
||||
mapped to file paths rather than treated as the opaque strings AWS S3
|
||||
allows. Keys must be in canonical path form: keys that contain `..` or
|
||||
`.` path segments, repeated slashes (`//`), or a leading or trailing
|
||||
slash are rejected with a `400 Bad Request` (`InvalidArgument`)
|
||||
instead of being normalised, since normalising them could alias two
|
||||
distinct keys to the same file or resolve a key outside its bucket.
|
||||
This matches the behaviour of other S3 servers such as MinIO.
|
||||
|
||||
## Supported operations
|
||||
|
||||
`serve s3` currently supports the following operations.
|
||||
@@ -18052,7 +18064,7 @@ Windows and `/dev/null` on Unix systems, then rclone will keep the
|
||||
configuration file in memory only.
|
||||
|
||||
You may see a log message "Config file not found - using defaults" if there is
|
||||
no configuration file. This can be supressed, e.g. if you are using rclone
|
||||
no configuration file. This can be suppressed, e.g. if you are using rclone
|
||||
entirely with [on the fly remotes](https://rclone.org/docs/#backend-path-to-dir), by using
|
||||
memory-only configuration file or by creating an empty configuration file, as
|
||||
described above.
|
||||
@@ -18664,7 +18676,7 @@ warnings and significant events.
|
||||
|
||||
See also the [logging](#logging) section.
|
||||
|
||||
### --windows-event-log LogLevel
|
||||
### --windows-event-log-level LogLevel
|
||||
|
||||
If this is configured (the default is `OFF`) then logs of this level
|
||||
and above will be logged to the Windows event log in **addition** to
|
||||
@@ -18680,15 +18692,15 @@ and `Error`. If enabled we map rclone levels like this.
|
||||
|
||||
Rclone will declare its log source as "rclone" if it is has enough
|
||||
permissions to create the registry key needed. If not then logs will
|
||||
appear as "Application". You can run `rclone version --windows-event-log DEBUG`
|
||||
appear as "Application". You can run `rclone version --windows-event-log-level DEBUG`
|
||||
once as administrator to create the registry key in advance.
|
||||
|
||||
**Note** that the `--windows-event-log` level must be greater (more
|
||||
**Note** that the `--windows-event-log-level` level must be greater (more
|
||||
severe) than or equal to the `--log-level`. For example to log DEBUG
|
||||
to a log file but ERRORs to the event log you would use
|
||||
|
||||
```text
|
||||
--log-file rclone.log --log-level DEBUG --windows-event-log ERROR
|
||||
--log-file rclone.log --log-level DEBUG --windows-event-log-level ERROR
|
||||
```
|
||||
|
||||
This option is only supported Windows platforms.
|
||||
@@ -20470,7 +20482,7 @@ y/e/d>
|
||||
## Configuring by copying the config file
|
||||
|
||||
Rclone stores all of its configuration in a single file. This can easily be
|
||||
copied to configure a remote rclone (although some backends does not support
|
||||
copied to configure a remote rclone (although some backends do not support
|
||||
reusing the same configuration, consult your backend documentation to be
|
||||
sure).
|
||||
|
||||
@@ -21640,6 +21652,43 @@ or [use HTTP directly](#api-http).
|
||||
If you just want to run a remote control then see the [rcd](https://rclone.org/commands/rclone_rcd/)
|
||||
command.
|
||||
|
||||
## Security {#security}
|
||||
|
||||
**Access to the rc API is equivalent to shell access as the user running
|
||||
rclone.** Treat the rc port as you would an interactive login on the host.
|
||||
|
||||
Any caller who can reach the API (and pass authentication, if it is enabled)
|
||||
can, among other things:
|
||||
|
||||
- **Run OS commands** as the rclone user. `core/command` re-executes the rclone
|
||||
binary with arbitrary arguments, and several backend options shell out to
|
||||
programs, so even creating a remote with `config/create` can lead to command
|
||||
execution.
|
||||
- **Read and write any file** reachable by the rclone process, by pointing
|
||||
`operations/*` or `sync/*` at a `local` remote (or via `--rc-files` /
|
||||
`--rc-serve`). Writing arbitrary files as the rclone user is itself a route to
|
||||
code execution.
|
||||
- **Read back stored credentials.** rclone configs routinely hold cloud-provider
|
||||
secrets. `config/dump` and friends expose them, so a compromise of the rc
|
||||
reaches every configured backend.
|
||||
- **Change rclone's runtime behaviour** with `options/set`, **manage remotes**
|
||||
with `config/*`, and **stop the process** with `core/quit`.
|
||||
|
||||
There is currently no per-endpoint capability or scope system: authentication is
|
||||
all-or-nothing. Granting any access grants all of the above.
|
||||
|
||||
Consequently:
|
||||
|
||||
- **Do not bind the rc to a network address you do not control.** The default
|
||||
bind is loopback (`localhost:5572`); keep it there unless you have a specific
|
||||
reason to change it.
|
||||
- **Do not use `--rc-no-auth` on a non-loopback bind.** It disables
|
||||
authentication on the endpoints that access remotes — see
|
||||
[`--rc-no-auth`](#--rc-no-auth).
|
||||
- **Use authentication and TLS** (`--rc-user`/`--rc-pass` or `--rc-htpasswd`,
|
||||
plus `--rc-cert`/`--rc-key`) whenever the port is reachable by anyone you do
|
||||
not fully trust, and raise `--rc-min-tls-version`.
|
||||
|
||||
## Supported parameters
|
||||
|
||||
### --rc
|
||||
@@ -21703,8 +21752,26 @@ so you can browse to `http://127.0.0.1:5572/` or `http://127.0.0.1:5572/*`
|
||||
to see a listing of the remotes. Objects may be requested from
|
||||
remotes using this syntax `http://127.0.0.1:5572/[remote:path]/path/to/object`
|
||||
|
||||
Unless the rc server has authentication configured (`--rc-user`/`--rc-pass`
|
||||
or `--rc-htpasswd`) or the `--rc-no-auth` flag is set, only remotes already
|
||||
present in the config file may be served this way. Inline remotes (e.g.
|
||||
`[:webdav,url=...:]`), connection string parameters and bare local paths are
|
||||
rejected, since instantiating them from an unauthenticated request could run
|
||||
commands or read arbitrary local files.
|
||||
|
||||
Default Off.
|
||||
|
||||
### global.* connection string options and the rc
|
||||
|
||||
Remotes instantiated by the rc do not let [connection
|
||||
string](https://rclone.org/docs/#connection-strings) `global.*` options change rclone's
|
||||
process-wide configuration. Remotes created directly on the command
|
||||
line or defined in the config file are unaffected.
|
||||
|
||||
A `global.*` option still takes effect for the individual backend it
|
||||
is set on (exactly like an `override.*` option), it just does not leak
|
||||
into the global config for the rest of the process.
|
||||
|
||||
### --rc-serve-no-modtime
|
||||
|
||||
Set this flag to skip reading the modification time (can speed things up).
|
||||
@@ -24985,7 +25052,7 @@ Flags for general networking and HTTP stuff.
|
||||
--tpslimit float Limit HTTP transactions per second to this
|
||||
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
|
||||
--use-cookies Enable session cookiejar
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.4")
|
||||
```
|
||||
|
||||
|
||||
@@ -25177,7 +25244,7 @@ Flags to control the Remote Control API.
|
||||
Flags to control the Metrics HTTP endpoint..
|
||||
|
||||
```
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind metrics server to
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind server to
|
||||
--metrics-allow-origin string Origin which cross-domain request (CORS) can be executed from
|
||||
--metrics-baseurl string Prefix for URLs - leave blank for root
|
||||
--metrics-cert string TLS PEM key (concatenation of certificate and CA certificate)
|
||||
@@ -25383,7 +25450,7 @@ Backend-only flags (these can be set in the config file also).
|
||||
--drime-list-chunk int Number of items to list in each call (default 1000)
|
||||
--drime-root-folder-id string ID of the root folder
|
||||
--drime-upload-concurrency int Concurrency for multipart uploads and copies (default 4)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 5Mi)
|
||||
--drime-workspace-id string Account ID
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded
|
||||
--drive-allow-import-name-change Allow the filetype to change when uploading Google docs
|
||||
@@ -25669,8 +25736,10 @@ Backend-only flags (these can be set in the config file also).
|
||||
--local-case-sensitive Force the filesystem to report itself as case sensitive
|
||||
--local-description string Description of the remote
|
||||
--local-encoding Encoding The encoding for the backend (default Slash,Dot)
|
||||
--local-fatal-if-no-space Make out-of-space errors fatal during transfers
|
||||
--local-hashes CommaSepList Comma separated list of supported checksum types
|
||||
--local-links Translate symlinks to/from regular files with a '.rclonelink' extension for the local backend
|
||||
--local-metadata-restore-special-bits Restore the setuid, setgid and sticky bits from metadata
|
||||
--local-no-check-updated Don't check to see if the files change during upload
|
||||
--local-no-clone Disable reflink cloning for server-side copies
|
||||
--local-no-preallocate Disable preallocation of disk space for transferred files
|
||||
@@ -27197,8 +27266,8 @@ internals of the generated export file.) Therefore, bisync automatically skips
|
||||
`--download-hash` for files with a size less than 0.
|
||||
|
||||
See also: [`Hasher`](https://rclone.org/hasher/) backend,
|
||||
[`cryptcheck`](https://rclone.org/commands/rclone_cryptcheck/) command, [`rclone check
|
||||
--download`](https://rclone.org/commands/rclone_check/) option,
|
||||
[`cryptcheck`](https://rclone.org/commands/rclone_cryptcheck/) command,
|
||||
[`rclone check --download`](https://rclone.org/commands/rclone_check/) option,
|
||||
[`md5sum`](https://rclone.org/commands/rclone_md5sum/) command
|
||||
|
||||
### --max-delete
|
||||
@@ -27772,9 +27841,28 @@ encodings.)
|
||||
The following backends have known issues that need more investigation:
|
||||
|
||||
<!--- start list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->
|
||||
- `TestDropbox` (`dropbox`)
|
||||
- [`TestBisyncRemoteRemote/normalization`](https://pub.rclone.org/integration-tests/current/dropbox-cmd.bisync-TestDropbox-1.txt)
|
||||
- Updated: 2026-05-01-010013
|
||||
- `TestGoogleCloudStorage,directory_markers` (`googlecloudstorage`)
|
||||
- [`TestBisyncRemoteLocal/all_changed`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/backupdir`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/basic`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/changes`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/check_access`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [79 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- `TestGoogleCloudStorage` (`googlecloudstorage`)
|
||||
- [`TestBisyncRemoteLocal/all_changed`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/backupdir`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/basic`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/changes`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/check_access`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [79 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- `TestHuaweiDrive` (`huaweidrive`)
|
||||
- [`TestBisyncRemoteLocal/ext_paths`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncRemoteLocal/extended_filenames`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncRemoteLocal/normalization`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncLocalRemote/ext_paths`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncLocalRemote/extended_filenames`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [4 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- Updated: 2026-07-08-010014
|
||||
<!--- end list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->
|
||||
|
||||
The following backends either have not been tested recently or have known issues
|
||||
@@ -27880,6 +27968,11 @@ is to rename it to the same name on both sides. (As of `rclone v1.64`,
|
||||
a `--resync` is no longer required after doing so, as bisync will automatically
|
||||
detect that Path1 and Path2 are in agreement.)
|
||||
|
||||
Note that although the flag --track-renames ensures that renamed/moved files won't
|
||||
be deleted and uploaded again, they are still counted as deleted files for purposes
|
||||
of the --max-delete flag (as this check happens before the rename detection
|
||||
operation). See [this issue](https://github.com/rclone/rclone/issues/8685).
|
||||
|
||||
### `--fast-list` used by default
|
||||
|
||||
Unlike most other rclone commands, bisync uses [`--fast-list`](https://rclone.org/docs/#fast-list)
|
||||
@@ -28618,6 +28711,11 @@ about *Unison* and synchronization in general.
|
||||
|
||||
## Changelog
|
||||
|
||||
### `v1.74.2`
|
||||
|
||||
- Fixed an issue causing `--conflict-loser pathname` to produce unexpected
|
||||
behavior if using a non-default `--conflict-resolve` value.
|
||||
|
||||
### `v1.74`
|
||||
|
||||
- Added several missing `rc` parameters.
|
||||
@@ -30500,6 +30598,9 @@ Properties:
|
||||
- "eu-south-1"
|
||||
- EU South 1
|
||||
- Provider: Fastly
|
||||
- "eu-west-1"
|
||||
- EU West 1
|
||||
- Provider: Fastly
|
||||
- "jp-central-1"
|
||||
- JP Central 1
|
||||
- Provider: Fastly
|
||||
@@ -30512,9 +30613,15 @@ Properties:
|
||||
- "us-east"
|
||||
- US East
|
||||
- Provider: Fastly
|
||||
- "us-east-1"
|
||||
- US East 1
|
||||
- Provider: Fastly
|
||||
- "us-west"
|
||||
- US West
|
||||
- Provider: Fastly
|
||||
- "us-west-1"
|
||||
- US West 1
|
||||
- Provider: Fastly
|
||||
- "global"
|
||||
- Global
|
||||
- Provider: FileLu
|
||||
@@ -31087,6 +31194,9 @@ Properties:
|
||||
- "eu-south-1.object.fastlystorage.app"
|
||||
- EU South 1
|
||||
- Provider: Fastly
|
||||
- "eu-west-1.object.fastlystorage.app"
|
||||
- EU West 1
|
||||
- Provider: Fastly
|
||||
- "jp-central-1.object.fastlystorage.app"
|
||||
- JP Central 1
|
||||
- Provider: Fastly
|
||||
@@ -31099,9 +31209,15 @@ Properties:
|
||||
- "us-east.object.fastlystorage.app"
|
||||
- US East
|
||||
- Provider: Fastly
|
||||
- "us-east-1.object.fastlystorage.app"
|
||||
- US East 1
|
||||
- Provider: Fastly
|
||||
- "us-west.object.fastlystorage.app"
|
||||
- US West
|
||||
- Provider: Fastly
|
||||
- "us-west-1.object.fastlystorage.app"
|
||||
- US West 1
|
||||
- Provider: Fastly
|
||||
- "s5lu.com"
|
||||
- Global FileLu S5 endpoint
|
||||
- Provider: FileLu
|
||||
@@ -31485,17 +31601,38 @@ Properties:
|
||||
- "br-ne1.magaluobjects.com"
|
||||
- Fortaleza, CE (BR), br-ne1
|
||||
- Provider: Magalu
|
||||
- "s3.eu-amsterdam.megas4.com"
|
||||
- Mega S4 Amsterdam
|
||||
- Provider: Mega
|
||||
- "s3.eu-luxembourg.megas4.com"
|
||||
- Mega S4 Luxembourg
|
||||
- Provider: Mega
|
||||
- "s3.eu-paris.megas4.com"
|
||||
- Mega S4 Paris
|
||||
- Provider: Mega
|
||||
- "s3.eu-barcelona.megas4.com"
|
||||
- Mega S4 Barcelona
|
||||
- Provider: Mega
|
||||
- "s3.ca-montreal.megas4.com"
|
||||
- Mega S4 Montreal
|
||||
- Provider: Mega
|
||||
- "s3.ca-vancouver.megas4.com"
|
||||
- Mega S4 Vancouver
|
||||
- Provider: Mega
|
||||
- "s3.ap-tokyo.megas4.com"
|
||||
- Mega S4 Tokyo
|
||||
- Provider: Mega
|
||||
- "s3.eu-central-1.s4.mega.io"
|
||||
- Mega S4 eu-central-1 (Amsterdam)
|
||||
- Mega S4 eu-central-1 (Amsterdam, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.eu-central-2.s4.mega.io"
|
||||
- Mega S4 eu-central-2 (Bettembourg)
|
||||
- Mega S4 eu-central-2 (Bettembourg, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.ca-central-1.s4.mega.io"
|
||||
- Mega S4 ca-central-1 (Montreal)
|
||||
- Mega S4 ca-central-1 (Montreal, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.ca-west-1.s4.mega.io"
|
||||
- Mega S4 ca-west-1 (Vancouver)
|
||||
- Mega S4 ca-west-1 (Vancouver, legacy)
|
||||
- Provider: Mega
|
||||
- "oos.eu-west-2.outscale.com"
|
||||
- Outscale EU West 2 (Paris)
|
||||
@@ -32700,7 +32837,7 @@ Cutoff for switching to multipart copy.
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 5 GiB.
|
||||
The minimum is 1 byte and the maximum is 5 GiB.
|
||||
|
||||
Properties:
|
||||
|
||||
@@ -32851,7 +32988,7 @@ If true use path style access if false use virtual hosted style.
|
||||
|
||||
If this is true (the default) then rclone will use path style access,
|
||||
if false then rclone will use virtual path style. See [the AWS S3
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html)
|
||||
for more info.
|
||||
|
||||
Some providers (e.g. AWS, Aliyun OSS, Netease COS, or Tencent COS) require this set to
|
||||
@@ -40204,7 +40341,7 @@ start and finish the upload) and another 2 requests for each chunk:
|
||||
/b2api/v1/b2_finish_large_file
|
||||
```
|
||||
|
||||
#### Versions
|
||||
### Viewing versions with --b2-versions
|
||||
|
||||
Versions can be viewed with the `--b2-versions` flag. When it is set
|
||||
rclone will show and act on older versions of files. For example
|
||||
@@ -43918,9 +44055,9 @@ This uses a 32 byte (256 bit key) key derived from the user password.
|
||||
1 MiB (1048576 bytes) file will encrypt to
|
||||
|
||||
- 32 bytes header
|
||||
- 16 chunks of 65568 bytes
|
||||
- 16 chunks of 65552 bytes
|
||||
|
||||
1049120 bytes total (a 0.05% overhead). This is the overhead for big
|
||||
1048864 bytes total (a 0.03% overhead). This is the overhead for big
|
||||
files.
|
||||
|
||||
### Name encryption
|
||||
@@ -44796,7 +44933,7 @@ Properties:
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_DRIME_UPLOAD_CUTOFF
|
||||
- Type: SizeSuffix
|
||||
- Default: 200Mi
|
||||
- Default: 5Mi
|
||||
|
||||
#### --drime-chunk-size
|
||||
|
||||
@@ -48443,9 +48580,14 @@ correct root to use itself.
|
||||
However you can set this to restrict rclone to a specific folder
|
||||
hierarchy or to access data within the "Computers" tab on the drive
|
||||
web interface (where files from Google's Backup and Sync desktop
|
||||
program go).
|
||||
program go). However, this is not the easiest method.
|
||||
Instead, you can create a shortcut to a folder in "Computers" and
|
||||
place it in "My Drive." To do so, right-click the folder, select
|
||||
"Organize", and then choose "Add shortcut" in the Google Drive web interface.
|
||||
Once you add the shortcut to "My Drive," rclone will display the folder, allowing you to interact with it.
|
||||
See also [Shortcuts](#shortcuts).
|
||||
|
||||
In order to do this you will have to find the `Folder ID` of the
|
||||
If, however, you choose to change your root folder, you will have to find the `Folder ID` of the
|
||||
directory you wish rclone to display. This will be the last segment
|
||||
of the URL when you open the relevant folder in the drive web
|
||||
interface.
|
||||
@@ -48456,7 +48598,11 @@ in the browser, then you use `1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh` as
|
||||
the `root_folder_id` in the config.
|
||||
|
||||
**NB** folders under the "Computers" tab seem to be read only (drive
|
||||
gives a 500 error) when using rclone.
|
||||
gives a 500 error) when using rclone, but they are also read-only in the Google Drive web interface,
|
||||
likely because this folder is managed by the Google Drive Desktop tool.
|
||||
Top-level folders in "Computers" are read-only, but you can interact
|
||||
with folders contained within them using rclone.
|
||||
These are the folders for which you can create a shortcut as well, as described above.
|
||||
|
||||
There doesn't appear to be an API to discover the folder IDs of the
|
||||
"Computers" tab - please contact us if you know otherwise!
|
||||
@@ -48753,7 +48899,7 @@ Drive, the size of all files in the Trash and the space used by other
|
||||
Google services such as Gmail. This command does not take any path
|
||||
arguments.
|
||||
|
||||
#### Import/Export of google documents
|
||||
### Import/Export of google documents
|
||||
|
||||
Google documents can be exported from and uploaded to Google Drive.
|
||||
|
||||
@@ -50280,6 +50426,23 @@ after a week, which can be annoying to refresh constantly. If, for whatever
|
||||
reason, a short grant time is not a problem, then keeping the application in
|
||||
testing mode would also be sufficient.
|
||||
|
||||
In most cases verification is not actually required. Google [exempts a number of
|
||||
app categories](https://support.google.com/cloud/answer/13464323) from mandatory
|
||||
verification, including:
|
||||
|
||||
- **Personal Use apps**: if the app is for your personal use (fewer than 100
|
||||
users), you and your limited number of users can keep using it without
|
||||
verification - you will just need to click through the "unverified app"
|
||||
warning screen during sign-in. Verification is only required if you want to
|
||||
grow your user base beyond 100 users.
|
||||
- **Development/Testing/Staging apps**: apps in development, testing or staging
|
||||
mode are not subject to verification (but are limited to the 100-user cap and
|
||||
the unverified-app warning until verified).
|
||||
|
||||
So for typical personal rclone use you can leave the app unverified, accept the
|
||||
warning screen, and publish it (rather than leaving it in "Testing") to avoid
|
||||
the weekly grant expiry described above.
|
||||
|
||||
(Thanks to @balazer on github for these instructions.)
|
||||
|
||||
Sometimes, creation of an OAuth consent in Google API Console fails due to an
|
||||
@@ -54082,6 +54245,8 @@ as described [below](#traditional):
|
||||
- MediaMarkt
|
||||
- MediaMarkt Cloud (mediamarkt.jottacloud.com)
|
||||
- Let's Go Cloud (letsgo.jotta.cloud)
|
||||
- Phonero
|
||||
- Phonero Sky (sky.phonero.no)
|
||||
|
||||
Paths are specified as `remote:path`
|
||||
|
||||
@@ -69790,6 +69955,31 @@ backends and the VFS.
|
||||
|
||||
Note that this flag is incompatible with `-copy-links` / `-L`.
|
||||
|
||||
#### Symlink targets and the destination
|
||||
|
||||
When rclone recreates a `.rclonelink` file as a symlink on local storage,
|
||||
the symlink can point anywhere - including, with an absolute path or one
|
||||
using `../` - to a location outside the directory you are copying into. This
|
||||
is normal - rclone reproduces whatever target the link had, so backups
|
||||
round-trip faithfully.
|
||||
|
||||
What rclone will **not** do is *write through* such a link. If a remote you
|
||||
are copying from contains both a symlink and a file or directory that would
|
||||
be placed inside it - for example a `dir.rclonelink` pointing somewhere
|
||||
outside the destination, alongside a `dir/file.txt` - rclone refuses to
|
||||
follow the symlink when writing `dir/file.txt`. The offending file is
|
||||
skipped with an error, the rest of the transfer continues, and the skipped
|
||||
file is counted in the error summary printed at the end of the run.
|
||||
|
||||
This protects you from a malicious or compromised remote using `-l` /
|
||||
`--links` to plant a symlink and then write through it to somewhere outside
|
||||
your destination. Ordinary symlink round-trips, and symlinks that stay
|
||||
inside the destination, are unaffected.
|
||||
|
||||
If you have intentionally pre-created a symlinked directory inside your
|
||||
destination and want rclone to write into the directory it points at, do
|
||||
not use `-l` / `--links` for that copy, or remove the symlink first.
|
||||
|
||||
### Restricting filesystems with --one-file-system
|
||||
|
||||
Normally rclone will recurse through filesystems as mounted.
|
||||
@@ -70106,6 +70296,47 @@ Properties:
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --local-metadata-restore-special-bits
|
||||
|
||||
Restore the setuid, setgid and sticky bits from metadata.
|
||||
|
||||
When restoring metadata with --metadata rclone applies the "mode" from
|
||||
the source. By default rclone applies only the permission bits and
|
||||
strips the setuid, setgid and sticky bits.
|
||||
|
||||
The "mode" comes from the source remote which may not be trusted.
|
||||
Restoring a setuid or setgid bit onto freshly written,
|
||||
source-controlled content can plant a setuid binary, which is dangerous
|
||||
in particular when restoring from an untrusted source while running as
|
||||
root. For this reason these bits are not restored by default.
|
||||
|
||||
If you trust the source and want the setuid, setgid and sticky bits
|
||||
restored - for example when restoring a system backup made by rclone -
|
||||
set this flag.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: metadata_restore_special_bits
|
||||
- Env Var: RCLONE_LOCAL_METADATA_RESTORE_SPECIAL_BITS
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --local-fatal-if-no-space
|
||||
|
||||
Make out-of-space errors fatal during transfers.
|
||||
|
||||
When enabled, an ENOSPC error during a write returns a fatal error so
|
||||
that rclone aborts rather than retrying the operation. Useful for
|
||||
backup scripts that should halt loudly on a full disk rather than spin
|
||||
retrying.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: fatal_if_no_space
|
||||
- Env Var: RCLONE_LOCAL_FATAL_IF_NO_SPACE
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --local-time-type
|
||||
|
||||
Set what kind of time is returned.
|
||||
@@ -70192,6 +70423,13 @@ supported by all file systems) under the "user.*" prefix.
|
||||
|
||||
Metadata is supported on files and directories.
|
||||
|
||||
When restoring metadata with `--metadata` rclone applies the
|
||||
"mode", "uid" and "gid" from the source. These come from the source
|
||||
remote which may not be trusted, so restoring metadata as root from an
|
||||
untrusted source can change file ownership and is not recommended. The
|
||||
setuid, setgid and sticky bits are not restored by default - see the
|
||||
`--local-metadata-restore-special-bits` flag.
|
||||
|
||||
Here are the possible system metadata items for the local backend.
|
||||
|
||||
| Name | Help | Type | Example | Read Only |
|
||||
@@ -70245,6 +70483,185 @@ Options:
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.74.4 - 2026-07-08
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.3...v1.74.4)
|
||||
|
||||
- Bug Fixes
|
||||
- accounting
|
||||
- Fix goroutine leak in ResetCounters (Nick Craig-Wood)
|
||||
- Fix goroutine leak in NewStatsGroup for zero-transfer rc jobs (Sanjays2402)
|
||||
- archive extract: Fix path traversal letting archives escape the destination CVE-2026-59732 (Nick Craig-Wood)
|
||||
- build
|
||||
- Fix multiple CVEs by upgrading to go1.26.5 (Nick Craig-Wood)
|
||||
- CVE-2026-39822: os: Root escape via symlink plus trailing slash
|
||||
- CVE-2026-42505: crypto/tls: Encrypted Client Hello privacy leak
|
||||
- Update golang.org/x/image to v0.43.0 to fix image decoding vulnerabilities (Nick Craig-Wood)
|
||||
- CVE-2026-46604: panic decoding a TIFF image with an out-of-bounds strip offset
|
||||
- CVE-2026-46602: unbounded memory use from lack of a limit on TIFF tile sizes
|
||||
- CVE-2026-46601: panic on a WEBP VP8 alpha channel size mismatch
|
||||
- CVE-2026-33813: panic decoding a large WEBP image on 32-bit platforms
|
||||
- cmd/mount2
|
||||
- Fix NFS file creation by implementing Mknod (Sandy Luppino)
|
||||
- Fix ESTALE over NFS by reporting stable inode numbers (Sandy Luppino)
|
||||
- Fix NFS directory listings by supporting non-zero Seekdir offsets (Sandy Luppino)
|
||||
- completion: Fix powershell completion corrupting non-ASCII names (Yash Anil)
|
||||
- doc fixes (Bryan Stenson, Castronaut, Filippo, Gaurav, happysnaker, Jan-Philipp Reßler, Nick Craig-Wood, user77)
|
||||
- filter: Fix `--files-from` copy stopping at the first unreadable file (Nick Craig-Wood)
|
||||
- fs
|
||||
- Fix command line flag being ignored when set to its default value (Nick Craig-Wood)
|
||||
- Fix negative offset when a suffix Range request exceeds object size (Amit Mishra)
|
||||
- gui: Update embedded release to 1.1.10 (Nick Craig-Wood)
|
||||
- ncdu: Fix duplicated keystrokes on Windows by pinning tcell to v2.9.0 (Nick Craig-Wood)
|
||||
- serve restic: Fix `--private-repos` isolation bypass CVE-2026-59733 (Nick Craig-Wood)
|
||||
- serve s3
|
||||
- Fix spurious 404 on HEAD/GET during VFS writeback (max)
|
||||
- Fix path traversal letting clients see files in the root GHSA-8v25-v8p6-qf7v (Nick Craig-Wood)
|
||||
- serve webdav: Fix MOVE overwrite failing without Overwrite header (Sanjay Santhanam)
|
||||
- serve/http: Fix `--disable-zip` so it works over rc (Nick Craig-Wood)
|
||||
- VFS
|
||||
- Fix hang reopening a file during the handle-caching grace period (Nick Craig-Wood)
|
||||
- Local
|
||||
- Stop `--links` symlinks escaping the destination directory CVE-2026-54572 (Nick Craig-Wood)
|
||||
- Don't restore setuid/setgid/sticky bits from metadata by default GHSA-945v-v9p3-v5xw (Nick Craig-Wood)
|
||||
- Drive
|
||||
- Warn when non-exportable Google documents are skipped (Nick Craig-Wood)
|
||||
- Fix stray %!(EXTRA) in unexportable google document log message (Nick Craig-Wood)
|
||||
- Warn when using rclone's shared client_id (Nick Craig-Wood)
|
||||
- Filelu
|
||||
- Fix recursive listing path handling and file filtering (kingston125)
|
||||
- Googlephotos
|
||||
- Warn when using rclone's shared client_id (Nick Craig-Wood)
|
||||
- Mega
|
||||
- Wait for server events after upload, delete and move (Nick Craig-Wood)
|
||||
- Fix hard deleted files reappearing in listings (Nick Craig-Wood)
|
||||
- S3
|
||||
- Remove session token on cross-host redirects (IceLocke)
|
||||
- Strip STS security token on same-host HTTPS->HTTP redirect GHSA-cf44-9pgv-m4xc (Nick Craig-Wood)
|
||||
- Fix error mapping in GetObject to match HeadObject (lewoberst)
|
||||
- Correct documented `copy_cutoff` minimum to 1 byte (max)
|
||||
- Fix mounting a prefix failing with 403 when HEAD is not permitted (Nick Craig-Wood)
|
||||
- Smb
|
||||
- Fix for IBM iSeries and signature verification (dithwick)
|
||||
- WebDAV
|
||||
- Fix mixed property statuses in multi-status responses (nako-ruru)
|
||||
|
||||
## v1.74.3 - 2026-06-05
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.2...v1.74.3)
|
||||
|
||||
- Bug Fixes
|
||||
- rc
|
||||
- Fix unauthenticated command execution via `--rc-serve` inline remotes CVE-2026-49980 (Nick Craig-Wood)
|
||||
- Stop `global.*` connection string options changing config CVE-2026-49980 (Nick Craig-Wood)
|
||||
- build: Fix multiple CVEs by upgrading to go1.26.4 (Nick Craig-Wood)
|
||||
- CVE-2026-42504: mime: quadratic complexity in WordDecoder.DecodeHeader
|
||||
- CVE-2026-42507: net/textproto: arbitrary input are included in errors without any escaping
|
||||
- CVE-2026-27145: crypto/x509: split candidate hostname only once
|
||||
- log: Fix wrong source `file:line` in JSON logs from release builds (Nick Craig-Wood)
|
||||
- mount2: Fix empty directory listings on re-read (Janne Beate Bakeng)
|
||||
- serve s3: Fix multipart `ListParts` pagination returning wrong part numbers (Nick Craig-Wood)
|
||||
- serve sftp
|
||||
- Fix file corruption when a client resumes an upload (Nick Craig-Wood)
|
||||
- Fix truncate request being silently ignored (Nick Craig-Wood)
|
||||
- Local
|
||||
- Fix `getXattr` returning empty map instead of nil (Leon Brocard)
|
||||
- Drime
|
||||
- Fix server-side copy and move failing with Cloudflare 520 error (Nick Craig-Wood)
|
||||
- Fix files being uploaded to the wrong directory (Nick Craig-Wood)
|
||||
- Remove duplicate upload_cutoff config option (Nick Craig-Wood)
|
||||
- Fix directory rename leaving the renamed folder empty in VFS (Nick Craig-Wood)
|
||||
- Drive
|
||||
- Fix server-side move failing on shared drives with duplicate dirs (Nick Craig-Wood)
|
||||
- Iclouddrive
|
||||
- Fix ADP/PCS cookie acquisition for iCloud Drive (Yakov Till)
|
||||
- Fix "Index has invalid data" error listing iCloud Photos (Nick Craig-Wood)
|
||||
|
||||
## v1.74.2 - 2026-05-22
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.1...v1.74.2)
|
||||
|
||||
- Bug Fixes
|
||||
- build
|
||||
- Update golang.org/x/net to v0.55.0 to address:
|
||||
- CVE-2026-42506: html: incorrect handling of namespaced elements in foreign content
|
||||
- CVE-2026-39821: idna: failure to reject ASCII-only Punycode-encoded labels
|
||||
- CVE-2026-42502: html: incorrect handling of HTML elements in foreign content
|
||||
- CVE-2026-25680: html: denial of service when parsing arbitrary HTML
|
||||
- CVE-2026-25681: html: incorrect handling of character references in DOCTYPE nodes
|
||||
- CVE-2026-27136: html: duplicate attributes can cause XSS
|
||||
- Update golang.org/x/crypto to v0.52.0 to address:
|
||||
- CVE-2026-46598: ssh/agent: pathological inputs can lead to client panic
|
||||
- CVE-2026-46597: ssh: byte arithmetic causes underflow and panic
|
||||
- CVE-2026-39828: ssh: bypass of certificate restrictions
|
||||
- CVE-2026-39835: ssh: server panic during CheckHostKey/Authenticate
|
||||
- CVE-2026-39833: ssh/agent: key constraints not enforced
|
||||
- CVE-2026-39832: ssh/agent: agent constraints dropped when forwarding keys
|
||||
- CVE-2026-39827: ssh: memory leak when rejecting channels can lead to DoS
|
||||
- CVE-2026-39830: ssh: client can cause server deadlock on unexpected responses
|
||||
- CVE-2026-39829: ssh: pathological RSA/DSA parameters may cause DoS
|
||||
- CVE-2026-39831: ssh: bypass of FIDO/U2F security keys physical interaction
|
||||
- CVE-2026-39834: ssh: infinite loop on large channel writes
|
||||
- CVE-2026-42508: ssh/knownhosts: auth bypass via unenforced @revoked status
|
||||
- CVE-2026-46595: ssh: VerifiedPublicKeyCallback permissions skip enforcement
|
||||
- Update golang.org/x/image to v0.41.0 to address:
|
||||
- CVE-2026-42500: bmp: panic when reading out of bound palette index
|
||||
- CVE-2026-33809: tiff: excessive resource consumption in PackBits decompression
|
||||
- Update golang.org/x/sys to version v0.45.0 to address:
|
||||
- CVE-2026-39824: windows: integer overflow in NewNTUnicodeString
|
||||
- Update github.com/go-git/go-billy/v5 to 5.9.0 to fix CVE-2026-44740
|
||||
- bisync: Fix --conflict-loser pathname with --conflict-resolve newer (nielash)
|
||||
- gui: Update embedded release to 1.1.8 (Nick Craig-Wood)
|
||||
- lib/http: Replace deprecated h2c.NewHandler with http.Server.Protocols (Nick Craig-Wood)
|
||||
- rc: Remove duplicate metrics_addr option registration (Nick Craig-Wood)
|
||||
- vfs/vfscache: Fix silent write failure when mounting with remote:. (Lucky945H)
|
||||
- doc fixes (FTCHD, Iizuki, Leon Brocard, Nick Craig-Wood)
|
||||
- Drime
|
||||
- Fix file doesn't exists error when trying to delete (John Volk)
|
||||
- Fix 500 errors when listing shared folders (Alvinwylim)
|
||||
- Jottacloud
|
||||
- Support whitelabel service Phonero Sky (Tore Anderson)
|
||||
- Protondrive
|
||||
- Fix corrupted on transfer: sha1 hashes differ (William Tange)
|
||||
- S3
|
||||
- Add new MEGA S4 endpoints on megas4.com including Asia-Pacific region (Nick Craig-Wood)
|
||||
- WebDAV
|
||||
- Honour auth_redirect on listAll PROPFIND (Sai Asish Y)
|
||||
|
||||
## v1.74.1 - 2026-05-08
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.0...v1.74.1)
|
||||
|
||||
- Bug Fixes
|
||||
- bisync: Fix retryable without `--resync` error message when `--resync` has a critical failure (Gustavo V. F.)
|
||||
- build
|
||||
- Fix multiple CVEs by upgrading to go1.26.3 (Nick Craig-Wood)
|
||||
- CVE-2026-42501: cmd/go: malicious module proxy can bypass checksum database
|
||||
- CVE-2026-39825: net/http/httputil: ReverseProxy forwards queries with more than urlmaxqueryparams parameters
|
||||
- CVE-2026-39836: net: panic in Dial and LookupPort when handling NUL byte on Windows
|
||||
- CVE-2026-42499: net/mail: quadratic string concatenation in consumePhrase
|
||||
- CVE-2026-39820: net/mail: quadratic string concatentation in consumeComment
|
||||
- CVE-2026-39819: cmd/go: "go bug" follows symlinks in predictable temporary filenames
|
||||
- CVE-2026-39817: cmd/go: "go tool pack" does not sanitize output paths
|
||||
- CVE-2026-33814: net/http: infinite loop in HTTP/2 transport when given bad SETTINGS_MAX_FRAME_SIZE
|
||||
- CVE-2026-39826: html/template: escaper bypass leads to XSS
|
||||
- CVE-2026-33811: net: crash when handling long CNAME response
|
||||
- CVE-2026-39823: html/template: bypass of meta content URL escaping causes XSS
|
||||
- Update golang.org/x/net to v0.53.0 to fix CVE-2026-33814 (Nick Craig-Wood)
|
||||
- cmd/serve/s3: Return object listings in key order (Leon Brocard)
|
||||
- Cloudinary
|
||||
- Fix retrying every error and fix pacer sleep units (Nick Craig-Wood)
|
||||
- Drime
|
||||
- Fix large file uploads landing in drive root instead of configured folder (Nick Craig-Wood)
|
||||
- Fix uploads of 100..200M files (Nick Craig-Wood)
|
||||
- Protondrive
|
||||
- Route HTTP through rclone's transport (Nick Craig-Wood)
|
||||
- Route library logging through rclone's logger (Nick Craig-Wood)
|
||||
- Fix segfault when copying files missing revision metadata (Nick Craig-Wood)
|
||||
- S3
|
||||
- Fix STS call per request by caching AssumeRole credentials (Nick Craig-Wood)
|
||||
- Add new Fastly Object Storage regions (Leon Brocard)
|
||||
|
||||
## v1.74.0 - 2026-05-01
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.73.0...v1.74.0)
|
||||
@@ -70745,7 +71162,7 @@ Options:
|
||||
- [OVHcloud Object Storage](https://rclone.org/s3/#ovhcloud) (Florent Vennetier)
|
||||
- [Zata](https://rclone.org/s3/#Zata) ($@M@RTH_)
|
||||
- New Features
|
||||
- Allow [global config to be overriden or set on backend creation](https://rclone.org/docs/#globalconfig) (Nick Craig-Wood)
|
||||
- Allow [global config to be overridden or set on backend creation](https://rclone.org/docs/#globalconfig) (Nick Craig-Wood)
|
||||
- bisync: Promoted from beta to stable (nielash)
|
||||
- build
|
||||
- Update to go1.25 and make go1.24 the minimum required version (Nick Craig-Wood)
|
||||
@@ -70812,7 +71229,7 @@ Options:
|
||||
- This allows `--ca-cert`, `--client-cert` etc to be used.
|
||||
- This also allows `override.ca_cert = XXX` to be used in the config file.
|
||||
- Googlephotos
|
||||
- Added warning for Google Photos compatability-fixes (raider13209)
|
||||
- Added warning for Google Photos compatibility-fixes (raider13209)
|
||||
- Imagekit
|
||||
- Return correct error when attempting to upload zero length files (Nick Craig-Wood)
|
||||
- Don't low level retry uploads (Nick Craig-Wood)
|
||||
@@ -72169,7 +72586,7 @@ instead of of `--size-only`, when `check` is not available.
|
||||
- Add `--drive-env-auth` to get IAM credentials from runtime (Peter Brunner)
|
||||
- Update drive service account guide (Juang, Yi-Lin)
|
||||
- Fix change notify picking up files outside the root (Nick Craig-Wood)
|
||||
- Fix trailing slash mis-identificaton of folder as file (Nick Craig-Wood)
|
||||
- Fix trailing slash mis-identification of folder as file (Nick Craig-Wood)
|
||||
- Fix incorrect remote after Update on object (Nick Craig-Wood)
|
||||
- Dropbox
|
||||
- Implement `--dropbox-pacer-min-sleep` flag (Nick Craig-Wood)
|
||||
|
||||
Generated
+546
-37
@@ -1,6 +1,6 @@
|
||||
rclone(1) User Manual
|
||||
Nick Craig-Wood
|
||||
May 01, 2026
|
||||
Jul 08, 2026
|
||||
|
||||
NAME
|
||||
|
||||
@@ -4632,10 +4632,10 @@ Examples:
|
||||
// Output: stories/The Quick Brown Fox!.txt
|
||||
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20260501
|
||||
// Output: stories/The Quick Brown Fox!-20260708
|
||||
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2026-05-01 0208PM
|
||||
// Output: stories/The Quick Brown Fox!-2026-07-08 0504PM
|
||||
|
||||
rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,regex=[\\.\\w]/ab"
|
||||
// Output: ababababababab/ababab ababababab ababababab ababab!abababab
|
||||
@@ -4837,8 +4837,9 @@ Synopsis
|
||||
If source:path is a file or directory then it copies it to a file or
|
||||
directory named dest:path.
|
||||
|
||||
This can be used to upload single files to other than their current
|
||||
name. If the source is a directory then it acts exactly like the copy
|
||||
This can be used to copy a single file to a destination with a name
|
||||
different from its source - for example, uploading and renaming in one
|
||||
step. If the source is a directory then it acts exactly like the copy
|
||||
command.
|
||||
|
||||
So
|
||||
@@ -12661,6 +12662,17 @@ Versioning is not currently supported.
|
||||
Metadata will only be saved in memory other than the rclone mtime
|
||||
metadata which will be set as the modification time of the file.
|
||||
|
||||
Object names
|
||||
|
||||
serve s3 stores objects as files in the backend, so object keys are
|
||||
mapped to file paths rather than treated as the opaque strings AWS S3
|
||||
allows. Keys must be in canonical path form: keys that contain .. or .
|
||||
path segments, repeated slashes (//), or a leading or trailing slash are
|
||||
rejected with a 400 Bad Request (InvalidArgument) instead of being
|
||||
normalised, since normalising them could alias two distinct keys to the
|
||||
same file or resolve a key outside its bucket. This matches the
|
||||
behaviour of other S3 servers such as MinIO.
|
||||
|
||||
Supported operations
|
||||
|
||||
serve s3 currently supports the following operations.
|
||||
@@ -16361,7 +16373,7 @@ notfound, or the os null device represented by value NUL on Windows and
|
||||
in memory only.
|
||||
|
||||
You may see a log message "Config file not found - using defaults" if
|
||||
there is no configuration file. This can be supressed, e.g. if you are
|
||||
there is no configuration file. This can be suppressed, e.g. if you are
|
||||
using rclone entirely with on the fly remotes, by using memory-only
|
||||
configuration file or by creating an empty configuration file, as
|
||||
described above.
|
||||
@@ -16947,7 +16959,7 @@ ERROR is equivalent to -q. It only outputs error messages.
|
||||
|
||||
See also the logging section.
|
||||
|
||||
--windows-event-log LogLevel
|
||||
--windows-event-log-level LogLevel
|
||||
|
||||
If this is configured (the default is OFF) then logs of this level and
|
||||
above will be logged to the Windows event log in addition to the normal
|
||||
@@ -16965,14 +16977,14 @@ Error. If enabled we map rclone levels like this.
|
||||
Rclone will declare its log source as "rclone" if it is has enough
|
||||
permissions to create the registry key needed. If not then logs will
|
||||
appear as "Application". You can run
|
||||
rclone version --windows-event-log DEBUG once as administrator to create
|
||||
the registry key in advance.
|
||||
rclone version --windows-event-log-level DEBUG once as administrator to
|
||||
create the registry key in advance.
|
||||
|
||||
Note that the --windows-event-log level must be greater (more severe)
|
||||
than or equal to the --log-level. For example to log DEBUG to a log file
|
||||
but ERRORs to the event log you would use
|
||||
Note that the --windows-event-log-level level must be greater (more
|
||||
severe) than or equal to the --log-level. For example to log DEBUG to a
|
||||
log file but ERRORs to the event log you would use
|
||||
|
||||
--log-file rclone.log --log-level DEBUG --windows-event-log ERROR
|
||||
--log-file rclone.log --log-level DEBUG --windows-event-log-level ERROR
|
||||
|
||||
This option is only supported Windows platforms.
|
||||
|
||||
@@ -18694,7 +18706,7 @@ Then back to the headless machine, paste in the code.
|
||||
Configuring by copying the config file
|
||||
|
||||
Rclone stores all of its configuration in a single file. This can easily
|
||||
be copied to configure a remote rclone (although some backends does not
|
||||
be copied to configure a remote rclone (although some backends do not
|
||||
support reusing the same configuration, consult your backend
|
||||
documentation to be sure).
|
||||
|
||||
@@ -19775,6 +19787,44 @@ directly.
|
||||
|
||||
If you just want to run a remote control then see the rcd command.
|
||||
|
||||
Security
|
||||
|
||||
Access to the rc API is equivalent to shell access as the user running
|
||||
rclone. Treat the rc port as you would an interactive login on the host.
|
||||
|
||||
Any caller who can reach the API (and pass authentication, if it is
|
||||
enabled) can, among other things:
|
||||
|
||||
- Run OS commands as the rclone user. core/command re-executes the
|
||||
rclone binary with arbitrary arguments, and several backend options
|
||||
shell out to programs, so even creating a remote with config/create
|
||||
can lead to command execution.
|
||||
- Read and write any file reachable by the rclone process, by pointing
|
||||
operations/* or sync/* at a local remote (or via --rc-files /
|
||||
--rc-serve). Writing arbitrary files as the rclone user is itself a
|
||||
route to code execution.
|
||||
- Read back stored credentials. rclone configs routinely hold
|
||||
cloud-provider secrets. config/dump and friends expose them, so a
|
||||
compromise of the rc reaches every configured backend.
|
||||
- Change rclone's runtime behaviour with options/set, manage remotes
|
||||
with config/*, and stop the process with core/quit.
|
||||
|
||||
There is currently no per-endpoint capability or scope system:
|
||||
authentication is all-or-nothing. Granting any access grants all of the
|
||||
above.
|
||||
|
||||
Consequently:
|
||||
|
||||
- Do not bind the rc to a network address you do not control. The
|
||||
default bind is loopback (localhost:5572); keep it there unless you
|
||||
have a specific reason to change it.
|
||||
- Do not use --rc-no-auth on a non-loopback bind. It disables
|
||||
authentication on the endpoints that access remotes — see
|
||||
--rc-no-auth.
|
||||
- Use authentication and TLS (--rc-user/--rc-pass or --rc-htpasswd,
|
||||
plus --rc-cert/--rc-key) whenever the port is reachable by anyone
|
||||
you do not fully trust, and raise --rc-min-tls-version.
|
||||
|
||||
Supported parameters
|
||||
|
||||
--rc
|
||||
@@ -19838,8 +19888,26 @@ can browse to http://127.0.0.1:5572/ or http://127.0.0.1:5572/* to see a
|
||||
listing of the remotes. Objects may be requested from remotes using this
|
||||
syntax http://127.0.0.1:5572/[remote:path]/path/to/object
|
||||
|
||||
Unless the rc server has authentication configured (--rc-user/--rc-pass
|
||||
or --rc-htpasswd) or the --rc-no-auth flag is set, only remotes already
|
||||
present in the config file may be served this way. Inline remotes (e.g.
|
||||
[:webdav,url=...:]), connection string parameters and bare local paths
|
||||
are rejected, since instantiating them from an unauthenticated request
|
||||
could run commands or read arbitrary local files.
|
||||
|
||||
Default Off.
|
||||
|
||||
global.* connection string options and the rc
|
||||
|
||||
Remotes instantiated by the rc do not let connection string global.*
|
||||
options change rclone's process-wide configuration. Remotes created
|
||||
directly on the command line or defined in the config file are
|
||||
unaffected.
|
||||
|
||||
A global.* option still takes effect for the individual backend it is
|
||||
set on (exactly like an override.* option), it just does not leak into
|
||||
the global config for the rest of the process.
|
||||
|
||||
--rc-serve-no-modtime
|
||||
|
||||
Set this flag to skip reading the modification time (can speed things
|
||||
@@ -23140,7 +23208,7 @@ Flags for general networking and HTTP stuff.
|
||||
--tpslimit float Limit HTTP transactions per second to this
|
||||
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
|
||||
--use-cookies Enable session cookiejar
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.4")
|
||||
|
||||
Performance
|
||||
|
||||
@@ -23305,7 +23373,7 @@ Metrics
|
||||
|
||||
Flags to control the Metrics HTTP endpoint..
|
||||
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind metrics server to
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind server to
|
||||
--metrics-allow-origin string Origin which cross-domain request (CORS) can be executed from
|
||||
--metrics-baseurl string Prefix for URLs - leave blank for root
|
||||
--metrics-cert string TLS PEM key (concatenation of certificate and CA certificate)
|
||||
@@ -23508,7 +23576,7 @@ Backend-only flags (these can be set in the config file also).
|
||||
--drime-list-chunk int Number of items to list in each call (default 1000)
|
||||
--drime-root-folder-id string ID of the root folder
|
||||
--drime-upload-concurrency int Concurrency for multipart uploads and copies (default 4)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 5Mi)
|
||||
--drime-workspace-id string Account ID
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded
|
||||
--drive-allow-import-name-change Allow the filetype to change when uploading Google docs
|
||||
@@ -23794,8 +23862,10 @@ Backend-only flags (these can be set in the config file also).
|
||||
--local-case-sensitive Force the filesystem to report itself as case sensitive
|
||||
--local-description string Description of the remote
|
||||
--local-encoding Encoding The encoding for the backend (default Slash,Dot)
|
||||
--local-fatal-if-no-space Make out-of-space errors fatal during transfers
|
||||
--local-hashes CommaSepList Comma separated list of supported checksum types
|
||||
--local-links Translate symlinks to/from regular files with a '.rclonelink' extension for the local backend
|
||||
--local-metadata-restore-special-bits Restore the setuid, setgid and sticky bits from metadata
|
||||
--local-no-check-updated Don't check to see if the files change during upload
|
||||
--local-no-clone Disable reflink cloning for server-side copies
|
||||
--local-no-preallocate Disable preallocation of disk space for transferred files
|
||||
@@ -25839,9 +25909,28 @@ disallowed special characters and filename encodings.)
|
||||
|
||||
The following backends have known issues that need more investigation:
|
||||
|
||||
- TestDropbox (dropbox)
|
||||
- TestBisyncRemoteRemote/normalization
|
||||
- Updated: 2026-05-01-010013
|
||||
- TestGoogleCloudStorage,directory_markers (googlecloudstorage)
|
||||
- TestBisyncRemoteLocal/all_changed
|
||||
- TestBisyncRemoteLocal/backupdir
|
||||
- TestBisyncRemoteLocal/basic
|
||||
- TestBisyncRemoteLocal/changes
|
||||
- TestBisyncRemoteLocal/check_access
|
||||
- 79 more
|
||||
- TestGoogleCloudStorage (googlecloudstorage)
|
||||
- TestBisyncRemoteLocal/all_changed
|
||||
- TestBisyncRemoteLocal/backupdir
|
||||
- TestBisyncRemoteLocal/basic
|
||||
- TestBisyncRemoteLocal/changes
|
||||
- TestBisyncRemoteLocal/check_access
|
||||
- 79 more
|
||||
- TestHuaweiDrive (huaweidrive)
|
||||
- TestBisyncRemoteLocal/ext_paths
|
||||
- TestBisyncRemoteLocal/extended_filenames
|
||||
- TestBisyncRemoteLocal/normalization
|
||||
- TestBisyncLocalRemote/ext_paths
|
||||
- TestBisyncLocalRemote/extended_filenames
|
||||
- 4 more
|
||||
- Updated: 2026-07-08-010014
|
||||
|
||||
The following backends either have not been tested recently or have
|
||||
known issues that are deemed unfixable for the time being:
|
||||
@@ -25944,6 +26033,11 @@ directory is to rename it to the same name on both sides. (As of
|
||||
rclone v1.64, a --resync is no longer required after doing so, as bisync
|
||||
will automatically detect that Path1 and Path2 are in agreement.)
|
||||
|
||||
Note that although the flag --track-renames ensures that renamed/moved
|
||||
files won't be deleted and uploaded again, they are still counted as
|
||||
deleted files for purposes of the --max-delete flag (as this check
|
||||
happens before the rename detection operation). See this issue.
|
||||
|
||||
--fast-list used by default
|
||||
|
||||
Unlike most other rclone commands, bisync uses --fast-list by default,
|
||||
@@ -26682,6 +26776,11 @@ Unison and synchronization in general.
|
||||
|
||||
Changelog
|
||||
|
||||
v1.74.2
|
||||
|
||||
- Fixed an issue causing --conflict-loser pathname to produce
|
||||
unexpected behavior if using a non-default --conflict-resolve value.
|
||||
|
||||
v1.74
|
||||
|
||||
- Added several missing rc parameters.
|
||||
@@ -28535,6 +28634,9 @@ Properties:
|
||||
- "eu-south-1"
|
||||
- EU South 1
|
||||
- Provider: Fastly
|
||||
- "eu-west-1"
|
||||
- EU West 1
|
||||
- Provider: Fastly
|
||||
- "jp-central-1"
|
||||
- JP Central 1
|
||||
- Provider: Fastly
|
||||
@@ -28547,9 +28649,15 @@ Properties:
|
||||
- "us-east"
|
||||
- US East
|
||||
- Provider: Fastly
|
||||
- "us-east-1"
|
||||
- US East 1
|
||||
- Provider: Fastly
|
||||
- "us-west"
|
||||
- US West
|
||||
- Provider: Fastly
|
||||
- "us-west-1"
|
||||
- US West 1
|
||||
- Provider: Fastly
|
||||
- "global"
|
||||
- Global
|
||||
- Provider: FileLu
|
||||
@@ -29124,6 +29232,9 @@ Properties:
|
||||
- "eu-south-1.object.fastlystorage.app"
|
||||
- EU South 1
|
||||
- Provider: Fastly
|
||||
- "eu-west-1.object.fastlystorage.app"
|
||||
- EU West 1
|
||||
- Provider: Fastly
|
||||
- "jp-central-1.object.fastlystorage.app"
|
||||
- JP Central 1
|
||||
- Provider: Fastly
|
||||
@@ -29136,9 +29247,15 @@ Properties:
|
||||
- "us-east.object.fastlystorage.app"
|
||||
- US East
|
||||
- Provider: Fastly
|
||||
- "us-east-1.object.fastlystorage.app"
|
||||
- US East 1
|
||||
- Provider: Fastly
|
||||
- "us-west.object.fastlystorage.app"
|
||||
- US West
|
||||
- Provider: Fastly
|
||||
- "us-west-1.object.fastlystorage.app"
|
||||
- US West 1
|
||||
- Provider: Fastly
|
||||
- "s5lu.com"
|
||||
- Global FileLu S5 endpoint
|
||||
- Provider: FileLu
|
||||
@@ -29522,17 +29639,38 @@ Properties:
|
||||
- "br-ne1.magaluobjects.com"
|
||||
- Fortaleza, CE (BR), br-ne1
|
||||
- Provider: Magalu
|
||||
- "s3.eu-amsterdam.megas4.com"
|
||||
- Mega S4 Amsterdam
|
||||
- Provider: Mega
|
||||
- "s3.eu-luxembourg.megas4.com"
|
||||
- Mega S4 Luxembourg
|
||||
- Provider: Mega
|
||||
- "s3.eu-paris.megas4.com"
|
||||
- Mega S4 Paris
|
||||
- Provider: Mega
|
||||
- "s3.eu-barcelona.megas4.com"
|
||||
- Mega S4 Barcelona
|
||||
- Provider: Mega
|
||||
- "s3.ca-montreal.megas4.com"
|
||||
- Mega S4 Montreal
|
||||
- Provider: Mega
|
||||
- "s3.ca-vancouver.megas4.com"
|
||||
- Mega S4 Vancouver
|
||||
- Provider: Mega
|
||||
- "s3.ap-tokyo.megas4.com"
|
||||
- Mega S4 Tokyo
|
||||
- Provider: Mega
|
||||
- "s3.eu-central-1.s4.mega.io"
|
||||
- Mega S4 eu-central-1 (Amsterdam)
|
||||
- Mega S4 eu-central-1 (Amsterdam, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.eu-central-2.s4.mega.io"
|
||||
- Mega S4 eu-central-2 (Bettembourg)
|
||||
- Mega S4 eu-central-2 (Bettembourg, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.ca-central-1.s4.mega.io"
|
||||
- Mega S4 ca-central-1 (Montreal)
|
||||
- Mega S4 ca-central-1 (Montreal, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.ca-west-1.s4.mega.io"
|
||||
- Mega S4 ca-west-1 (Vancouver)
|
||||
- Mega S4 ca-west-1 (Vancouver, legacy)
|
||||
- Provider: Mega
|
||||
- "oos.eu-west-2.outscale.com"
|
||||
- Outscale EU West 2 (Paris)
|
||||
@@ -30769,7 +30907,7 @@ Cutoff for switching to multipart copy.
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 5 GiB.
|
||||
The minimum is 1 byte and the maximum is 5 GiB.
|
||||
|
||||
Properties:
|
||||
|
||||
@@ -37963,7 +38101,7 @@ start and finish the upload) and another 2 requests for each chunk:
|
||||
/b2api/v1/b2_upload_part/
|
||||
/b2api/v1/b2_finish_large_file
|
||||
|
||||
Versions
|
||||
Viewing versions with --b2-versions
|
||||
|
||||
Versions can be viewed with the --b2-versions flag. When it is set
|
||||
rclone will show and act on older versions of files. For example
|
||||
@@ -41578,9 +41716,9 @@ Examples
|
||||
1 MiB (1048576 bytes) file will encrypt to
|
||||
|
||||
- 32 bytes header
|
||||
- 16 chunks of 65568 bytes
|
||||
- 16 chunks of 65552 bytes
|
||||
|
||||
1049120 bytes total (a 0.05% overhead). This is the overhead for big
|
||||
1048864 bytes total (a 0.03% overhead). This is the overhead for big
|
||||
files.
|
||||
|
||||
Name encryption
|
||||
@@ -42405,7 +42543,7 @@ Properties:
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_DRIME_UPLOAD_CUTOFF
|
||||
- Type: SizeSuffix
|
||||
- Default: 200Mi
|
||||
- Default: 5Mi
|
||||
|
||||
--drime-chunk-size
|
||||
|
||||
@@ -45883,11 +46021,17 @@ root to use itself.
|
||||
However you can set this to restrict rclone to a specific folder
|
||||
hierarchy or to access data within the "Computers" tab on the drive web
|
||||
interface (where files from Google's Backup and Sync desktop program
|
||||
go).
|
||||
go). However, this is not the easiest method. Instead, you can create a
|
||||
shortcut to a folder in "Computers" and place it in "My Drive." To do
|
||||
so, right-click the folder, select "Organize", and then choose "Add
|
||||
shortcut" in the Google Drive web interface. Once you add the shortcut
|
||||
to "My Drive," rclone will display the folder, allowing you to interact
|
||||
with it. See also Shortcuts.
|
||||
|
||||
In order to do this you will have to find the Folder ID of the directory
|
||||
you wish rclone to display. This will be the last segment of the URL
|
||||
when you open the relevant folder in the drive web interface.
|
||||
If, however, you choose to change your root folder, you will have to
|
||||
find the Folder ID of the directory you wish rclone to display. This
|
||||
will be the last segment of the URL when you open the relevant folder in
|
||||
the drive web interface.
|
||||
|
||||
So if the folder you want rclone to use has a URL which looks like
|
||||
https://drive.google.com/drive/folders/1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh
|
||||
@@ -45895,7 +46039,12 @@ in the browser, then you use 1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh as the
|
||||
root_folder_id in the config.
|
||||
|
||||
NB folders under the "Computers" tab seem to be read only (drive gives a
|
||||
500 error) when using rclone.
|
||||
500 error) when using rclone, but they are also read-only in the Google
|
||||
Drive web interface, likely because this folder is managed by the Google
|
||||
Drive Desktop tool. Top-level folders in "Computers" are read-only, but
|
||||
you can interact with folders contained within them using rclone. These
|
||||
are the folders for which you can create a shortcut as well, as
|
||||
described above.
|
||||
|
||||
There doesn't appear to be an API to discover the folder IDs of the
|
||||
"Computers" tab - please contact us if you know otherwise!
|
||||
@@ -47732,6 +47881,22 @@ which can be annoying to refresh constantly. If, for whatever reason, a
|
||||
short grant time is not a problem, then keeping the application in
|
||||
testing mode would also be sufficient.
|
||||
|
||||
In most cases verification is not actually required. Google exempts a
|
||||
number of app categories from mandatory verification, including:
|
||||
|
||||
- Personal Use apps: if the app is for your personal use (fewer than
|
||||
100 users), you and your limited number of users can keep using it
|
||||
without verification - you will just need to click through the
|
||||
"unverified app" warning screen during sign-in. Verification is only
|
||||
required if you want to grow your user base beyond 100 users.
|
||||
- Development/Testing/Staging apps: apps in development, testing or
|
||||
staging mode are not subject to verification (but are limited to the
|
||||
100-user cap and the unverified-app warning until verified).
|
||||
|
||||
So for typical personal rclone use you can leave the app unverified,
|
||||
accept the warning screen, and publish it (rather than leaving it in
|
||||
"Testing") to avoid the weekly grant expiry described above.
|
||||
|
||||
(Thanks to @balazer on github for these instructions.)
|
||||
|
||||
Sometimes, creation of an OAuth consent in Google API Console fails due
|
||||
@@ -51596,6 +51761,8 @@ setup as described below:
|
||||
- MediaMarkt
|
||||
- MediaMarkt Cloud (mediamarkt.jottacloud.com)
|
||||
- Let's Go Cloud (letsgo.jotta.cloud)
|
||||
- Phonero
|
||||
- Phonero Sky (sky.phonero.no)
|
||||
|
||||
Paths are specified as remote:path
|
||||
|
||||
@@ -66879,6 +67046,32 @@ VFS.
|
||||
|
||||
Note that this flag is incompatible with -copy-links / -L.
|
||||
|
||||
Symlink targets and the destination
|
||||
|
||||
When rclone recreates a .rclonelink file as a symlink on local storage,
|
||||
the symlink can point anywhere - including, with an absolute path or one
|
||||
using ../ - to a location outside the directory you are copying into.
|
||||
This is normal - rclone reproduces whatever target the link had, so
|
||||
backups round-trip faithfully.
|
||||
|
||||
What rclone will not do is write through such a link. If a remote you
|
||||
are copying from contains both a symlink and a file or directory that
|
||||
would be placed inside it - for example a dir.rclonelink pointing
|
||||
somewhere outside the destination, alongside a dir/file.txt - rclone
|
||||
refuses to follow the symlink when writing dir/file.txt. The offending
|
||||
file is skipped with an error, the rest of the transfer continues, and
|
||||
the skipped file is counted in the error summary printed at the end of
|
||||
the run.
|
||||
|
||||
This protects you from a malicious or compromised remote using -l /
|
||||
--links to plant a symlink and then write through it to somewhere
|
||||
outside your destination. Ordinary symlink round-trips, and symlinks
|
||||
that stay inside the destination, are unaffected.
|
||||
|
||||
If you have intentionally pre-created a symlinked directory inside your
|
||||
destination and want rclone to write into the directory it points at, do
|
||||
not use -l / --links for that copy, or remove the symlink first.
|
||||
|
||||
Restricting filesystems with --one-file-system
|
||||
|
||||
Normally rclone will recurse through filesystems as mounted.
|
||||
@@ -67188,6 +67381,47 @@ Properties:
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
--local-metadata-restore-special-bits
|
||||
|
||||
Restore the setuid, setgid and sticky bits from metadata.
|
||||
|
||||
When restoring metadata with --metadata rclone applies the "mode" from
|
||||
the source. By default rclone applies only the permission bits and
|
||||
strips the setuid, setgid and sticky bits.
|
||||
|
||||
The "mode" comes from the source remote which may not be trusted.
|
||||
Restoring a setuid or setgid bit onto freshly written, source-controlled
|
||||
content can plant a setuid binary, which is dangerous in particular when
|
||||
restoring from an untrusted source while running as root. For this
|
||||
reason these bits are not restored by default.
|
||||
|
||||
If you trust the source and want the setuid, setgid and sticky bits
|
||||
restored - for example when restoring a system backup made by rclone -
|
||||
set this flag.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: metadata_restore_special_bits
|
||||
- Env Var: RCLONE_LOCAL_METADATA_RESTORE_SPECIAL_BITS
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
--local-fatal-if-no-space
|
||||
|
||||
Make out-of-space errors fatal during transfers.
|
||||
|
||||
When enabled, an ENOSPC error during a write returns a fatal error so
|
||||
that rclone aborts rather than retrying the operation. Useful for backup
|
||||
scripts that should halt loudly on a full disk rather than spin
|
||||
retrying.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: fatal_if_no_space
|
||||
- Env Var: RCLONE_LOCAL_FATAL_IF_NO_SPACE
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
--local-time-type
|
||||
|
||||
Set what kind of time is returned.
|
||||
@@ -67274,6 +67508,13 @@ supported by all file systems) under the "user.*" prefix.
|
||||
|
||||
Metadata is supported on files and directories.
|
||||
|
||||
When restoring metadata with --metadata rclone applies the "mode", "uid"
|
||||
and "gid" from the source. These come from the source remote which may
|
||||
not be trusted, so restoring metadata as root from an untrusted source
|
||||
can change file ownership and is not recommended. The setuid, setgid and
|
||||
sticky bits are not restored by default - see the
|
||||
--local-metadata-restore-special-bits flag.
|
||||
|
||||
Here are the possible system metadata items for the local backend.
|
||||
|
||||
---------------------------------------------------------------------------------------------------
|
||||
@@ -67336,6 +67577,274 @@ Options:
|
||||
|
||||
Changelog
|
||||
|
||||
v1.74.4 - 2026-07-08
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- accounting
|
||||
- Fix goroutine leak in ResetCounters (Nick Craig-Wood)
|
||||
- Fix goroutine leak in NewStatsGroup for zero-transfer rc
|
||||
jobs (Sanjays2402)
|
||||
- archive extract: Fix path traversal letting archives escape the
|
||||
destination CVE-2026-59732 (Nick Craig-Wood)
|
||||
- build
|
||||
- Fix multiple CVEs by upgrading to go1.26.5 (Nick Craig-Wood)
|
||||
- CVE-2026-39822: os: Root escape via symlink plus
|
||||
trailing slash
|
||||
- CVE-2026-42505: crypto/tls: Encrypted Client Hello
|
||||
privacy leak
|
||||
- Update golang.org/x/image to v0.43.0 to fix image decoding
|
||||
vulnerabilities (Nick Craig-Wood)
|
||||
- CVE-2026-46604: panic decoding a TIFF image with an
|
||||
out-of-bounds strip offset
|
||||
- CVE-2026-46602: unbounded memory use from lack of a
|
||||
limit on TIFF tile sizes
|
||||
- CVE-2026-46601: panic on a WEBP VP8 alpha channel size
|
||||
mismatch
|
||||
- CVE-2026-33813: panic decoding a large WEBP image on
|
||||
32-bit platforms
|
||||
- cmd/mount2
|
||||
- Fix NFS file creation by implementing Mknod (Sandy Luppino)
|
||||
- Fix ESTALE over NFS by reporting stable inode numbers (Sandy
|
||||
Luppino)
|
||||
- Fix NFS directory listings by supporting non-zero Seekdir
|
||||
offsets (Sandy Luppino)
|
||||
- completion: Fix powershell completion corrupting non-ASCII names
|
||||
(Yash Anil)
|
||||
- doc fixes (Bryan Stenson, Castronaut, Filippo, Gaurav,
|
||||
happysnaker, Jan-Philipp Reßler, Nick Craig-Wood, user77)
|
||||
- filter: Fix --files-from copy stopping at the first unreadable
|
||||
file (Nick Craig-Wood)
|
||||
- fs
|
||||
- Fix command line flag being ignored when set to its default
|
||||
value (Nick Craig-Wood)
|
||||
- Fix negative offset when a suffix Range request exceeds
|
||||
object size (Amit Mishra)
|
||||
- gui: Update embedded release to 1.1.10 (Nick Craig-Wood)
|
||||
- ncdu: Fix duplicated keystrokes on Windows by pinning tcell to
|
||||
v2.9.0 (Nick Craig-Wood)
|
||||
- serve restic: Fix --private-repos isolation bypass
|
||||
CVE-2026-59733 (Nick Craig-Wood)
|
||||
- serve s3
|
||||
- Fix spurious 404 on HEAD/GET during VFS writeback (max)
|
||||
- Fix path traversal letting clients see files in the root
|
||||
GHSA-8v25-v8p6-qf7v (Nick Craig-Wood)
|
||||
- serve webdav: Fix MOVE overwrite failing without Overwrite
|
||||
header (Sanjay Santhanam)
|
||||
- serve/http: Fix --disable-zip so it works over rc (Nick
|
||||
Craig-Wood)
|
||||
- VFS
|
||||
- Fix hang reopening a file during the handle-caching grace period
|
||||
(Nick Craig-Wood)
|
||||
- Local
|
||||
- Stop --links symlinks escaping the destination directory
|
||||
CVE-2026-54572 (Nick Craig-Wood)
|
||||
- Don't restore setuid/setgid/sticky bits from metadata by default
|
||||
GHSA-945v-v9p3-v5xw (Nick Craig-Wood)
|
||||
- Drive
|
||||
- Warn when non-exportable Google documents are skipped (Nick
|
||||
Craig-Wood)
|
||||
- Fix stray %!(EXTRA) in unexportable google document log message
|
||||
(Nick Craig-Wood)
|
||||
- Warn when using rclone's shared client_id (Nick Craig-Wood)
|
||||
- Filelu
|
||||
- Fix recursive listing path handling and file filtering
|
||||
(kingston125)
|
||||
- Googlephotos
|
||||
- Warn when using rclone's shared client_id (Nick Craig-Wood)
|
||||
- Mega
|
||||
- Wait for server events after upload, delete and move (Nick
|
||||
Craig-Wood)
|
||||
- Fix hard deleted files reappearing in listings (Nick Craig-Wood)
|
||||
- S3
|
||||
- Remove session token on cross-host redirects (IceLocke)
|
||||
- Strip STS security token on same-host HTTPS->HTTP redirect
|
||||
GHSA-cf44-9pgv-m4xc (Nick Craig-Wood)
|
||||
- Fix error mapping in GetObject to match HeadObject (lewoberst)
|
||||
- Correct documented copy_cutoff minimum to 1 byte (max)
|
||||
- Fix mounting a prefix failing with 403 when HEAD is not
|
||||
permitted (Nick Craig-Wood)
|
||||
- Smb
|
||||
- Fix for IBM iSeries and signature verification (dithwick)
|
||||
- WebDAV
|
||||
- Fix mixed property statuses in multi-status responses
|
||||
(nako-ruru)
|
||||
|
||||
v1.74.3 - 2026-06-05
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- rc
|
||||
- Fix unauthenticated command execution via --rc-serve inline
|
||||
remotes CVE-2026-49980 (Nick Craig-Wood)
|
||||
- Stop global.* connection string options changing config
|
||||
CVE-2026-49980 (Nick Craig-Wood)
|
||||
- build: Fix multiple CVEs by upgrading to go1.26.4 (Nick
|
||||
Craig-Wood)
|
||||
- CVE-2026-42504: mime: quadratic complexity in
|
||||
WordDecoder.DecodeHeader
|
||||
- CVE-2026-42507: net/textproto: arbitrary input are included
|
||||
in errors without any escaping
|
||||
- CVE-2026-27145: crypto/x509: split candidate hostname only
|
||||
once
|
||||
- log: Fix wrong source file:line in JSON logs from release builds
|
||||
(Nick Craig-Wood)
|
||||
- mount2: Fix empty directory listings on re-read (Janne Beate
|
||||
Bakeng)
|
||||
- serve s3: Fix multipart ListParts pagination returning wrong
|
||||
part numbers (Nick Craig-Wood)
|
||||
- serve sftp
|
||||
- Fix file corruption when a client resumes an upload (Nick
|
||||
Craig-Wood)
|
||||
- Fix truncate request being silently ignored (Nick
|
||||
Craig-Wood)
|
||||
- Local
|
||||
- Fix getXattr returning empty map instead of nil (Leon Brocard)
|
||||
- Drime
|
||||
- Fix server-side copy and move failing with Cloudflare 520 error
|
||||
(Nick Craig-Wood)
|
||||
- Fix files being uploaded to the wrong directory (Nick
|
||||
Craig-Wood)
|
||||
- Remove duplicate upload_cutoff config option (Nick Craig-Wood)
|
||||
- Fix directory rename leaving the renamed folder empty in VFS
|
||||
(Nick Craig-Wood)
|
||||
- Drive
|
||||
- Fix server-side move failing on shared drives with duplicate
|
||||
dirs (Nick Craig-Wood)
|
||||
- Iclouddrive
|
||||
- Fix ADP/PCS cookie acquisition for iCloud Drive (Yakov Till)
|
||||
- Fix "Index has invalid data" error listing iCloud Photos (Nick
|
||||
Craig-Wood)
|
||||
|
||||
v1.74.2 - 2026-05-22
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- build
|
||||
- Update golang.org/x/net to v0.55.0 to address:
|
||||
- CVE-2026-42506: html: incorrect handling of namespaced
|
||||
elements in foreign content
|
||||
- CVE-2026-39821: idna: failure to reject ASCII-only
|
||||
Punycode-encoded labels
|
||||
- CVE-2026-42502: html: incorrect handling of HTML
|
||||
elements in foreign content
|
||||
- CVE-2026-25680: html: denial of service when parsing
|
||||
arbitrary HTML
|
||||
- CVE-2026-25681: html: incorrect handling of character
|
||||
references in DOCTYPE nodes
|
||||
- CVE-2026-27136: html: duplicate attributes can cause XSS
|
||||
- Update golang.org/x/crypto to v0.52.0 to address:
|
||||
- CVE-2026-46598: ssh/agent: pathological inputs can lead
|
||||
to client panic
|
||||
- CVE-2026-46597: ssh: byte arithmetic causes underflow
|
||||
and panic
|
||||
- CVE-2026-39828: ssh: bypass of certificate restrictions
|
||||
- CVE-2026-39835: ssh: server panic during
|
||||
CheckHostKey/Authenticate
|
||||
- CVE-2026-39833: ssh/agent: key constraints not enforced
|
||||
- CVE-2026-39832: ssh/agent: agent constraints dropped
|
||||
when forwarding keys
|
||||
- CVE-2026-39827: ssh: memory leak when rejecting channels
|
||||
can lead to DoS
|
||||
- CVE-2026-39830: ssh: client can cause server deadlock on
|
||||
unexpected responses
|
||||
- CVE-2026-39829: ssh: pathological RSA/DSA parameters may
|
||||
cause DoS
|
||||
- CVE-2026-39831: ssh: bypass of FIDO/U2F security keys
|
||||
physical interaction
|
||||
- CVE-2026-39834: ssh: infinite loop on large channel
|
||||
writes
|
||||
- CVE-2026-42508: ssh/knownhosts: auth bypass via
|
||||
unenforced @revoked status
|
||||
- CVE-2026-46595: ssh: VerifiedPublicKeyCallback
|
||||
permissions skip enforcement
|
||||
- Update golang.org/x/image to v0.41.0 to address:
|
||||
- CVE-2026-42500: bmp: panic when reading out of bound
|
||||
palette index
|
||||
- CVE-2026-33809: tiff: excessive resource consumption in
|
||||
PackBits decompression
|
||||
- Update golang.org/x/sys to version v0.45.0 to address:
|
||||
- CVE-2026-39824: windows: integer overflow in
|
||||
NewNTUnicodeString
|
||||
- Update github.com/go-git/go-billy/v5 to 5.9.0 to fix
|
||||
CVE-2026-44740
|
||||
- bisync: Fix --conflict-loser pathname with
|
||||
--conflict-resolve newer (nielash)
|
||||
- gui: Update embedded release to 1.1.8 (Nick Craig-Wood)
|
||||
- lib/http: Replace deprecated h2c.NewHandler with
|
||||
http.Server.Protocols (Nick Craig-Wood)
|
||||
- rc: Remove duplicate metrics_addr option registration (Nick
|
||||
Craig-Wood)
|
||||
- vfs/vfscache: Fix silent write failure when mounting with
|
||||
remote:. (Lucky945H)
|
||||
- doc fixes (FTCHD, Iizuki, Leon Brocard, Nick Craig-Wood)
|
||||
- Drime
|
||||
- Fix file doesn't exists error when trying to delete (John Volk)
|
||||
- Fix 500 errors when listing shared folders (Alvinwylim)
|
||||
- Jottacloud
|
||||
- Support whitelabel service Phonero Sky (Tore Anderson)
|
||||
- Protondrive
|
||||
- Fix corrupted on transfer: sha1 hashes differ (William Tange)
|
||||
- S3
|
||||
- Add new MEGA S4 endpoints on megas4.com including Asia-Pacific
|
||||
region (Nick Craig-Wood)
|
||||
- WebDAV
|
||||
- Honour auth_redirect on listAll PROPFIND (Sai Asish Y)
|
||||
|
||||
v1.74.1 - 2026-05-08
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- bisync: Fix retryable without --resync error message when
|
||||
--resync has a critical failure (Gustavo V. F.)
|
||||
- build
|
||||
- Fix multiple CVEs by upgrading to go1.26.3 (Nick Craig-Wood)
|
||||
- CVE-2026-42501: cmd/go: malicious module proxy can
|
||||
bypass checksum database
|
||||
- CVE-2026-39825: net/http/httputil: ReverseProxy forwards
|
||||
queries with more than urlmaxqueryparams parameters
|
||||
- CVE-2026-39836: net: panic in Dial and LookupPort when
|
||||
handling NUL byte on Windows
|
||||
- CVE-2026-42499: net/mail: quadratic string concatenation
|
||||
in consumePhrase
|
||||
- CVE-2026-39820: net/mail: quadratic string
|
||||
concatentation in consumeComment
|
||||
- CVE-2026-39819: cmd/go: "go bug" follows symlinks in
|
||||
predictable temporary filenames
|
||||
- CVE-2026-39817: cmd/go: "go tool pack" does not sanitize
|
||||
output paths
|
||||
- CVE-2026-33814: net/http: infinite loop in HTTP/2
|
||||
transport when given bad SETTINGS_MAX_FRAME_SIZE
|
||||
- CVE-2026-39826: html/template: escaper bypass leads to
|
||||
XSS
|
||||
- CVE-2026-33811: net: crash when handling long CNAME
|
||||
response
|
||||
- CVE-2026-39823: html/template: bypass of meta content
|
||||
URL escaping causes XSS
|
||||
- Update golang.org/x/net to v0.53.0 to fix CVE-2026-33814
|
||||
(Nick Craig-Wood)
|
||||
- cmd/serve/s3: Return object listings in key order (Leon Brocard)
|
||||
- Cloudinary
|
||||
- Fix retrying every error and fix pacer sleep units (Nick
|
||||
Craig-Wood)
|
||||
- Drime
|
||||
- Fix large file uploads landing in drive root instead of
|
||||
configured folder (Nick Craig-Wood)
|
||||
- Fix uploads of 100..200M files (Nick Craig-Wood)
|
||||
- Protondrive
|
||||
- Route HTTP through rclone's transport (Nick Craig-Wood)
|
||||
- Route library logging through rclone's logger (Nick Craig-Wood)
|
||||
- Fix segfault when copying files missing revision metadata (Nick
|
||||
Craig-Wood)
|
||||
- S3
|
||||
- Fix STS call per request by caching AssumeRole credentials (Nick
|
||||
Craig-Wood)
|
||||
- Add new Fastly Object Storage regions (Leon Brocard)
|
||||
|
||||
v1.74.0 - 2026-05-01
|
||||
|
||||
See commits
|
||||
@@ -67972,7 +68481,7 @@ See commits
|
||||
- OVHcloud Object Storage (Florent Vennetier)
|
||||
- Zata ($@M@RTH_)
|
||||
- New Features
|
||||
- Allow global config to be overriden or set on backend creation
|
||||
- Allow global config to be overridden or set on backend creation
|
||||
(Nick Craig-Wood)
|
||||
- bisync: Promoted from beta to stable (nielash)
|
||||
- build
|
||||
@@ -68072,7 +68581,7 @@ See commits
|
||||
- This also allows override.ca_cert = XXX to be used in the
|
||||
config file.
|
||||
- Googlephotos
|
||||
- Added warning for Google Photos compatability-fixes
|
||||
- Added warning for Google Photos compatibility-fixes
|
||||
(raider13209)
|
||||
- Imagekit
|
||||
- Return correct error when attempting to upload zero length files
|
||||
@@ -69958,7 +70467,7 @@ See commits
|
||||
- Update drive service account guide (Juang, Yi-Lin)
|
||||
- Fix change notify picking up files outside the root (Nick
|
||||
Craig-Wood)
|
||||
- Fix trailing slash mis-identificaton of folder as file (Nick
|
||||
- Fix trailing slash mis-identification of folder as file (Nick
|
||||
Craig-Wood)
|
||||
- Fix incorrect remote after Update on object (Nick Craig-Wood)
|
||||
- Dropbox
|
||||
|
||||
+5
-2
@@ -43,7 +43,10 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
Early in the next release cycle update the dependencies.
|
||||
|
||||
- Review any pinned packages in go.mod and remove if possible
|
||||
- Active Pins
|
||||
- `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
|
||||
- See: [#9539](https://github.com/rclone/rclone/issues/9539)
|
||||
- Review Active Pins in go.mod and remove if possible
|
||||
- `make updatedirect`
|
||||
- `make GOTAGS=cmount`
|
||||
- `make compiletest`
|
||||
@@ -192,7 +195,7 @@ this will be done already.
|
||||
Now
|
||||
|
||||
- git co ${BASE_TAG}-stable
|
||||
- git cherry-pick any fixes
|
||||
- `git cherry-pick -x` any fixes
|
||||
- make startstable
|
||||
- Do the steps as above
|
||||
- git co master
|
||||
|
||||
@@ -34,6 +34,13 @@ import (
|
||||
"github.com/zeebo/blake3"
|
||||
)
|
||||
|
||||
// Pacer settings - tuned to match other rclone backends
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Cloudinary shouldn't have a trailing dot if there is no path
|
||||
func cldPathDir(somePath string) string {
|
||||
if somePath == "" || somePath == "." {
|
||||
@@ -189,7 +196,7 @@ func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (f
|
||||
root: root,
|
||||
opt: *opt,
|
||||
cld: cld,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
srv: rest.NewClient(client),
|
||||
}
|
||||
|
||||
@@ -586,21 +593,18 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
// Cloudinary returns rate-limit windows as "Try again on YYYY-MM-DD HH:MM:SS UTC"
|
||||
// embedded in the error message - honour it as a retry-after.
|
||||
if err != nil {
|
||||
tryAgain := "Try again on "
|
||||
if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
|
||||
layout := "2006-01-02 15:04:05 UTC"
|
||||
dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
|
||||
timestamp, err2 := time.Parse(layout, dateStr)
|
||||
if err2 == nil {
|
||||
if timestamp, err2 := time.Parse(layout, dateStr); err2 == nil {
|
||||
return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
|
||||
}
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "Retrying API error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ type MultiPartCreateRequest struct {
|
||||
Mime string `json:"mime"`
|
||||
Size int64 `json:"size"`
|
||||
Extension string `json:"extension"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
ParentID json.Number `json:"parentId"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
@@ -222,7 +222,7 @@ type MultiPartEntriesRequest struct {
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
ClientExtension string `json:"clientExtension"`
|
||||
ParentID json.Number `json:"parent_id"`
|
||||
ParentID json.Number `json:"parentId"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
WorkspaceID string `json:"workspaceId,omitempty"`
|
||||
}
|
||||
|
||||
+88
-17
@@ -54,7 +54,7 @@ const (
|
||||
rootURL = baseURL + "api/v1"
|
||||
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
|
||||
minChunkSize = fs.SizeSuffix(1024 * 1024 * 5)
|
||||
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
|
||||
defaultUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024) // as per https://docs.drime.cloud/uploads-guide
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -146,14 +146,6 @@ and these uploads do not fully utilize your bandwidth, then increasing
|
||||
this may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: `Cutoff for switching to chunked upload.
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -278,6 +270,11 @@ var retryErrorCodes = []int{
|
||||
503, // Service Unavailable
|
||||
504, // Gateway Timeout
|
||||
509, // Bandwidth Limit Exceeded
|
||||
520, // Cloudflare: Web server returns an unknown error
|
||||
521, // Cloudflare: Web server is down
|
||||
522, // Cloudflare: Connection timed out
|
||||
523, // Cloudflare: Origin is unreachable
|
||||
524, // Cloudflare: A timeout occurred
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this resp and err
|
||||
@@ -527,7 +524,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
if dirID != "" {
|
||||
opts.Parameters.Add("parentIds", dirID)
|
||||
opts.Parameters.Add("folderId", dirID)
|
||||
}
|
||||
if directoriesOnly {
|
||||
opts.Parameters.Add("type", api.ItemTypeFolder)
|
||||
@@ -709,6 +706,11 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
// If Drime returns 422 with invalid entry ID, the object is already gone.
|
||||
// Map this to a standard not-found error
|
||||
if strings.Contains(err.Error(), "entry ids is invalid") {
|
||||
return fs.ErrorObjectNotFound
|
||||
}
|
||||
return fmt.Errorf("failed to delete item: %w", err)
|
||||
}
|
||||
// Check the individual result codes also
|
||||
@@ -782,10 +784,16 @@ func (f *Fs) patch(ctx context.Context, id, attribute string, value string) (ite
|
||||
Name: value,
|
||||
}
|
||||
var result api.UpdateItemResponse
|
||||
// The drime origin returns a malformed response (seen by Cloudflare as
|
||||
// a 520) for a literal PUT to this endpoint, so use a POST with Laravel
|
||||
// method spoofing instead - the API routes it to the same handler.
|
||||
opts := rest.Opts{
|
||||
Method: "PUT",
|
||||
Method: "POST",
|
||||
Path: "/file-entries/" + id,
|
||||
Parameters: url.Values{},
|
||||
ExtraHeaders: map[string]string{
|
||||
"X-HTTP-Method-Override": "PUT",
|
||||
},
|
||||
}
|
||||
if f.opt.WorkspaceID != "" {
|
||||
opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
|
||||
@@ -919,6 +927,48 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
return dstObj, nil
|
||||
}
|
||||
|
||||
// countDirChildren returns the number of direct children of dirID.
|
||||
func (f *Fs) countDirChildren(ctx context.Context, dirID string) (int, error) {
|
||||
n := 0
|
||||
_, err := f.listAll(ctx, dirID, false, false, "", func(*api.Item) bool {
|
||||
n++
|
||||
return false
|
||||
})
|
||||
return n, err
|
||||
}
|
||||
|
||||
// waitForDirChildren polls dirID until it lists at least want direct
|
||||
// children, or the timeout elapses. Drime's folder rename has visible
|
||||
// eventual-consistency on the children listing - the rename returns
|
||||
// success immediately but the renamed folder lists as empty for a
|
||||
// short window - so callers that rename a folder use this to wait for
|
||||
// the listing to settle before returning (see issue #9450).
|
||||
func (f *Fs) waitForDirChildren(ctx context.Context, dirID string, want int) error {
|
||||
const timeout = 30 * time.Second
|
||||
deadline := time.Now().Add(timeout)
|
||||
backoff := 100 * time.Millisecond
|
||||
for {
|
||||
got, err := f.countDirChildren(ctx, dirID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if got >= want {
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("timed out waiting for directory listing to settle: got %d, want %d", got, want)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
if backoff < 2*time.Second {
|
||||
backoff *= 2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DirMove moves src, srcRemote to this remote at dstRemote
|
||||
// using server-side move operations.
|
||||
//
|
||||
@@ -939,11 +989,25 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
return err
|
||||
}
|
||||
|
||||
// Count children before the move so we can wait for the server's
|
||||
// children listing to reflect the rename - see waitForDirChildren.
|
||||
preCount, err := f.countDirChildren(ctx, srcID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DirMove: count source children: %w", err)
|
||||
}
|
||||
|
||||
// Do the move
|
||||
_, err = f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
info, err := f.moveTo(ctx, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if preCount > 0 {
|
||||
if err := f.waitForDirChildren(ctx, info.ID.String(), preCount); err != nil {
|
||||
fs.Logf(f, "DirMove: %v (continuing)", err)
|
||||
}
|
||||
}
|
||||
|
||||
srcFs.dirCache.FlushDir(srcRemote)
|
||||
return nil
|
||||
}
|
||||
@@ -1096,6 +1160,11 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
// Send just the leaf as relativePath, matching the single-part /uploads
|
||||
// convention. The file is placed by parentId; sending an absolute path
|
||||
// here makes the server build folders from it and drop "0" path segments.
|
||||
relPath := f.opt.Enc.FromStandardName(leaf)
|
||||
|
||||
// Temporary Object under construction
|
||||
o := &Object{
|
||||
fs: f,
|
||||
@@ -1129,7 +1198,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
Size: createSize,
|
||||
Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
|
||||
ParentID: json.Number(directoryID),
|
||||
RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
|
||||
RelativePath: relPath,
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
@@ -1156,8 +1225,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
if ext == "" {
|
||||
ext = "bin"
|
||||
}
|
||||
rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
|
||||
|
||||
chunkWriter := &drimeChunkWriter{
|
||||
uploadID: resp.UploadID,
|
||||
key: resp.Key,
|
||||
@@ -1170,7 +1237,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
mime: mime,
|
||||
extension: ext,
|
||||
parentID: json.Number(directoryID),
|
||||
relativePath: rel,
|
||||
relativePath: relPath,
|
||||
}
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(chunkSize),
|
||||
@@ -1530,7 +1597,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Debugf(o, "Removing old object on successful upload")
|
||||
deleteErr := o.fs.deleteObject(ctx, id)
|
||||
if deleteErr != nil {
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
if errors.Is(deleteErr, fs.ErrorObjectNotFound) {
|
||||
fs.Debugf(o, "Old object already deleted, safely ignoring")
|
||||
} else {
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
+16
-4
@@ -144,6 +144,7 @@ var (
|
||||
_importFormats map[string][]string // allowed import MIME type conversions
|
||||
templatesOnce sync.Once // parse link templates only once
|
||||
_linkTemplates map[string]*template.Template // available link types
|
||||
gdocsWarnOnce sync.Once // warn once about skipped non-exportable Google documents
|
||||
)
|
||||
|
||||
// rwChoices type for fs.Bits
|
||||
@@ -1302,6 +1303,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
|
||||
return nil, fmt.Errorf("failed to create client from environment: %w", err)
|
||||
}
|
||||
} else {
|
||||
oauthutil.SharedClientIDWarning(name, "Google Drive", "https://rclone.org/drive/#making-your-own-client-id", m)
|
||||
oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create oauth client: %w", err)
|
||||
@@ -1686,6 +1688,9 @@ func (f *Fs) newObjectWithExportInfo(
|
||||
// If item MimeType is in the ExportFormats then it is a google doc
|
||||
if !isDocument {
|
||||
fs.Debugf(remote, "Ignoring unknown document type %q", info.MimeType)
|
||||
gdocsWarnOnce.Do(func() {
|
||||
fs.Logf(remote, "Skipping unexportable google document %q. Use --drive-show-all-gdocs to include them in server side copy and move", info.MimeType)
|
||||
})
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
if extension == "" {
|
||||
@@ -3034,11 +3039,18 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
remote = remote[:len(remote)-len(ext)]
|
||||
}
|
||||
|
||||
_, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// Find the ID of the parent to remove the file from.
|
||||
var srcParentID string
|
||||
if len(srcObj.parents) == 1 {
|
||||
srcParentID = srcObj.parents[0]
|
||||
} else {
|
||||
var err error
|
||||
_, srcParentID, err = srcObj.fs.dirCache.FindPath(ctx, src.Remote(), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcParentID = actualID(srcParentID)
|
||||
}
|
||||
srcParentID = actualID(srcParentID)
|
||||
|
||||
// Temporary Object under construction
|
||||
dstInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
|
||||
|
||||
@@ -681,6 +681,83 @@ func (f *Fs) InternalTestSingleQuoteFolder(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestIntegration/FsMkdir/FsPutFiles/Internal/MoveDuplicateParent
|
||||
//
|
||||
// Check that moving a file removes it from its real parent even when there
|
||||
// are two directories with the same name and the directory cache resolves the
|
||||
// path to the wrong duplicate. Otherwise the file ends up with two parents
|
||||
// which fails on shared drives with a teamDrivesParentLimit error (#9472).
|
||||
func (f *Fs) InternalTestMoveDuplicateParent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if f.Features().Move == nil {
|
||||
t.Skip("Move not supported")
|
||||
}
|
||||
|
||||
rootID, err := f.dirCache.RootID(ctx, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
const dupName = "dup9472"
|
||||
const dstName = "dst9472"
|
||||
const fileName = "michael.txt"
|
||||
|
||||
// Create the first "dup9472" directory - this is the one the directory
|
||||
// cache will resolve the path to.
|
||||
dirA, err := f.createDir(ctx, rootID, dupName, nil)
|
||||
require.NoError(t, err)
|
||||
f.dirCache.Put(dupName, dirA.Id)
|
||||
|
||||
// Create a second directory with the SAME name and parent: a duplicate
|
||||
// the directory cache does not know about.
|
||||
dirB, err := f.createDir(ctx, rootID, dupName, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Destination directory
|
||||
dirDst, err := f.createDir(ctx, rootID, dstName, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
_ = f.delete(ctx, dirA.Id, false)
|
||||
_ = f.delete(ctx, dirB.Id, false)
|
||||
_ = f.delete(ctx, dirDst.Id, false)
|
||||
f.dirCache.Flush()
|
||||
}()
|
||||
|
||||
// Upload a file directly into dirB (the duplicate the cache doesn't know).
|
||||
createInfo := &drive.File{
|
||||
Name: fileName,
|
||||
Parents: []string{dirB.Id},
|
||||
}
|
||||
var fileInfo *drive.File
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
fileInfo, err = f.svc.Files.Create(createInfo).
|
||||
Media(strings.NewReader("9472 duplicate parent test")).
|
||||
Fields(f.getFileFields(ctx)).
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{dirB.Id}, fileInfo.Parents)
|
||||
|
||||
// Sanity check: the directory cache resolves the path to dirA, not the
|
||||
// dirB the file actually lives in.
|
||||
gotDirID, err := f.dirCache.FindDir(ctx, dupName, false)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, dirA.Id, gotDirID, "directory cache should resolve to the wrong duplicate")
|
||||
|
||||
// Build the source object pointing at the file in dirB and move it.
|
||||
srcObj, err := f.newObjectWithInfo(ctx, dupName+"/"+fileName, fileInfo)
|
||||
require.NoError(t, err)
|
||||
|
||||
dstObj, err := f.Move(ctx, srcObj, dstName+"/"+fileName)
|
||||
require.NoError(t, err)
|
||||
|
||||
// The moved file must have exactly one parent: the destination.
|
||||
movedInfo, err := f.getFile(ctx, dstObj.(fs.IDer).ID(), "id,parents")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{dirDst.Id}, movedInfo.Parents)
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
// These tests all depend on each other so run them as nested tests
|
||||
t.Run("DocumentImport", func(t *testing.T) {
|
||||
@@ -701,6 +778,7 @@ func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Query", f.InternalTestQuery)
|
||||
t.Run("AgeQuery", f.InternalTestAgeQuery)
|
||||
t.Run("SingleQuoteFolder", f.InternalTestSingleQuoteFolder)
|
||||
t.Run("MoveDuplicateParent", f.InternalTestMoveDuplicateParent)
|
||||
t.Run("ShouldRetry", f.InternalTestShouldRetry)
|
||||
}
|
||||
|
||||
|
||||
+15
-17
@@ -222,38 +222,36 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fldMap := map[string]bool{}
|
||||
for _, folder := range result.Result.Folders {
|
||||
fldMap[folder.FldID.String()] = true
|
||||
if f.root == "" && dir == "" && strings.Contains(folder.Path, "/") {
|
||||
continue
|
||||
}
|
||||
|
||||
paths := strings.Split(folder.Path, fullPath+"/")
|
||||
remote := paths[0]
|
||||
if len(paths) > 1 {
|
||||
remote = paths[1]
|
||||
folderPath := strings.Trim(folder.Path, "/")
|
||||
root := strings.Trim(f.root, "/")
|
||||
|
||||
remotePathWithoutRoot := folderPath
|
||||
|
||||
if root != "" {
|
||||
if remotePathWithoutRoot == root {
|
||||
continue
|
||||
}
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, root+"/")
|
||||
}
|
||||
|
||||
if strings.Contains(remote, "/") {
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
|
||||
|
||||
if remotePathWithoutRoot == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
pathsWithoutRoot := strings.Split(folder.Path, "/"+f.root+"/")
|
||||
remotePathWithoutRoot := pathsWithoutRoot[0]
|
||||
if len(pathsWithoutRoot) > 1 {
|
||||
remotePathWithoutRoot = pathsWithoutRoot[1]
|
||||
}
|
||||
remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
|
||||
entries = append(entries, fs.NewDir(remotePathWithoutRoot, time.Now()))
|
||||
}
|
||||
|
||||
for _, file := range result.Result.Files {
|
||||
if _, ok := fldMap[file.FldID.String()]; ok {
|
||||
continue
|
||||
}
|
||||
remote := path.Join(dir, file.Name)
|
||||
// trim leading slashes
|
||||
remote = strings.TrimPrefix(remote, "/")
|
||||
|
||||
obj := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
|
||||
@@ -349,6 +349,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
oauthutil.SharedClientIDWarning(name, "Google Photos", "https://rclone.org/googlephotos/#making-your-own-client-id", m)
|
||||
|
||||
baseClient := fshttp.NewClient(ctx)
|
||||
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
|
||||
if err != nil {
|
||||
|
||||
@@ -25,6 +25,13 @@ const (
|
||||
authEndpoint = "https://idmsa.apple.com/appleauth/auth"
|
||||
)
|
||||
|
||||
// Webservice keys in AccountInfo.Webservices map
|
||||
const (
|
||||
WsDrive = "drivews"
|
||||
WsDocs = "docws"
|
||||
WsPhotos = "ckdatabasews"
|
||||
)
|
||||
|
||||
type sessionSave func(*Session)
|
||||
|
||||
// Client defines the client configuration
|
||||
@@ -32,6 +39,7 @@ type Client struct {
|
||||
appleID string
|
||||
password string
|
||||
remoteName string // rclone remote name, used for cache namespacing
|
||||
pcsWSKey string // webservice key for PCS cookie scoping (e.g. WsDrive, WsPhotos)
|
||||
srv *rest.Client
|
||||
Session *Session
|
||||
sessionSaveCallback sessionSave
|
||||
@@ -41,11 +49,14 @@ type Client struct {
|
||||
}
|
||||
|
||||
// New creates a new iCloud API client and initializes its HTTP session
|
||||
func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string) (*Client, error) {
|
||||
// pcsWSKey scopes PCS cookie acquisition to the caller's webservice (WsDrive, WsPhotos);
|
||||
// empty string skips PCS entirely
|
||||
func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string, pcsWSKey string) (*Client, error) {
|
||||
icloud := &Client{
|
||||
appleID: strings.ToLower(appleID), // Apple SRP requires lowercase in client-side proof
|
||||
password: password,
|
||||
remoteName: filepath.Base(remoteName),
|
||||
pcsWSKey: pcsWSKey,
|
||||
srv: rest.NewClient(fshttp.NewClient(context.Background())),
|
||||
Session: NewSession(),
|
||||
sessionSaveCallback: sessionSaveCallback,
|
||||
@@ -75,8 +86,8 @@ func (c *Client) DriveService() (*DriveService, error) {
|
||||
func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
|
||||
resp, err = c.Session.Request(ctx, opts, request, response)
|
||||
if err != nil && resp != nil {
|
||||
// try to reauth
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 421 {
|
||||
// 401/421 = session expired, 423 = missing PCS cookies (ADP)
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 421 || resp.StatusCode == 423 {
|
||||
err = c.Authenticate(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -95,6 +106,24 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, respo
|
||||
func (c *Client) Authenticate(ctx context.Context) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if err := c.authenticateSession(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Ensure PCS cookies after any successful auth path (ADP accounts)
|
||||
acquired, err := c.Session.ensurePCSCookies(ctx, c.pcsWSKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if acquired && c.sessionSaveCallback != nil {
|
||||
c.sessionSaveCallback(c.Session)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// authenticateSession establishes a valid session via the cheapest available path
|
||||
func (c *Client) authenticateSession(ctx context.Context) error {
|
||||
// Skip /validate round-trip when saved session has cookies + service endpoints
|
||||
// Native client behavior: use cached session, reauth lazily on 401/421
|
||||
if c.Session.Cookies != nil && len(c.Session.AccountInfo.Webservices) > 0 {
|
||||
|
||||
@@ -33,7 +33,7 @@ type DriveService struct {
|
||||
|
||||
// NewDriveService creates a new DriveService instance.
|
||||
func NewDriveService(icloud *Client) (*DriveService, error) {
|
||||
return &DriveService{icloud: icloud, RootID: "FOLDER::com.apple.CloudDocs::root", endpoint: icloud.Session.AccountInfo.Webservices["drivews"].URL, docsEndpoint: icloud.Session.AccountInfo.Webservices["docws"].URL}, nil
|
||||
return &DriveService{icloud: icloud, RootID: "FOLDER::com.apple.CloudDocs::root", endpoint: icloud.Session.AccountInfo.Webservices[WsDrive].URL, docsEndpoint: icloud.Session.AccountInfo.Webservices[WsDocs].URL}, nil
|
||||
}
|
||||
|
||||
// GetItemByDriveID retrieves a DriveItem by its Drive ID.
|
||||
|
||||
@@ -510,7 +510,7 @@ func (album *Album) SetTestPhotoCache(cache map[string]*Photo) {
|
||||
|
||||
// NewPhotosService creates a new PhotosService instance
|
||||
func NewPhotosService(ctx context.Context, client *Client, pacer *fs.Pacer, shouldRetry ShouldRetryFunc) (*PhotosService, error) {
|
||||
service, exists := client.Session.AccountInfo.Webservices["ckdatabasews"]
|
||||
service, exists := client.Session.AccountInfo.Webservices[WsPhotos]
|
||||
if !exists || service.Status != "active" {
|
||||
return nil, fmt.Errorf("ckdatabasews service not available")
|
||||
}
|
||||
@@ -605,6 +605,13 @@ func (ps *PhotosService) discoverLibraries(ctx context.Context) (*libraryDiscove
|
||||
continue
|
||||
}
|
||||
name := zone.ZoneID.ZoneName
|
||||
// Only PrimarySync and SharedSync-* are photo libraries. Other
|
||||
// zones (e.g. CMM-* shared-album zones) appear in changes/database
|
||||
// but have no CPLAlbumByPositionLive index, so querying their
|
||||
// albums returns BAD_REQUEST / "Index has invalid data".
|
||||
if name != "PrimarySync" && !strings.HasPrefix(name, "SharedSync") {
|
||||
continue
|
||||
}
|
||||
// SharedSync-* found in private takes precedence over shared
|
||||
if _, exists := result.libraries[name]; exists {
|
||||
continue
|
||||
@@ -2313,7 +2320,7 @@ func (ps *PhotosService) requestWithReauth(ctx context.Context, makeOpts func()
|
||||
reauthDone := false
|
||||
return ps.pacer.Call(func() (bool, error) {
|
||||
resp, err := ps.client.Session.Request(ctx, makeOpts(), data, response)
|
||||
if !reauthDone && err != nil && resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 421) {
|
||||
if !reauthDone && err != nil && resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 421 || resp.StatusCode == 423) {
|
||||
reauthDone = true
|
||||
if authErr := ps.client.Authenticate(ctx); authErr != nil {
|
||||
return false, authErr
|
||||
|
||||
@@ -495,7 +495,7 @@ func (s *Session) getSRPAuthHeaders() map[string]string {
|
||||
return headers
|
||||
}
|
||||
|
||||
// AuthWithToken authenticates the session
|
||||
// AuthWithToken authenticates the session with the account login endpoint
|
||||
func (s *Session) AuthWithToken(ctx context.Context) error {
|
||||
values := map[string]any{
|
||||
"accountCountryCode": s.AccountCountry,
|
||||
@@ -522,86 +522,110 @@ func (s *Session) AuthWithToken(ctx context.Context) error {
|
||||
fs.Debugf(nil, "iclouddrive: accountLogin response cookies: %v", cookieDebugSummaries(resp.Cookies()))
|
||||
fs.Debugf(nil, "iclouddrive: session cookie jar after accountLogin: %v", cookieJarDebugSummaries(s.Cookies))
|
||||
|
||||
// Acquire PCS cookies if Advanced Data Protection is enabled
|
||||
if ws := s.AccountInfo.Webservices["ckdatabasews"]; ws != nil && ws.PcsRequired {
|
||||
fs.Debugf(nil, "iclouddrive: ADP detected (pcsRequired=true)")
|
||||
if s.hasPCSCookies() {
|
||||
fs.Debugf(nil, "iclouddrive: PCS cookies already present, skipping acquisition")
|
||||
} else {
|
||||
if err := s.acquirePCSCookies(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(nil, "iclouddrive: no ADP (pcsRequired=false)")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasPCSCookies checks if the required PCS cookies for Photos are already present
|
||||
func (s *Session) hasPCSCookies() bool {
|
||||
var hasPhotos, hasSharing bool
|
||||
for _, c := range s.Cookies {
|
||||
switch c.Name {
|
||||
case "X-APPLE-WEBAUTH-PCS-Photos":
|
||||
hasPhotos = true
|
||||
case "X-APPLE-WEBAUTH-PCS-Sharing":
|
||||
hasSharing = true
|
||||
}
|
||||
}
|
||||
return hasPhotos && hasSharing
|
||||
type pcsService struct {
|
||||
wsKey string
|
||||
appName string
|
||||
cookies []string
|
||||
}
|
||||
|
||||
// acquirePCSCookies requests PCS cookies for ADP-enabled accounts
|
||||
// pcsServices lists all services that need PCS cookies
|
||||
// appName values from icloud.com JS (Build 2616/19)
|
||||
var pcsServices = []pcsService{
|
||||
{WsPhotos, "photos", []string{"X-APPLE-WEBAUTH-PCS-Photos", "X-APPLE-WEBAUTH-PCS-Sharing"}},
|
||||
{WsDrive, "iclouddrive", []string{"X-APPLE-WEBAUTH-PCS-Documents"}},
|
||||
}
|
||||
|
||||
func (s *Session) hasPCSCookiesFor(names []string) bool {
|
||||
for _, name := range names {
|
||||
if !slices.ContainsFunc(s.Cookies, func(c *http.Cookie) bool { return c.Name == name }) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ensurePCSCookies checks whether the session needs PCS cookies for the given
|
||||
// webservice and acquires them if missing, returning true if new cookies were acquired
|
||||
func (s *Session) ensurePCSCookies(ctx context.Context, pcsWSKey string) (bool, error) {
|
||||
if pcsWSKey == "" {
|
||||
return false, nil
|
||||
}
|
||||
for _, pcs := range pcsServices {
|
||||
if pcs.wsKey != pcsWSKey {
|
||||
continue
|
||||
}
|
||||
ws := s.AccountInfo.Webservices[pcs.wsKey]
|
||||
if ws == nil || !ws.PcsRequired {
|
||||
return false, nil
|
||||
}
|
||||
if s.hasPCSCookiesFor(pcs.cookies) {
|
||||
return false, nil
|
||||
}
|
||||
fs.Debugf(nil, "iclouddrive: ADP detected, acquiring PCS cookies for %s", pcs.appName)
|
||||
if err := s.acquirePCSCookiesFor(ctx, pcs.appName, pcs.cookies); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// acquirePCSCookiesFor requests PCS cookies for a specific service on ADP-enabled accounts
|
||||
// May require user approval on a trusted device (polls every 10s, max 5 min)
|
||||
func (s *Session) acquirePCSCookies(ctx context.Context) error {
|
||||
fs.Logf(nil, "iclouddrive: Advanced Data Protection enabled, requesting PCS cookies")
|
||||
const maxAttempts = 30 // 30 * 10s = 5 minutes max
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
fs.Debugf(nil, "iclouddrive: requestPCS outgoing cookies: %v", cookieJarDebugSummaries(s.Cookies))
|
||||
func (s *Session) acquirePCSCookiesFor(ctx context.Context, appName string, cookies []string) error {
|
||||
fs.Logf(nil, "iclouddrive: Advanced Data Protection enabled, requesting PCS cookies for %s", appName)
|
||||
const maxAttempts = 30
|
||||
for range maxAttempts {
|
||||
fs.Debugf(nil, "iclouddrive: requestPCS(%s) outgoing cookies: %v", appName, cookieJarDebugSummaries(s.Cookies))
|
||||
values := map[string]any{
|
||||
"appName": "photos",
|
||||
"appName": appName,
|
||||
"derivedFromUserAction": true,
|
||||
}
|
||||
body, err := IntoReader(values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("requestPCS: %w", err)
|
||||
return fmt.Errorf("requestPCS(%s): %w", appName, err)
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/requestPCS",
|
||||
ExtraHeaders: s.GetHeaders(map[string]string{}),
|
||||
RootURL: setupEndpoint,
|
||||
Body: body,
|
||||
}
|
||||
opts.Body = body
|
||||
var pcsResp struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
resp, err := s.Request(ctx, opts, nil, &pcsResp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("requestPCS: %w", err)
|
||||
return fmt.Errorf("requestPCS(%s): %w", appName, err)
|
||||
}
|
||||
fs.Debugf(nil, "iclouddrive: requestPCS response cookies: %v", cookieDebugSummaries(resp.Cookies()))
|
||||
fs.Debugf(nil, "iclouddrive: requestPCS response: status=%q message=%q cookies=%d",
|
||||
pcsResp.Status, pcsResp.Message, len(resp.Cookies()))
|
||||
fs.Debugf(nil, "iclouddrive: requestPCS(%s) response: status=%q message=%q cookies=%d %v",
|
||||
appName, pcsResp.Status, pcsResp.Message, len(resp.Cookies()), cookieDebugSummaries(resp.Cookies()))
|
||||
if pcsResp.Status == "success" {
|
||||
if !s.hasPCSCookies() {
|
||||
return fmt.Errorf("requestPCS: server returned success but PCS cookies missing")
|
||||
if !s.hasPCSCookiesFor(cookies) {
|
||||
var missing []string
|
||||
for _, name := range cookies {
|
||||
if !slices.ContainsFunc(s.Cookies, func(c *http.Cookie) bool { return c.Name == name }) {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("requestPCS(%s): server returned success but cookies still missing: %v", appName, missing)
|
||||
}
|
||||
fs.Logf(nil, "iclouddrive: PCS cookies acquired")
|
||||
fs.Logf(nil, "iclouddrive: PCS cookies acquired for %s", appName)
|
||||
return nil
|
||||
}
|
||||
// Device consent required - poll until approved
|
||||
fs.Logf(nil, "iclouddrive: waiting for device approval for PCS (%s)", pcsResp.Message)
|
||||
fs.Logf(nil, "iclouddrive: waiting for device approval for PCS/%s (%s)", appName, pcsResp.Message)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("requestPCS: timed out waiting for device approval after 5 minutes")
|
||||
return fmt.Errorf("requestPCS(%s): timed out waiting for device approval after 5 minutes", appName)
|
||||
}
|
||||
|
||||
// RequestPushNotification explicitly requests a push notification to trusted devices
|
||||
|
||||
@@ -96,7 +96,7 @@ func resumeConfigClient(m configmap.Mapper, appleid, password, trustToken, clien
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config")
|
||||
icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config", "")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -280,7 +280,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
|
||||
// Force fresh SRP authentication - ignore stale trust token and cookies
|
||||
// so that reconnect always prompts for 2FA
|
||||
m.Set(configAuthSession, "")
|
||||
icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config")
|
||||
icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -410,7 +410,8 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
|
||||
|
||||
// newICloudClient parses options, authenticates, and returns a ready client
|
||||
// Shared between NewFs (Drive) and NewFsPhotos (Photos) to avoid duplication
|
||||
func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*api.Client, *Options, error) {
|
||||
// pcsWSKey scopes PCS cookie acquisition to the caller's service
|
||||
func newICloudClient(ctx context.Context, name string, m configmap.Mapper, pcsWSKey string) (*api.Client, *Options, error) {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
@@ -442,6 +443,7 @@ func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*api
|
||||
cookies,
|
||||
callback,
|
||||
name,
|
||||
pcsWSKey,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
@@ -677,7 +677,7 @@ func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bo
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
icloud, opt, err := newICloudClient(ctx, name, m)
|
||||
icloud, opt, err := newICloudClient(ctx, name, m, api.WsDrive)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ type PhotosObject struct {
|
||||
|
||||
// NewFsPhotos constructs an Fs for Photos from the path, container:path
|
||||
func NewFsPhotos(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
icloud, opt, err := newICloudClient(ctx, name, m)
|
||||
icloud, opt, err := newICloudClient(ctx, name, m, api.WsPhotos)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -91,6 +91,7 @@ func getServices() []service {
|
||||
{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
|
||||
{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"onlime", "Onlime (Denmark)", "cloud-auth.onlime.dk", "onlime_wl", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"phonero", "Phonero Sky", "sky.phonero.no", "phonero", "desktop-win", []string{"openid", "offline_access"}},
|
||||
{"tele2", "Tele2 Cloud (Sweden)", "mittcloud-auth.tele2.se", "comhem", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"telia_no", "Telia Sky (Norway)", "sky-auth.telia.no", "get", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"telia_se", "Telia Cloud (Sweden)", "cloud-auth.telia.se", "telia_se", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
|
||||
+140
-32
@@ -78,6 +78,13 @@ User metadata is stored as extended attributes (which may not be
|
||||
supported by all file systems) under the "user.*" prefix.
|
||||
|
||||
Metadata is supported on files and directories.
|
||||
|
||||
When restoring metadata with ` + "`--metadata`" + ` rclone applies the
|
||||
"mode", "uid" and "gid" from the source. These come from the source
|
||||
remote which may not be trusted, so restoring metadata as root from an
|
||||
untrusted source can change file ownership and is not recommended. The
|
||||
setuid, setgid and sticky bits are not restored by default - see the
|
||||
` + "`--local-metadata-restore-special-bits`" + ` flag.
|
||||
`,
|
||||
},
|
||||
Options: []fs.Option{
|
||||
@@ -280,6 +287,37 @@ enabled, rclone will no longer update the modtime after copying a file.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
{
|
||||
Name: "metadata_restore_special_bits",
|
||||
Help: `Restore the setuid, setgid and sticky bits from metadata.
|
||||
|
||||
When restoring metadata with --metadata rclone applies the "mode" from
|
||||
the source. By default rclone applies only the permission bits and
|
||||
strips the setuid, setgid and sticky bits.
|
||||
|
||||
The "mode" comes from the source remote which may not be trusted.
|
||||
Restoring a setuid or setgid bit onto freshly written,
|
||||
source-controlled content can plant a setuid binary, which is dangerous
|
||||
in particular when restoring from an untrusted source while running as
|
||||
root. For this reason these bits are not restored by default.
|
||||
|
||||
If you trust the source and want the setuid, setgid and sticky bits
|
||||
restored - for example when restoring a system backup made by rclone -
|
||||
set this flag.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
{
|
||||
Name: "fatal_if_no_space",
|
||||
Help: `Make out-of-space errors fatal during transfers.
|
||||
|
||||
When enabled, an ENOSPC error during a write returns a fatal error so
|
||||
that rclone aborts rather than retrying the operation. Useful for
|
||||
backup scripts that should halt loudly on a full disk rather than spin
|
||||
retrying.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
{
|
||||
Name: "time_type",
|
||||
Help: `Set what kind of time is returned.
|
||||
@@ -336,23 +374,24 @@ only useful for reading.
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
FollowSymlinks bool `config:"copy_links"`
|
||||
TranslateSymlinks bool `config:"links"`
|
||||
SkipSymlinks bool `config:"skip_links"`
|
||||
SkipSpecials bool `config:"skip_specials"`
|
||||
UTFNorm bool `config:"unicode_normalization"`
|
||||
NoCheckUpdated bool `config:"no_check_updated"`
|
||||
NoUNC bool `config:"nounc"`
|
||||
OneFileSystem bool `config:"one_file_system"`
|
||||
CaseSensitive bool `config:"case_sensitive"`
|
||||
CaseInsensitive bool `config:"case_insensitive"`
|
||||
NoPreAllocate bool `config:"no_preallocate"`
|
||||
NoSparse bool `config:"no_sparse"`
|
||||
NoSetModTime bool `config:"no_set_modtime"`
|
||||
TimeType timeType `config:"time_type"`
|
||||
Hashes fs.CommaSepList `config:"hashes"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
NoClone bool `config:"no_clone"`
|
||||
FollowSymlinks bool `config:"copy_links"`
|
||||
TranslateSymlinks bool `config:"links"`
|
||||
SkipSymlinks bool `config:"skip_links"`
|
||||
SkipSpecials bool `config:"skip_specials"`
|
||||
UTFNorm bool `config:"unicode_normalization"`
|
||||
NoCheckUpdated bool `config:"no_check_updated"`
|
||||
NoUNC bool `config:"nounc"`
|
||||
OneFileSystem bool `config:"one_file_system"`
|
||||
CaseSensitive bool `config:"case_sensitive"`
|
||||
CaseInsensitive bool `config:"case_insensitive"`
|
||||
NoPreAllocate bool `config:"no_preallocate"`
|
||||
NoSparse bool `config:"no_sparse"`
|
||||
NoSetModTime bool `config:"no_set_modtime"`
|
||||
TimeType timeType `config:"time_type"`
|
||||
Hashes fs.CommaSepList `config:"hashes"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
NoClone bool `config:"no_clone"`
|
||||
MetadataRestoreSpecial bool `config:"metadata_restore_special_bits"`
|
||||
}
|
||||
|
||||
// Fs represents a local filesystem rooted at root
|
||||
@@ -778,7 +817,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
// Mkdir creates the directory if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
localPath := f.localPath(dir)
|
||||
err := file.MkdirAll(localPath, 0777)
|
||||
err := f.mkdirAll(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1395,10 +1434,84 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// osRoot opens an *os.Root anchored at f.root, together with localPath
|
||||
// expressed relative to it. The caller must Close the returned root.
|
||||
//
|
||||
// When translating symlinks (-l/--links) all writes go through an os.Root so a
|
||||
// symlink planted by a malicious source can never be traversed to escape the
|
||||
// destination (CWE-59, GHSA-cf44-9pgv-m4xc).
|
||||
//
|
||||
// f.root is the trusted destination the user chose, so it is created (and
|
||||
// followed if it is itself a symlink) before being opened.
|
||||
func (f *Fs) osRoot(localPath string) (root *os.Root, rel string, err error) {
|
||||
rel, err = filepath.Rel(f.root, localPath)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if err = file.MkdirAll(f.root, 0777); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
root, err = os.OpenRoot(f.root)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return root, rel, nil
|
||||
}
|
||||
|
||||
// mkdirAll makes localPath and any missing parents. When translating
|
||||
// symlinks it does so through os.Root so directory creation can't be
|
||||
// redirected through a planted symlink out of the destination.
|
||||
func (f *Fs) mkdirAll(localPath string) (err error) {
|
||||
if !f.opt.TranslateSymlinks {
|
||||
return file.MkdirAll(localPath, 0777)
|
||||
}
|
||||
root, rel, err := f.osRoot(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fs.CheckClose(root, &err)
|
||||
if rel == "." {
|
||||
return nil // the root itself, already created by linkRoot
|
||||
}
|
||||
return root.MkdirAll(rel, 0777)
|
||||
}
|
||||
|
||||
// openFile opens localPath for writing. When translating symlinks it goes
|
||||
// through os.Root so a symlink planted at the path, or at any parent, is never
|
||||
// followed out of the destination. The returned file is independent of the
|
||||
// root, which is closed before returning.
|
||||
func (f *Fs) openFile(localPath string, flags int, perm os.FileMode) (fi *os.File, err error) {
|
||||
if !f.opt.TranslateSymlinks {
|
||||
return file.OpenFile(localPath, flags, perm)
|
||||
}
|
||||
root, rel, err := f.osRoot(localPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fs.CheckClose(root, &err)
|
||||
return root.OpenFile(rel, flags, perm)
|
||||
}
|
||||
|
||||
// symlink creates a symlink with the given target at localPath, removing
|
||||
// any existing file or symlink there first. It goes through os.Root, which
|
||||
// creates the link verbatim (the target may point anywhere, preserving a
|
||||
// faithful backup) but refuses to create it through a planted symlink, and
|
||||
// won't remove a directory in the way.
|
||||
func (f *Fs) symlink(target, localPath string) (err error) {
|
||||
root, rel, err := f.osRoot(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fs.CheckClose(root, &err)
|
||||
if err := root.Remove(rel); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return root.Symlink(target, rel)
|
||||
}
|
||||
|
||||
// mkdirAll makes all the directories needed to store the object
|
||||
func (o *Object) mkdirAll() error {
|
||||
dir := filepath.Dir(o.path)
|
||||
return file.MkdirAll(dir, 0777)
|
||||
return o.fs.mkdirAll(filepath.Dir(o.path))
|
||||
}
|
||||
|
||||
type nopWriterCloser struct {
|
||||
@@ -1440,13 +1553,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// If it is a translated link, just read in the contents, and
|
||||
// then create a symlink
|
||||
if !o.translatedLink {
|
||||
f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
f, err := o.fs.openFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
if err != nil {
|
||||
if runtime.GOOS == "windows" && os.IsPermission(err) {
|
||||
// If permission denied on Windows might be trying to update a
|
||||
// hidden file, in which case try opening without CREATE
|
||||
// See: https://stackoverflow.com/questions/13215716/ioerror-errno-13-permission-denied-when-trying-to-open-hidden-file-in-w-mod
|
||||
f, err = file.OpenFile(o.path, os.O_WRONLY|os.O_TRUNC, 0666)
|
||||
f, err = o.fs.openFile(o.path, os.O_WRONLY|os.O_TRUNC, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1483,15 +1596,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
if o.translatedLink {
|
||||
if err == nil {
|
||||
// Remove any current symlink or file, if one exists
|
||||
if _, err := os.Lstat(o.path); err == nil {
|
||||
if removeErr := os.Remove(o.path); removeErr != nil {
|
||||
fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
|
||||
return removeErr
|
||||
}
|
||||
}
|
||||
// Use the contents for the copied object to create a symlink
|
||||
err = os.Symlink(symlinkData.String(), o.path)
|
||||
// Use the contents of the copied object to create a symlink,
|
||||
// without following or creating it through a planted symlink
|
||||
// (CWE-59). Any existing file or symlink at the path is replaced.
|
||||
err = o.fs.symlink(symlinkData.String(), o.path)
|
||||
}
|
||||
|
||||
// only continue if symlink creation succeeded
|
||||
@@ -1555,7 +1663,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
|
||||
return nil, errors.New("can't open a symlink for random writing")
|
||||
}
|
||||
|
||||
out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
out, err := f.openFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -204,6 +205,140 @@ func TestSymlinkError(t *testing.T) {
|
||||
assert.Equal(t, errLinksAndCopyLinks, err)
|
||||
}
|
||||
|
||||
// putLink writes target as a translated link object (name + ".rclonelink") on f.
|
||||
func putLink(ctx context.Context, f fs.Fs, name, target string) error {
|
||||
in := bytes.NewBufferString(target)
|
||||
src := object.NewStaticObjectInfo(name+fs.LinkSuffix, fstest.Time("2001-02-03T04:05:10Z"), int64(len(target)), true, nil, nil)
|
||||
_, err := f.Put(ctx, in, src)
|
||||
return err
|
||||
}
|
||||
|
||||
// putFile writes content as a regular object on f.
|
||||
func putFile(ctx context.Context, f fs.Fs, remote, content string) error {
|
||||
in := bytes.NewBufferString(content)
|
||||
src := object.NewStaticObjectInfo(remote, fstest.Time("2001-02-03T04:05:10Z"), int64(len(content)), true, nil, nil)
|
||||
_, err := f.Put(ctx, in, src)
|
||||
return err
|
||||
}
|
||||
|
||||
// linksMode puts f into "-l/--links" mode, as if --links or the backend
|
||||
// links=true option were set.
|
||||
func linksMode(f *Fs) {
|
||||
f.opt.FollowSymlinks = false
|
||||
f.opt.TranslateSymlinks = true
|
||||
f.lstat = os.Lstat
|
||||
}
|
||||
|
||||
// TestSymlinkEscapeWriteThroughBlocked mirrors the GHSA-cf44-9pgv-m4xc PoC: a
|
||||
// malicious --links source serves "pwn.rclonelink" whose body is a path outside
|
||||
// the destination, plus a sibling "pwn/authkeys" that sorts after it and would
|
||||
// be written through the planted symlink. rclone reproduces the symlink (a
|
||||
// faithful backup of the source) but must refuse to write through it, so
|
||||
// nothing lands outside the destination (CWE-59).
|
||||
func TestSymlinkEscapeWriteThroughBlocked(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// A directory outside the destination the attacker wants to write into
|
||||
evil := t.TempDir()
|
||||
evilFile := filepath.Join(evil, "authkeys")
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
// The symlink is reproduced faithfully, pointing outside the destination.
|
||||
require.NoError(t, putLink(ctx, f, "pwn", evil))
|
||||
link := filepath.Join(f.root, "pwn")
|
||||
fi, err := os.Lstat(link)
|
||||
require.NoError(t, err)
|
||||
require.True(t, fi.Mode()&os.ModeSymlink != 0, "symlink should be reproduced faithfully")
|
||||
target, err := os.Readlink(link)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, evil, target)
|
||||
|
||||
// But writing the sibling object through it must be refused.
|
||||
err = putFile(ctx, f, "pwn/authkeys", "PWNED")
|
||||
require.Error(t, err, "writing through a planted symlink should be refused")
|
||||
|
||||
// Nothing escaped the destination.
|
||||
_, err = os.Stat(evilFile)
|
||||
require.True(t, os.IsNotExist(err), "a file escaped the destination into %q", evilFile)
|
||||
}
|
||||
|
||||
// TestSymlinkEscapeNestedBlocked covers the chained variant: an in-tree symlink
|
||||
// "evil" -> "." (the destination root) is created, then "evil/pwn" -> outside
|
||||
// is planted through it, then a write nested under that. Every component is
|
||||
// re-validated against the root, so the write-through is refused and nothing
|
||||
// escapes.
|
||||
func TestSymlinkEscapeNestedBlocked(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
evil := t.TempDir()
|
||||
evilFile := filepath.Join(evil, "authkeys")
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
require.NoError(t, putLink(ctx, f, "evil", "."))
|
||||
require.NoError(t, putLink(ctx, f, "evil/pwn", evil))
|
||||
|
||||
err := putFile(ctx, f, "evil/pwn/authkeys", "PWNED")
|
||||
require.Error(t, err, "writing through a nested planted symlink should be refused")
|
||||
|
||||
_, err = os.Stat(evilFile)
|
||||
require.True(t, os.IsNotExist(err), "a file escaped the destination into %q", evilFile)
|
||||
}
|
||||
|
||||
// TestSymlinkEscapeConcurrent races symlink creation against the sibling write
|
||||
// for many pairs at once, exercising the time-of-check/time-of-use window.
|
||||
// os.Root resolves relative to a directory file descriptor, so whatever the
|
||||
// interleaving nothing may escape the destination.
|
||||
func TestSymlinkEscapeConcurrent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
evil := t.TempDir()
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
const pairs = 50
|
||||
var wg sync.WaitGroup
|
||||
for i := range pairs {
|
||||
name := fmt.Sprintf("pwn%d", i)
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); _ = putLink(ctx, f, name, evil) }()
|
||||
go func() { defer wg.Done(); _ = putFile(ctx, f, name+"/authkeys", "PWNED") }()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
entries, err := os.ReadDir(evil)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, entries, "files escaped the destination into %q", evil)
|
||||
}
|
||||
|
||||
// TestSymlinkInTreeWriteThroughWorks checks the fix doesn't break legitimate
|
||||
// use: an in-tree symlink to a sibling directory can still be created and
|
||||
// written through, since that write stays inside the destination.
|
||||
func TestSymlinkInTreeWriteThroughWorks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
r := fstest.NewRun(t)
|
||||
f := r.Flocal.(*Fs)
|
||||
linksMode(f)
|
||||
|
||||
require.NoError(t, putFile(ctx, f, "sub/keep.txt", "hello"))
|
||||
require.NoError(t, putLink(ctx, f, "link", "sub"))
|
||||
|
||||
require.NoError(t, putFile(ctx, f, "link/file.txt", "world"))
|
||||
|
||||
// The write landed in the real sibling directory, inside the destination.
|
||||
got, err := os.ReadFile(filepath.Join(f.root, "sub", "file.txt"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "world", string(got))
|
||||
}
|
||||
|
||||
func TestHashWithTypeNone(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
@@ -471,6 +606,59 @@ func testMetadata(t *testing.T, r *fstest.Run, o *Object, when time.Time) {
|
||||
})
|
||||
}
|
||||
|
||||
// Check that the setuid, setgid and sticky bits from "mode" metadata are
|
||||
// stripped by default and only restored with --local-metadata-restore-special-bits.
|
||||
//
|
||||
// See: https://github.com/rclone/rclone/security/advisories/GHSA-945v-v9p3-v5xw
|
||||
func TestMetadataSpecialBits(t *testing.T) {
|
||||
switch runtime.GOOS {
|
||||
case "windows", "plan9", "js":
|
||||
t.Skip("mode metadata is not applied on this OS")
|
||||
}
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
const filePath = "setuid.bin"
|
||||
r.WriteFile(filePath, "payload", time.Now())
|
||||
f := r.Flocal.(*Fs)
|
||||
|
||||
obj, err := f.NewObject(ctx, filePath)
|
||||
require.NoError(t, err)
|
||||
o := obj.(*Object)
|
||||
osPath := filepath.Join(f.root, filePath)
|
||||
|
||||
statMode := func() os.FileMode {
|
||||
t.Helper()
|
||||
fi, err := os.Stat(osPath)
|
||||
require.NoError(t, err)
|
||||
return fi.Mode()
|
||||
}
|
||||
|
||||
// "40000755" is Go's os.FileMode layout for setuid|0755 - a value a real
|
||||
// unix st_mode can never produce, so it can only come from an
|
||||
// attacker-controlled source remote.
|
||||
const setuidMode = "40000755"
|
||||
|
||||
t.Run("StrippedByDefault", func(t *testing.T) {
|
||||
require.NoError(t, os.Chmod(osPath, 0644))
|
||||
require.NoError(t, o.writeMetadataToFile(fs.Metadata{"mode": setuidMode}))
|
||||
mode := statMode()
|
||||
assert.Equal(t, os.FileMode(0755), mode.Perm())
|
||||
assert.Zero(t, mode&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky),
|
||||
fmt.Sprintf("special bits should be stripped by default, got %v", mode))
|
||||
})
|
||||
|
||||
t.Run("RestoredWithFlag", func(t *testing.T) {
|
||||
f.opt.MetadataRestoreSpecial = true
|
||||
defer func() { f.opt.MetadataRestoreSpecial = false }()
|
||||
require.NoError(t, os.Chmod(osPath, 0644))
|
||||
require.NoError(t, o.writeMetadataToFile(fs.Metadata{"mode": setuidMode}))
|
||||
mode := statMode()
|
||||
assert.Equal(t, os.FileMode(0755), mode.Perm())
|
||||
assert.NotZero(t, mode&os.ModeSetuid,
|
||||
fmt.Sprintf("setuid bit should be restored with the flag, got %v", mode))
|
||||
})
|
||||
}
|
||||
|
||||
func TestFilter(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
|
||||
@@ -140,15 +140,22 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
|
||||
if mode >= 0 {
|
||||
umode := uint(mode)
|
||||
if umode <= math.MaxUint32 {
|
||||
fileMode := os.FileMode(umode)
|
||||
// fileMode comes from the source, which may be untrusted, so
|
||||
// by default apply only the permission bits and strip the
|
||||
// setuid, setgid and sticky bits.
|
||||
if !o.fs.opt.MetadataRestoreSpecial {
|
||||
fileMode = fileMode.Perm()
|
||||
}
|
||||
if o.translatedLink {
|
||||
if haveLChmod {
|
||||
err = lChmod(o.path, os.FileMode(umode))
|
||||
err = lChmod(o.path, fileMode)
|
||||
} else {
|
||||
fs.Debugf(o, "Unable to set mode %v on a symlink on this OS", os.FileMode(umode))
|
||||
fs.Debugf(o, "Unable to set mode %v on a symlink on this OS", fileMode)
|
||||
err = nil
|
||||
}
|
||||
} else {
|
||||
err = os.Chmod(o.path, os.FileMode(umode))
|
||||
err = os.Chmod(o.path, fileMode)
|
||||
}
|
||||
if err != nil {
|
||||
outErr = fmt.Errorf("failed to change permissions: %w", err)
|
||||
|
||||
@@ -81,6 +81,10 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) {
|
||||
}
|
||||
metadata[k] = string(v)
|
||||
}
|
||||
// Return nil if all xattrs were filtered out (e.g. com.apple.provenance on macOS)
|
||||
if len(metadata) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
|
||||
+12
-2
@@ -770,6 +770,8 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
|
||||
return fmt.Errorf("server-side move failed to lookup src parent dir: %w", err)
|
||||
}
|
||||
|
||||
waitEvent := f.srv.WaitEventsStart()
|
||||
|
||||
// move the object into its new directory if required
|
||||
if srcDirNode != dstDirNode && srcDirNode.GetHash() != dstDirNode.GetHash() {
|
||||
//log.Printf("move src %p %q dst %p %q", srcDirNode, srcDirNode.GetName(), dstDirNode, dstDirNode.GetName())
|
||||
@@ -782,8 +784,6 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
|
||||
}
|
||||
}
|
||||
|
||||
waitEvent := f.srv.WaitEventsStart()
|
||||
|
||||
// rename the object if required
|
||||
if srcLeaf != dstLeaf {
|
||||
//log.Printf("rename %q to %q", srcLeaf, dstLeaf)
|
||||
@@ -1216,6 +1216,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
waitEvent := o.fs.srv.WaitEventsStart()
|
||||
|
||||
// Finish the upload
|
||||
var info *mega.Node
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
@@ -1235,15 +1237,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
o.info = nil
|
||||
}
|
||||
|
||||
// Wait for the server to confirm the new file
|
||||
o.fs.srv.WaitEvents(waitEvent, eventWaitTime)
|
||||
|
||||
return o.setMetaData(info)
|
||||
}
|
||||
|
||||
// Remove an object
|
||||
func (o *Object) Remove(ctx context.Context) error {
|
||||
waitEvent := o.fs.srv.WaitEventsStart()
|
||||
|
||||
err := o.fs.deleteNode(ctx, o.info)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Remove object failed: %w", err)
|
||||
}
|
||||
|
||||
// Wait for the server to confirm the deletion
|
||||
o.fs.srv.WaitEvents(waitEvent, eventWaitTime)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package mega
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// InternalTestGhostAfterRemove checks that a file removed in a long-running
|
||||
// process disappears from listings straight away rather than lingering as a
|
||||
// ghost until go-mega's next event poll reconciles the in-memory tree.
|
||||
//
|
||||
// It puts, removes and lists in a tight loop to make any lingering entry
|
||||
// show up reliably, exercising whichever delete mode the remote is
|
||||
// configured with (hard_delete or the default trash).
|
||||
func (f *Fs) InternalTestGhostAfterRemove(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
dir := "ghost-test"
|
||||
require.NoError(t, f.Mkdir(ctx, dir))
|
||||
defer func() { _ = f.Rmdir(ctx, dir) }()
|
||||
|
||||
contents := []byte("ghost test contents")
|
||||
for i := range 10 {
|
||||
remote := dir + "/test.txt"
|
||||
src := object.NewStaticObjectInfo(remote, time.Now(), int64(len(contents)), true, nil, nil)
|
||||
obj, err := f.Put(ctx, bytes.NewReader(contents), src)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, obj.Remove(ctx))
|
||||
|
||||
// The file must be gone from the listing straight away - if the
|
||||
// in-memory tree isn't settled it lingers as a ghost.
|
||||
entries, err := f.List(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
names = append(names, entry.Remote())
|
||||
}
|
||||
assert.NotContains(t, names, remote, "file should be gone immediately after remove (iteration %d)", i)
|
||||
}
|
||||
}
|
||||
|
||||
// InternalTest dispatches the backend specific internal tests
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("GhostAfterRemove", f.InternalTestGhostAfterRemove)
|
||||
}
|
||||
|
||||
var _ fstests.InternalTester = (*Fs)(nil)
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
@@ -420,6 +421,16 @@ func isDecimalString(value string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// protonLogger adapts rclone's fs.Debugf/Logf/Errorf into the
|
||||
// resty.Logger / common.Logger shape expected by Proton-API-Bridge and
|
||||
// go-proton-api so that library output participates in -v / -vv levels
|
||||
// and is captured by --log-file.
|
||||
type protonLogger struct{ f *Fs }
|
||||
|
||||
func (l protonLogger) Errorf(format string, v ...interface{}) { fs.Errorf(l.f, format, v...) }
|
||||
func (l protonLogger) Warnf(format string, v ...interface{}) { fs.Logf(l.f, format, v...) }
|
||||
func (l protonLogger) Debugf(format string, v ...interface{}) { fs.Debugf(l.f, format, v...) }
|
||||
|
||||
func newProtonDrive(ctx context.Context, f *Fs, opt *Options, m configmap.Mapper) (*protonDriveAPI.ProtonDrive, error) {
|
||||
config := protonDriveAPI.NewDefaultConfig()
|
||||
config.AppVersion = opt.AppVersion
|
||||
@@ -428,6 +439,15 @@ func newProtonDrive(ctx context.Context, f *Fs, opt *Options, m configmap.Mapper
|
||||
}
|
||||
config.UserAgent = f.ci.UserAgent // opt.UserAgent
|
||||
|
||||
// Route HTTP through rclone's transport so global flags such as
|
||||
// --dump headers, --no-check-certificate, --user-agent, --bind,
|
||||
// --ca-cert and --header all take effect against Proton Drive.
|
||||
config.Transport = fshttp.NewTransport(ctx)
|
||||
|
||||
// Route bridge and go-proton-api log output through rclone's
|
||||
// logging system so it honours -v / -vv and --log-file.
|
||||
config.Logger = protonLogger{f: f}
|
||||
|
||||
config.ReplaceExistingDraft = opt.ReplaceExistingDraft
|
||||
config.EnableCaching = opt.EnableCaching
|
||||
|
||||
@@ -949,7 +969,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
}
|
||||
|
||||
if o.digests != nil {
|
||||
return *o.digests, nil
|
||||
return strings.ToLower(*o.digests), nil
|
||||
}
|
||||
|
||||
// sha1 not cached: we fetch and try to obtain the sha1 of the link
|
||||
@@ -999,7 +1019,7 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
fs.FixRangeOption(options, *o.originalSize)
|
||||
fs.FixRangeOption(options, o.Size())
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options { // if the caller passes in nil for options, it will become array of nil
|
||||
switch x := option.(type) {
|
||||
|
||||
@@ -4,20 +4,26 @@ region:
|
||||
au-east-1: AU East 1
|
||||
eu-central: EU Central
|
||||
eu-south-1: EU South 1
|
||||
eu-west-1: EU West 1
|
||||
jp-central-1: JP Central 1
|
||||
uk-east-1: UK East 1
|
||||
us-central-1: US Central 1
|
||||
us-east: US East
|
||||
us-east-1: US East 1
|
||||
us-west: US West
|
||||
us-west-1: US West 1
|
||||
endpoint:
|
||||
au-east-1.object.fastlystorage.app: AU East 1
|
||||
eu-central.object.fastlystorage.app: EU Central
|
||||
eu-south-1.object.fastlystorage.app: EU South 1
|
||||
eu-west-1.object.fastlystorage.app: EU West 1
|
||||
jp-central-1.object.fastlystorage.app: JP Central 1
|
||||
uk-east-1.object.fastlystorage.app: UK East 1
|
||||
us-central-1.object.fastlystorage.app: US Central 1
|
||||
us-east.object.fastlystorage.app: US East
|
||||
us-east-1.object.fastlystorage.app: US East 1
|
||||
us-west.object.fastlystorage.app: US West
|
||||
us-west-1.object.fastlystorage.app: US West 1
|
||||
quirks:
|
||||
force_path_style: true
|
||||
use_already_exists: false
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
name: Mega
|
||||
description: MEGA S4 Object Storage
|
||||
endpoint:
|
||||
s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam)
|
||||
s3.eu-central-2.s4.mega.io: Mega S4 eu-central-2 (Bettembourg)
|
||||
s3.ca-central-1.s4.mega.io: Mega S4 ca-central-1 (Montreal)
|
||||
s3.ca-west-1.s4.mega.io: Mega S4 ca-west-1 (Vancouver)
|
||||
s3.eu-amsterdam.megas4.com: Mega S4 Amsterdam
|
||||
s3.eu-luxembourg.megas4.com: Mega S4 Luxembourg
|
||||
s3.eu-paris.megas4.com: Mega S4 Paris
|
||||
s3.eu-barcelona.megas4.com: Mega S4 Barcelona
|
||||
s3.ca-montreal.megas4.com: Mega S4 Montreal
|
||||
s3.ca-vancouver.megas4.com: Mega S4 Vancouver
|
||||
s3.ap-tokyo.megas4.com: Mega S4 Tokyo
|
||||
s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam, legacy)
|
||||
s3.eu-central-2.s4.mega.io: Mega S4 eu-central-2 (Bettembourg, legacy)
|
||||
s3.ca-central-1.s4.mega.io: Mega S4 ca-central-1 (Montreal, legacy)
|
||||
s3.ca-west-1.s4.mega.io: Mega S4 ca-west-1 (Vancouver, legacy)
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
list_version: 2
|
||||
|
||||
+72
-11
@@ -286,7 +286,7 @@ large file of a known size to stay below this number of chunks limit.
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
The minimum is 1 byte and the maximum is 5 GiB.`,
|
||||
Default: fs.SizeSuffix(maxSizeForCopy),
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -371,7 +371,7 @@ this may help to speed up the transfers.`,
|
||||
|
||||
If this is true (the default) then rclone will use path style access,
|
||||
if false then rclone will use virtual path style. See [the AWS S3
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html)
|
||||
for more info.
|
||||
|
||||
Some providers (e.g. AWS, Aliyun OSS, Netease COS, or Tencent COS) require this set to
|
||||
@@ -1351,10 +1351,34 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
|
||||
}
|
||||
})
|
||||
return &http.Client{
|
||||
Transport: t,
|
||||
Transport: t,
|
||||
CheckRedirect: s3CheckRedirect,
|
||||
}
|
||||
}
|
||||
|
||||
func s3CheckRedirect(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 {
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
if s3RedirectCrossesHost(req, via) {
|
||||
req.Header.Del("X-Amz-Security-Token")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func s3RedirectCrossesHost(req *http.Request, via []*http.Request) bool {
|
||||
if len(via) == 0 {
|
||||
return false
|
||||
}
|
||||
scheme, host := via[0].URL.Scheme, via[0].URL.Host
|
||||
for _, redirect := range via[1:] {
|
||||
if redirect.URL.Host != host || redirect.URL.Scheme != scheme {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return host != req.URL.Host || scheme != req.URL.Scheme
|
||||
}
|
||||
|
||||
// Fixup the request if needed.
|
||||
//
|
||||
// Google Cloud Storage alters the Accept-Encoding header, which
|
||||
@@ -1522,8 +1546,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
|
||||
}
|
||||
}
|
||||
|
||||
// Create AssumeRole credentials provider
|
||||
awsConfig.Credentials = stscreds.NewAssumeRoleProvider(stsClient, opt.RoleARN, assumeRoleOptions)
|
||||
// Create AssumeRole credentials provider, wrapped in a
|
||||
// CredentialsCache so we don't call AssumeRole on every
|
||||
// request.
|
||||
awsConfig.Credentials = aws.NewCredentialsCache(stscreds.NewAssumeRoleProvider(stsClient, opt.RoleARN, assumeRoleOptions))
|
||||
}
|
||||
|
||||
provider = loadProvider(opt.Provider)
|
||||
@@ -1897,20 +1923,52 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
newRoot, leaf := path.Split(oldRoot)
|
||||
f.setRoot(newRoot)
|
||||
_, err := f.NewObject(ctx, leaf)
|
||||
if errors.Is(err, fs.ErrorObjectNotFound) {
|
||||
switch {
|
||||
case err == nil:
|
||||
// It is a file so return an fs which points to the parent
|
||||
return f, fs.ErrorIsFile
|
||||
case errors.Is(err, fs.ErrorObjectNotFound):
|
||||
// File doesn't exist or is a directory so return old f
|
||||
f.setRoot(oldRoot)
|
||||
return f, nil
|
||||
default:
|
||||
// We couldn't HEAD the object so now attempt to list it
|
||||
hasChildren, listErr := f.hasChildren(ctx, leaf)
|
||||
if listErr != nil {
|
||||
fs.Debugf(f, "Couldn't check %q for children after HEAD failed (%v): %v", oldRoot, err, listErr)
|
||||
}
|
||||
// If it listed and has children it must be a directory
|
||||
if hasChildren {
|
||||
f.setRoot(oldRoot)
|
||||
return f, nil
|
||||
}
|
||||
// If it has no children it is either a file or an empty directory. We can't
|
||||
// tell these two cases apart. We choose file which is more likely, and
|
||||
// return an fs which points to the parent.
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// return an error with an fs which points to the parent
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// hasChildren reports whether the directory dir contains any objects.
|
||||
func (f *Fs) hasChildren(ctx context.Context, dir string) (found bool, err error) {
|
||||
bucket, directory := f.split(dir)
|
||||
err = f.list(ctx, listOpt{
|
||||
bucket: bucket,
|
||||
directory: directory,
|
||||
prefix: f.rootDirectory,
|
||||
recurse: true,
|
||||
}, func(remote string, object *types.Object, versionID *string, isDirectory bool) error {
|
||||
found = true
|
||||
return errEndList // stop after the first object
|
||||
})
|
||||
if err != nil && err != fs.ErrorDirNotFound {
|
||||
return false, err
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// getMetaDataListing gets the metadata from the object unconditionally from the listing
|
||||
//
|
||||
// This is needed to find versioned objects from their paths.
|
||||
@@ -4335,6 +4393,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if statusCode := getHTTPStatusCode(err); statusCode == http.StatusNotFound || statusCode == http.StatusMethodNotAllowed {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
@@ -144,6 +144,31 @@ func (f *Fs) InternalTestNoHead(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestHasChildren(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("has-children/file.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
assert.NoError(t, obj.Remove(ctx))
|
||||
}()
|
||||
|
||||
// A prefix with an object under it is a directory
|
||||
found, err := f.hasChildren(ctx, "has-children")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found, "prefix with an object should report children")
|
||||
|
||||
// The object itself is a file so has no children
|
||||
found, err = f.hasChildren(ctx, "has-children/file.txt")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found, "a file should not report children")
|
||||
|
||||
// A path which doesn't exist has no children
|
||||
found, err = f.hasChildren(ctx, "has-children/does-not-exist")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found, "a missing path should not report children")
|
||||
}
|
||||
|
||||
func TestVersionLess(t *testing.T) {
|
||||
key1 := "key1"
|
||||
key2 := "key2"
|
||||
@@ -785,6 +810,7 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Metadata", f.InternalTestMetadata)
|
||||
t.Run("NoHead", f.InternalTestNoHead)
|
||||
t.Run("HasChildren", f.InternalTestHasChildren)
|
||||
t.Run("Versions", f.InternalTestVersions)
|
||||
t.Run("ObjectLock", f.InternalTestObjectLock)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ package s3
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -11,6 +12,8 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
"github.com/rclone/rclone/lib/bucket"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -22,6 +25,211 @@ func SetupS3Test(t *testing.T) (context.Context, *Options, *http.Client) {
|
||||
return ctx, opt, client
|
||||
}
|
||||
|
||||
func TestClientRemovesSecurityTokenOnCrossHostRedirect(t *testing.T) {
|
||||
ctx, _, client := SetupS3Test(t)
|
||||
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Empty(t, r.Header.Get("X-Amz-Security-Token"))
|
||||
assert.Equal(t, "date", r.Header.Get("X-Amz-Date"))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "token", r.Header.Get("X-Amz-Security-Token"))
|
||||
http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer initialServer.Close()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, initialServer.URL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Security-Token", "token")
|
||||
req.Header.Set("X-Amz-Date", "date")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
}
|
||||
|
||||
func TestClientDoesNotRestoreSecurityTokenAfterCrossHostRedirect(t *testing.T) {
|
||||
ctx, _, client := SetupS3Test(t)
|
||||
|
||||
redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/middle":
|
||||
assert.Empty(t, r.Header.Get("X-Amz-Security-Token"))
|
||||
assert.Equal(t, "date", r.Header.Get("X-Amz-Date"))
|
||||
http.Redirect(w, r, "/final", http.StatusTemporaryRedirect)
|
||||
case "/final":
|
||||
assert.Empty(t, r.Header.Get("X-Amz-Security-Token"))
|
||||
assert.Equal(t, "date", r.Header.Get("X-Amz-Date"))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer redirectServer.Close()
|
||||
|
||||
initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "token", r.Header.Get("X-Amz-Security-Token"))
|
||||
http.Redirect(w, r, redirectServer.URL+"/middle", http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer initialServer.Close()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, initialServer.URL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Security-Token", "token")
|
||||
req.Header.Set("X-Amz-Date", "date")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
}
|
||||
|
||||
func TestClientKeepsSecurityTokenOnSameHostRedirect(t *testing.T) {
|
||||
ctx, _, client := SetupS3Test(t)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
assert.Equal(t, "token", r.Header.Get("X-Amz-Security-Token"))
|
||||
http.Redirect(w, r, "/redirected", http.StatusTemporaryRedirect)
|
||||
case "/redirected":
|
||||
assert.Equal(t, "token", r.Header.Get("X-Amz-Security-Token"))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Security-Token", "token")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
assert.NoError(t, resp.Body.Close())
|
||||
}
|
||||
|
||||
func TestRedirectCrossesHost(t *testing.T) {
|
||||
mustReq := func(method, url string) *http.Request {
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
require.NoError(t, err)
|
||||
return req
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
via []string
|
||||
req string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "SameHost",
|
||||
via: []string{"https://bucket.example.com/"},
|
||||
req: "https://bucket.example.com/redirected",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "DifferentHost",
|
||||
via: []string{"https://bucket.example.com/"},
|
||||
req: "https://evil.example.com/redirected",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "SchemeDowngradeSameHost",
|
||||
via: []string{"https://bucket.example.com/"},
|
||||
req: "http://bucket.example.com/redirected",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "SchemeDowngradeMidChain",
|
||||
via: []string{"https://bucket.example.com/", "http://bucket.example.com/middle"},
|
||||
req: "http://bucket.example.com/final",
|
||||
want: true,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
via := make([]*http.Request, len(test.via))
|
||||
for i, url := range test.via {
|
||||
via[i] = mustReq(http.MethodGet, url)
|
||||
}
|
||||
got := s3RedirectCrossesHost(mustReq(http.MethodGet, test.req), via)
|
||||
assert.Equal(t, test.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientStopsAfterTenRedirects(t *testing.T) {
|
||||
_, _, client := SetupS3Test(t)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, r.URL.String(), http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
resp, err := client.Get(server.URL)
|
||||
if resp != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "stopped after 10 redirects")
|
||||
}
|
||||
|
||||
func TestObjectNotFoundMapping(t *testing.T) {
|
||||
ctx, opt, client := SetupS3Test(t)
|
||||
gotHead, gotGet := false, false
|
||||
|
||||
// Return 404 for all requests.
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
gotHead = true
|
||||
case http.MethodGet:
|
||||
gotGet = true
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
opt.Endpoint = server.URL
|
||||
opt.ForcePathStyle = true
|
||||
opt.Region = "us-east-1"
|
||||
opt.AccessKeyID = "id"
|
||||
opt.SecretAccessKey = "secret"
|
||||
c, _, err := s3Connection(ctx, opt, client)
|
||||
require.NoError(t, err)
|
||||
|
||||
f := &Fs{
|
||||
name: "s3test",
|
||||
opt: *opt,
|
||||
ctx: ctx,
|
||||
c: c,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
|
||||
cache: bucket.NewCache(),
|
||||
}
|
||||
f.setRoot("bucket")
|
||||
|
||||
// HEAD path: NewObject reads metadata via HeadObject.
|
||||
_, headErr := f.NewObject(ctx, "missing.txt")
|
||||
require.True(t, gotHead, "server should have received a HEAD request")
|
||||
assert.ErrorIs(t, headErr, fs.ErrorObjectNotFound)
|
||||
|
||||
// GET path: Object.Open issues a GetObject.
|
||||
o := &Object{fs: f, remote: "missing.txt"}
|
||||
in, getErr := o.Open(ctx)
|
||||
if in != nil {
|
||||
_ = in.Close()
|
||||
}
|
||||
require.True(t, gotGet, "server should have received a GET request")
|
||||
assert.ErrorIs(t, getErr, fs.ErrorObjectNotFound)
|
||||
|
||||
assert.Equal(t, headErr, getErr, "HeadObject and GetObject should map a 404 to the same error")
|
||||
}
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
opt := &fstests.Opt{
|
||||
|
||||
@@ -98,21 +98,19 @@ func (p *Prop) Code() int {
|
||||
return code
|
||||
}
|
||||
|
||||
// StatusOK examines the Status and returns an OK flag
|
||||
// Parse a status of the form "HTTP/1.1 2xx"
|
||||
var parseStatus2XX = regexp.MustCompile(`^HTTP/[\d.]+\s+2\d{2}`)
|
||||
|
||||
// StatusOK returns true if there is at least one 2XX response.
|
||||
func (p *Prop) StatusOK() bool {
|
||||
// Fetch status code as int
|
||||
c := p.Code()
|
||||
|
||||
// Assume OK if no statuses received
|
||||
if c == -1 {
|
||||
if len(p.Status) == 0 {
|
||||
return true
|
||||
}
|
||||
if c == 0 {
|
||||
return false
|
||||
}
|
||||
if c >= 200 && c < 300 {
|
||||
return true
|
||||
|
||||
for _, statusStr := range p.Status {
|
||||
if parseStatus2XX.MatchString(statusStr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPropStatusOK(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status []string
|
||||
want bool
|
||||
}{
|
||||
{"Empty status", []string{}, true},
|
||||
{"Single 200 OK", []string{"HTTP/1.1 200 OK"}, true},
|
||||
{"Single 404 Not Found", []string{"HTTP/1.1 404 Not Found"}, false},
|
||||
{"Mixed status with 404 first", []string{"HTTP/1.1 404 Not Found", "HTTP/1.1 200 OK"}, true},
|
||||
{"Mixed status with 200 first", []string{"HTTP/1.1 200 OK", "HTTP/1.1 404 Not Found"}, true},
|
||||
{"Multiple non-2xx statuses", []string{"HTTP/1.1 404 Not Found", "HTTP/1.1 403 Forbidden"}, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
p := &Prop{Status: tt.status}
|
||||
if got := p.StatusOK(); got != tt.want {
|
||||
t.Errorf("Prop.StatusOK() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -796,6 +796,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
||||
ExtraHeaders: map[string]string{
|
||||
"Depth": depth,
|
||||
},
|
||||
AuthRedirect: f.opt.AuthRedirect, // allow redirects to preserve Auth
|
||||
}
|
||||
if f.hasOCMD5 || f.hasOCSHA1 {
|
||||
opts.Body = bytes.NewBuffer(owncloudProps)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -81,6 +82,44 @@ func TestHeaders(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestListAllAuthRedirect checks auth_redirect is honoured on listAll PROPFIND.
|
||||
func TestListAllAuthRedirect(t *testing.T) {
|
||||
var targetAuth string
|
||||
var targetHits int
|
||||
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
targetHits++
|
||||
targetAuth = r.Header.Get("Authorization")
|
||||
_, err := fmt.Fprint(w, `<d:multistatus xmlns:d="DAV:"></d:multistatus>`)
|
||||
require.NoError(t, err)
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
// Redirect via a different hostname so net/http strips Authorization on cross-host redirect.
|
||||
targetURL := strings.Replace(target.URL, "127.0.0.1", "localhost", 1)
|
||||
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, targetURL+r.URL.Path, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer source.Close()
|
||||
|
||||
configfile.Install()
|
||||
m := configmap.Simple{
|
||||
"type": "webdav",
|
||||
"url": source.URL,
|
||||
"user": "alice",
|
||||
"pass": obscure.MustObscure("secret"),
|
||||
"auth_redirect": "true",
|
||||
}
|
||||
|
||||
f, err := webdav.NewFs(context.Background(), remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _ = f.List(context.Background(), "")
|
||||
|
||||
assert.GreaterOrEqual(t, targetHits, 1, "redirect target should receive the request")
|
||||
assert.NotEmpty(t, targetAuth, "Authorization header should be preserved across redirect")
|
||||
}
|
||||
|
||||
// TestReservedCharactersInPathAreEscaped verifies that reserved characters
|
||||
// like semicolons and equals signs in file paths are percent-encoded in
|
||||
// HTTP requests to the WebDAV server (RFC 3986 compliance).
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package archive_test
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -170,6 +172,60 @@ func TestIntegration(t *testing.T) {
|
||||
testArchive(t)
|
||||
}
|
||||
|
||||
// craftZip builds an in-memory zip archive with the given entries, returning
|
||||
// it as a string. The entry names are written verbatim so they can include
|
||||
// crafted path traversal sequences that the archive create command would
|
||||
// never produce from real files.
|
||||
func craftZip(t *testing.T, entries []struct{ name, content string }) string {
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for _, e := range entries {
|
||||
w, err := zw.CreateHeader(&zip.FileHeader{Name: e.name, Method: zip.Store})
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write([]byte(e.content))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, zw.Close())
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// TestExtractPathTraversal checks that a crafted archive whose entries try to
|
||||
// escape the destination directory with ".." components is rejected and does
|
||||
// not write anything outside the destination (GHSA-4vr5-p2gc-h23p).
|
||||
func TestExtractPathTraversal(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
entry string
|
||||
}{
|
||||
{name: "forward slash", entry: "../escaped.txt"},
|
||||
{name: "backslash", entry: `..\escaped.txt`},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
|
||||
zipData := craftZip(t, []struct{ name, content string }{
|
||||
{"safe.txt", "safe content"},
|
||||
{tc.entry, "escaped content"},
|
||||
})
|
||||
r.WriteObject(ctx, "malicious.zip", zipData, t1)
|
||||
|
||||
// Extract into a subdirectory so a successful escape would land
|
||||
// in "safe/escaped.txt", a sibling of the "safe/prefix" target.
|
||||
err := extract.ArchiveExtract(ctx, r.Flocal, "safe/prefix", r.Fremote, "malicious.zip")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "..")
|
||||
|
||||
// Nothing should have escaped the destination directory.
|
||||
for _, escaped := range []string{"safe/escaped.txt", "escaped.txt"} {
|
||||
_, err := r.Flocal.NewObject(ctx, escaped)
|
||||
assert.ErrorIs(t, err, fs.ErrorObjectNotFound, escaped)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemory(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("skipping as -remote is set")
|
||||
|
||||
@@ -150,19 +150,13 @@ func ArchiveExtract(ctx context.Context, dst fs.Fs, dstDir string, src fs.Fs, sr
|
||||
}
|
||||
// extract files
|
||||
err = ex.Extract(ctx, in, func(ctx context.Context, f archives.FileInfo) error {
|
||||
remote := f.NameInArchive
|
||||
// Strip leading "./" from archive paths. Tar files created with
|
||||
// relative paths (e.g. "tar -czf archive.tar.gz .") use "./" prefixed
|
||||
// entries. Without stripping, rclone encodes the "." as a full-width
|
||||
// dot character creating a spurious directory. We only strip "./"
|
||||
// specifically to avoid enabling path traversal attacks via "../".
|
||||
remote = strings.TrimPrefix(remote, "./")
|
||||
// If the entry was exactly "./" (the root dir), skip it
|
||||
if remote == "" && f.IsDir() {
|
||||
return nil
|
||||
remote, err := destPath(f.NameInArchive, dstDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dstDir != "" {
|
||||
remote = path.Join(dstDir, remote)
|
||||
// Skip the archive root entry ("./") which has no name of its own.
|
||||
if remote == "" {
|
||||
return nil
|
||||
}
|
||||
// check if file should be extracted
|
||||
if !fi.Include(remote, f.Size(), f.ModTime(), fs.Metadata{}) {
|
||||
@@ -203,3 +197,38 @@ func ArchiveExtract(ctx context.Context, dst fs.Fs, dstDir string, src fs.Fs, sr
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// destPath maps an archive entry name onto its destination remote within
|
||||
// dstDir, returning an error if the name is unsafe.
|
||||
//
|
||||
// Archive entry names are attacker controlled. A leading "./" is stripped:
|
||||
// tar archives created with relative paths (e.g. "tar -czf archive.tar.gz .")
|
||||
// use "./" prefixed entries and, without stripping, rclone would encode the
|
||||
// "." as a full-width dot character creating a spurious directory.
|
||||
//
|
||||
// Any entry with a ".." path component is then rejected to prevent a path
|
||||
// traversal ("Zip Slip") attack: path.Join collapses "..", so an entry such
|
||||
// as "../escaped.txt" joined onto "dir" would resolve to "escaped.txt" and be
|
||||
// written outside the selected destination directory. Both "/" and "\" are
|
||||
// treated as separators when looking for ".." segments: archive names should
|
||||
// use "/", but a crafted archive may use "\", which the local backend treats
|
||||
// as a path separator on Windows.
|
||||
//
|
||||
// The returned remote is empty for the archive root entry ("./"), which the
|
||||
// caller should skip.
|
||||
func destPath(nameInArchive, dstDir string) (string, error) {
|
||||
remote := strings.TrimPrefix(nameInArchive, "./")
|
||||
isSeparator := func(r rune) bool { return r == '/' || r == '\\' }
|
||||
for _, segment := range strings.FieldsFunc(remote, isSeparator) {
|
||||
if segment == ".." {
|
||||
return "", fmt.Errorf("refusing to extract archive entry %q with a %q path component", nameInArchive, "..")
|
||||
}
|
||||
}
|
||||
if remote == "" {
|
||||
return "", nil
|
||||
}
|
||||
if dstDir != "" {
|
||||
remote = path.Join(dstDir, remote)
|
||||
}
|
||||
return remote, nil
|
||||
}
|
||||
@@ -3,17 +3,19 @@
|
||||
package extract
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestStripDotSlashPrefix(t *testing.T) {
|
||||
func TestDestPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
dstDir string
|
||||
expected string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "strip leading dot-slash from file",
|
||||
@@ -36,12 +38,13 @@ func TestStripDotSlashPrefix(t *testing.T) {
|
||||
expected: "dir/file.txt",
|
||||
},
|
||||
{
|
||||
name: "dot-dot-slash NOT stripped (path traversal safety)",
|
||||
input: "../etc/passwd",
|
||||
expected: "../etc/passwd",
|
||||
name: "joined onto destination directory",
|
||||
input: "file.txt",
|
||||
dstDir: "safe/prefix",
|
||||
expected: "safe/prefix/file.txt",
|
||||
},
|
||||
{
|
||||
name: "dot-slash directory entry becomes empty",
|
||||
name: "archive root entry skipped",
|
||||
input: "./",
|
||||
expected: "",
|
||||
},
|
||||
@@ -50,12 +53,49 @@ func TestStripDotSlashPrefix(t *testing.T) {
|
||||
input: "././file.txt",
|
||||
expected: "./file.txt",
|
||||
},
|
||||
{
|
||||
name: "leading dot-dot rejected",
|
||||
input: "../etc/passwd",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "leading dot-dot rejected with destination",
|
||||
input: "../escaped.txt",
|
||||
dstDir: "safe/prefix",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "interior dot-dot rejected",
|
||||
input: "dir/../../escaped.txt",
|
||||
dstDir: "safe/prefix",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "trailing dot-dot rejected",
|
||||
input: "dir/..",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "backslash dot-dot rejected",
|
||||
input: `..\escaped.txt`,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "nested backslash dot-dot rejected",
|
||||
input: `dir\..\..\escaped.txt`,
|
||||
dstDir: "safe/prefix",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// This mirrors the stripping logic in ArchiveExtract
|
||||
got := strings.TrimPrefix(tc.input, "./")
|
||||
got, err := destPath(tc.input, tc.dstDir)
|
||||
if tc.wantErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expected, got)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -196,7 +196,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
|
||||
}
|
||||
|
||||
if b.critical {
|
||||
if b.retryable && b.opt.Resilient {
|
||||
if b.retryable && b.opt.Resilient && !b.opt.Resync {
|
||||
fs.Errorf(nil, Color(terminal.RedFg, "Bisync critical error: %v"), err)
|
||||
fs.Error(nil, Color(terminal.YellowFg, "Bisync aborted. Error is retryable without --resync due to --resilient mode."))
|
||||
} else {
|
||||
|
||||
@@ -165,6 +165,16 @@ func (b *bisyncRun) resolve(ctxMove context.Context, path1, path2, file, alias s
|
||||
},
|
||||
}
|
||||
|
||||
if b.opt.ConflictLoser == ConflictLoserPathname {
|
||||
switch winningPath {
|
||||
case 1: // keep path1, rename path2
|
||||
r.path1.newName = r.path1.oldName
|
||||
case 2: // keep path2, rename path1
|
||||
r.path2.newName = r.path2.oldName
|
||||
}
|
||||
// case 0 keeps default
|
||||
}
|
||||
|
||||
// handle auto-numbering
|
||||
// note that we still queue copies for both files, whether or not we renamed
|
||||
// we also set these for ConflictLoserDelete in case there is no winner.
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
"file2.txt"
|
||||
"file1.txt.path2"
|
||||
+3
-1
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-25T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -8,3 +8,5 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt.path2"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file2.txt"
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-25T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -8,4 +8,4 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 33 - - 2005-01-02T00:00:00.000000000+0000 "file2.txt"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file2.txt"
|
||||
+1
-1
@@ -1,5 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -7,4 +8,3 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
|
||||
+3
-1
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-25T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -8,3 +8,5 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt.path2"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file2.txt"
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2005-01-02T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
|
||||
+1
-1
@@ -1,5 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -7,4 +8,3 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
|
||||
@@ -305,3 +305,56 @@ INFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mP
|
||||
INFO : Updating listings
|
||||
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
|
||||
INFO : [32mBisync successful[0m
|
||||
|
||||
[36m(48) :[0m [34mtest --conflict-resolve newer --conflict-loser pathname --conflict-suffix path[0m
|
||||
[36m(49) :[0m [34mtouch-glob 2004-07-23 {datadir/} file1R.txt[0m
|
||||
[36m(50) :[0m [34mcopy-as {datadir/}file1R.txt {path2/} file1.txt[0m
|
||||
[36m(51) :[0m [34mtouch-glob 2004-07-25 {datadir/} file1L.txt[0m
|
||||
[36m(52) :[0m [34mcopy-as {datadir/}file1L.txt {path1/} file1.txt[0m
|
||||
|
||||
[36m(53) :[0m [34mtest bisync run with --conflict-resolve=newer --conflict-loser=pathname --conflict-suffix=path[0m
|
||||
|
||||
[36m(54) :[0m [34mbisync conflict-resolve=newer conflict-loser=pathname conflict-suffix=path[0m
|
||||
INFO : [2mSetting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.[0m
|
||||
INFO : Bisyncing with Comparison Settings:
|
||||
{
|
||||
"Modtime": true,
|
||||
"Size": true,
|
||||
"Checksum": false,
|
||||
"NoSlowHash": false,
|
||||
"SlowHashSyncOnly": false,
|
||||
"DownloadHash": false
|
||||
}
|
||||
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
|
||||
INFO : Building Path1 and Path2 listings
|
||||
INFO : Path1 checking for diffs
|
||||
INFO : - [36mPath1[0m [35m[33mFile changed: [35mtime (older)[0m[0m[0m - [36mfile1.txt[0m
|
||||
INFO : - [36mPath1[0m [35m[32mFile is new[0m[0m - [36mfile2.txt[0m
|
||||
INFO : Path1: 2 changes: [32m 1 new[0m, [33m 1 modified[0m, [31m 0 deleted[0m
|
||||
INFO : ([33mModified[0m: [36m 0 newer[0m, [34m 1 older[0m)
|
||||
INFO : Path2 checking for diffs
|
||||
INFO : - [34mPath2[0m [35m[33mFile changed: [35mtime (older)[0m[0m[0m - [36mfile1.txt[0m
|
||||
INFO : - [34mPath2[0m [35m[32mFile is new[0m[0m - [36mfile2.txt[0m
|
||||
INFO : Path2: 2 changes: [32m 1 new[0m, [33m 1 modified[0m, [31m 0 deleted[0m
|
||||
INFO : ([33mModified[0m: [36m 0 newer[0m, [34m 1 older[0m)
|
||||
INFO : Applying changes
|
||||
INFO : Checking potential conflicts...
|
||||
ERROR : file1.txt: {hashtype} differ
|
||||
NOTICE: {path2String}: 1 differences found
|
||||
NOTICE: {path2String}: 1 errors while checking
|
||||
NOTICE: {path2String}: 1 matching files
|
||||
INFO : Finished checking the potential conflicts. 1 differences found
|
||||
NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfile1.txt[0m
|
||||
INFO : file1.txt: Path1 is newer. Path1: 2004-07-25 00:00:00 +0000 UTC, Path2: 2004-07-23 00:00:00 +0000 UTC, Difference: 48h0m0s
|
||||
INFO : file1.txt: [32mThe winner is: Path1[0m
|
||||
NOTICE: - [36mPath1[0m [35mNot renaming Path1 copy, as it was determined the winner[0m - [36m{path1/}file1.txt[0m
|
||||
NOTICE: - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{path2/}file1.txt[0m
|
||||
NOTICE: - [34mPath2[0m [35mRenaming Path2 copy[0m - [36m{path2/}file1.txt.path2[0m
|
||||
NOTICE: - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file1.txt.path2[0m
|
||||
NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfile2.txt[0m
|
||||
INFO : Files are equal! Skipping: file2.txt
|
||||
INFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m
|
||||
INFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m
|
||||
INFO : Updating listings
|
||||
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
|
||||
INFO : [32mBisync successful[0m
|
||||
+11
-1
@@ -64,4 +64,14 @@ copy-as {datadir/}file1L.txt {path1/} file1.txt
|
||||
copy-as {datadir/}file1L.txt {path2/} file2.txt
|
||||
|
||||
test bisync run with --conflict-resolve=newer --conflict-loser=delete
|
||||
bisync conflict-resolve=newer conflict-loser=delete
|
||||
bisync conflict-resolve=newer conflict-loser=delete
|
||||
|
||||
test --conflict-resolve newer --conflict-loser pathname --conflict-suffix path
|
||||
touch-glob 2004-07-23 {datadir/} file1R.txt
|
||||
copy-as {datadir/}file1R.txt {path2/} file1.txt
|
||||
touch-glob 2004-07-25 {datadir/} file1L.txt
|
||||
copy-as {datadir/}file1L.txt {path1/} file1.txt
|
||||
|
||||
test bisync run with --conflict-resolve=newer --conflict-loser=pathname --conflict-suffix=path
|
||||
# result should be -- "file1.txt", "file1.txt.path2"
|
||||
bisync conflict-resolve=newer conflict-loser=pathname conflict-suffix=path
|
||||
@@ -29,8 +29,9 @@ var commandDefinition = &cobra.Command{
|
||||
Long: `If source:path is a file or directory then it copies it to a file or
|
||||
directory named dest:path.
|
||||
|
||||
This can be used to upload single files to other than their current
|
||||
name. If the source is a directory then it acts exactly like the
|
||||
This can be used to copy a single file to a destination with a name
|
||||
different from its source - for example, uploading and renaming in one
|
||||
step. If the source is a directory then it acts exactly like the
|
||||
[copy](/commands/rclone_copy/) command.
|
||||
|
||||
So
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package genautocomplete
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -13,6 +15,34 @@ func init() {
|
||||
completionDefinition.AddCommand(powershellCommandDefinition)
|
||||
}
|
||||
|
||||
// powerShellInvokeLine is the line in the Cobra generated PowerShell completion
|
||||
// script that captures rclone's output through a pipeline.
|
||||
const powerShellInvokeLine = `Invoke-Expression -OutVariable out "$RequestComp" 2>&1 | Out-Null`
|
||||
|
||||
// powerShellUTF8Fix forces the captured output to be decoded as UTF-8. When
|
||||
// PowerShell captures a child process' stdout through a pipeline it decodes the
|
||||
// bytes using [Console]::OutputEncoding, which on non-UTF-8 systems (for
|
||||
// example PowerShell 5.1 on a Windows install with an OEM code page such as
|
||||
// CP852) corrupts the UTF-8 that rclone emits. Setting the encoding to UTF-8 is
|
||||
// safe on PowerShell 7+, where it is already the default.
|
||||
const powerShellUTF8Fix = `[Console]::OutputEncoding = [System.Text.Encoding]::UTF8`
|
||||
|
||||
// patchPowerShellCompletion injects the UTF-8 output encoding fix immediately
|
||||
// before the Invoke-Expression call in the Cobra generated PowerShell
|
||||
// completion script. If the expected line is not found (for example because the
|
||||
// upstream Cobra template changed), the script is returned unmodified so we
|
||||
// never emit a corrupted completion script.
|
||||
func patchPowerShellCompletion(script string) string {
|
||||
idx := strings.Index(script, powerShellInvokeLine)
|
||||
if idx == -1 {
|
||||
return script
|
||||
}
|
||||
// Reuse the indentation of the Invoke-Expression line for the inserted line.
|
||||
lineStart := strings.LastIndex(script[:idx], "\n") + 1
|
||||
indent := script[lineStart:idx]
|
||||
return script[:lineStart] + indent + powerShellUTF8Fix + "\n" + script[lineStart:]
|
||||
}
|
||||
|
||||
var powershellCommandDefinition = &cobra.Command{
|
||||
Use: "powershell [output_file]",
|
||||
Short: `Output powershell completion script for rclone.`,
|
||||
@@ -30,15 +60,18 @@ to your powershell profile.
|
||||
If output_file is "-" or missing, then the output will be written to stdout.`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(0, 1, command, args)
|
||||
var buf bytes.Buffer
|
||||
if err := cmd.Root.GenPowerShellCompletion(&buf); err != nil {
|
||||
fs.Fatal(nil, fmt.Sprint(err))
|
||||
}
|
||||
script := patchPowerShellCompletion(buf.String())
|
||||
if len(args) == 0 || (len(args) > 0 && args[0] == "-") {
|
||||
err := cmd.Root.GenPowerShellCompletion(os.Stdout)
|
||||
if err != nil {
|
||||
if _, err := os.Stdout.WriteString(script); err != nil {
|
||||
fs.Fatal(nil, fmt.Sprint(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
err := cmd.Root.GenPowerShellCompletionFile(args[0])
|
||||
if err != nil {
|
||||
if err := os.WriteFile(args[0], []byte(script), 0644); err != nil {
|
||||
fs.Fatal(nil, fmt.Sprint(err))
|
||||
}
|
||||
},
|
||||
|
||||
@@ -2,6 +2,7 @@ package genautocomplete
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -107,3 +108,58 @@ func TestCompletionFishStdout(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, string(output))
|
||||
}
|
||||
|
||||
func TestCompletionPowershell(t *testing.T) {
|
||||
tempFile, err := os.CreateTemp("", "completion_powershell")
|
||||
assert.NoError(t, err)
|
||||
defer func() {
|
||||
_ = tempFile.Close()
|
||||
_ = os.Remove(tempFile.Name())
|
||||
}()
|
||||
|
||||
powershellCommandDefinition.Run(powershellCommandDefinition, []string{tempFile.Name()})
|
||||
|
||||
bs, err := os.ReadFile(tempFile.Name())
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, string(bs))
|
||||
// The generated script must force UTF-8 output decoding so that non-ASCII
|
||||
// remote names are not corrupted on non-UTF-8 PowerShell hosts.
|
||||
assert.Contains(t, string(bs), powerShellUTF8Fix)
|
||||
}
|
||||
|
||||
func TestCompletionPowershellStdout(t *testing.T) {
|
||||
originalStdout := os.Stdout
|
||||
tempFile, err := os.CreateTemp("", "completion_powershell")
|
||||
assert.NoError(t, err)
|
||||
defer func() {
|
||||
_ = tempFile.Close()
|
||||
_ = os.Remove(tempFile.Name())
|
||||
}()
|
||||
|
||||
os.Stdout = tempFile
|
||||
defer func() { os.Stdout = originalStdout }()
|
||||
|
||||
powershellCommandDefinition.Run(powershellCommandDefinition, []string{"-"})
|
||||
|
||||
output, err := os.ReadFile(tempFile.Name())
|
||||
assert.NoError(t, err)
|
||||
assert.NotEmpty(t, string(output))
|
||||
assert.Contains(t, string(output), powerShellUTF8Fix)
|
||||
}
|
||||
|
||||
func TestPatchPowerShellCompletion(t *testing.T) {
|
||||
t.Run("injects the encoding fix before the invoke line", func(t *testing.T) {
|
||||
script := "before\n " + powerShellInvokeLine + "\nafter\n"
|
||||
got := patchPowerShellCompletion(script)
|
||||
// The fix is inserted on its own line, sharing the indentation of the
|
||||
// invoke line, immediately before it.
|
||||
want := "before\n " + powerShellUTF8Fix + "\n " + powerShellInvokeLine + "\nafter\n"
|
||||
assert.Equal(t, want, got)
|
||||
assert.Less(t, strings.Index(got, powerShellUTF8Fix), strings.Index(got, powerShellInvokeLine))
|
||||
})
|
||||
|
||||
t.Run("leaves the script unchanged when the invoke line is absent", func(t *testing.T) {
|
||||
script := "some other script\nwithout the expected line\n"
|
||||
assert.Equal(t, script, patchPowerShellCompletion(script))
|
||||
})
|
||||
}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
1.1.7
|
||||
1.1.10
|
||||
Binary file not shown.
@@ -75,6 +75,7 @@ func setAttr(node vfs.Node, attr *fuse.Attr) {
|
||||
vfs := node.VFS()
|
||||
attr.Owner.Gid = vfs.Opt.GID
|
||||
attr.Owner.Uid = vfs.Opt.UID
|
||||
attr.Ino = node.Inode()
|
||||
attr.Mode = getMode(node)
|
||||
attr.Size = Size
|
||||
attr.Nlink = 1
|
||||
|
||||
+56
-4
@@ -201,7 +201,7 @@ func (n *Node) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (ino
|
||||
// out.SetAttrTimeout(dt time.Duration)
|
||||
n.fsys.setEntryOut(vfsNode, out)
|
||||
|
||||
return n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode}), 0
|
||||
return n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode, Ino: newNode.node.Inode()}), 0
|
||||
}
|
||||
|
||||
var _ = (fusefs.NodeLookuper)((*Node)(nil))
|
||||
@@ -271,7 +271,26 @@ func (ds *dirStream) Next() (de fuse.DirEntry, errno syscall.Errno) {
|
||||
func (ds *dirStream) Close() {
|
||||
}
|
||||
|
||||
// Seekdir implements fusefs.FileSeekdirer so go-fuse can reposition the
|
||||
// directory stream. The kernel calls this both to rewind (lseek(fd, 0,
|
||||
// SEEK_SET) before a second getdents) and, for a kernel-NFS-exported mount,
|
||||
// to resume from a previously returned directory cookie: nfsd is stateless,
|
||||
// so it opens a fresh handle and seeks to the last offset on every readdir
|
||||
// continuation. Handling only offset 0 made go-fuse return ENOTSUP for those
|
||||
// resumes, so any listing spanning more than one readdir batch failed over NFS.
|
||||
//
|
||||
// dirStream is a snapshot taken at Readdir time and go-fuse assigns each entry
|
||||
// a sequential offset in Next() order (the entry yielded at index i carries
|
||||
// offset i+1), so repositioning to off means the next entry is the one at
|
||||
// index off. Values past the end clamp to EOF via HasNext.
|
||||
// See: https://github.com/hanwen/go-fuse/issues/549
|
||||
func (ds *dirStream) Seekdir(_ context.Context, off uint64) syscall.Errno {
|
||||
ds.i = int(off)
|
||||
return 0
|
||||
}
|
||||
|
||||
var _ fusefs.DirStream = (*dirStream)(nil)
|
||||
var _ fusefs.FileSeekdirer = (*dirStream)(nil)
|
||||
|
||||
// Readdir opens a stream of directory entries.
|
||||
//
|
||||
@@ -324,7 +343,7 @@ func (n *Node) Mkdir(ctx context.Context, name string, mode uint32, out *fuse.En
|
||||
}
|
||||
newNode := newNode(n.fsys, newDir)
|
||||
n.fsys.setEntryOut(newNode.node, out)
|
||||
newInode := n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode})
|
||||
newInode := n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode, Ino: newNode.node.Inode()})
|
||||
return newInode, 0
|
||||
}
|
||||
|
||||
@@ -366,12 +385,45 @@ func (n *Node) Create(ctx context.Context, name string, flags uint32, mode uint3
|
||||
n.fsys.setEntryOut(vfsNode, out)
|
||||
newNode := newNode(n.fsys, vfsNode)
|
||||
fs.Debugf(nil, "attr=%#v", out.Attr)
|
||||
newInode := n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode})
|
||||
newInode := n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode, Ino: newNode.node.Inode()})
|
||||
return newInode, fh, 0, 0
|
||||
}
|
||||
|
||||
var _ = (fusefs.NodeCreater)((*Node)(nil))
|
||||
|
||||
// Mknod creates a regular file. The kernel NFS server creates regular files
|
||||
// with MKNOD (it creates-then-opens, so vfs_create routes through fuse_create
|
||||
// which sends FUSE_MKNOD when there is no open intent), so without this an
|
||||
// NFS-exported mount fails every file creation with ENOTSUPP. Local and SMB
|
||||
// clients create via Create (FUSE_CREATE) and are unaffected. This mirrors the
|
||||
// cmd/mount (bazil) backend, which implements mknod for the same reason
|
||||
// (see #2115).
|
||||
//
|
||||
// Device/special files are not supported by the VFS, so a non-zero rdev is
|
||||
// rejected. Mknod returns no open handle (unlike Create), so the handle Create
|
||||
// opens is flushed (to instantiate the file and surface any I/O error) and
|
||||
// released before returning.
|
||||
func (n *Node) Mknod(ctx context.Context, name string, mode uint32, rdev uint32, out *fuse.EntryOut) (node *fusefs.Inode, errno syscall.Errno) {
|
||||
defer log.Trace(n, "name=%q, mode=%#o, rdev=%d", name, mode, rdev)("node=%v, errno=%v", &node, &errno)
|
||||
if rdev != 0 {
|
||||
fs.Errorf(n, "Can't create device node %q", name)
|
||||
return nil, syscall.EIO
|
||||
}
|
||||
node, fh, _, errno := n.Create(ctx, name, uint32(os.O_CREATE|os.O_WRONLY), mode, out)
|
||||
if errno != 0 {
|
||||
return nil, errno
|
||||
}
|
||||
if fh != nil {
|
||||
if errno := fh.(fusefs.FileFlusher).Flush(ctx); errno != 0 {
|
||||
return nil, errno
|
||||
}
|
||||
_ = fh.(fusefs.FileReleaser).Release(ctx)
|
||||
}
|
||||
return node, 0
|
||||
}
|
||||
|
||||
var _ = (fusefs.NodeMknoder)((*Node)(nil))
|
||||
|
||||
// Unlink should remove a child from this directory. If the
|
||||
// return status is OK, the Inode is removed as child in the
|
||||
// FS tree automatically. Default is to return EROFS.
|
||||
@@ -482,7 +534,7 @@ func (n *Node) Symlink(ctx context.Context, target, name string, out *fuse.Entry
|
||||
|
||||
n.fsys.setEntryOut(vfsNode, out)
|
||||
newNode := newNode(n.fsys, vfsNode)
|
||||
newInode := n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode})
|
||||
newInode := n.NewInode(ctx, newNode, fusefs.StableAttr{Mode: out.Attr.Mode, Ino: newNode.node.Inode()})
|
||||
|
||||
return newInode, 0
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
//go:build linux || (darwin && amd64)
|
||||
|
||||
package mount2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDirStreamSeekdir checks that Seekdir repositions the snapshot directory
|
||||
// stream to an arbitrary offset. A kernel-NFS-exported mount needs this for
|
||||
// readdir continuation: the stateless server opens a fresh handle and seeks to
|
||||
// the last returned cookie on every batch. Offset 0 must still reset to the
|
||||
// start (the rewind / re-read case) and offsets at or past the end must clamp
|
||||
// to EOF.
|
||||
func TestDirStreamSeekdir(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
// 3 real entries plus the synthesized "." and ".." => 5 entries total,
|
||||
// occupying internal indices 0..4 (go-fuse offsets 1..5). HasNext is true
|
||||
// while i < len(nodes)+2, i.e. i < 5.
|
||||
ds := &dirStream{nodes: make([]os.FileInfo, 3)}
|
||||
|
||||
for _, tc := range []struct {
|
||||
off uint64
|
||||
wantI int
|
||||
wantNext bool
|
||||
}{
|
||||
{0, 0, true}, // rewind to start (the re-read case)
|
||||
{2, 2, true}, // resume at the first real entry
|
||||
{4, 4, true}, // last entry
|
||||
{5, 5, false}, // exactly at end => EOF
|
||||
{99, 99, false}, // past end => clamped to EOF via HasNext
|
||||
} {
|
||||
if errno := ds.Seekdir(ctx, tc.off); errno != 0 {
|
||||
t.Fatalf("Seekdir(%d) returned errno %v, want 0", tc.off, errno)
|
||||
}
|
||||
if ds.i != tc.wantI {
|
||||
t.Errorf("Seekdir(%d): i = %d, want %d", tc.off, ds.i, tc.wantI)
|
||||
}
|
||||
if got := ds.HasNext(); got != tc.wantNext {
|
||||
t.Errorf("Seekdir(%d): HasNext() = %v, want %v", tc.off, got, tc.wantNext)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,11 @@ import (
|
||||
)
|
||||
|
||||
// OptionsInfo describes the Options in use
|
||||
var OptionsInfo = fs.Options{}.
|
||||
var OptionsInfo = fs.Options{{
|
||||
Name: "disable_zip",
|
||||
Default: false,
|
||||
Help: "Disable zip download of directories",
|
||||
}}.
|
||||
Add(libhttp.ConfigInfo).
|
||||
Add(libhttp.AuthConfigInfo).
|
||||
Add(libhttp.TemplateConfigInfo)
|
||||
@@ -45,7 +49,7 @@ type Options struct {
|
||||
Auth libhttp.AuthConfig
|
||||
HTTP libhttp.Config
|
||||
Template libhttp.TemplateConfig
|
||||
DisableZip bool
|
||||
DisableZip bool `config:"disable_zip"`
|
||||
}
|
||||
|
||||
// DefaultOpt is the default values used for Options
|
||||
@@ -74,7 +78,6 @@ func init() {
|
||||
flags.AddFlagsFromOptions(flagSet, "", OptionsInfo)
|
||||
vfsflags.AddFlags(flagSet)
|
||||
proxyflags.AddFlags(flagSet)
|
||||
flagSet.BoolVar(&Opt.DisableZip, "disable-zip", false, "Disable zip download of directories")
|
||||
cmdserve.Command.AddCommand(Command)
|
||||
cmdserve.AddRc("http", func(ctx context.Context, f fs.Fs, in rc.Params) (cmdserve.Handle, error) {
|
||||
// Read VFS Opts
|
||||
|
||||
@@ -244,6 +244,12 @@ func WithRemote(next http.Handler) http.Handler {
|
||||
urlpath = r.URL.Path
|
||||
}
|
||||
urlpath = strings.Trim(urlpath, "/")
|
||||
// Reject any non-canonical path, in particular one containing ".."
|
||||
// traversal elements.
|
||||
if urlpath != "" && path.Clean(urlpath) != urlpath {
|
||||
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
parts := matchData.FindStringSubmatch(urlpath)
|
||||
// if no data directory, layout is flat
|
||||
if parts != nil {
|
||||
|
||||
@@ -75,4 +75,16 @@ func TestResticPrivateRepositories(t *testing.T) {
|
||||
checkRequest(t, router.ServeHTTP, req, []wantFunc{wantCode(http.StatusForbidden)})
|
||||
}
|
||||
|
||||
// Reaching another user's repo via a ".." traversal in the path must be
|
||||
// rejected even though the {userID} segment matches the authenticated user
|
||||
// (GHSA-fqj9-69pf-6pjg).
|
||||
reqs = []*http.Request{
|
||||
newAuthenticatedRequest(t, "GET", "/test/../other_user/config", nil, opt.Auth.BasicUser, opt.Auth.BasicPass),
|
||||
newAuthenticatedRequest(t, "POST", "/test/../other_user/config", strings.NewReader("evil"), opt.Auth.BasicUser, opt.Auth.BasicPass),
|
||||
newAuthenticatedRequest(t, "DELETE", "/test/../other_user/config", nil, opt.Auth.BasicUser, opt.Auth.BasicPass),
|
||||
}
|
||||
for _, req := range reqs {
|
||||
checkRequest(t, router.ServeHTTP, req, []wantFunc{wantCode(http.StatusBadRequest)})
|
||||
}
|
||||
|
||||
}
|
||||
+43
-19
@@ -117,7 +117,10 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin
|
||||
return nil, gofakes3.BucketNotFound(bucketName)
|
||||
}
|
||||
|
||||
fp := path.Join(bucketName, objectName)
|
||||
fp, err := bucketObjectPath(bucketName, objectName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, err := _vfs.Stat(fp)
|
||||
if err != nil {
|
||||
return nil, gofakes3.KeyNotFound(objectName)
|
||||
@@ -127,18 +130,24 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin
|
||||
return nil, gofakes3.KeyNotFound(objectName)
|
||||
}
|
||||
|
||||
// node.DirEntry() is nil while the file is still being uploaded to the
|
||||
// backing remote (e.g. just after a multipart upload, before the VFS
|
||||
// writeback completes). In that window the file already exists in the VFS
|
||||
// and is returned by ListBucket, so serve its metadata from the node
|
||||
// rather than returning a spurious 404. getFileHashByte already falls back
|
||||
// to hashing the VFS cache when the backing object is not available yet.
|
||||
entry := node.DirEntry()
|
||||
if entry == nil {
|
||||
return nil, gofakes3.KeyNotFound(objectName)
|
||||
}
|
||||
|
||||
fobj := entry.(fs.Object)
|
||||
size := node.Size()
|
||||
hash := getFileHashByte(fobj, b.s.etagHashType)
|
||||
hash := getFileHashByte(node, b.s.etagHashType)
|
||||
|
||||
mimeType := fs.MimeTypeFromName(objectName)
|
||||
if fobj, ok := entry.(fs.Object); ok {
|
||||
mimeType = fs.MimeType(context.Background(), fobj)
|
||||
}
|
||||
|
||||
meta := map[string]string{
|
||||
"Last-Modified": formatHeaderTime(node.ModTime()),
|
||||
"Content-Type": fs.MimeType(context.Background(), fobj),
|
||||
"Content-Type": mimeType,
|
||||
}
|
||||
|
||||
if val, ok := b.meta.Load(fp); ok {
|
||||
@@ -166,7 +175,10 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
|
||||
return nil, gofakes3.BucketNotFound(bucketName)
|
||||
}
|
||||
|
||||
fp := path.Join(bucketName, objectName)
|
||||
fp, err := bucketObjectPath(bucketName, objectName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node, err := _vfs.Stat(fp)
|
||||
if err != nil {
|
||||
return nil, gofakes3.KeyNotFound(objectName)
|
||||
@@ -176,16 +188,14 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
|
||||
return nil, gofakes3.KeyNotFound(objectName)
|
||||
}
|
||||
|
||||
// As in HeadObject, node.DirEntry() may be nil while the file is still
|
||||
// being written back to the backing remote. The data is readable from the
|
||||
// VFS cache via file.Open regardless, so serve it instead of 404ing.
|
||||
entry := node.DirEntry()
|
||||
if entry == nil {
|
||||
return nil, gofakes3.KeyNotFound(objectName)
|
||||
}
|
||||
|
||||
fobj := entry.(fs.Object)
|
||||
file := node.(*vfs.File)
|
||||
|
||||
size := node.Size()
|
||||
hash := getFileHashByte(fobj, b.s.etagHashType)
|
||||
hash := getFileHashByte(node, b.s.etagHashType)
|
||||
|
||||
in, err := file.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
@@ -211,9 +221,14 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
|
||||
rdr = limitReadCloser(rdr, in.Close, rnge.Length)
|
||||
}
|
||||
|
||||
mimeType := fs.MimeTypeFromName(objectName)
|
||||
if fobj, ok := entry.(fs.Object); ok {
|
||||
mimeType = fs.MimeType(context.Background(), fobj)
|
||||
}
|
||||
|
||||
meta := map[string]string{
|
||||
"Last-Modified": formatHeaderTime(node.ModTime()),
|
||||
"Content-Type": fs.MimeType(context.Background(), fobj),
|
||||
"Content-Type": mimeType,
|
||||
}
|
||||
|
||||
if val, ok := b.meta.Load(fp); ok {
|
||||
@@ -301,7 +316,10 @@ func (b *s3Backend) PutObject(
|
||||
return result, gofakes3.BucketNotFound(bucketName)
|
||||
}
|
||||
|
||||
fp := path.Join(bucketName, objectName)
|
||||
fp, err := bucketObjectPath(bucketName, objectName)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
objectDir := path.Dir(fp)
|
||||
// _, err = db.fs.Stat(objectDir)
|
||||
// if err == vfs.ENOENT {
|
||||
@@ -394,7 +412,10 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
|
||||
return gofakes3.BucketNotFound(bucketName)
|
||||
}
|
||||
|
||||
fp := path.Join(bucketName, objectName)
|
||||
fp, err := bucketObjectPath(bucketName, objectName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// S3 does not report an error when attempting to delete a key that does not exist, so
|
||||
// we need to skip IsNotExist errors.
|
||||
if err := _vfs.Remove(fp); err != nil && !os.IsNotExist(err) {
|
||||
@@ -465,7 +486,10 @@ func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
fp := path.Join(srcBucket, srcKey)
|
||||
fp, err := bucketObjectPath(srcBucket, srcKey)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if srcBucket == dstBucket && srcKey == dstKey {
|
||||
b.meta.Store(fp, meta)
|
||||
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/gofakes3"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/cmd/serve/proxy"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newTestBackend serves a root directory containing a bucket directory, an
|
||||
// object inside that bucket and a root-level file that is not part of any
|
||||
// bucket. It returns the backend and the serve root path.
|
||||
func newTestBackend(t *testing.T) (*s3Backend, string) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "bucket"), 0777))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(root, "root-secret.txt"), []byte(rootSecret), 0666))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(root, "bucket", "object.txt"), []byte("normal object"), 0666))
|
||||
|
||||
f, err := fs.NewFs(ctx, root)
|
||||
require.NoError(t, err)
|
||||
|
||||
opt := Opt
|
||||
opt.HTTP.ListenAddr = []string{endpoint}
|
||||
w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
|
||||
require.NoError(t, err)
|
||||
|
||||
return newBackend(w).(*s3Backend), root
|
||||
}
|
||||
|
||||
const rootSecret = "ROOT_LEVEL_SECRET_MARKER"
|
||||
|
||||
// TestPathTraversal checks that dot-dot segments in an object key cannot
|
||||
// escape the bucket namespace to read, overwrite or create files outside the
|
||||
// selected bucket.
|
||||
func TestPathTraversal(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("Get", func(t *testing.T) {
|
||||
b, _ := newTestBackend(t)
|
||||
_, err := b.GetObject(ctx, "bucket", "../root-secret.txt", nil)
|
||||
assert.Error(t, err, "GET with dot-dot key must not read the root-level file")
|
||||
})
|
||||
|
||||
t.Run("Head", func(t *testing.T) {
|
||||
b, _ := newTestBackend(t)
|
||||
_, err := b.HeadObject(ctx, "bucket", "../root-secret.txt")
|
||||
assert.Error(t, err, "HEAD with dot-dot key must not find the root-level file")
|
||||
})
|
||||
|
||||
t.Run("Put", func(t *testing.T) {
|
||||
b, root := newTestBackend(t)
|
||||
const payload = "OVERWRITTEN_BY_DOTDOT"
|
||||
_, err := b.PutObject(ctx, "bucket", "../root-secret.txt", map[string]string{}, strings.NewReader(payload), int64(len(payload)))
|
||||
assert.Error(t, err, "PUT with dot-dot key must not overwrite the root-level file")
|
||||
got, readErr := os.ReadFile(filepath.Join(root, "root-secret.txt"))
|
||||
require.NoError(t, readErr)
|
||||
assert.Equal(t, rootSecret, string(got), "root-level file must be unchanged")
|
||||
})
|
||||
|
||||
// path.Dir of the joined path can re-introduce a traversal, so check that a
|
||||
// rejected PUT does not create directories outside the bucket as a side
|
||||
// effect of making the object's parent directory.
|
||||
t.Run("PutNoSideEffectDir", func(t *testing.T) {
|
||||
b, root := newTestBackend(t)
|
||||
_, err := b.PutObject(ctx, "bucket", "../sneaky-dir/f.txt", map[string]string{}, strings.NewReader("x"), 1)
|
||||
assert.Error(t, err, "PUT with dot-dot key must be rejected")
|
||||
_, statErr := os.Stat(filepath.Join(root, "sneaky-dir"))
|
||||
assert.True(t, os.IsNotExist(statErr), "no directory must be created outside the bucket")
|
||||
})
|
||||
|
||||
// A legitimate object inside the bucket must still be readable.
|
||||
t.Run("LegitimateObject", func(t *testing.T) {
|
||||
b, _ := newTestBackend(t)
|
||||
obj, err := b.GetObject(ctx, "bucket", "object.txt", nil)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = obj.Contents.Close() }()
|
||||
contents, err := io.ReadAll(obj.Contents)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "normal object", string(contents))
|
||||
})
|
||||
|
||||
// A legitimate nested object must still be writable, creating its parent
|
||||
// directory inside the bucket.
|
||||
t.Run("LegitimateNestedPut", func(t *testing.T) {
|
||||
b, root := newTestBackend(t)
|
||||
const payload = "nested"
|
||||
_, err := b.PutObject(ctx, "bucket", "sub/dir/file.txt", map[string]string{}, strings.NewReader(payload), int64(len(payload)))
|
||||
require.NoError(t, err)
|
||||
got, readErr := os.ReadFile(filepath.Join(root, "bucket", "sub", "dir", "file.txt"))
|
||||
require.NoError(t, readErr)
|
||||
assert.Equal(t, payload, string(got))
|
||||
})
|
||||
}
|
||||
|
||||
// TestBucketObjectPath unit-tests the bucket/key join used by serve s3. Object
|
||||
// keys are opaque, so non-canonical keys (containing "..", ".", "//", or
|
||||
// leading/trailing slashes) are rejected rather than normalised.
|
||||
func TestBucketObjectPath(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
bucket, key string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"bucket", "object.txt", "bucket/object.txt", false},
|
||||
{"bucket", "a/b/c.txt", "bucket/a/b/c.txt", false},
|
||||
{"bucket", "a/../b.txt", "", true}, // distinct from b.txt, not normalised
|
||||
{"bucket", "a/./b.txt", "", true},
|
||||
{"bucket", "a//b.txt", "", true},
|
||||
{"bucket", "/leading", "", true},
|
||||
{"bucket", "trailing/", "", true},
|
||||
{"bucket", "", "", true},
|
||||
{"bucket", "../root-secret.txt", "", true},
|
||||
{"bucket", "../../etc/passwd", "", true},
|
||||
{"bucket", "../otherbucket/x", "", true},
|
||||
{"bucket", "a/../../escape", "", true},
|
||||
{"bucket", "..", "", true},
|
||||
} {
|
||||
got, err := bucketObjectPath(test.bucket, test.key)
|
||||
if test.wantErr {
|
||||
assert.Error(t, err, "bucket=%q key=%q", test.bucket, test.key)
|
||||
assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrInvalidArgument), "want 400 InvalidArgument for key=%q, got %v", test.key, err)
|
||||
} else {
|
||||
require.NoError(t, err, "bucket=%q key=%q", test.bucket, test.key)
|
||||
assert.Equal(t, test.want, got, "bucket=%q key=%q", test.bucket, test.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketDirPath unit-tests the directory-prefix join: the empty prefix
|
||||
// addresses the bucket root and a trailing slash is ignored, but traversal and
|
||||
// other non-canonical prefixes are still rejected.
|
||||
func TestBucketDirPath(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
bucket, dir string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{"bucket", "", "bucket", false}, // empty prefix addresses the bucket root
|
||||
{"bucket", "dir", "bucket/dir", false},
|
||||
{"bucket", "dir/", "", true}, // trailing slash not normalised away
|
||||
{"bucket", "a/b", "bucket/a/b", false},
|
||||
{"bucket", "../x", "", true},
|
||||
{"bucket", "a//b", "", true},
|
||||
{"bucket", "..", "", true},
|
||||
} {
|
||||
got, err := bucketDirPath(test.bucket, test.dir)
|
||||
if test.wantErr {
|
||||
assert.Error(t, err, "bucket=%q dir=%q", test.bucket, test.dir)
|
||||
} else {
|
||||
require.NoError(t, err, "bucket=%q dir=%q", test.bucket, test.dir)
|
||||
assert.Equal(t, test.want, got, "bucket=%q dir=%q", test.bucket, test.dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,8 +8,12 @@ import (
|
||||
"github.com/rclone/rclone/vfs"
|
||||
)
|
||||
|
||||
func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {
|
||||
fp := path.Join(bucket, fdPath)
|
||||
func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucketName, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error {
|
||||
fp, err := bucketDirPath(bucketName, fdPath)
|
||||
if err != nil {
|
||||
// A listing prefix that can't be represented as a path matches nothing.
|
||||
return gofakes3.ErrNoSuchKey
|
||||
}
|
||||
|
||||
dirEntries, err := getDirEntries(fp, _vfs)
|
||||
if err != nil {
|
||||
@@ -32,7 +36,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucket, fdPath, name string, addPr
|
||||
response.AddPrefix(prefixWithTrailingSlash)
|
||||
continue
|
||||
}
|
||||
err := b.entryListR(_vfs, bucket, path.Join(fdPath, object), "", false, response)
|
||||
err := b.entryListR(_vfs, bucketName, path.Join(fdPath, object), "", false, response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -13,9 +13,9 @@ func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPa
|
||||
sort.Slice(list.CommonPrefixes, func(i, j int) bool {
|
||||
return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix
|
||||
})
|
||||
// sort by modtime
|
||||
// sort by key name
|
||||
sort.Slice(list.Contents, func(i, j int) bool {
|
||||
return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time)
|
||||
return list.Contents[i].Key < list.Contents[j].Key
|
||||
})
|
||||
tokens := page.MaxKeys
|
||||
if tokens == 0 {
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/gofakes3"
|
||||
)
|
||||
|
||||
func TestPagerSortsContentsByKey(t *testing.T) {
|
||||
list := gofakes3.NewObjectList()
|
||||
list.Add(&gofakes3.Content{
|
||||
Key: "b.txt",
|
||||
LastModified: gofakes3.NewContentTime(time.Unix(100, 0)),
|
||||
})
|
||||
list.Add(&gofakes3.Content{
|
||||
Key: "a.txt",
|
||||
LastModified: gofakes3.NewContentTime(time.Unix(200, 0)),
|
||||
})
|
||||
|
||||
got, err := (&s3Backend{}).pager(list, gofakes3.ListBucketPage{MaxKeys: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Contents) != 2 {
|
||||
t.Fatalf("expected 2 contents, got %d", len(got.Contents))
|
||||
}
|
||||
|
||||
if got.Contents[0].Key != "a.txt" || got.Contents[1].Key != "b.txt" {
|
||||
t.Fatalf("expected lexicographic key order [a.txt b.txt], got [%s %s]", got.Contents[0].Key, got.Contents[1].Key)
|
||||
}
|
||||
}
|
||||
@@ -131,6 +131,17 @@ Versioning is not currently supported.
|
||||
Metadata will only be saved in memory other than the rclone `mtime`
|
||||
metadata which will be set as the modification time of the file.
|
||||
|
||||
### Object names
|
||||
|
||||
`serve s3` stores objects as files in the backend, so object keys are
|
||||
mapped to file paths rather than treated as the opaque strings AWS S3
|
||||
allows. Keys must be in canonical path form: keys that contain `..` or
|
||||
`.` path segments, repeated slashes (`//`), or a leading or trailing
|
||||
slash are rejected with a `400 Bad Request` (`InvalidArgument`)
|
||||
instead of being normalised, since normalising them could alias two
|
||||
distinct keys to the same file or resolve a key outside its bucket.
|
||||
This matches the behaviour of other S3 servers such as MinIO.
|
||||
|
||||
### Supported operations
|
||||
|
||||
`serve s3` currently supports the following operations.
|
||||
|
||||
@@ -86,6 +86,50 @@ func getFileHash(node any, hashType hash.Type) string {
|
||||
return hash
|
||||
}
|
||||
|
||||
// canonicalKey reports whether key is a non-empty path in canonical form: it
|
||||
// is equal to the path it cleans to, so it has no ".", ".." or empty
|
||||
// (repeated, leading or trailing slash) segments.
|
||||
//
|
||||
// The key is anchored with a leading "/" before cleaning so that a leading
|
||||
// ".." cannot silently escape upwards and still compare equal.
|
||||
func canonicalKey(key string) bool {
|
||||
return key != "" && "/"+key == path.Clean("/"+key)
|
||||
}
|
||||
|
||||
// errInvalidObjectName is returned for object keys that cannot be represented
|
||||
// as a backend path, for example because they contain "..", "." or "//"
|
||||
// segments. Like MinIO, this is reported as a 400 Bad Request rather than
|
||||
// silently resolving the key to a different object (or one outside the
|
||||
// bucket).
|
||||
func errInvalidObjectName(key string) error {
|
||||
return gofakes3.ErrorMessagef(gofakes3.ErrInvalidArgument, "Object name contains unsupported characters: %q", key)
|
||||
}
|
||||
|
||||
// bucketObjectPath joins the bucket name and object key into a backend path.
|
||||
//
|
||||
// S3 object keys are opaque, so rclone treats "dir/../file" and "file" as
|
||||
// distinct keys and refuses to normalise one into the other. Keys that are
|
||||
// not in canonical form (or that would escape the bucket) are rejected with
|
||||
// errInvalidObjectName rather than resolved.
|
||||
func bucketObjectPath(bucketName, objectName string) (string, error) {
|
||||
if !canonicalKey(objectName) {
|
||||
return "", errInvalidObjectName(objectName)
|
||||
}
|
||||
return path.Join(bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// bucketDirPath joins the bucket name and a directory path (a listing prefix)
|
||||
// into a backend path. It is like bucketObjectPath except that the empty path
|
||||
// is allowed and addresses the bucket root. Every other non-canonical path,
|
||||
// including one with a trailing slash, is rejected - it is not normalised, so
|
||||
// "dir" and "dir/" are not treated as the same directory.
|
||||
func bucketDirPath(bucketName, dirName string) (string, error) {
|
||||
if dirName == "" {
|
||||
return bucketName, nil
|
||||
}
|
||||
return bucketObjectPath(bucketName, dirName)
|
||||
}
|
||||
|
||||
func prefixParser(p *gofakes3.Prefix) (path, remaining string) {
|
||||
idx := strings.LastIndexByte(p.Prefix, '/')
|
||||
if idx < 0 {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Regression test for #8188: HEAD/GET during the VFS writeback window.
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/rclone/rclone/cmd/serve/proxy"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestHeadGetDuringWriteback uploads an object and immediately HEADs and GETs it
|
||||
// while it is still in the VFS write-back window — i.e. before the dirty file is
|
||||
// flushed to the backing remote, so node.DirEntry() is still nil. Previously
|
||||
// HeadObject/GetObject returned a spurious 404 (KeyNotFound) in that window even
|
||||
// though ListBucket already reported the object. A long --vfs-write-back keeps
|
||||
// the window open deterministically. Regression test for #8188.
|
||||
func TestHeadGetDuringWriteback(t *testing.T) {
|
||||
fstest.Initialise()
|
||||
ctx := context.Background()
|
||||
f, err := fs.NewFs(ctx, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
const bucket = "test"
|
||||
require.NoError(t, f.Mkdir(ctx, bucket))
|
||||
|
||||
keyid := random.String(16)
|
||||
keysec := random.String(16)
|
||||
opt := Opt
|
||||
opt.AuthKey = []string{fmt.Sprintf("%s,%s", keyid, keysec)}
|
||||
opt.HTTP.ListenAddr = []string{endpoint}
|
||||
|
||||
// Cache writes and hold the writeback so DirEntry() stays nil for the HEAD/GET.
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.CacheMode = vfscommon.CacheModeWrites
|
||||
vfsOpt.WriteBack = fs.Duration(2 * time.Minute)
|
||||
|
||||
w, err := newServer(ctx, f, &opt, &vfsOpt, &proxy.Opt)
|
||||
require.NoError(t, err)
|
||||
go func() { _ = w.Serve() }()
|
||||
t.Cleanup(func() { _ = w.Shutdown() })
|
||||
|
||||
u, err := url.Parse(w.server.URLs()[0])
|
||||
require.NoError(t, err)
|
||||
client, err := minio.New(u.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(keyid, keysec, ""),
|
||||
Secure: false,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const object = "writeback.txt"
|
||||
want := []byte("hello writeback window")
|
||||
_, err = client.PutObject(ctx, bucket, object, bytes.NewReader(want), int64(len(want)), minio.PutObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// HEAD must not 404 while the object is still being written back.
|
||||
info, err := client.StatObject(ctx, bucket, object, minio.StatObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(want)), info.Size)
|
||||
|
||||
// GET must return the body from the VFS cache, not 404.
|
||||
rc, err := client.GetObject(ctx, bucket, object, minio.GetObjectOptions{})
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rc)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rc.Close())
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
@@ -38,7 +38,22 @@ func (v vfsHandler) Fileread(r *sftp.Request) (io.ReaderAt, error) {
|
||||
}
|
||||
|
||||
func (v vfsHandler) Filewrite(r *sftp.Request) (io.WriterAt, error) {
|
||||
file, err := v.OpenFile(r.Filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
|
||||
// Respect the flags requested in the SFTP OPEN packet
|
||||
p := r.Pflags()
|
||||
flags := os.O_WRONLY
|
||||
if p.Append {
|
||||
flags |= os.O_APPEND
|
||||
}
|
||||
if p.Creat {
|
||||
flags |= os.O_CREATE
|
||||
}
|
||||
if p.Trunc {
|
||||
flags |= os.O_TRUNC
|
||||
}
|
||||
if p.Excl {
|
||||
flags |= os.O_EXCL
|
||||
}
|
||||
file, err := v.OpenFile(r.Filepath, flags, 0777)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -49,7 +64,18 @@ func (v vfsHandler) Filecmd(r *sftp.Request) error {
|
||||
switch r.Method {
|
||||
case "Setstat":
|
||||
attr := r.Attributes()
|
||||
if attr.Mtime != 0 {
|
||||
flags := r.AttrFlags()
|
||||
// A size attribute is a request to truncate the file
|
||||
if flags.Size {
|
||||
node, err := v.Stat(r.Filepath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := node.Truncate(int64(attr.Size)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if flags.Acmodtime {
|
||||
modTime := time.Unix(int64(attr.Mtime), 0)
|
||||
err := v.Chtimes(r.Filepath, modTime, modTime)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
// Test the SFTP serve handler against a real server and client.
|
||||
//
|
||||
// We skip tests on platforms with troublesome character mappings
|
||||
|
||||
//go:build !windows && !darwin && !plan9
|
||||
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/cmd/serve/proxy"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/vfs/vfscommon"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// startTestServer starts an sftp server serving a temporary local directory
|
||||
// with the given VFS options and returns a connected sftp client.
|
||||
func startTestServer(t *testing.T, vfsOpt *vfscommon.Options) *sftp.Client {
|
||||
ctx := context.Background()
|
||||
|
||||
f, err := fs.NewFs(ctx, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
opt := Opt
|
||||
opt.ListenAddr = testBindAddress
|
||||
opt.User = testUser
|
||||
opt.Pass = testPass
|
||||
|
||||
w, err := newServer(ctx, f, &opt, vfsOpt, &proxy.Opt)
|
||||
require.NoError(t, err)
|
||||
go func() {
|
||||
_ = w.Serve()
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
assert.NoError(t, w.Shutdown())
|
||||
})
|
||||
|
||||
clientConfig := &ssh.ClientConfig{
|
||||
User: testUser,
|
||||
Auth: []ssh.AuthMethod{ssh.Password(testPass)},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
}
|
||||
conn, err := ssh.Dial("tcp", w.Addr().String(), clientConfig)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
_ = conn.Close()
|
||||
})
|
||||
|
||||
client, err := sftp.NewClient(conn)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
_ = client.Close()
|
||||
})
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
// Test that re-opening a file for write without the truncate flag preserves
|
||||
// the data already written, rather than zeroing the start of the file.
|
||||
//
|
||||
// This reproduces a corruption seen with WinSCP "Process in Background", which
|
||||
// resumes an upload on a second connection by re-opening the partial file
|
||||
// without SSH_FXF_TRUNC and writing from the offset it had reached.
|
||||
func TestFilewriteResumeNoTruncate(t *testing.T) {
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.CacheMode = vfscommon.CacheModeWrites
|
||||
client := startTestServer(t, &vfsOpt)
|
||||
|
||||
const (
|
||||
fileName = "file.bin"
|
||||
total = 1024 * 1024 // 1 MiB
|
||||
split = 700 * 1024 // where the upload is "backgrounded"
|
||||
)
|
||||
|
||||
// Deterministic non-zero contents so a zeroed hole is easy to spot
|
||||
contents := make([]byte, total)
|
||||
for i := range contents {
|
||||
contents[i] = byte(i%251 + 1)
|
||||
}
|
||||
|
||||
// First connection: write the first part with truncate, as a fresh upload
|
||||
f, err := client.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
|
||||
require.NoError(t, err)
|
||||
_, err = f.Write(contents[:split])
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
// Resume: re-open WITHOUT truncate and write the remainder at its offset
|
||||
f, err = client.OpenFile(fileName, os.O_WRONLY|os.O_CREATE)
|
||||
require.NoError(t, err)
|
||||
_, err = f.WriteAt(contents[split:], int64(split))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
// Read the file back and check it is intact - no zeroed prefix
|
||||
rd, err := client.Open(fileName)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rd)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rd.Close())
|
||||
|
||||
require.Equal(t, total, len(got), "wrong file size")
|
||||
assert.Equal(t, contents, got, "file contents corrupted on resumed upload")
|
||||
}
|
||||
|
||||
// Test that opening a file for write with the truncate flag does truncate any
|
||||
// existing data, which is the normal overwrite case for most clients.
|
||||
func TestFilewriteTruncate(t *testing.T) {
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.CacheMode = vfscommon.CacheModeWrites
|
||||
client := startTestServer(t, &vfsOpt)
|
||||
|
||||
const fileName = "file.bin"
|
||||
|
||||
// Write some initial long contents
|
||||
require.NoError(t, writeFile(client, fileName, strings.Repeat("A", 1024)))
|
||||
|
||||
// Overwrite with shorter contents using truncate
|
||||
const newContents = "hello"
|
||||
require.NoError(t, writeFile(client, fileName, newContents))
|
||||
|
||||
rd, err := client.Open(fileName)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rd)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rd.Close())
|
||||
|
||||
assert.Equal(t, newContents, string(got))
|
||||
}
|
||||
|
||||
// Test that a SETSTAT request with a size attribute truncates the file, rather
|
||||
// than being silently ignored.
|
||||
func TestSetstatTruncate(t *testing.T) {
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.CacheMode = vfscommon.CacheModeWrites
|
||||
client := startTestServer(t, &vfsOpt)
|
||||
|
||||
const fileName = "file.bin"
|
||||
|
||||
// Write a file and then truncate it via FSETSTAT (a size attribute) on the
|
||||
// open handle, the way a client setting the final size of an upload does.
|
||||
f, err := client.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
|
||||
require.NoError(t, err)
|
||||
_, err = f.Write([]byte(strings.Repeat("A", 1024)))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Truncate(10))
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
fi, err := client.Stat(fileName)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(10), fi.Size(), "file not truncated to requested size")
|
||||
|
||||
rd, err := client.Open(fileName)
|
||||
require.NoError(t, err)
|
||||
got, err := io.ReadAll(rd)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rd.Close())
|
||||
assert.Equal(t, strings.Repeat("A", 10), string(got))
|
||||
}
|
||||
|
||||
// writeFile writes contents to fileName via the client truncating any existing
|
||||
// data, the way a normal upload does.
|
||||
func writeFile(client *sftp.Client, fileName, contents string) error {
|
||||
f, err := client.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write([]byte(contents)); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
@@ -391,6 +391,12 @@ func (w *WebDAV) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
|
||||
w.serveDir(rw, r, remote)
|
||||
return
|
||||
}
|
||||
// Work around bug in x/net/webdav which handles Overwrite incorrectly
|
||||
// See: https://github.com/golang/go/issues/66059
|
||||
// Remove when the above bug is fixed.
|
||||
if (r.Method == "COPY" || r.Method == "MOVE") && r.Header.Get("Overwrite") == "" {
|
||||
r.Header.Set("Overwrite", "T")
|
||||
}
|
||||
// Add URL Prefix back to path since webdavhandler needs to
|
||||
// return absolute references.
|
||||
r.URL.Path = w.opt.HTTP.BaseURL + r.URL.Path
|
||||
|
||||
@@ -271,3 +271,89 @@ func TestRc(t *testing.T) {
|
||||
"vfs_cache_mode": "off",
|
||||
})
|
||||
}
|
||||
|
||||
// startWritableServer starts a webdav server backed by a fresh temp
|
||||
// directory and returns the server URL. It is used by the Overwrite tests
|
||||
// which need to exercise mutating verbs such as MKCOL and MOVE.
|
||||
func startWritableServer(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
f, err := fs.NewFs(context.Background(), t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
opt := Opt
|
||||
opt.HTTP.ListenAddr = []string{testBindAddress}
|
||||
|
||||
w, err := newWebDAV(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)
|
||||
require.NoError(t, err)
|
||||
go func() {
|
||||
require.NoError(t, w.Serve())
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
assert.NoError(t, w.Shutdown())
|
||||
})
|
||||
|
||||
return w.server.URLs()[0]
|
||||
}
|
||||
|
||||
func mkcol(t *testing.T, baseURL, path string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest("MKCOL", baseURL+path, nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
_ = resp.Body.Close()
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode, "MKCOL %s", path)
|
||||
}
|
||||
|
||||
// TestMoveDefaultsToOverwrite is a regression test for
|
||||
// https://github.com/rclone/rclone/issues/9496
|
||||
//
|
||||
// RFC 4918 section 10.6 requires that when the Overwrite header is omitted
|
||||
// from a COPY or MOVE request, the resource MUST behave as if Overwrite: T
|
||||
// had been sent. The upstream golang.org/x/net/webdav library mis-handles
|
||||
// the MOVE case (see https://github.com/golang/go/issues/66059), so rclone
|
||||
// normalises the header before delegating so the default matches the RFC.
|
||||
func TestMoveDefaultsToOverwrite(t *testing.T) {
|
||||
testURL := startWritableServer(t)
|
||||
|
||||
mkcol(t, testURL, "dir1")
|
||||
mkcol(t, testURL, "dir2")
|
||||
|
||||
// MOVE without Overwrite header: per RFC 4918 the default is T, so the
|
||||
// existing destination must be replaced and the server must return 2xx.
|
||||
req, err := http.NewRequest("MOVE", testURL+"dir2", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Destination", testURL+"dir1")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
assert.NotEqual(t, http.StatusPreconditionFailed, resp.StatusCode,
|
||||
"MOVE without Overwrite header must not return 412; RFC 4918 default is Overwrite: T")
|
||||
assert.True(t, resp.StatusCode >= 200 && resp.StatusCode < 300,
|
||||
"expected 2xx, got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// TestMoveOverwriteFalseStillRejects ensures the rclone normalisation only
|
||||
// fills in a missing Overwrite header and never overrides an explicit
|
||||
// Overwrite: F sent by the client.
|
||||
func TestMoveOverwriteFalseStillRejects(t *testing.T) {
|
||||
testURL := startWritableServer(t)
|
||||
|
||||
mkcol(t, testURL, "dir1")
|
||||
mkcol(t, testURL, "dir2")
|
||||
|
||||
req, err := http.NewRequest("MOVE", testURL+"dir2", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Destination", testURL+"dir1")
|
||||
req.Header.Set("Overwrite", "F")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
assert.Equal(t, http.StatusPreconditionFailed, resp.StatusCode,
|
||||
"MOVE with explicit Overwrite: F must still return 412 when destination exists")
|
||||
}
|
||||
+1
-1
@@ -314,7 +314,7 @@ start and finish the upload) and another 2 requests for each chunk:
|
||||
/b2api/v1/b2_finish_large_file
|
||||
```
|
||||
|
||||
#### Versions
|
||||
### Viewing versions with --b2-versions
|
||||
|
||||
Versions can be viewed with the `--b2-versions` flag. When it is set
|
||||
rclone will show and act on older versions of files. For example
|
||||
|
||||
+38
-9
@@ -477,9 +477,9 @@ their checksums would change from run to run (due to small variances in the
|
||||
internals of the generated export file.) Therefore, bisync automatically skips
|
||||
`--download-hash` for files with a size less than 0.
|
||||
|
||||
See also: [`Hasher`](https://rclone.org/hasher/) backend,
|
||||
[`cryptcheck`](/commands/rclone_cryptcheck/) command, [`rclone check
|
||||
--download`](/commands/rclone_check/) option,
|
||||
See also: [`Hasher`](/hasher/) backend,
|
||||
[`cryptcheck`](/commands/rclone_cryptcheck/) command,
|
||||
[`rclone check --download`](/commands/rclone_check/) option,
|
||||
[`md5sum`](/commands/rclone_md5sum/) command
|
||||
|
||||
### --max-delete
|
||||
@@ -1053,9 +1053,28 @@ encodings.)
|
||||
The following backends have known issues that need more investigation:
|
||||
|
||||
<!--- start list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->
|
||||
- `TestDropbox` (`dropbox`)
|
||||
- [`TestBisyncRemoteRemote/normalization`](https://pub.rclone.org/integration-tests/current/dropbox-cmd.bisync-TestDropbox-1.txt)
|
||||
- Updated: 2026-05-01-010013
|
||||
- `TestGoogleCloudStorage,directory_markers` (`googlecloudstorage`)
|
||||
- [`TestBisyncRemoteLocal/all_changed`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/backupdir`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/basic`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/changes`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [`TestBisyncRemoteLocal/check_access`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage,directory_markers-1.txt)
|
||||
- [79 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- `TestGoogleCloudStorage` (`googlecloudstorage`)
|
||||
- [`TestBisyncRemoteLocal/all_changed`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/backupdir`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/basic`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/changes`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [`TestBisyncRemoteLocal/check_access`](https://pub.rclone.org/integration-tests/current/googlecloudstorage-cmd.bisync-TestGoogleCloudStorage-1.txt)
|
||||
- [79 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- `TestHuaweiDrive` (`huaweidrive`)
|
||||
- [`TestBisyncRemoteLocal/ext_paths`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncRemoteLocal/extended_filenames`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncRemoteLocal/normalization`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncLocalRemote/ext_paths`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [`TestBisyncLocalRemote/extended_filenames`](https://pub.rclone.org/integration-tests/current/huaweidrive-cmd.bisync-TestHuaweiDrive-1.txt)
|
||||
- [4 more](https://pub.rclone.org/integration-tests/current/)
|
||||
- Updated: 2026-07-08-010014
|
||||
<!--- end list_failures - DO NOT EDIT THIS SECTION - use make commanddocs --->
|
||||
|
||||
The following backends either have not been tested recently or have known issues
|
||||
@@ -1120,8 +1139,8 @@ a mechanism to mark files as needing to be internally rechecked next time, for
|
||||
added safety. It should therefore no longer be necessary to sync only at quiet
|
||||
times -- however, note that an error can still occur if a file happens to change
|
||||
at the exact moment it's being read/written by bisync (same as would happen in
|
||||
`rclone sync`.) (See also: [`--ignore-checksum`](https://rclone.org/docs/#ignore-checksum),
|
||||
[`--local-no-check-updated`](https://rclone.org/local/#local-no-check-updated))
|
||||
`rclone sync`.) (See also: [`--ignore-checksum`](/docs/#ignore-checksum),
|
||||
[`--local-no-check-updated`](/local/#local-no-check-updated))
|
||||
|
||||
### Empty directories
|
||||
|
||||
@@ -1161,6 +1180,11 @@ is to rename it to the same name on both sides. (As of `rclone v1.64`,
|
||||
a `--resync` is no longer required after doing so, as bisync will automatically
|
||||
detect that Path1 and Path2 are in agreement.)
|
||||
|
||||
Note that although the flag --track-renames ensures that renamed/moved files won't
|
||||
be deleted and uploaded again, they are still counted as deleted files for purposes
|
||||
of the --max-delete flag (as this check happens before the rename detection
|
||||
operation). See [this issue](https://github.com/rclone/rclone/issues/8685).
|
||||
|
||||
### `--fast-list` used by default
|
||||
|
||||
Unlike most other rclone commands, bisync uses [`--fast-list`](/docs/#fast-list)
|
||||
@@ -1494,7 +1518,7 @@ and in most cases it's probably not what you want!
|
||||
|
||||
To bisync Google Docs as URL shortcut links (in a manner similar to "Drive for
|
||||
Desktop"), use: `--drive-export-formats url` (or
|
||||
[alternatives](https://rclone.org/drive/#exportformats:~:text=available%20Google%20Documents.-,Extension,macOS,-Standard%20options).)
|
||||
[alternatives](/drive/#exportformats:~:text=available%20Google%20Documents.-,Extension,macOS,-Standard%20options).)
|
||||
|
||||
Note that these link files cannot be edited on the non-drive side -- you will
|
||||
get errors if you try to sync an edited link file back to drive. They CAN be
|
||||
@@ -1899,6 +1923,11 @@ about *Unison* and synchronization in general.
|
||||
|
||||
## Changelog
|
||||
|
||||
### `v1.74.2`
|
||||
|
||||
- Fixed an issue causing `--conflict-loser pathname` to produce unexpected
|
||||
behavior if using a non-default `--conflict-resolve` value.
|
||||
|
||||
### `v1.74`
|
||||
|
||||
- Added several missing `rc` parameters.
|
||||
|
||||
+182
-3
@@ -6,6 +6,185 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.74.4 - 2026-07-08
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.3...v1.74.4)
|
||||
|
||||
- Bug Fixes
|
||||
- accounting
|
||||
- Fix goroutine leak in ResetCounters (Nick Craig-Wood)
|
||||
- Fix goroutine leak in NewStatsGroup for zero-transfer rc jobs (Sanjays2402)
|
||||
- archive extract: Fix path traversal letting archives escape the destination CVE-2026-59732 (Nick Craig-Wood)
|
||||
- build
|
||||
- Fix multiple CVEs by upgrading to go1.26.5 (Nick Craig-Wood)
|
||||
- CVE-2026-39822: os: Root escape via symlink plus trailing slash
|
||||
- CVE-2026-42505: crypto/tls: Encrypted Client Hello privacy leak
|
||||
- Update golang.org/x/image to v0.43.0 to fix image decoding vulnerabilities (Nick Craig-Wood)
|
||||
- CVE-2026-46604: panic decoding a TIFF image with an out-of-bounds strip offset
|
||||
- CVE-2026-46602: unbounded memory use from lack of a limit on TIFF tile sizes
|
||||
- CVE-2026-46601: panic on a WEBP VP8 alpha channel size mismatch
|
||||
- CVE-2026-33813: panic decoding a large WEBP image on 32-bit platforms
|
||||
- cmd/mount2
|
||||
- Fix NFS file creation by implementing Mknod (Sandy Luppino)
|
||||
- Fix ESTALE over NFS by reporting stable inode numbers (Sandy Luppino)
|
||||
- Fix NFS directory listings by supporting non-zero Seekdir offsets (Sandy Luppino)
|
||||
- completion: Fix powershell completion corrupting non-ASCII names (Yash Anil)
|
||||
- doc fixes (Bryan Stenson, Castronaut, Filippo, Gaurav, happysnaker, Jan-Philipp Reßler, Nick Craig-Wood, user77)
|
||||
- filter: Fix `--files-from` copy stopping at the first unreadable file (Nick Craig-Wood)
|
||||
- fs
|
||||
- Fix command line flag being ignored when set to its default value (Nick Craig-Wood)
|
||||
- Fix negative offset when a suffix Range request exceeds object size (Amit Mishra)
|
||||
- gui: Update embedded release to 1.1.10 (Nick Craig-Wood)
|
||||
- ncdu: Fix duplicated keystrokes on Windows by pinning tcell to v2.9.0 (Nick Craig-Wood)
|
||||
- serve restic: Fix `--private-repos` isolation bypass CVE-2026-59733 (Nick Craig-Wood)
|
||||
- serve s3
|
||||
- Fix spurious 404 on HEAD/GET during VFS writeback (max)
|
||||
- Fix path traversal letting clients see files in the root GHSA-8v25-v8p6-qf7v (Nick Craig-Wood)
|
||||
- serve webdav: Fix MOVE overwrite failing without Overwrite header (Sanjay Santhanam)
|
||||
- serve/http: Fix `--disable-zip` so it works over rc (Nick Craig-Wood)
|
||||
- VFS
|
||||
- Fix hang reopening a file during the handle-caching grace period (Nick Craig-Wood)
|
||||
- Local
|
||||
- Stop `--links` symlinks escaping the destination directory CVE-2026-54572 (Nick Craig-Wood)
|
||||
- Don't restore setuid/setgid/sticky bits from metadata by default GHSA-945v-v9p3-v5xw (Nick Craig-Wood)
|
||||
- Drive
|
||||
- Warn when non-exportable Google documents are skipped (Nick Craig-Wood)
|
||||
- Fix stray %!(EXTRA) in unexportable google document log message (Nick Craig-Wood)
|
||||
- Warn when using rclone's shared client_id (Nick Craig-Wood)
|
||||
- Filelu
|
||||
- Fix recursive listing path handling and file filtering (kingston125)
|
||||
- Googlephotos
|
||||
- Warn when using rclone's shared client_id (Nick Craig-Wood)
|
||||
- Mega
|
||||
- Wait for server events after upload, delete and move (Nick Craig-Wood)
|
||||
- Fix hard deleted files reappearing in listings (Nick Craig-Wood)
|
||||
- S3
|
||||
- Remove session token on cross-host redirects (IceLocke)
|
||||
- Strip STS security token on same-host HTTPS->HTTP redirect GHSA-cf44-9pgv-m4xc (Nick Craig-Wood)
|
||||
- Fix error mapping in GetObject to match HeadObject (lewoberst)
|
||||
- Correct documented `copy_cutoff` minimum to 1 byte (max)
|
||||
- Fix mounting a prefix failing with 403 when HEAD is not permitted (Nick Craig-Wood)
|
||||
- Smb
|
||||
- Fix for IBM iSeries and signature verification (dithwick)
|
||||
- WebDAV
|
||||
- Fix mixed property statuses in multi-status responses (nako-ruru)
|
||||
|
||||
## v1.74.3 - 2026-06-05
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.2...v1.74.3)
|
||||
|
||||
- Bug Fixes
|
||||
- rc
|
||||
- Fix unauthenticated command execution via `--rc-serve` inline remotes CVE-2026-49980 (Nick Craig-Wood)
|
||||
- Stop `global.*` connection string options changing config CVE-2026-49980 (Nick Craig-Wood)
|
||||
- build: Fix multiple CVEs by upgrading to go1.26.4 (Nick Craig-Wood)
|
||||
- CVE-2026-42504: mime: quadratic complexity in WordDecoder.DecodeHeader
|
||||
- CVE-2026-42507: net/textproto: arbitrary input are included in errors without any escaping
|
||||
- CVE-2026-27145: crypto/x509: split candidate hostname only once
|
||||
- log: Fix wrong source `file:line` in JSON logs from release builds (Nick Craig-Wood)
|
||||
- mount2: Fix empty directory listings on re-read (Janne Beate Bakeng)
|
||||
- serve s3: Fix multipart `ListParts` pagination returning wrong part numbers (Nick Craig-Wood)
|
||||
- serve sftp
|
||||
- Fix file corruption when a client resumes an upload (Nick Craig-Wood)
|
||||
- Fix truncate request being silently ignored (Nick Craig-Wood)
|
||||
- Local
|
||||
- Fix `getXattr` returning empty map instead of nil (Leon Brocard)
|
||||
- Drime
|
||||
- Fix server-side copy and move failing with Cloudflare 520 error (Nick Craig-Wood)
|
||||
- Fix files being uploaded to the wrong directory (Nick Craig-Wood)
|
||||
- Remove duplicate upload_cutoff config option (Nick Craig-Wood)
|
||||
- Fix directory rename leaving the renamed folder empty in VFS (Nick Craig-Wood)
|
||||
- Drive
|
||||
- Fix server-side move failing on shared drives with duplicate dirs (Nick Craig-Wood)
|
||||
- Iclouddrive
|
||||
- Fix ADP/PCS cookie acquisition for iCloud Drive (Yakov Till)
|
||||
- Fix "Index has invalid data" error listing iCloud Photos (Nick Craig-Wood)
|
||||
|
||||
## v1.74.2 - 2026-05-22
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.1...v1.74.2)
|
||||
|
||||
- Bug Fixes
|
||||
- build
|
||||
- Update golang.org/x/net to v0.55.0 to address:
|
||||
- CVE-2026-42506: html: incorrect handling of namespaced elements in foreign content
|
||||
- CVE-2026-39821: idna: failure to reject ASCII-only Punycode-encoded labels
|
||||
- CVE-2026-42502: html: incorrect handling of HTML elements in foreign content
|
||||
- CVE-2026-25680: html: denial of service when parsing arbitrary HTML
|
||||
- CVE-2026-25681: html: incorrect handling of character references in DOCTYPE nodes
|
||||
- CVE-2026-27136: html: duplicate attributes can cause XSS
|
||||
- Update golang.org/x/crypto to v0.52.0 to address:
|
||||
- CVE-2026-46598: ssh/agent: pathological inputs can lead to client panic
|
||||
- CVE-2026-46597: ssh: byte arithmetic causes underflow and panic
|
||||
- CVE-2026-39828: ssh: bypass of certificate restrictions
|
||||
- CVE-2026-39835: ssh: server panic during CheckHostKey/Authenticate
|
||||
- CVE-2026-39833: ssh/agent: key constraints not enforced
|
||||
- CVE-2026-39832: ssh/agent: agent constraints dropped when forwarding keys
|
||||
- CVE-2026-39827: ssh: memory leak when rejecting channels can lead to DoS
|
||||
- CVE-2026-39830: ssh: client can cause server deadlock on unexpected responses
|
||||
- CVE-2026-39829: ssh: pathological RSA/DSA parameters may cause DoS
|
||||
- CVE-2026-39831: ssh: bypass of FIDO/U2F security keys physical interaction
|
||||
- CVE-2026-39834: ssh: infinite loop on large channel writes
|
||||
- CVE-2026-42508: ssh/knownhosts: auth bypass via unenforced @revoked status
|
||||
- CVE-2026-46595: ssh: VerifiedPublicKeyCallback permissions skip enforcement
|
||||
- Update golang.org/x/image to v0.41.0 to address:
|
||||
- CVE-2026-42500: bmp: panic when reading out of bound palette index
|
||||
- CVE-2026-33809: tiff: excessive resource consumption in PackBits decompression
|
||||
- Update golang.org/x/sys to version v0.45.0 to address:
|
||||
- CVE-2026-39824: windows: integer overflow in NewNTUnicodeString
|
||||
- Update github.com/go-git/go-billy/v5 to 5.9.0 to fix CVE-2026-44740
|
||||
- bisync: Fix --conflict-loser pathname with --conflict-resolve newer (nielash)
|
||||
- gui: Update embedded release to 1.1.8 (Nick Craig-Wood)
|
||||
- lib/http: Replace deprecated h2c.NewHandler with http.Server.Protocols (Nick Craig-Wood)
|
||||
- rc: Remove duplicate metrics_addr option registration (Nick Craig-Wood)
|
||||
- vfs/vfscache: Fix silent write failure when mounting with remote:. (Lucky945H)
|
||||
- doc fixes (FTCHD, Iizuki, Leon Brocard, Nick Craig-Wood)
|
||||
- Drime
|
||||
- Fix file doesn't exists error when trying to delete (John Volk)
|
||||
- Fix 500 errors when listing shared folders (Alvinwylim)
|
||||
- Jottacloud
|
||||
- Support whitelabel service Phonero Sky (Tore Anderson)
|
||||
- Protondrive
|
||||
- Fix corrupted on transfer: sha1 hashes differ (William Tange)
|
||||
- S3
|
||||
- Add new MEGA S4 endpoints on megas4.com including Asia-Pacific region (Nick Craig-Wood)
|
||||
- WebDAV
|
||||
- Honour auth_redirect on listAll PROPFIND (Sai Asish Y)
|
||||
|
||||
## v1.74.1 - 2026-05-08
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.74.0...v1.74.1)
|
||||
|
||||
- Bug Fixes
|
||||
- bisync: Fix retryable without `--resync` error message when `--resync` has a critical failure (Gustavo V. F.)
|
||||
- build
|
||||
- Fix multiple CVEs by upgrading to go1.26.3 (Nick Craig-Wood)
|
||||
- CVE-2026-42501: cmd/go: malicious module proxy can bypass checksum database
|
||||
- CVE-2026-39825: net/http/httputil: ReverseProxy forwards queries with more than urlmaxqueryparams parameters
|
||||
- CVE-2026-39836: net: panic in Dial and LookupPort when handling NUL byte on Windows
|
||||
- CVE-2026-42499: net/mail: quadratic string concatenation in consumePhrase
|
||||
- CVE-2026-39820: net/mail: quadratic string concatentation in consumeComment
|
||||
- CVE-2026-39819: cmd/go: "go bug" follows symlinks in predictable temporary filenames
|
||||
- CVE-2026-39817: cmd/go: "go tool pack" does not sanitize output paths
|
||||
- CVE-2026-33814: net/http: infinite loop in HTTP/2 transport when given bad SETTINGS_MAX_FRAME_SIZE
|
||||
- CVE-2026-39826: html/template: escaper bypass leads to XSS
|
||||
- CVE-2026-33811: net: crash when handling long CNAME response
|
||||
- CVE-2026-39823: html/template: bypass of meta content URL escaping causes XSS
|
||||
- Update golang.org/x/net to v0.53.0 to fix CVE-2026-33814 (Nick Craig-Wood)
|
||||
- cmd/serve/s3: Return object listings in key order (Leon Brocard)
|
||||
- Cloudinary
|
||||
- Fix retrying every error and fix pacer sleep units (Nick Craig-Wood)
|
||||
- Drime
|
||||
- Fix large file uploads landing in drive root instead of configured folder (Nick Craig-Wood)
|
||||
- Fix uploads of 100..200M files (Nick Craig-Wood)
|
||||
- Protondrive
|
||||
- Route HTTP through rclone's transport (Nick Craig-Wood)
|
||||
- Route library logging through rclone's logger (Nick Craig-Wood)
|
||||
- Fix segfault when copying files missing revision metadata (Nick Craig-Wood)
|
||||
- S3
|
||||
- Fix STS call per request by caching AssumeRole credentials (Nick Craig-Wood)
|
||||
- Add new Fastly Object Storage regions (Leon Brocard)
|
||||
|
||||
## v1.74.0 - 2026-05-01
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.73.0...v1.74.0)
|
||||
@@ -506,7 +685,7 @@ description: "Rclone Changelog"
|
||||
- [OVHcloud Object Storage](/s3/#ovhcloud) (Florent Vennetier)
|
||||
- [Zata](/s3/#Zata) ($@M@RTH_)
|
||||
- New Features
|
||||
- Allow [global config to be overriden or set on backend creation](/docs/#globalconfig) (Nick Craig-Wood)
|
||||
- Allow [global config to be overridden or set on backend creation](/docs/#globalconfig) (Nick Craig-Wood)
|
||||
- bisync: Promoted from beta to stable (nielash)
|
||||
- build
|
||||
- Update to go1.25 and make go1.24 the minimum required version (Nick Craig-Wood)
|
||||
@@ -573,7 +752,7 @@ description: "Rclone Changelog"
|
||||
- This allows `--ca-cert`, `--client-cert` etc to be used.
|
||||
- This also allows `override.ca_cert = XXX` to be used in the config file.
|
||||
- Googlephotos
|
||||
- Added warning for Google Photos compatability-fixes (raider13209)
|
||||
- Added warning for Google Photos compatibility-fixes (raider13209)
|
||||
- Imagekit
|
||||
- Return correct error when attempting to upload zero length files (Nick Craig-Wood)
|
||||
- Don't low level retry uploads (Nick Craig-Wood)
|
||||
@@ -1930,7 +2109,7 @@ instead of of `--size-only`, when `check` is not available.
|
||||
- Add `--drive-env-auth` to get IAM credentials from runtime (Peter Brunner)
|
||||
- Update drive service account guide (Juang, Yi-Lin)
|
||||
- Fix change notify picking up files outside the root (Nick Craig-Wood)
|
||||
- Fix trailing slash mis-identificaton of folder as file (Nick Craig-Wood)
|
||||
- Fix trailing slash mis-identification of folder as file (Nick Craig-Wood)
|
||||
- Fix incorrect remote after Update on object (Nick Craig-Wood)
|
||||
- Dropbox
|
||||
- Implement `--dropbox-pacer-min-sleep` flag (Nick Craig-Wood)
|
||||
|
||||
@@ -232,7 +232,7 @@ rclone [flags]
|
||||
--drime-list-chunk int Number of items to list in each call (default 1000)
|
||||
--drime-root-folder-id string ID of the root folder
|
||||
--drime-upload-concurrency int Concurrency for multipart uploads and copies (default 4)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 5Mi)
|
||||
--drime-workspace-id string Account ID
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded
|
||||
--drive-allow-import-name-change Allow the filetype to change when uploading Google docs
|
||||
@@ -558,8 +558,10 @@ rclone [flags]
|
||||
--local-case-sensitive Force the filesystem to report itself as case sensitive
|
||||
--local-description string Description of the remote
|
||||
--local-encoding Encoding The encoding for the backend (default Slash,Dot)
|
||||
--local-fatal-if-no-space Make out-of-space errors fatal during transfers
|
||||
--local-hashes CommaSepList Comma separated list of supported checksum types
|
||||
--local-links Translate symlinks to/from regular files with a '.rclonelink' extension for the local backend
|
||||
--local-metadata-restore-special-bits Restore the setuid, setgid and sticky bits from metadata
|
||||
--local-no-check-updated Don't check to see if the files change during upload
|
||||
--local-no-clone Disable reflink cloning for server-side copies
|
||||
--local-no-preallocate Disable preallocation of disk space for transferred files
|
||||
@@ -624,7 +626,7 @@ rclone [flags]
|
||||
--metadata-include-from stringArray Read metadata include patterns from file (use - to read from stdin)
|
||||
--metadata-mapper SpaceSepList Program to run to transforming metadata before upload
|
||||
--metadata-set stringArray Add metadata key=value when uploading
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind metrics server to
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind server to
|
||||
--metrics-allow-origin string Origin which cross-domain request (CORS) can be executed from
|
||||
--metrics-baseurl string Prefix for URLs - leave blank for root
|
||||
--metrics-cert string TLS PEM key (concatenation of certificate and CA certificate)
|
||||
@@ -1094,7 +1096,7 @@ rclone [flags]
|
||||
--use-json-log Use json log format
|
||||
--use-mmap Use mmap allocator (see docs)
|
||||
--use-server-modtime Use server modified time instead of object metadata
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.4")
|
||||
-v, --verbose count Print lots more stuff (repeat for more)
|
||||
-V, --version Print the version number
|
||||
--webdav-auth-redirect Preserve authentication on redirect
|
||||
|
||||
@@ -231,12 +231,12 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,command=e
|
||||
|
||||
```console
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20260501
|
||||
// Output: stories/The Quick Brown Fox!-20260708
|
||||
```
|
||||
|
||||
```console
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2026-05-01 0208PM
|
||||
// Output: stories/The Quick Brown Fox!-2026-07-08 0504PM
|
||||
```
|
||||
|
||||
```console
|
||||
|
||||
@@ -13,8 +13,9 @@ Copy files from source to dest, skipping identical files.
|
||||
If source:path is a file or directory then it copies it to a file or
|
||||
directory named dest:path.
|
||||
|
||||
This can be used to upload single files to other than their current
|
||||
name. If the source is a directory then it acts exactly like the
|
||||
This can be used to copy a single file to a destination with a name
|
||||
different from its source - for example, uploading and renaming in one
|
||||
step. If the source is a directory then it acts exactly like the
|
||||
[copy](/commands/rclone_copy/) command.
|
||||
|
||||
So
|
||||
|
||||
@@ -144,6 +144,17 @@ Versioning is not currently supported.
|
||||
Metadata will only be saved in memory other than the rclone `mtime`
|
||||
metadata which will be set as the modification time of the file.
|
||||
|
||||
## Object names
|
||||
|
||||
`serve s3` stores objects as files in the backend, so object keys are
|
||||
mapped to file paths rather than treated as the opaque strings AWS S3
|
||||
allows. Keys must be in canonical path form: keys that contain `..` or
|
||||
`.` path segments, repeated slashes (`//`), or a leading or trailing
|
||||
slash are rejected with a `400 Bad Request` (`InvalidArgument`)
|
||||
instead of being normalised, since normalising them could alias two
|
||||
distinct keys to the same file or resolve a key outside its bucket.
|
||||
This matches the behaviour of other S3 servers such as MinIO.
|
||||
|
||||
## Supported operations
|
||||
|
||||
`serve s3` currently supports the following operations.
|
||||
|
||||
@@ -787,9 +787,9 @@ This uses a 32 byte (256 bit key) key derived from the user password.
|
||||
1 MiB (1048576 bytes) file will encrypt to
|
||||
|
||||
- 32 bytes header
|
||||
- 16 chunks of 65568 bytes
|
||||
- 16 chunks of 65552 bytes
|
||||
|
||||
1049120 bytes total (a 0.05% overhead). This is the overhead for big
|
||||
1048864 bytes total (a 0.03% overhead). This is the overhead for big
|
||||
files.
|
||||
|
||||
### Name encryption
|
||||
|
||||
@@ -1212,7 +1212,7 @@ Windows and `/dev/null` on Unix systems, then rclone will keep the
|
||||
configuration file in memory only.
|
||||
|
||||
You may see a log message "Config file not found - using defaults" if there is
|
||||
no configuration file. This can be supressed, e.g. if you are using rclone
|
||||
no configuration file. This can be suppressed, e.g. if you are using rclone
|
||||
entirely with [on the fly remotes](/docs/#backend-path-to-dir), by using
|
||||
memory-only configuration file or by creating an empty configuration file, as
|
||||
described above.
|
||||
@@ -1824,7 +1824,7 @@ warnings and significant events.
|
||||
|
||||
See also the [logging](#logging) section.
|
||||
|
||||
### --windows-event-log LogLevel
|
||||
### --windows-event-log-level LogLevel
|
||||
|
||||
If this is configured (the default is `OFF`) then logs of this level
|
||||
and above will be logged to the Windows event log in **addition** to
|
||||
@@ -1840,15 +1840,15 @@ and `Error`. If enabled we map rclone levels like this.
|
||||
|
||||
Rclone will declare its log source as "rclone" if it is has enough
|
||||
permissions to create the registry key needed. If not then logs will
|
||||
appear as "Application". You can run `rclone version --windows-event-log DEBUG`
|
||||
appear as "Application". You can run `rclone version --windows-event-log-level DEBUG`
|
||||
once as administrator to create the registry key in advance.
|
||||
|
||||
**Note** that the `--windows-event-log` level must be greater (more
|
||||
**Note** that the `--windows-event-log-level` level must be greater (more
|
||||
severe) than or equal to the `--log-level`. For example to log DEBUG
|
||||
to a log file but ERRORs to the event log you would use
|
||||
|
||||
```text
|
||||
--log-file rclone.log --log-level DEBUG --windows-event-log ERROR
|
||||
--log-file rclone.log --log-level DEBUG --windows-event-log-level ERROR
|
||||
```
|
||||
|
||||
This option is only supported Windows platforms.
|
||||
|
||||
@@ -234,7 +234,7 @@ Properties:
|
||||
- Config: upload_cutoff
|
||||
- Env Var: RCLONE_DRIME_UPLOAD_CUTOFF
|
||||
- Type: SizeSuffix
|
||||
- Default: 200Mi
|
||||
- Default: 5Mi
|
||||
|
||||
#### --drime-chunk-size
|
||||
|
||||
|
||||
+30
-4
@@ -186,9 +186,14 @@ correct root to use itself.
|
||||
However you can set this to restrict rclone to a specific folder
|
||||
hierarchy or to access data within the "Computers" tab on the drive
|
||||
web interface (where files from Google's Backup and Sync desktop
|
||||
program go).
|
||||
program go). However, this is not the easiest method.
|
||||
Instead, you can create a shortcut to a folder in "Computers" and
|
||||
place it in "My Drive." To do so, right-click the folder, select
|
||||
"Organize", and then choose "Add shortcut" in the Google Drive web interface.
|
||||
Once you add the shortcut to "My Drive," rclone will display the folder, allowing you to interact with it.
|
||||
See also [Shortcuts](#shortcuts).
|
||||
|
||||
In order to do this you will have to find the `Folder ID` of the
|
||||
If, however, you choose to change your root folder, you will have to find the `Folder ID` of the
|
||||
directory you wish rclone to display. This will be the last segment
|
||||
of the URL when you open the relevant folder in the drive web
|
||||
interface.
|
||||
@@ -199,7 +204,11 @@ in the browser, then you use `1XyfxxxxxxxxxxxxxxxxxxxxxxxxxKHCh` as
|
||||
the `root_folder_id` in the config.
|
||||
|
||||
**NB** folders under the "Computers" tab seem to be read only (drive
|
||||
gives a 500 error) when using rclone.
|
||||
gives a 500 error) when using rclone, but they are also read-only in the Google Drive web interface,
|
||||
likely because this folder is managed by the Google Drive Desktop tool.
|
||||
Top-level folders in "Computers" are read-only, but you can interact
|
||||
with folders contained within them using rclone.
|
||||
These are the folders for which you can create a shortcut as well, as described above.
|
||||
|
||||
There doesn't appear to be an API to discover the folder IDs of the
|
||||
"Computers" tab - please contact us if you know otherwise!
|
||||
@@ -496,7 +505,7 @@ Drive, the size of all files in the Trash and the space used by other
|
||||
Google services such as Gmail. This command does not take any path
|
||||
arguments.
|
||||
|
||||
#### Import/Export of google documents
|
||||
### Import/Export of google documents
|
||||
|
||||
Google documents can be exported from and uploaded to Google Drive.
|
||||
|
||||
@@ -2023,6 +2032,23 @@ after a week, which can be annoying to refresh constantly. If, for whatever
|
||||
reason, a short grant time is not a problem, then keeping the application in
|
||||
testing mode would also be sufficient.
|
||||
|
||||
In most cases verification is not actually required. Google [exempts a number of
|
||||
app categories](https://support.google.com/cloud/answer/13464323) from mandatory
|
||||
verification, including:
|
||||
|
||||
- **Personal Use apps**: if the app is for your personal use (fewer than 100
|
||||
users), you and your limited number of users can keep using it without
|
||||
verification - you will just need to click through the "unverified app"
|
||||
warning screen during sign-in. Verification is only required if you want to
|
||||
grow your user base beyond 100 users.
|
||||
- **Development/Testing/Staging apps**: apps in development, testing or staging
|
||||
mode are not subject to verification (but are limited to the 100-user cap and
|
||||
the unverified-app warning until verified).
|
||||
|
||||
So for typical personal rclone use you can leave the app unverified, accept the
|
||||
warning screen, and publish it (rather than leaving it in "Testing") to avoid
|
||||
the weekly grant expiry described above.
|
||||
|
||||
(Thanks to @balazer on github for these instructions.)
|
||||
|
||||
Sometimes, creation of an OAuth consent in Google API Console fails due to an
|
||||
|
||||
@@ -121,7 +121,7 @@ Flags for general networking and HTTP stuff.
|
||||
--tpslimit float Limit HTTP transactions per second to this
|
||||
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
|
||||
--use-cookies Enable session cookiejar
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.74.4")
|
||||
```
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ Flags to control the Remote Control API.
|
||||
Flags to control the Metrics HTTP endpoint..
|
||||
|
||||
```
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind metrics server to
|
||||
--metrics-addr stringArray IPaddress:Port or :Port to bind server to
|
||||
--metrics-allow-origin string Origin which cross-domain request (CORS) can be executed from
|
||||
--metrics-baseurl string Prefix for URLs - leave blank for root
|
||||
--metrics-cert string TLS PEM key (concatenation of certificate and CA certificate)
|
||||
@@ -519,7 +519,7 @@ Backend-only flags (these can be set in the config file also).
|
||||
--drime-list-chunk int Number of items to list in each call (default 1000)
|
||||
--drime-root-folder-id string ID of the root folder
|
||||
--drime-upload-concurrency int Concurrency for multipart uploads and copies (default 4)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi)
|
||||
--drime-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 5Mi)
|
||||
--drime-workspace-id string Account ID
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded
|
||||
--drive-allow-import-name-change Allow the filetype to change when uploading Google docs
|
||||
@@ -805,8 +805,10 @@ Backend-only flags (these can be set in the config file also).
|
||||
--local-case-sensitive Force the filesystem to report itself as case sensitive
|
||||
--local-description string Description of the remote
|
||||
--local-encoding Encoding The encoding for the backend (default Slash,Dot)
|
||||
--local-fatal-if-no-space Make out-of-space errors fatal during transfers
|
||||
--local-hashes CommaSepList Comma separated list of supported checksum types
|
||||
--local-links Translate symlinks to/from regular files with a '.rclonelink' extension for the local backend
|
||||
--local-metadata-restore-special-bits Restore the setuid, setgid and sticky bits from metadata
|
||||
--local-no-check-updated Don't check to see if the files change during upload
|
||||
--local-no-clone Disable reflink cloning for server-side copies
|
||||
--local-no-preallocate Disable preallocation of disk space for transferred files
|
||||
|
||||
@@ -30,6 +30,8 @@ as described [below](#traditional):
|
||||
- MediaMarkt
|
||||
- MediaMarkt Cloud (mediamarkt.jottacloud.com)
|
||||
- Let's Go Cloud (letsgo.jotta.cloud)
|
||||
- Phonero
|
||||
- Phonero Sky (sky.phonero.no)
|
||||
|
||||
Paths are specified as `remote:path`
|
||||
|
||||
|
||||
@@ -296,6 +296,31 @@ backends and the VFS.
|
||||
|
||||
Note that this flag is incompatible with `-copy-links` / `-L`.
|
||||
|
||||
#### Symlink targets and the destination
|
||||
|
||||
When rclone recreates a `.rclonelink` file as a symlink on local storage,
|
||||
the symlink can point anywhere - including, with an absolute path or one
|
||||
using `../` - to a location outside the directory you are copying into. This
|
||||
is normal - rclone reproduces whatever target the link had, so backups
|
||||
round-trip faithfully.
|
||||
|
||||
What rclone will **not** do is *write through* such a link. If a remote you
|
||||
are copying from contains both a symlink and a file or directory that would
|
||||
be placed inside it - for example a `dir.rclonelink` pointing somewhere
|
||||
outside the destination, alongside a `dir/file.txt` - rclone refuses to
|
||||
follow the symlink when writing `dir/file.txt`. The offending file is
|
||||
skipped with an error, the rest of the transfer continues, and the skipped
|
||||
file is counted in the error summary printed at the end of the run.
|
||||
|
||||
This protects you from a malicious or compromised remote using `-l` /
|
||||
`--links` to plant a symlink and then write through it to somewhere outside
|
||||
your destination. Ordinary symlink round-trips, and symlinks that stay
|
||||
inside the destination, are unaffected.
|
||||
|
||||
If you have intentionally pre-created a symlinked directory inside your
|
||||
destination and want rclone to write into the directory it points at, do
|
||||
not use `-l` / `--links` for that copy, or remove the symlink first.
|
||||
|
||||
### Restricting filesystems with --one-file-system
|
||||
|
||||
Normally rclone will recurse through filesystems as mounted.
|
||||
@@ -612,6 +637,47 @@ Properties:
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --local-metadata-restore-special-bits
|
||||
|
||||
Restore the setuid, setgid and sticky bits from metadata.
|
||||
|
||||
When restoring metadata with --metadata rclone applies the "mode" from
|
||||
the source. By default rclone applies only the permission bits and
|
||||
strips the setuid, setgid and sticky bits.
|
||||
|
||||
The "mode" comes from the source remote which may not be trusted.
|
||||
Restoring a setuid or setgid bit onto freshly written,
|
||||
source-controlled content can plant a setuid binary, which is dangerous
|
||||
in particular when restoring from an untrusted source while running as
|
||||
root. For this reason these bits are not restored by default.
|
||||
|
||||
If you trust the source and want the setuid, setgid and sticky bits
|
||||
restored - for example when restoring a system backup made by rclone -
|
||||
set this flag.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: metadata_restore_special_bits
|
||||
- Env Var: RCLONE_LOCAL_METADATA_RESTORE_SPECIAL_BITS
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --local-fatal-if-no-space
|
||||
|
||||
Make out-of-space errors fatal during transfers.
|
||||
|
||||
When enabled, an ENOSPC error during a write returns a fatal error so
|
||||
that rclone aborts rather than retrying the operation. Useful for
|
||||
backup scripts that should halt loudly on a full disk rather than spin
|
||||
retrying.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: fatal_if_no_space
|
||||
- Env Var: RCLONE_LOCAL_FATAL_IF_NO_SPACE
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --local-time-type
|
||||
|
||||
Set what kind of time is returned.
|
||||
@@ -698,6 +764,13 @@ supported by all file systems) under the "user.*" prefix.
|
||||
|
||||
Metadata is supported on files and directories.
|
||||
|
||||
When restoring metadata with `--metadata` rclone applies the
|
||||
"mode", "uid" and "gid" from the source. These come from the source
|
||||
remote which may not be trusted, so restoring metadata as root from an
|
||||
untrusted source can change file ownership and is not recommended. The
|
||||
setuid, setgid and sticky bits are not restored by default - see the
|
||||
`--local-metadata-restore-special-bits` flag.
|
||||
|
||||
Here are the possible system metadata items for the local backend.
|
||||
|
||||
| Name | Help | Type | Example | Read Only |
|
||||
|
||||
@@ -15,6 +15,43 @@ or [use HTTP directly](#api-http).
|
||||
If you just want to run a remote control then see the [rcd](/commands/rclone_rcd/)
|
||||
command.
|
||||
|
||||
## Security {#security}
|
||||
|
||||
**Access to the rc API is equivalent to shell access as the user running
|
||||
rclone.** Treat the rc port as you would an interactive login on the host.
|
||||
|
||||
Any caller who can reach the API (and pass authentication, if it is enabled)
|
||||
can, among other things:
|
||||
|
||||
- **Run OS commands** as the rclone user. `core/command` re-executes the rclone
|
||||
binary with arbitrary arguments, and several backend options shell out to
|
||||
programs, so even creating a remote with `config/create` can lead to command
|
||||
execution.
|
||||
- **Read and write any file** reachable by the rclone process, by pointing
|
||||
`operations/*` or `sync/*` at a `local` remote (or via `--rc-files` /
|
||||
`--rc-serve`). Writing arbitrary files as the rclone user is itself a route to
|
||||
code execution.
|
||||
- **Read back stored credentials.** rclone configs routinely hold cloud-provider
|
||||
secrets. `config/dump` and friends expose them, so a compromise of the rc
|
||||
reaches every configured backend.
|
||||
- **Change rclone's runtime behaviour** with `options/set`, **manage remotes**
|
||||
with `config/*`, and **stop the process** with `core/quit`.
|
||||
|
||||
There is currently no per-endpoint capability or scope system: authentication is
|
||||
all-or-nothing. Granting any access grants all of the above.
|
||||
|
||||
Consequently:
|
||||
|
||||
- **Do not bind the rc to a network address you do not control.** The default
|
||||
bind is loopback (`localhost:5572`); keep it there unless you have a specific
|
||||
reason to change it.
|
||||
- **Do not use `--rc-no-auth` on a non-loopback bind.** It disables
|
||||
authentication on the endpoints that access remotes — see
|
||||
[`--rc-no-auth`](#--rc-no-auth).
|
||||
- **Use authentication and TLS** (`--rc-user`/`--rc-pass` or `--rc-htpasswd`,
|
||||
plus `--rc-cert`/`--rc-key`) whenever the port is reachable by anyone you do
|
||||
not fully trust, and raise `--rc-min-tls-version`.
|
||||
|
||||
## Supported parameters
|
||||
|
||||
### --rc
|
||||
@@ -78,8 +115,26 @@ so you can browse to `http://127.0.0.1:5572/` or `http://127.0.0.1:5572/*`
|
||||
to see a listing of the remotes. Objects may be requested from
|
||||
remotes using this syntax `http://127.0.0.1:5572/[remote:path]/path/to/object`
|
||||
|
||||
Unless the rc server has authentication configured (`--rc-user`/`--rc-pass`
|
||||
or `--rc-htpasswd`) or the `--rc-no-auth` flag is set, only remotes already
|
||||
present in the config file may be served this way. Inline remotes (e.g.
|
||||
`[:webdav,url=...:]`), connection string parameters and bare local paths are
|
||||
rejected, since instantiating them from an unauthenticated request could run
|
||||
commands or read arbitrary local files.
|
||||
|
||||
Default Off.
|
||||
|
||||
### global.* connection string options and the rc
|
||||
|
||||
Remotes instantiated by the rc do not let [connection
|
||||
string](/docs/#connection-strings) `global.*` options change rclone's
|
||||
process-wide configuration. Remotes created directly on the command
|
||||
line or defined in the config file are unaffected.
|
||||
|
||||
A `global.*` option still takes effect for the individual backend it
|
||||
is set on (exactly like an `override.*` option), it just does not leak
|
||||
into the global config for the rest of the process.
|
||||
|
||||
### --rc-serve-no-modtime
|
||||
|
||||
Set this flag to skip reading the modification time (can speed things up).
|
||||
|
||||
@@ -75,7 +75,7 @@ y/e/d>
|
||||
## Configuring by copying the config file
|
||||
|
||||
Rclone stores all of its configuration in a single file. This can easily be
|
||||
copied to configure a remote rclone (although some backends does not support
|
||||
copied to configure a remote rclone (although some backends do not support
|
||||
reusing the same configuration, consult your backend documentation to be
|
||||
sure).
|
||||
|
||||
|
||||
+45
-6
@@ -1273,6 +1273,9 @@ Properties:
|
||||
- "eu-south-1"
|
||||
- EU South 1
|
||||
- Provider: Fastly
|
||||
- "eu-west-1"
|
||||
- EU West 1
|
||||
- Provider: Fastly
|
||||
- "jp-central-1"
|
||||
- JP Central 1
|
||||
- Provider: Fastly
|
||||
@@ -1285,9 +1288,15 @@ Properties:
|
||||
- "us-east"
|
||||
- US East
|
||||
- Provider: Fastly
|
||||
- "us-east-1"
|
||||
- US East 1
|
||||
- Provider: Fastly
|
||||
- "us-west"
|
||||
- US West
|
||||
- Provider: Fastly
|
||||
- "us-west-1"
|
||||
- US West 1
|
||||
- Provider: Fastly
|
||||
- "global"
|
||||
- Global
|
||||
- Provider: FileLu
|
||||
@@ -1860,6 +1869,9 @@ Properties:
|
||||
- "eu-south-1.object.fastlystorage.app"
|
||||
- EU South 1
|
||||
- Provider: Fastly
|
||||
- "eu-west-1.object.fastlystorage.app"
|
||||
- EU West 1
|
||||
- Provider: Fastly
|
||||
- "jp-central-1.object.fastlystorage.app"
|
||||
- JP Central 1
|
||||
- Provider: Fastly
|
||||
@@ -1872,9 +1884,15 @@ Properties:
|
||||
- "us-east.object.fastlystorage.app"
|
||||
- US East
|
||||
- Provider: Fastly
|
||||
- "us-east-1.object.fastlystorage.app"
|
||||
- US East 1
|
||||
- Provider: Fastly
|
||||
- "us-west.object.fastlystorage.app"
|
||||
- US West
|
||||
- Provider: Fastly
|
||||
- "us-west-1.object.fastlystorage.app"
|
||||
- US West 1
|
||||
- Provider: Fastly
|
||||
- "s5lu.com"
|
||||
- Global FileLu S5 endpoint
|
||||
- Provider: FileLu
|
||||
@@ -2258,17 +2276,38 @@ Properties:
|
||||
- "br-ne1.magaluobjects.com"
|
||||
- Fortaleza, CE (BR), br-ne1
|
||||
- Provider: Magalu
|
||||
- "s3.eu-amsterdam.megas4.com"
|
||||
- Mega S4 Amsterdam
|
||||
- Provider: Mega
|
||||
- "s3.eu-luxembourg.megas4.com"
|
||||
- Mega S4 Luxembourg
|
||||
- Provider: Mega
|
||||
- "s3.eu-paris.megas4.com"
|
||||
- Mega S4 Paris
|
||||
- Provider: Mega
|
||||
- "s3.eu-barcelona.megas4.com"
|
||||
- Mega S4 Barcelona
|
||||
- Provider: Mega
|
||||
- "s3.ca-montreal.megas4.com"
|
||||
- Mega S4 Montreal
|
||||
- Provider: Mega
|
||||
- "s3.ca-vancouver.megas4.com"
|
||||
- Mega S4 Vancouver
|
||||
- Provider: Mega
|
||||
- "s3.ap-tokyo.megas4.com"
|
||||
- Mega S4 Tokyo
|
||||
- Provider: Mega
|
||||
- "s3.eu-central-1.s4.mega.io"
|
||||
- Mega S4 eu-central-1 (Amsterdam)
|
||||
- Mega S4 eu-central-1 (Amsterdam, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.eu-central-2.s4.mega.io"
|
||||
- Mega S4 eu-central-2 (Bettembourg)
|
||||
- Mega S4 eu-central-2 (Bettembourg, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.ca-central-1.s4.mega.io"
|
||||
- Mega S4 ca-central-1 (Montreal)
|
||||
- Mega S4 ca-central-1 (Montreal, legacy)
|
||||
- Provider: Mega
|
||||
- "s3.ca-west-1.s4.mega.io"
|
||||
- Mega S4 ca-west-1 (Vancouver)
|
||||
- Mega S4 ca-west-1 (Vancouver, legacy)
|
||||
- Provider: Mega
|
||||
- "oos.eu-west-2.outscale.com"
|
||||
- Outscale EU West 2 (Paris)
|
||||
@@ -3473,7 +3512,7 @@ Cutoff for switching to multipart copy.
|
||||
Any files larger than this that need to be server-side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 5 GiB.
|
||||
The minimum is 1 byte and the maximum is 5 GiB.
|
||||
|
||||
Properties:
|
||||
|
||||
@@ -3624,7 +3663,7 @@ If true use path style access if false use virtual hosted style.
|
||||
|
||||
If this is true (the default) then rclone will use path style access,
|
||||
if false then rclone will use virtual path style. See [the AWS S3
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html)
|
||||
for more info.
|
||||
|
||||
Some providers (e.g. AWS, Aliyun OSS, Netease COS, or Tencent COS) require this set to
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
title: "Sponsor"
|
||||
description: "Sponsor the rclone project."
|
||||
type: page
|
||||
notoc: true
|
||||
aliases:
|
||||
- donate
|
||||
---
|
||||
@@ -21,12 +22,14 @@ Thank you to our sponsors:
|
||||
{{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}}
|
||||
{{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
|
||||
{{< sponsor src="/img/logos/rcloneui.svg" width="300" height="200" title="Visit our sponsor RcloneUI" link="https://github.com/rclone-ui/rclone-ui">}}
|
||||
{{< sponsor src="/img/logos/data-raven.svg" width="300" height="200" title="Data Raven: rclone, managed. Unlimited transfers, no per-GB fees." link="https://dataraven.io/r/rclone">}}
|
||||
{{< sponsor src="/img/logos/shade.svg" width="300" height="200" title="Visit our sponsor Shade" link="https://shade.inc">}}
|
||||
{{< sponsor src="/img/logos/filelu-rclone.svg" width="300" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}}
|
||||
{{< sponsor src="/img/logos/torbox.png" width="200" height="200" title="Visit our sponsor TORBOX" link="https://www.torbox.app/">}}
|
||||
{{< sponsor src="/img/logos/spectra-logic.svg" width="300" height="200" title="Visit our sponsor Spectra Logic" link="https://spectralogic.com/">}}
|
||||
{{< sponsor src="/img/logos/servercore.svg" width="300" height="200" title="Visit our sponsor servercore" link="https://servercore.com/services/object-storage/?utm_source=rclone.org&utm_medium=referral&utm_campaign=cloud-s3_rclone_231025_paid">}}
|
||||
{{< sponsor src="/img/logos/exchangerate-api.png" width="300" height="200" title="Visit our sponsor ExchangeRate-API" link="https://www.exchangerate-api.com/">}}
|
||||
{{< sponsor src="/img/logos/impossible-cloud.svg" width="300" height="200" title="Visit our sponsor Impossible Cloud" link="https://www.impossiblecloud.com/">}}
|
||||
|
||||
<!-- markdownlint-restore -->
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
backend: protondrive
|
||||
name: Proton Drive
|
||||
tier: Tier 5
|
||||
maintainers: None
|
||||
tier: Tier 4
|
||||
maintainers: Community
|
||||
features_score: 4
|
||||
integration_tests: Failing
|
||||
integration_tests: Passing
|
||||
data_integrity: Hash
|
||||
performance: High
|
||||
adoption: Often used
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
</div>
|
||||
|
||||
{{if .IsHome}}
|
||||
<div id="silver-sponsors">
|
||||
<div class="card">
|
||||
<div class="card-header">Silver Sponsor</div>
|
||||
<div class="card-body">
|
||||
@@ -32,6 +33,25 @@
|
||||
<a href="https://shade.inc/" target="_blank" rel="noopener" title="Visit rclone's sponsor Shade"><img style="max-width: 100%; height: auto;" src="/img/logos/shade.svg"></a><br />
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-header">Silver Sponsor</div>
|
||||
<div class="card-body">
|
||||
<a href="https://dataraven.io/r/rclone" target="_blank" rel="noopener" title="Data Raven: rclone, managed. Unlimited transfers, no per-GB fees."><img style="max-width: 100%; height: auto;" src="/img/logos/data-raven.svg"></a><br />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
(function() {
|
||||
var container = document.getElementById('silver-sponsors');
|
||||
if (!container) return;
|
||||
var cards = Array.prototype.slice.call(container.children);
|
||||
for (var i = cards.length - 1; i > 0; i--) {
|
||||
var j = Math.floor(Math.random() * (i + 1));
|
||||
var tmp = cards[i]; cards[i] = cards[j]; cards[j] = tmp;
|
||||
}
|
||||
cards.forEach(function(c) { container.appendChild(c); });
|
||||
}());
|
||||
</script>
|
||||
{{end}}
|
||||
|
||||
<div class="card">
|
||||
|
||||
@@ -1 +1 @@
|
||||
v1.74.0
|
||||
v1.74.4
|
||||
Vendored
+7
-4
@@ -513,11 +513,14 @@
|
||||
|
||||
// Determine what to copy
|
||||
var text;
|
||||
var pre = btn.closest("pre");
|
||||
if (pre) {
|
||||
var preWrap = btn.closest(".pre-wrap");
|
||||
if (preWrap) {
|
||||
// Code block: copy the code element text (excludes button)
|
||||
var code = pre.querySelector("code");
|
||||
text = (code || pre).textContent;
|
||||
var pre = preWrap.querySelector("pre");
|
||||
if (pre) {
|
||||
var code = pre.querySelector("code");
|
||||
text = (code || pre).textContent;
|
||||
}
|
||||
} else {
|
||||
// Input field: copy value of the preceding input
|
||||
var input = btn.previousElementSibling;
|
||||
|
||||
@@ -730,9 +730,15 @@ func (s *StatsInfo) ResetCounters() {
|
||||
s.startedTransfers = nil
|
||||
s.oldDuration = 0
|
||||
|
||||
// Only restart the average loop if it was running. Otherwise
|
||||
// ResetCounters would spawn a goroutine that pins the StatsInfo,
|
||||
// leaking memory when called on stats that never started the loop.
|
||||
wasStarted := s.average.started
|
||||
s._stopAverageLoop()
|
||||
s.average = averageValues{}
|
||||
s._startAverageLoop()
|
||||
if wasStarted {
|
||||
s._startAverageLoop()
|
||||
}
|
||||
}
|
||||
|
||||
// ResetErrors sets the errors count to 0 and resets lastError, fatalError and retryError
|
||||
|
||||
@@ -301,9 +301,16 @@ func GlobalStats() *StatsInfo {
|
||||
}
|
||||
|
||||
// NewStatsGroup creates new stats under named group.
|
||||
//
|
||||
// The averageLoop is intentionally NOT started here: it is started on
|
||||
// demand by NewTransfer / NewTransferRemoteSize when real transfer
|
||||
// activity begins and stopped by DoneTransferring when the last
|
||||
// transfer completes. Starting it eagerly leaked one goroutine per
|
||||
// group that never performed a transfer (issue #9567) - e.g. rcd
|
||||
// daemons driven by frequent rc sync/move calls where each call gets
|
||||
// a fresh job/N group and the source is often empty.
|
||||
func NewStatsGroup(ctx context.Context, group string) *StatsInfo {
|
||||
stats := NewStats(ctx)
|
||||
stats.startAverageLoop()
|
||||
stats.group = group
|
||||
groups.set(ctx, group, stats)
|
||||
return stats
|
||||
|
||||
@@ -271,3 +271,42 @@ func percentDiff(start, end uint64) uint64 {
|
||||
}
|
||||
return (diff * 100) / start
|
||||
}
|
||||
|
||||
// Regression for #9567: NewStatsGroup used to start the averageLoop
|
||||
// goroutine unconditionally at group creation. When an rcd daemon was
|
||||
// driven by many rc sync/move calls that transferred zero files, each
|
||||
// call created a fresh job/N group whose averageLoop was started but
|
||||
// never stopped (nothing ever went transferring->empty), leaking one
|
||||
// goroutine per call. The loop must only start on demand from
|
||||
// NewTransfer / NewTransferRemoteSize, and it must still stop when the
|
||||
// last transfer completes.
|
||||
func TestNewStatsGroupDoesNotStartAverageLoop(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
defer func() {
|
||||
groups = newStatsGroups()
|
||||
}()
|
||||
|
||||
stats := NewStatsGroup(ctx, "test-group-9567")
|
||||
|
||||
stats.mu.RLock()
|
||||
started := stats.average.started
|
||||
stats.mu.RUnlock()
|
||||
assert.False(t, started,
|
||||
"NewStatsGroup must not start averageLoop for a zero-transfer group (issue #9567)")
|
||||
|
||||
// The loop must still start on demand when a real transfer is added,
|
||||
// and stop cleanly once the last transfer completes.
|
||||
tr := stats.NewTransferRemoteSize("regression-9567", 0, nil, nil)
|
||||
stats.mu.RLock()
|
||||
startedAfterTransfer := stats.average.started
|
||||
stats.mu.RUnlock()
|
||||
assert.True(t, startedAfterTransfer,
|
||||
"averageLoop should start on demand when a transfer is added")
|
||||
|
||||
tr.Done(ctx, nil)
|
||||
stats.mu.RLock()
|
||||
startedAfterDone := stats.average.started
|
||||
stats.mu.RUnlock()
|
||||
assert.False(t, startedAfterDone,
|
||||
"averageLoop should stop when the last transfer is done")
|
||||
}
|
||||
Loaded 100 of 134 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user