Compare commits
197
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b22fe9811c | ||
|
|
ce64f4edb8 | ||
|
|
f3986e2bb7 | ||
|
|
f0961f4a8d | ||
|
|
139ccc7ffd | ||
|
|
b3ddec50c6 | ||
|
|
3cd224e03f | ||
|
|
52af9caa14 | ||
|
|
b2cb7f89c1 | ||
|
|
2292ba0cd4 | ||
|
|
07c87327ae | ||
|
|
6cc0d1a0b0 | ||
|
|
7a5e01b015 | ||
|
|
bf111df84a | ||
|
|
0b73e2b858 | ||
|
|
e0f210cdc4 | ||
|
|
1c06bb9272 | ||
|
|
b2e1aa65dc | ||
|
|
3a414d8262 | ||
|
|
24054305ce | ||
|
|
076fb2bc53 | ||
|
|
e1fd3725bc | ||
|
|
5780a30c30 | ||
|
|
17c2db7289 | ||
|
|
44c56ed481 | ||
|
|
ebca82f7db | ||
|
|
e9d280b793 | ||
|
|
4f93bd482f | ||
|
|
b6c68882e5 | ||
|
|
23dc9d0515 | ||
|
|
be7c0f31a4 | ||
|
|
3ac9b75a83 | ||
|
|
5e9cb39084 | ||
|
|
87d0b59a51 | ||
|
|
76596b6727 | ||
|
|
aa031c51cc | ||
|
|
45eb93c3af | ||
|
|
18899a58f3 | ||
|
|
7c3909589c | ||
|
|
d6377a283d | ||
|
|
56b7d7500e | ||
|
|
7400a811fd | ||
|
|
bd62357998 | ||
|
|
306fb0a304 | ||
|
|
9c67942986 | ||
|
|
0812cee9e9 | ||
|
|
7447f23d66 | ||
|
|
77954e5cd2 | ||
|
|
b30cf8fbe7 | ||
|
|
d0c469c3c0 | ||
|
|
6b67be9d48 | ||
|
|
d2b8b73ea3 | ||
|
|
7c56eff1a7 | ||
|
|
55da1abb23 | ||
|
|
b8bfcbc0c0 | ||
|
|
96f0979b66 | ||
|
|
a55ad46201 | ||
|
|
a71c6f0459 | ||
|
|
96760f1c14 | ||
|
|
8e60792d96 | ||
|
|
c385d8586a | ||
|
|
328ac017c1 | ||
|
|
8502532c22 | ||
|
|
3e9e29ba8f | ||
|
|
9cb329809d | ||
|
|
6d0bca0fc8 | ||
|
|
49650db8af | ||
|
|
9b7f960a24 | ||
|
|
18aa4b2f29 | ||
|
|
8e9ea05a67 | ||
|
|
1f44b2ee93 | ||
|
|
90028ab3da | ||
|
|
d45f66f840 | ||
|
|
4942541f02 | ||
|
|
ae5d388ea3 | ||
|
|
a1ad9b3f46 | ||
|
|
79379faeac | ||
|
|
9dedb12b9d | ||
|
|
53a571abf1 | ||
|
|
8688785a7a | ||
|
|
f1a5e44d50 | ||
|
|
65ef7d8e6c | ||
|
|
c744949d91 | ||
|
|
94e7adaeba | ||
|
|
f191448b0d | ||
|
|
c5df1ddf4b | ||
|
|
9e3e68d00c | ||
|
|
08490972a5 | ||
|
|
06aa958ad6 | ||
|
|
67e5f435c6 | ||
|
|
e76a30471a | ||
|
|
2b570c788c | ||
|
|
9b889684b0 | ||
|
|
48a6584baf | ||
|
|
58a51c2ce9 | ||
|
|
b715ad2a97 | ||
|
|
e12c250705 | ||
|
|
513d2fc17e | ||
|
|
3be3347e86 | ||
|
|
bf55d5e6d3 | ||
|
|
dd5250ca55 | ||
|
|
cf11c8bbd9 | ||
|
|
bbd7297b33 | ||
|
|
7b8994ab32 | ||
|
|
9f2edc3077 | ||
|
|
771267d713 | ||
|
|
5f791079fc | ||
|
|
7b9ac79ab4 | ||
|
|
8ddccc1285 | ||
|
|
55afa13921 | ||
|
|
a08b48adaa | ||
|
|
5337b14739 | ||
|
|
3ad0178b5b | ||
|
|
1a924aa746 | ||
|
|
33996381df | ||
|
|
299e7274f8 | ||
|
|
c5b4745cb9 | ||
|
|
560882de2c | ||
|
|
acf887b464 | ||
|
|
082031cc85 | ||
|
|
40b064993e | ||
|
|
d15f1142ef | ||
|
|
0b4c21fbda | ||
|
|
3658470022 | ||
|
|
9623850979 | ||
|
|
e863f751f0 | ||
|
|
a2ce8b04f3 | ||
|
|
bfd650b428 | ||
|
|
16591fdc21 | ||
|
|
c1f5df2090 | ||
|
|
0fa16e8638 | ||
|
|
d58ce7b1b7 | ||
|
|
f757e6c0a7 | ||
|
|
20eaad4b6d | ||
|
|
1ac0c3eecb | ||
|
|
561f20f83f | ||
|
|
e7deebdf40 | ||
|
|
cb9bdf629c | ||
|
|
1f3770a57f | ||
|
|
829b06d39b | ||
|
|
0db3e7a2a0 | ||
|
|
654df91375 | ||
|
|
f079361a01 | ||
|
|
3677b7ff77 | ||
|
|
4a00a4dc4b | ||
|
|
2610beb18d | ||
|
|
35e4f60548 | ||
|
|
839a3b92e0 | ||
|
|
492e0bd616 | ||
|
|
d4894db765 | ||
|
|
8d09f16681 | ||
|
|
550889c096 | ||
|
|
896961404b | ||
|
|
5eabfc74f4 | ||
|
|
787d033376 | ||
|
|
db258d404a | ||
|
|
3cdf680a15 | ||
|
|
bc57110920 | ||
|
|
133eae4d5b | ||
|
|
24c40463eb | ||
|
|
e9fddaabeb | ||
|
|
c49015552c | ||
|
|
ffdd043b95 | ||
|
|
434edba275 | ||
|
|
da7a3d8d64 | ||
|
|
2362b4ca50 | ||
|
|
5e3d783b3e | ||
|
|
7a63990df2 | ||
|
|
7ca667d35d | ||
|
|
1e22af8358 | ||
|
|
589d45a0bf | ||
|
|
d5deec3534 | ||
|
|
53c19e04ba | ||
|
|
e7fadc7e3d | ||
|
|
7dfc5b3076 | ||
|
|
523a29a4e9 | ||
|
|
a3e1312d9d | ||
|
|
e987d4f351 | ||
|
|
639bd88959 | ||
|
|
bfa2fbb78e | ||
|
|
69ccbacf30 | ||
|
|
5d6690eb20 | ||
|
|
c7b8654b17 | ||
|
|
906b27d8b9 | ||
|
|
e233f8b0a7 | ||
|
|
17a547a980 | ||
|
|
cdb3ecb0fb | ||
|
|
2d7f84576a | ||
|
|
ab234dcfd7 | ||
|
|
c5bcfbac8f | ||
|
|
88ed8e2b43 | ||
|
|
b097454f83 | ||
|
|
2f890114b6 | ||
|
|
78a7d9bb8e | ||
|
|
72c561d209 | ||
|
|
b864c4f9c9 | ||
|
|
bb78eb8ab2 |
No files matched your search
@@ -19,6 +19,9 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
@@ -34,7 +37,7 @@ jobs:
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.3'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -45,14 +48,14 @@ jobs:
|
||||
|
||||
- job_name: linux_386
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.3'
|
||||
goarch: 386
|
||||
gotags: cmount
|
||||
quicktest: true
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.3'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -61,14 +64,14 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.3'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.3'
|
||||
gotags: cmount
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
@@ -78,7 +81,7 @@ jobs:
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '~1.26.0'
|
||||
go: '~1.26.3'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
@@ -224,10 +227,10 @@ jobs:
|
||||
id: setup-go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.26.0'
|
||||
go-version: '~1.26.3'
|
||||
check-latest: true
|
||||
cache: false
|
||||
|
||||
|
||||
- name: Cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
@@ -315,7 +318,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '~1.26.0'
|
||||
go-version: '~1.26.3'
|
||||
|
||||
- name: Set global environment variables
|
||||
run: |
|
||||
|
||||
@@ -18,6 +18,10 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build-image:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request')
|
||||
@@ -92,7 +96,7 @@ jobs:
|
||||
# There's no way around this, because "ImageOS" is only available to
|
||||
# processes, but the setup-go action uses it in its key.
|
||||
id: imageos
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
@@ -100,7 +104,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
|
||||
with:
|
||||
@@ -122,10 +126,10 @@ jobs:
|
||||
type=raw,value=beta,enable={{is_default_branch}}
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Load Go Build Cache for Docker
|
||||
id: go-cache
|
||||
@@ -148,7 +152,7 @@ jobs:
|
||||
skip-extraction: ${{ steps.go-cache.outputs.cache-hit }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
@@ -159,7 +163,7 @@ jobs:
|
||||
|
||||
- name: Build and Publish Image Digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
file: Dockerfile
|
||||
context: .
|
||||
@@ -210,7 +214,7 @@ jobs:
|
||||
|
||||
- name: Extract Metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
uses: docker/metadata-action@v6
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
|
||||
with:
|
||||
@@ -261,16 +265,16 @@ jobs:
|
||||
env.write(f"ANNOTATIONS={annotations_string}\n")
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
# This is the user that triggered the Workflow. In this case, it will
|
||||
|
||||
@@ -14,6 +14,9 @@ on:
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build_docker_volume_plugin:
|
||||
if: inputs.manual || github.repository == 'rclone/rclone'
|
||||
|
||||
@@ -4,6 +4,9 @@ on:
|
||||
issues:
|
||||
types: [labeled]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -3,6 +3,9 @@ on:
|
||||
release:
|
||||
types: [released]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
default: true
|
||||
|
||||
# Use specific styles, to be consistent accross all documents.
|
||||
# Use specific styles, to be consistent across all documents.
|
||||
# Default is to accept any as long as it is consistent within the same document.
|
||||
heading-style: # MD003
|
||||
style: atx
|
||||
@@ -48,12 +48,12 @@ link-fragments: false # MD051
|
||||
# https://gohugo.io/content-management/syntax-highlighting/#languages
|
||||
# https://docs.github.com//get-started/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#syntax-highlighting
|
||||
# In addition, we only want to allow identifiers (aliases) that correspond to
|
||||
# the same language in Hugo and GitHub, and preferrably also VSCode and other
|
||||
# the same language in Hugo and GitHub, and preferably also VSCode and other
|
||||
# commonly used tools, to avoid confusion. An example of this is that "shell"
|
||||
# by some are considered an identifier for shell scripts, i.e. an alias for
|
||||
# "sh", while others consider it an identifier for shell sessions, i.e. an
|
||||
# alias for "console". Although Hugo and GitHub in this case are consistent and
|
||||
# have choosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# have chosen the former, using "sh" instead, and not allowing use of "shell",
|
||||
# avoids the confusion entirely.
|
||||
fenced-code-language: # MD040
|
||||
allowed_languages:
|
||||
|
||||
+3
-4
@@ -627,10 +627,9 @@ See the [testing](#testing) section for more information on integration tests.
|
||||
|
||||
### Backend documentation
|
||||
|
||||
Add your backend to the docs - you'll need to pick an icon for it from
|
||||
[fontawesome](http://fontawesome.io/icons/). Keep lists of remotes in
|
||||
alphabetical order of full name of remote (e.g. `drive` is ordered as
|
||||
`Google Drive`) but with the local file system last.
|
||||
Add your backend to the docs. Keep lists of remotes in alphabetical
|
||||
order of full name of remote (e.g. `drive` is ordered as `Google
|
||||
Drive`) but with the local file system last.
|
||||
|
||||
First add a data file about your backend in
|
||||
`docs/data/backends/remote.yaml` - this is used to build the overview
|
||||
|
||||
@@ -23,6 +23,7 @@ Current active maintainers of rclone are:
|
||||
| nielash | @nielash | bisync |
|
||||
| Dan McArdle | @dmcardle | gitannex |
|
||||
| Sam Harrison | @childish-sambino | filescom |
|
||||
| Enduriel | @Enduriel | filen backend |
|
||||
|
||||
## This is a work in progress draft
|
||||
|
||||
|
||||
Generated
+5234
-2073
File diff suppressed because it is too large.
Load diff
Generated
+3185
-393
File diff suppressed because it is too large.
Load diff
@@ -45,7 +45,7 @@ LINTTAGS=--build-tags "$(GOTAGS)"
|
||||
endif
|
||||
LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
|
||||
|
||||
.PHONY: rclone test_all vars version
|
||||
.PHONY: rclone test_all vars version fetch-gui fetch-gui-and-commit
|
||||
|
||||
rclone:
|
||||
ifeq ($(GO_OS),windows)
|
||||
@@ -59,6 +59,12 @@ endif
|
||||
cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
|
||||
mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
|
||||
|
||||
fetch-gui:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh
|
||||
|
||||
fetch-gui-and-commit:
|
||||
$(SHELL) ./bin/fetch-gui-dist.sh --commit
|
||||
|
||||
test_all:
|
||||
go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
|
||||
|
||||
@@ -121,6 +127,25 @@ updatedirect:
|
||||
go get $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
go mod tidy
|
||||
|
||||
# Update direct dependencies only but won't update the `go` line in go.mod
|
||||
updatedirectnoupgrade:
|
||||
@GO_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
for mod in $$(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all); do \
|
||||
cp go.mod go.mod.bak && \
|
||||
cp go.sum go.sum.bak && \
|
||||
go get $$mod && \
|
||||
NEW_VERSION=$$(awk '/^go /{print $$2}' go.mod) && \
|
||||
if [ "$$NEW_VERSION" != "$$GO_VERSION" ]; then \
|
||||
echo "SKIPPING $$mod (requires go $$NEW_VERSION)"; \
|
||||
cp go.mod.bak go.mod; \
|
||||
cp go.sum.bak go.sum; \
|
||||
else \
|
||||
echo "updated $$mod"; \
|
||||
fi; \
|
||||
done && \
|
||||
rm -f go.mod.bak go.sum.bak && \
|
||||
go mod tidy
|
||||
|
||||
# Update direct and indirect dependencies and test dependencies
|
||||
update:
|
||||
go get -u -t ./...
|
||||
@@ -146,6 +171,7 @@ MANUAL.txt: MANUAL.md
|
||||
|
||||
commanddocs: rclone
|
||||
go generate ./lib/transform
|
||||
go generate ./cmd/bisync
|
||||
-@rmdir -p '$$HOME/.config/rclone'
|
||||
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/
|
||||
@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)
|
||||
|
||||
@@ -59,14 +59,17 @@ directories to and from different cloud storage providers.
|
||||
- Hetzner Object Storage [:page_facing_up:](https://rclone.org/s3/#hetzner)
|
||||
- Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
|
||||
- HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
|
||||
- Hitachi Content Platform (HCP) [:page_facing_up:](https://rclone.org/s3/#hcp)
|
||||
- HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
- Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
|
||||
- Huawei Drive [:page_facing_up:](https://rclone.org/huaweidrive/)
|
||||
- iCloud Drive [:page_facing_up:](https://rclone.org/iclouddrive/)
|
||||
- ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
|
||||
- Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
|
||||
- Internxt [:page_facing_up:](https://rclone.org/internxt/)
|
||||
- Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
- IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
- Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
|
||||
- Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
|
||||
- IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
|
||||
- Koofr [:page_facing_up:](https://rclone.org/koofr/)
|
||||
@@ -123,6 +126,7 @@ directories to and from different cloud storage providers.
|
||||
- Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
|
||||
- Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
|
||||
- Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
|
||||
- US3 Object Storage [:page_facing_up:](https://rclone.org/s3/#us3)
|
||||
- Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
|
||||
- WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
- Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
|
||||
|
||||
+25
-24
@@ -11,6 +11,7 @@ This file describes how to make the various kinds of releases
|
||||
|
||||
- git checkout master # see below for stable branch
|
||||
- git pull # IMPORTANT
|
||||
- make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
|
||||
- git status - make sure everything is checked in
|
||||
- Check GitHub actions build for master is Green
|
||||
- make test # see integration test server or run locally
|
||||
@@ -32,10 +33,11 @@ This file describes how to make the various kinds of releases
|
||||
- make sign_upload
|
||||
- make check_sign
|
||||
- make upload
|
||||
- make upload_test_website # check sponsors are correct/unchanged
|
||||
- make upload_website
|
||||
- make upload_github
|
||||
- make startdev # make startstable for stable branch
|
||||
- \# announce with forum post, twitter post, patreon post
|
||||
- \# announce with forum post, twitter post, and post to annoucements list
|
||||
|
||||
## Update dependencies
|
||||
|
||||
@@ -54,32 +56,18 @@ If the `make updatedirect` upgrades the version of go in the `go.mod`
|
||||
go 1.22.0
|
||||
```
|
||||
|
||||
then go to manual mode. `go1.22` here is the lowest supported version
|
||||
in the `go.mod`.
|
||||
then so this instead `make updatedirectnoupgrade`. This will try each
|
||||
upgrade individually. Consider making an upstream bug report for any
|
||||
`SKIPPING` lines as these only support the latest Go version not the
|
||||
previous version too.
|
||||
|
||||
If `make updatedirect` added a `toolchain` directive then remove it.
|
||||
We don't want to force a toolchain on our users. Linux packagers are
|
||||
often using a version of Go that is a few versions out of date.
|
||||
|
||||
```console
|
||||
go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
|
||||
go get -d $(cat /tmp/potential-upgrades)
|
||||
go mod tidy -go=1.22 -compat=1.22
|
||||
```
|
||||
|
||||
If the `go mod tidy` fails use the output from it to remove the
|
||||
package which can't be upgraded from `/tmp/potential-upgrades` when
|
||||
done
|
||||
|
||||
```console
|
||||
git co go.mod go.sum
|
||||
```
|
||||
|
||||
And try again.
|
||||
If `make updatedirect` or `make updatedirectnoupgrade` added a
|
||||
`toolchain` directive then remove it. We don't want to force a
|
||||
toolchain on our users. Linux packagers are often using a version of
|
||||
Go that is a few versions out of date.
|
||||
|
||||
Optionally upgrade the direct and indirect dependencies. This is very
|
||||
likely to fail if the manual method was used abve - in that case
|
||||
ignore it as it is too time consuming to fix.
|
||||
likely to fail if `make updatedirectnoupgrade` was used.
|
||||
|
||||
- `make update`
|
||||
- `make GOTAGS=cmount`
|
||||
@@ -97,6 +85,19 @@ build.
|
||||
Once it compiles locally, push it on a test branch and commit fixes
|
||||
until the tests pass.
|
||||
|
||||
### Pseudo versions
|
||||
|
||||
Go makes pseudo versions for untagged repos and repos not at a tag.
|
||||
The pseudo versions on repos that have been tagged before do not get
|
||||
updated automatically so need manually checking. These can be found with
|
||||
|
||||
```console
|
||||
grep -E '[0-9]{14}-[0-9a-f]{12}' go.mod | grep -v indirect | grep -v 'v0\.0\.0'
|
||||
```
|
||||
|
||||
These will need to be updated manually using the `go get ...@branch`
|
||||
syntax.
|
||||
|
||||
### Major versions
|
||||
|
||||
The above procedure will not upgrade major versions, so v2 to v3.
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/hdfs"
|
||||
_ "github.com/rclone/rclone/backend/hidrive"
|
||||
_ "github.com/rclone/rclone/backend/http"
|
||||
_ "github.com/rclone/rclone/backend/huaweidrive"
|
||||
_ "github.com/rclone/rclone/backend/iclouddrive"
|
||||
_ "github.com/rclone/rclone/backend/imagekit"
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
|
||||
@@ -36,7 +36,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (*Fs
|
||||
// FIXME vfs cache?
|
||||
// FIXME could factor out ReadFileHandle and just use that rather than the full VFS
|
||||
fs.Debugf(nil, "New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
VFS := vfs.New(wrappedFs, nil)
|
||||
VFS := vfs.New(ctx, wrappedFs, nil)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
|
||||
@@ -50,7 +50,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
fs.Debugf(nil, "Squashfs: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
|
||||
@@ -52,7 +52,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
|
||||
fs.Debugf(nil, "Zip: New: remote=%q, prefix=%q, root=%q", remote, prefix, root)
|
||||
vfsOpt := vfscommon.Opt
|
||||
vfsOpt.ReadWait = 0
|
||||
VFS := vfs.New(wrappedFs, &vfsOpt)
|
||||
VFS := vfs.New(ctx, wrappedFs, &vfsOpt)
|
||||
node, err := VFS.Stat(remote)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find %q archive: %w", remote, err)
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
@@ -283,6 +284,9 @@ type transporter struct {
|
||||
|
||||
// Make a new transporter
|
||||
func newTransporter(ctx context.Context) transporter {
|
||||
// Set the User-Agent to include the Microsoft Partner Network prefix
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.UserAgent = "APN/1.0 rclone/1.0 rclone/" + strings.TrimPrefix(fs.Version, "v")
|
||||
return transporter{
|
||||
RoundTripper: fshttp.NewTransport(ctx),
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/transferaccounter"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -67,6 +68,16 @@ const (
|
||||
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
|
||||
)
|
||||
|
||||
// setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
|
||||
// on every request. This prevents the Go HTTP transport from automatically
|
||||
// decompressing gzip-encoded blobs on download.
|
||||
type setAcceptEncodingGzip struct{}
|
||||
|
||||
func (p setAcceptEncodingGzip) Do(req *policy.Request) (*http.Response, error) {
|
||||
req.Raw().Header.Set("Accept-Encoding", "gzip")
|
||||
return req.Next()
|
||||
}
|
||||
|
||||
var (
|
||||
errCantUpdateArchiveTierBlobs = fserrors.NoRetryError(errors.New("can't update archive tier blob without --azureblob-archive-tier-delete"))
|
||||
|
||||
@@ -350,6 +361,19 @@ rclone does if you know the container exists already.
|
||||
Default: "",
|
||||
Exclusive: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "decompress",
|
||||
Help: `If set this will decompress gzip encoded objects.
|
||||
|
||||
It is possible to upload objects to Azure Blob Storage with "Content-Encoding: gzip"
|
||||
set. Normally rclone will download these files as compressed objects.
|
||||
|
||||
If this flag is set then rclone will decompress these files with
|
||||
"Content-Encoding: gzip" as they are received. This means that rclone
|
||||
can't check the size and hash but the file contents will be decompressed.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}}),
|
||||
})
|
||||
}
|
||||
@@ -373,6 +397,7 @@ type Options struct {
|
||||
NoCheckContainer bool `config:"no_check_container"`
|
||||
NoHeadObject bool `config:"no_head_object"`
|
||||
DeleteSnapshots string `config:"delete_snapshots"`
|
||||
Decompress bool `config:"decompress"`
|
||||
}
|
||||
|
||||
// Fs represents a remote azure server
|
||||
@@ -397,6 +422,8 @@ type Fs struct {
|
||||
copyToken *pacer.TokenDispenser // global multipart copy concurrency limiter
|
||||
publicAccess container.PublicAccessType // Container Public Access Level
|
||||
|
||||
warnCompressed sync.Once // warn once about compressed files
|
||||
|
||||
// user delegation cache
|
||||
userDelegationMu sync.Mutex
|
||||
userDelegation *service.UserDelegationCredential
|
||||
@@ -405,15 +432,16 @@ type Fs struct {
|
||||
|
||||
// Object describes an azure object
|
||||
type Object struct {
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
accessTier blob.AccessTier // Blob Access Tier
|
||||
meta map[string]string // blob metadata - take metadataMu when accessing
|
||||
tags map[string]string // blob tags
|
||||
fs *Fs // what this object is part of
|
||||
remote string // The remote path
|
||||
modTime time.Time // The modified time of the object if known
|
||||
md5 string // MD5 hash if known
|
||||
size int64 // Size of the object
|
||||
mimeType string // Content-Type of the object
|
||||
accessTier blob.AccessTier // Blob Access Tier
|
||||
meta map[string]string // blob metadata - take metadataMu when accessing
|
||||
tags map[string]string // blob tags
|
||||
contentEncoding *string // Content-Encoding of the object
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -634,6 +662,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
NewClientWithSharedKeyCredential: service.NewClientWithSharedKeyCredential,
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
// Override the automatic decompression in the transport
|
||||
// to download compressed files as-is
|
||||
policyClientOptions.PerCallPolicies = append(policyClientOptions.PerCallPolicies, setAcceptEncodingGzip{})
|
||||
options.ClientOptions = policyClientOptions
|
||||
},
|
||||
}
|
||||
@@ -2001,6 +2032,10 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
if t != hash.MD5 {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
// If decompressing, erase the hash
|
||||
if o.size < 0 {
|
||||
return "", nil
|
||||
}
|
||||
// Convert base64 encoded md5 into lower case hex
|
||||
if o.md5 == "" {
|
||||
return "", nil
|
||||
@@ -2126,6 +2161,13 @@ func (o *Object) decodeMetaDataFromPropertiesResponse(info *blob.GetPropertiesRe
|
||||
o.accessTier = blob.AccessTier(*info.AccessTier)
|
||||
}
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2178,6 +2220,13 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
|
||||
fs.Debugf(o, "Failed to find length in %q", contentRange)
|
||||
}
|
||||
}
|
||||
o.contentEncoding = info.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2216,6 +2265,13 @@ func (o *Object) decodeMetaDataFromBlob(info *container.BlobItem) (err error) {
|
||||
o.accessTier = *info.Properties.AccessTier
|
||||
}
|
||||
o.setMetadata(metadata)
|
||||
o.contentEncoding = info.Properties.ContentEncoding
|
||||
|
||||
// If decompressing then size and md5sum are unknown
|
||||
if o.fs.opt.Decompress && o.contentEncoding != nil && *o.contentEncoding == "gzip" {
|
||||
o.size = -1
|
||||
o.md5 = ""
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2391,6 +2447,17 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode metadata for download: %w", err)
|
||||
}
|
||||
|
||||
// Decompress body if necessary
|
||||
if downloadResponse.ContentEncoding != nil && *downloadResponse.ContentEncoding == "gzip" {
|
||||
if o.fs.opt.Decompress {
|
||||
return readers.NewGzipReader(downloadResponse.Body)
|
||||
}
|
||||
o.fs.warnCompressed.Do(func() {
|
||||
fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --azureblob-decompress to override")
|
||||
})
|
||||
}
|
||||
|
||||
return downloadResponse.Body, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
package azureblob
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -14,6 +17,7 @@ import (
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/object"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
@@ -150,10 +154,65 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
|
||||
require.NoError(t, dst.Remove(ctx))
|
||||
}
|
||||
|
||||
func gz(t *testing.T, s string) string {
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
_, err := zw.Write([]byte(s))
|
||||
require.NoError(t, err)
|
||||
err = zw.Close()
|
||||
require.NoError(t, err)
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func md5sum(t *testing.T, s string) string {
|
||||
hash := md5.Sum([]byte(s))
|
||||
return fmt.Sprintf("%x", hash)
|
||||
}
|
||||
|
||||
func (f *Fs) testGzipEncoding(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
original := random.String(1000)
|
||||
contents := gz(t, original)
|
||||
|
||||
item := fstest.NewItem("test-gzip", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
metadata := fs.Metadata{
|
||||
"content-encoding": "gzip",
|
||||
"content-type": "text/plain",
|
||||
}
|
||||
obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", metadata)
|
||||
defer func() {
|
||||
assert.NoError(t, obj.Remove(ctx))
|
||||
}()
|
||||
o := obj.(*Object)
|
||||
|
||||
// Test that the gzipped file we uploaded can be
|
||||
// downloaded with and without decompression
|
||||
checkDownload := func(wantContents string, wantSize int64, wantHash string) {
|
||||
gotContents := fstests.ReadObject(ctx, t, o, -1)
|
||||
assert.Equal(t, wantContents, gotContents)
|
||||
assert.Equal(t, wantSize, o.Size())
|
||||
gotHash, err := o.Hash(ctx, hash.MD5)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wantHash, gotHash)
|
||||
}
|
||||
|
||||
t.Run("NoDecompress", func(t *testing.T) {
|
||||
checkDownload(contents, int64(len(contents)), md5sum(t, contents))
|
||||
})
|
||||
t.Run("Decompress", func(t *testing.T) {
|
||||
f.opt.Decompress = true
|
||||
defer func() {
|
||||
f.opt.Decompress = false
|
||||
}()
|
||||
checkDownload(original, -1, "")
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
t.Run("Features", f.testFeatures)
|
||||
t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
|
||||
t.Run("Metadata", f.testMetadataPaths)
|
||||
t.Run("GzipEncoding", f.testGzipEncoding)
|
||||
}
|
||||
|
||||
// helper to read blob properties for an object
|
||||
|
||||
@@ -173,6 +173,10 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
|
||||
NewSharedKeyCredential: service.NewSharedKeyCredential,
|
||||
SetClientOptions: func(options *service.ClientOptions, policyClientOptions policy.ClientOptions) {
|
||||
options.ClientOptions = policyClientOptions
|
||||
// FileRequestIntent is required for TokenCredential (OAuth)
|
||||
// auth; the SDK omits it for shared-key / SAS.
|
||||
backup := service.ShareTokenIntentBackup
|
||||
options.FileRequestIntent = &backup
|
||||
},
|
||||
}
|
||||
res, err := auth.NewClient(ctx, conf, &opt.Options)
|
||||
|
||||
Vendored
+4
-3
@@ -563,9 +563,10 @@ Eg
|
||||
})
|
||||
|
||||
rc.Add(rc.Call{
|
||||
Path: "cache/stats",
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Path: "cache/stats",
|
||||
NoAuth: true,
|
||||
Fn: f.httpStats,
|
||||
Title: "Get cache stats",
|
||||
Help: `
|
||||
Show statistics for the cache remote.
|
||||
`,
|
||||
|
||||
@@ -34,6 +34,13 @@ import (
|
||||
"github.com/zeebo/blake3"
|
||||
)
|
||||
|
||||
// Pacer settings - tuned to match other rclone backends
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Cloudinary shouldn't have a trailing dot if there is no path
|
||||
func cldPathDir(somePath string) string {
|
||||
if somePath == "" || somePath == "." {
|
||||
@@ -189,7 +196,7 @@ func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (f
|
||||
root: root,
|
||||
opt: *opt,
|
||||
cld: cld,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1000), pacer.MaxSleep(10000), pacer.DecayConstant(2))),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
srv: rest.NewClient(client),
|
||||
}
|
||||
|
||||
@@ -586,21 +593,18 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
// Cloudinary returns rate-limit windows as "Try again on YYYY-MM-DD HH:MM:SS UTC"
|
||||
// embedded in the error message - honour it as a retry-after.
|
||||
if err != nil {
|
||||
tryAgain := "Try again on "
|
||||
if idx := strings.Index(err.Error(), tryAgain); idx != -1 {
|
||||
layout := "2006-01-02 15:04:05 UTC"
|
||||
dateStr := err.Error()[idx+len(tryAgain) : idx+len(tryAgain)+len(layout)]
|
||||
timestamp, err2 := time.Parse(layout, dateStr)
|
||||
if err2 == nil {
|
||||
if timestamp, err2 := time.Parse(layout, dateStr); err2 == nil {
|
||||
return true, fserrors.NewErrorRetryAfter(time.Until(timestamp))
|
||||
}
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "Retrying API error %v", err)
|
||||
return true, err
|
||||
}
|
||||
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ type User struct {
|
||||
Avatar string `json:"avatar"`
|
||||
ModelType string `json:"model_type"`
|
||||
OwnsEntry bool `json:"owns_entry"`
|
||||
EntryPermissions []any `json:"entry_permissions"`
|
||||
EntryPermissions any `json:"entry_permissions"`
|
||||
DisplayName string `json:"display_name"`
|
||||
}
|
||||
|
||||
@@ -77,9 +77,9 @@ type Listing struct {
|
||||
Data []Item `json:"data"`
|
||||
From int `json:"from"`
|
||||
LastPage int `json:"last_page"`
|
||||
NextPage int `json:"next_page"`
|
||||
NextPageURL string `json:"next_page_url"`
|
||||
PerPage int `json:"per_page"`
|
||||
PrevPage int `json:"prev_page"`
|
||||
PrevPageURL string `json:"prev_page_url"`
|
||||
To int `json:"to"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
@@ -238,6 +238,14 @@ type MultiPartAbort struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// FolderPathResponse is returned by GET /folders/{hash}/path
|
||||
//
|
||||
// Path is the breadcrumb from the drive root down to the requested folder.
|
||||
type FolderPathResponse struct {
|
||||
Status string `json:"status"`
|
||||
Path []Item `json:"path"`
|
||||
}
|
||||
|
||||
// SpaceUsageResponse is returned by GET /user/space-usage
|
||||
type SpaceUsageResponse struct {
|
||||
Used int64 `json:"used"`
|
||||
|
||||
+98
-21
@@ -15,6 +15,7 @@ should stay under that.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -54,7 +55,7 @@ const (
|
||||
rootURL = baseURL + "api/v1"
|
||||
maxUploadParts = 10000 // maximum allowed number of parts in a multi-part upload
|
||||
minChunkSize = fs.SizeSuffix(1024 * 1024 * 5)
|
||||
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
|
||||
defaultUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024) // as per https://docs.drime.cloud/uploads-guide
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -199,13 +200,16 @@ type Options struct {
|
||||
|
||||
// Fs represents a remote drime
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
srv *rest.Client // the connection to the server
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
pacer *fs.Pacer // pacer for API calls
|
||||
name string // name of this remote
|
||||
root string // the path we are working on
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
srv *rest.Client // the connection to the server
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
pacer *fs.Pacer // pacer for API calls
|
||||
absRootOnce *sync.Once // protects absRoot computation
|
||||
absRoot string // absolute path of f.root from drive root
|
||||
absRootErr error // error from computing absRoot, if any
|
||||
}
|
||||
|
||||
// Object describes a drime object
|
||||
@@ -330,6 +334,62 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
|
||||
return info, err
|
||||
}
|
||||
|
||||
// idToAbsolutePath returns the absolute path (from the user's drive root) of
|
||||
// the folder with the given ID.
|
||||
//
|
||||
// Drime exposes GET /folders/{hash}/path which returns the breadcrumb from
|
||||
// drive root down to the folder. The hash format is undocumented but
|
||||
// observed to be base64("<id>|") - every item drime returns has a `hash`
|
||||
// field of that shape (e.g. id 704791396 => "NzA0NzkxMzk2fA"). The docs
|
||||
// example "MTExMzQ0fHBhZA" decodes to "111344|pad", so a non-empty suffix
|
||||
// after the pipe is also accepted; if drime ever starts requiring a
|
||||
// specific suffix this will break.
|
||||
//
|
||||
// Returns "" for an empty/zero ID (drive root).
|
||||
func (f *Fs) idToAbsolutePath(ctx context.Context, id string) (string, error) {
|
||||
if id == "" || id == "0" {
|
||||
return "", nil
|
||||
}
|
||||
hash := base64.StdEncoding.EncodeToString([]byte(id + "|"))
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/folders/" + hash + "/path",
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
if f.opt.WorkspaceID != "" {
|
||||
opts.Parameters.Set("workspaceId", f.opt.WorkspaceID)
|
||||
}
|
||||
var result api.FolderPathResponse
|
||||
var resp *http.Response
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get folder path for %q: %w", id, err)
|
||||
}
|
||||
parts := make([]string, 0, len(result.Path))
|
||||
for _, item := range result.Path {
|
||||
parts = append(parts, f.opt.Enc.ToStandardName(item.Name))
|
||||
}
|
||||
return path.Join(parts...), nil
|
||||
}
|
||||
|
||||
// absoluteRoot returns the absolute path from the drive root to the Fs root.
|
||||
// Computed once and cached.
|
||||
func (f *Fs) absoluteRoot(ctx context.Context) (string, error) {
|
||||
f.absRootOnce.Do(func() {
|
||||
rootID, err := f.dirCache.RootID(ctx, false)
|
||||
if err != nil {
|
||||
f.absRootErr = err
|
||||
return
|
||||
}
|
||||
f.absRoot, f.absRootErr = f.idToAbsolutePath(ctx, rootID)
|
||||
})
|
||||
return f.absRoot, f.absRootErr
|
||||
}
|
||||
|
||||
// errorHandler parses a non 2xx error response into an error
|
||||
func errorHandler(resp *http.Response) error {
|
||||
body, err := rest.ReadBody(resp)
|
||||
@@ -370,11 +430,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
client := fshttp.NewClient(ctx)
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: rest.NewClient(client).SetRoot(rootURL),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: rest.NewClient(client).SetRoot(rootURL),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
absRootOnce: new(sync.Once),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
@@ -527,7 +588,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
if dirID != "" {
|
||||
opts.Parameters.Add("parentIds", dirID)
|
||||
opts.Parameters.Add("folderId", dirID)
|
||||
}
|
||||
if directoriesOnly {
|
||||
opts.Parameters.Add("type", api.ItemTypeFolder)
|
||||
@@ -565,10 +626,10 @@ OUTER:
|
||||
break OUTER
|
||||
}
|
||||
}
|
||||
if result.NextPage == 0 {
|
||||
if result.CurrentPage == result.LastPage {
|
||||
break
|
||||
}
|
||||
page = result.NextPage
|
||||
page = result.CurrentPage + 1
|
||||
}
|
||||
return found, err
|
||||
}
|
||||
@@ -709,6 +770,11 @@ func (f *Fs) deleteObject(ctx context.Context, id string) error {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
// If Drime returns 422 with invalid entry ID, the object is already gone.
|
||||
// Map this to a standard not-found error
|
||||
if strings.Contains(err.Error(), "entry ids is invalid") {
|
||||
return fs.ErrorObjectNotFound
|
||||
}
|
||||
return fmt.Errorf("failed to delete item: %w", err)
|
||||
}
|
||||
// Check the individual result codes also
|
||||
@@ -1096,6 +1162,15 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, nil, err
|
||||
}
|
||||
|
||||
// The /s3/multipart/create and /s3/entries endpoints interpret
|
||||
// relativePath as an absolute path from the drive root, not relative to
|
||||
// parent_id. Resolve our root's absolute path so we can build it.
|
||||
absRoot, err := f.absoluteRoot(ctx)
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to resolve absolute path of root: %w", err)
|
||||
}
|
||||
relPath := f.opt.Enc.FromStandardPath(path.Join(absRoot, remote))
|
||||
|
||||
// Temporary Object under construction
|
||||
o := &Object{
|
||||
fs: f,
|
||||
@@ -1129,7 +1204,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
Size: createSize,
|
||||
Extension: strings.TrimPrefix(path.Ext(leaf), `.`),
|
||||
ParentID: json.Number(directoryID),
|
||||
RelativePath: f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
|
||||
RelativePath: relPath,
|
||||
WorkspaceID: f.opt.WorkspaceID,
|
||||
}
|
||||
|
||||
@@ -1156,8 +1231,6 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
if ext == "" {
|
||||
ext = "bin"
|
||||
}
|
||||
rel := f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
|
||||
|
||||
chunkWriter := &drimeChunkWriter{
|
||||
uploadID: resp.UploadID,
|
||||
key: resp.Key,
|
||||
@@ -1170,7 +1243,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
mime: mime,
|
||||
extension: ext,
|
||||
parentID: json.Number(directoryID),
|
||||
relativePath: rel,
|
||||
relativePath: relPath,
|
||||
}
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(chunkSize),
|
||||
@@ -1530,7 +1603,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Debugf(o, "Removing old object on successful upload")
|
||||
deleteErr := o.fs.deleteObject(ctx, id)
|
||||
if deleteErr != nil {
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
if errors.Is(deleteErr, fs.ErrorObjectNotFound) {
|
||||
fs.Debugf(o, "Old object already deleted, safely ignoring")
|
||||
} else {
|
||||
err = fmt.Errorf("failed to delete existing object: %w", deleteErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
+30
-2
@@ -21,9 +21,11 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -47,7 +49,7 @@ func init() {
|
||||
},
|
||||
{
|
||||
Name: "api_key",
|
||||
Help: `API Key for your Filen account
|
||||
Help: `API Key for your Filen account
|
||||
|
||||
Get this using the Filen CLI export-api-key command
|
||||
You can download the Filen CLI from https://github.com/FilenCloudDienste/filen-cli`,
|
||||
@@ -161,6 +163,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
filen: filen,
|
||||
Enc: opt.Encoder,
|
||||
concurrency: opt.UploadConcurrency,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
|
||||
fileSystem.features = (&fs.Features{
|
||||
@@ -194,6 +197,13 @@ type Options struct {
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
}
|
||||
|
||||
// defaults copied from box
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
)
|
||||
|
||||
// Fs represents a virtual filesystem mounted on a specific root folder
|
||||
type Fs struct {
|
||||
name string
|
||||
@@ -202,6 +212,7 @@ type Fs struct {
|
||||
Enc encoder.MultiEncoder
|
||||
features *fs.Features
|
||||
concurrency int
|
||||
pacer *fs.Pacer
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
@@ -351,6 +362,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
type chunkWriter struct {
|
||||
sdk.FileUpload
|
||||
filen *sdk.Filen
|
||||
pacer *fs.Pacer
|
||||
bucketAndRegion chan client.V3UploadResponse
|
||||
chunkSize int64
|
||||
|
||||
@@ -362,6 +374,13 @@ type chunkWriter struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
return fserrors.ShouldRetry(err), err
|
||||
}
|
||||
|
||||
func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
|
||||
realChunkNumber := int64(chunkNumber) * (cw.chunkSize) / sdk.ChunkSize
|
||||
chunk := make([]byte, sdk.ChunkSize, sdk.ChunkSize+cw.EncryptionKey.Cipher.Overhead())
|
||||
@@ -414,7 +433,15 @@ func (cw *chunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader i
|
||||
if err != nil {
|
||||
return totalWritten, err
|
||||
}
|
||||
resp, err := cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
|
||||
var resp *client.V3UploadResponse
|
||||
err = cw.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = cw.filen.UploadChunk(ctx, &cw.FileUpload, realChunkNumber, chunkReadSlice)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return totalWritten, err
|
||||
}
|
||||
@@ -494,6 +521,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
|
||||
return info, &chunkWriter{
|
||||
FileUpload: *fu,
|
||||
filen: f.filen,
|
||||
pacer: f.pacer,
|
||||
chunkSize: chunkSize,
|
||||
bucketAndRegion: make(chan client.V3UploadResponse, 1),
|
||||
knownChunks: make(map[int64][]byte),
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
api文档
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-public-info-0000001050159641
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-aboutget-0000001050151684
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filescopy-0000001050151696
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filescreate-0000001050151686
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filescreatecontent-0000001050151688
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filescreatemultipart-0000001050151694
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filescreateresume-0000001050151690
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesdelete-0000001050153647
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesemptyrecycle-0000001050151698
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesgetmetadata-0000001050153645
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesgetcontent-0000001050153651
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-fileslist-0000001050153649
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesresume-0000001050153643
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filessubscribe-0000001050153637
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesupdate-0000001050153639
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesupdatepatchcontent-0000001050151700
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesupdatepatchmultipart-0000001050153653
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesupdatepatchresume-0000001050153641
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesupdateputcontent-0000001050151702
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesupdateputmultipart-0000001050151704
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-filesupdateputresume-0000001050151692
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-thumbnailget-0000001050151706
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-changesgetstartcursor-0000001050153659
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-changeslist-0000001050151710
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-changessubscribe-0000001050151708
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-channelsstop-0000001050151712
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentscreate-0000001050151716
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentsdelete-0000001050151718
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentsget-0000001050153665
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentslist-0000001050151714
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentsupdate-0000001050153667
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-repliescreate-0000001050151722
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-repliesdelete-0000001050151724
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-repliesget-0000001050153671
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-replieslist-0000001050151720
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-repliesupdate-0000001050153673
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-smallthumbnailget-0000001050151726
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentsdelete-0000001050188702
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentsget-0000001050430911
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-historyversionsgetcontent-0000001050747405
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentslist-0000001050188698
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-commentsupdate-0000001050190853
|
||||
https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-api-batchcallback-0000001050727110
|
||||
@@ -0,0 +1,255 @@
|
||||
// Package api provides types used by the Huawei Drive API.
|
||||
package api
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// About represents response to About.get endpoint
|
||||
type About struct {
|
||||
StorageQuota struct {
|
||||
UsedSpace string `json:"usedSpace"`
|
||||
UserCapacity string `json:"userCapacity"`
|
||||
} `json:"storageQuota"`
|
||||
MaxThumbnailSize int64 `json:"maxThumbnailSize"`
|
||||
NeedUpdate bool `json:"needUpdate"`
|
||||
MaxFileUploadSize string `json:"maxFileUploadSize"`
|
||||
Domain string `json:"domain"`
|
||||
Category string `json:"category"`
|
||||
User User `json:"user"`
|
||||
FolderLevel int `json:"folderLevel"`
|
||||
}
|
||||
|
||||
// User represents a user
|
||||
type User struct {
|
||||
PermissionID string `json:"permissionId"`
|
||||
DisplayName string `json:"displayName"`
|
||||
Me bool `json:"me"`
|
||||
Category string `json:"category"`
|
||||
}
|
||||
|
||||
// FileList represents response to Files.list endpoint
|
||||
type FileList struct {
|
||||
Files []File `json:"files"`
|
||||
NextPageToken string `json:"nextPageToken,omitempty"`
|
||||
Category string `json:"category"`
|
||||
}
|
||||
|
||||
// File represents a file or folder in Huawei Drive
|
||||
type File struct {
|
||||
ID string `json:"id"`
|
||||
FileName string `json:"fileName"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Category string `json:"category"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
CreatedTime time.Time `json:"createdTime"`
|
||||
EditedTime time.Time `json:"editedTime"`
|
||||
EditedByMeTime time.Time `json:"editedByMeTime,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Owners []User `json:"owners"`
|
||||
LastEditor User `json:"lastEditor"`
|
||||
Permissions []Permission `json:"permissions"`
|
||||
PermissionIDs []string `json:"permissionIds"`
|
||||
Capabilities FileCapabilities `json:"capabilities"`
|
||||
OwnedByMe bool `json:"ownedByMe"`
|
||||
EditedByMe bool `json:"editedByMe"`
|
||||
ViewedByMe bool `json:"viewedByMe"`
|
||||
HasShared bool `json:"hasShared"`
|
||||
Recycled bool `json:"recycled"`
|
||||
DirectlyRecycled bool `json:"directlyRecycled"`
|
||||
Favorite bool `json:"favorite"`
|
||||
ExistThumbnail bool `json:"existThumbnail"`
|
||||
ThumbnailVersion int64 `json:"thumbnailVersion"`
|
||||
IconDownloadLink string `json:"iconDownloadLink,omitempty"`
|
||||
ContentDownloadLink string `json:"contentDownloadLink,omitempty"`
|
||||
ContentVersion string `json:"contentVersion,omitempty"`
|
||||
LastHistoryVersionID string `json:"lastHistoryVersionId,omitempty"`
|
||||
OccupiedSpace int64 `json:"occupiedSpace,omitempty"`
|
||||
Version int64 `json:"version"`
|
||||
WritersHasSharePermission bool `json:"writersHasSharePermission"`
|
||||
WriterHasCopyPermission bool `json:"writerHasCopyPermission"`
|
||||
Containers []string `json:"containers"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
ContentExtras *ContentExtras `json:"contentExtras,omitempty"`
|
||||
}
|
||||
|
||||
// IsDir returns true if the file is a directory
|
||||
func (f *File) IsDir() bool {
|
||||
return f.MimeType == FolderMimeType
|
||||
}
|
||||
|
||||
// Permission represents file permission
|
||||
type Permission struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Role string `json:"role"`
|
||||
DisplayName string `json:"displayName,omitempty"`
|
||||
Deleted bool `json:"deleted"`
|
||||
AllowFileDiscovery bool `json:"allowFileDiscovery,omitempty"`
|
||||
Category string `json:"category"`
|
||||
}
|
||||
|
||||
// FileCapabilities represents capabilities for a file
|
||||
type FileCapabilities struct {
|
||||
DownloadPermission bool `json:"downloadPermission"`
|
||||
RenameFilePermission bool `json:"renameFilePermission"`
|
||||
CommentPermission bool `json:"commentPermission"`
|
||||
DeletePermission bool `json:"deletePermission"`
|
||||
ReadHistoryVersionPermission bool `json:"readHistoryVersionPermission"`
|
||||
RecyclePermission bool `json:"recyclePermission"`
|
||||
AddChildNodePermission bool `json:"addChildNodePermission"`
|
||||
UnrecyclePermission bool `json:"unrecyclePermission"`
|
||||
WriterHasChangeCopyPermission bool `json:"writerHasChangeCopyPermission"`
|
||||
CopyPermission bool `json:"copyPermission"`
|
||||
EditContentPermission bool `json:"editContentPermission"`
|
||||
RemoveChildNodePermission bool `json:"removeChildNodePermission"`
|
||||
EditPermission bool `json:"editPermission"`
|
||||
ListChildNodePermission bool `json:"listChildNodePermission"`
|
||||
ShareFilePermission bool `json:"shareFilePermission"`
|
||||
}
|
||||
|
||||
// ContentExtras represents additional content information
|
||||
type ContentExtras struct {
|
||||
Thumbnail *Thumbnail `json:"thumbnail,omitempty"`
|
||||
}
|
||||
|
||||
// Thumbnail represents thumbnail information
|
||||
type Thumbnail struct {
|
||||
Content string `json:"content"` // base64 encoded
|
||||
MimeType string `json:"mimeType"`
|
||||
}
|
||||
|
||||
// CreateFolderRequest represents request to create a folder
|
||||
type CreateFolderRequest struct {
|
||||
FileName string `json:"fileName"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Favorite bool `json:"favorite,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AutoRename int `json:"autoRename,omitempty"`
|
||||
}
|
||||
|
||||
// UpdateFileRequest represents request to update a file
|
||||
type UpdateFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
Favorite *bool `json:"favorite,omitempty"`
|
||||
Recycled *bool `json:"recycled,omitempty"`
|
||||
OriginalFilename string `json:"originalFilename,omitempty"`
|
||||
WriterHasCopyPermission *bool `json:"writerHasCopyPermission,omitempty"`
|
||||
WritersHasSharePermission *bool `json:"writersHasSharePermission,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
AddParentFolder []string `json:"addParentFolder,omitempty"`
|
||||
RemoveParentFolder []string `json:"removeParentFolder,omitempty"`
|
||||
CreatedTime *time.Time `json:"createdTime,omitempty"`
|
||||
EditedTime *time.Time `json:"editedTime,omitempty"`
|
||||
}
|
||||
|
||||
// CopyFileRequest represents request to copy a file
|
||||
type CopyFileRequest struct {
|
||||
FileName string `json:"fileName,omitempty"`
|
||||
MimeType string `json:"mimeType,omitempty"`
|
||||
ParentFolder []string `json:"parentFolder,omitempty"`
|
||||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||||
AppSettings map[string]interface{} `json:"appSettings,omitempty"`
|
||||
}
|
||||
|
||||
// ResumeUploadInitResponse represents response to resume upload init
|
||||
type ResumeUploadInitResponse struct {
|
||||
SliceSize int64 `json:"sliceSize"`
|
||||
}
|
||||
|
||||
// ErrorResponse represents an error response from the API
|
||||
type ErrorResponse struct {
|
||||
Error struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
// Constants
|
||||
const (
|
||||
// MIME types
|
||||
FolderMimeType = "application/vnd.huawei-apps.folder"
|
||||
|
||||
// Special folder names
|
||||
ApplicationDataFolder = "applicationData"
|
||||
|
||||
// Upload types
|
||||
UploadTypeMultipart = "multipart"
|
||||
UploadTypeContent = "content"
|
||||
UploadTypeResume = "resume"
|
||||
|
||||
// Form types
|
||||
FormContent = "content"
|
||||
FormJSON = "json"
|
||||
|
||||
// Categories
|
||||
CategoryDriveFile = "drive#file"
|
||||
CategoryDriveFileList = "drive#fileList"
|
||||
CategoryDriveAbout = "drive#about"
|
||||
CategoryDriveUser = "drive#user"
|
||||
|
||||
// Global API domain (always available)
|
||||
GlobalDriveAPI = "https://driveapis.cloud.huawei.com.cn"
|
||||
GlobalUploadAPI = "https://driveapis.cloud.huawei.com.cn"
|
||||
)
|
||||
|
||||
// DomainToRootURL maps the domain field from About:get to the regional API root URL.
|
||||
// The API returns the full domain name (e.g., "driveapis.cloud.huawei.com.cn").
|
||||
// If the domain matches the global API, no switch is needed.
|
||||
// See: https://developer.huawei.com/consumer/cn/doc/HMSCore-References/server-public-info-0000001050159641
|
||||
var DomainToRootURL = map[string]string{
|
||||
"china": "https://drive-drcn.cloud.dbankcloud.cn",
|
||||
"europe": "https://drive-dre.cloud.dbankcloud.cn",
|
||||
"aala": "https://drive-dra.cloud.dbankcloud.cn",
|
||||
"russia": "https://drive-drru.cloud.dbankcloud.cn",
|
||||
"drive-drcn.cloud.dbankcloud.cn": "https://drive-drcn.cloud.dbankcloud.cn",
|
||||
"drive-dre.cloud.dbankcloud.cn": "https://drive-dre.cloud.dbankcloud.cn",
|
||||
"drive-dra.cloud.dbankcloud.cn": "https://drive-dra.cloud.dbankcloud.cn",
|
||||
"drive-drru.cloud.dbankcloud.cn": "https://drive-drru.cloud.dbankcloud.cn",
|
||||
}
|
||||
|
||||
// GlobalDomains lists domain values that indicate the global API endpoint (no switch needed)
|
||||
var GlobalDomains = map[string]bool{
|
||||
"driveapis.cloud.huawei.com.cn": true,
|
||||
"drive.cloud.hicloud.com": true,
|
||||
}
|
||||
|
||||
// BoolPtr returns a pointer to a bool value (helper for UpdateFileRequest)
|
||||
func BoolPtr(b bool) *bool {
|
||||
return &b
|
||||
}
|
||||
|
||||
// StartCursor represents response to Changes.getStartCursor endpoint
|
||||
type StartCursor struct {
|
||||
Category string `json:"category"` // "drive#startCursor"
|
||||
StartCursor string `json:"startCursor"`
|
||||
}
|
||||
|
||||
// ChangeList represents response to Changes.list endpoint
|
||||
type ChangeList struct {
|
||||
Category string `json:"category"` // "drive#changeList"
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
NewStartCursor string `json:"newStartCursor,omitempty"`
|
||||
Changes []ChangeItem `json:"changes"`
|
||||
}
|
||||
|
||||
// ChangeItem represents a single change entry
|
||||
type ChangeItem struct {
|
||||
Category string `json:"category"` // "drive#change"
|
||||
Type string `json:"type"` // resource type, e.g. "File"
|
||||
ChangeType string `json:"changeType"` // change type, e.g. "File"
|
||||
Time string `json:"time"`
|
||||
Deleted bool `json:"deleted"`
|
||||
FileID string `json:"fileId"`
|
||||
File *File `json:"file,omitempty"`
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
||||
// Package api provides functionality for interacting with the iCloud API.
|
||||
// Package api provides functionality for interacting with the iCloud API
|
||||
package api
|
||||
|
||||
import (
|
||||
@@ -8,16 +8,19 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
const (
|
||||
baseEndpoint = "https://www.icloud.com"
|
||||
homeEndpoint = "https://www.icloud.com"
|
||||
setupEndpoint = "https://setup.icloud.com/setup/ws/1"
|
||||
authEndpoint = "https://idmsa.apple.com/appleauth/auth"
|
||||
)
|
||||
@@ -28,26 +31,21 @@ type sessionSave func(*Session)
|
||||
type Client struct {
|
||||
appleID string
|
||||
password string
|
||||
remoteName string // rclone remote name, used for cache namespacing
|
||||
srv *rest.Client
|
||||
Session *Session
|
||||
sessionSaveCallback sessionSave
|
||||
|
||||
drive *DriveService
|
||||
mu sync.Mutex // protects drive and Authenticate
|
||||
}
|
||||
|
||||
// New creates a new Client instance with the provided Apple ID, password, trust token, cookies, and session save callback.
|
||||
//
|
||||
// Parameters:
|
||||
// - appleID: the Apple ID of the user.
|
||||
// - password: the password of the user.
|
||||
// - trustToken: the trust token for the session.
|
||||
// - clientID: the client id for the session.
|
||||
// - cookies: the cookies for the session.
|
||||
// - sessionSaveCallback: the callback function to save the session.
|
||||
func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave) (*Client, error) {
|
||||
// New creates a new iCloud API client and initializes its HTTP session
|
||||
func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string) (*Client, error) {
|
||||
icloud := &Client{
|
||||
appleID: appleID,
|
||||
appleID: strings.ToLower(appleID), // Apple SRP requires lowercase in client-side proof
|
||||
password: password,
|
||||
remoteName: filepath.Base(remoteName),
|
||||
srv: rest.NewClient(fshttp.NewClient(context.Background())),
|
||||
Session: NewSession(),
|
||||
sessionSaveCallback: sessionSaveCallback,
|
||||
@@ -59,10 +57,12 @@ func New(appleID, password, trustToken string, clientID string, cookies []*http.
|
||||
return icloud, nil
|
||||
}
|
||||
|
||||
// DriveService returns the DriveService instance associated with the Client.
|
||||
// DriveService returns the DriveService instance, creating it on first call
|
||||
func (c *Client) DriveService() (*DriveService, error) {
|
||||
var err error
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.drive == nil {
|
||||
var err error
|
||||
c.drive, err = NewDriveService(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -71,11 +71,7 @@ func (c *Client) DriveService() (*DriveService, error) {
|
||||
return c.drive, nil
|
||||
}
|
||||
|
||||
// Request makes a request and retries it if the session is invalid.
|
||||
//
|
||||
// This function is the main entry point for making requests to the iCloud
|
||||
// API. If the initial request returns a 401 (Unauthorized), it will try to
|
||||
// reauthenticate and retry the request.
|
||||
// Request makes a request to the iCloud API, re-authenticating on 401/421
|
||||
func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
|
||||
resp, err = c.Session.Request(ctx, opts, request, response)
|
||||
if err != nil && resp != nil {
|
||||
@@ -89,48 +85,93 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, respo
|
||||
if c.Session.Requires2FA() {
|
||||
return nil, errors.New("trust token expired, please reauth")
|
||||
}
|
||||
return c.RequestNoReAuth(ctx, opts, request, response)
|
||||
return c.Session.Request(ctx, opts, request, response)
|
||||
}
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// RequestNoReAuth makes a request without re-authenticating.
|
||||
//
|
||||
// This function is useful when you have a session that is already
|
||||
// authenticated, but you need to make a request without triggering
|
||||
// a re-authentication.
|
||||
func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) {
|
||||
// Make the request without re-authenticating
|
||||
resp, err = c.Session.Request(ctx, opts, request, response)
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// Authenticate authenticates the client with the iCloud API.
|
||||
// Authenticate authenticates the client, reusing existing session if valid
|
||||
func (c *Client) Authenticate(ctx context.Context) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
// Skip /validate round-trip when saved session has cookies + service endpoints
|
||||
// Native client behavior: use cached session, reauth lazily on 401/421
|
||||
if c.Session.Cookies != nil && len(c.Session.AccountInfo.Webservices) > 0 {
|
||||
fs.Debugf(nil, "iclouddrive: reusing saved session")
|
||||
return nil
|
||||
}
|
||||
// Try loading cached service endpoints to avoid /validate round-trip (~5s)
|
||||
if c.Session.Cookies != nil && c.loadCachedWebservices() {
|
||||
fs.Debugf(nil, "iclouddrive: reusing session with cached endpoints")
|
||||
return nil
|
||||
}
|
||||
if c.Session.Cookies != nil {
|
||||
if err := c.Session.ValidateSession(ctx); err == nil {
|
||||
fs.Debugf("icloud", "Valid session, no need to reauth")
|
||||
fs.Debugf(nil, "iclouddrive: valid session, no need to reauth")
|
||||
c.saveCachedWebservices()
|
||||
return nil
|
||||
}
|
||||
c.Session.Cookies = nil
|
||||
}
|
||||
|
||||
fs.Debugf("icloud", "Authenticating as %s\n", c.appleID)
|
||||
fs.Debugf(nil, "iclouddrive: authenticating")
|
||||
err := c.Session.SignIn(ctx, c.appleID, c.password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If 2FA is required, skip AuthWithToken - caller must complete 2FA first
|
||||
if c.Session.Requires2FA() {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = c.Session.AuthWithToken(ctx)
|
||||
if err == nil {
|
||||
err = c.Session.AuthWithToken(ctx)
|
||||
if err == nil && c.sessionSaveCallback != nil {
|
||||
c.saveCachedWebservices()
|
||||
if c.sessionSaveCallback != nil {
|
||||
c.sessionSaveCallback(c.Session)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SignIn signs in the client using the provided context and credentials.
|
||||
func (c *Client) SignIn(ctx context.Context) error {
|
||||
return c.Session.SignIn(ctx, c.appleID, c.password)
|
||||
// loadCachedWebservices loads service endpoints from disk cache
|
||||
func (c *Client) loadCachedWebservices() bool {
|
||||
data, err := os.ReadFile(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName, "webservices.json"))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var ws map[string]*webService
|
||||
if err := json.Unmarshal(data, &ws); err != nil {
|
||||
return false
|
||||
}
|
||||
if len(ws) == 0 {
|
||||
return false
|
||||
}
|
||||
c.Session.AccountInfo.Webservices = ws
|
||||
return true
|
||||
}
|
||||
|
||||
// saveCachedWebservices persists service endpoints to disk
|
||||
func (c *Client) saveCachedWebservices() {
|
||||
if len(c.Session.AccountInfo.Webservices) == 0 {
|
||||
return
|
||||
}
|
||||
saveJSONCache(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName), "webservices.json", c.Session.AccountInfo.Webservices)
|
||||
}
|
||||
|
||||
// CacheDir returns the disk cache directory for this remote
|
||||
func (c *Client) CacheDir() string {
|
||||
return filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName)
|
||||
}
|
||||
|
||||
// ClearCacheDir removes all disk cache files for a remote
|
||||
func ClearCacheDir(remoteName string) {
|
||||
dir := filepath.Join(config.GetCacheDir(), cacheSubdir, filepath.Base(remoteName))
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
fs.Debugf(nil, "iclouddrive: failed to clear cache: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// IntoReader marshals the provided values into a JSON encoded reader
|
||||
@@ -148,19 +189,19 @@ type RequestError struct {
|
||||
Text string
|
||||
}
|
||||
|
||||
// Error satisfy the error interface.
|
||||
// Error satisfies the error interface
|
||||
func (e *RequestError) Error() string {
|
||||
return fmt.Sprintf("%s: %s", e.Text, e.Status)
|
||||
}
|
||||
|
||||
func newRequestError(Status string, Text string) *RequestError {
|
||||
func newRequestError(status string, text string) *RequestError {
|
||||
return &RequestError{
|
||||
Status: strings.ToLower(Status),
|
||||
Text: Text,
|
||||
Status: strings.ToLower(status),
|
||||
Text: text,
|
||||
}
|
||||
}
|
||||
|
||||
// newErr orf makes a new error from sprintf parameters.
|
||||
func newRequestErrorf(Status string, Text string, Parameters ...any) *RequestError {
|
||||
return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...))
|
||||
// newRequestErrorf makes a new error from sprintf parameters
|
||||
func newRequestErrorf(status string, text string, params ...any) *RequestError {
|
||||
return newRequestError(status, fmt.Sprintf(text, params...))
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+691
-200
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,40 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExtractHeadersMergesCookies(t *testing.T) {
|
||||
s := NewSession()
|
||||
s.Cookies = []*http.Cookie{{Name: "existing", Value: "old"}}
|
||||
|
||||
resp := &http.Response{Header: make(http.Header)}
|
||||
resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "existing", Value: "new"}).String())
|
||||
resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "fresh", Value: "value"}).String())
|
||||
resp.Header.Set("X-Apple-Session-Token", "session-token")
|
||||
|
||||
s.extractHeaders(resp)
|
||||
|
||||
require.Len(t, s.Cookies, 2)
|
||||
assert.Equal(t, "new", s.Cookies[0].Value)
|
||||
assert.Equal(t, "fresh", s.Cookies[1].Name)
|
||||
assert.Equal(t, "session-token", s.SessionToken)
|
||||
}
|
||||
|
||||
func TestExtractHeadersDeletesEmptyCookies(t *testing.T) {
|
||||
s := NewSession()
|
||||
s.Cookies = []*http.Cookie{{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: "stale"}, {Name: "keep", Value: "value"}}
|
||||
|
||||
resp := &http.Response{Header: make(http.Header)}
|
||||
resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: ""}).String())
|
||||
|
||||
s.extractHeaders(resp)
|
||||
|
||||
require.Len(t, s.Cookies, 1)
|
||||
assert.Equal(t, "keep", s.Cookies[0].Name)
|
||||
assert.Equal(t, "keep=value", s.GetCookieString())
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"crypto/pbkdf2"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"math/big"
|
||||
)
|
||||
|
||||
// RFC 5054 2048-bit group parameters for SRP-6a (g=2, SHA-256).
|
||||
var (
|
||||
srpN = mustParseBigHex(
|
||||
"AC6BDB41324A9A9BF166DE5E1389582FAF72B6651987EE07FC3192943DB56050" +
|
||||
"A37329CBB4A099ED8193E0757767A13DD52312AB4B03310DCD7F48A9DA04FD50" +
|
||||
"E8083969EDB767B0CF6095179A163AB3661A05FBD5FAAAE82918A9962F0B93B8" +
|
||||
"55F97993EC975EEAA80D740ADBF4FF747359D041D5C33EA71D281E446B14773B" +
|
||||
"CA97B43A23FB801676BD207A436C6481F1D2B9078717461A5B9D32E688F87748" +
|
||||
"544523B524B0D57D5EA77A2775D2ECFA032CFBDBF52FB37861602790" +
|
||||
"04E57AE6AF874E7303CE53299CCC041C7BC308D82A5698F3A8D0C38271AE35F8" +
|
||||
"E9DBFBB694B5C803D89F7AE435DE236D525F54759B65E372FCD68EF20FA7111F" +
|
||||
"9E4AFF73")
|
||||
srpG = big.NewInt(2)
|
||||
srpNLenBytes = 2048 / 8
|
||||
srpHashFunc = sha256.New
|
||||
)
|
||||
|
||||
func mustParseBigHex(s string) *big.Int {
|
||||
b, ok := new(big.Int).SetString(s, 16)
|
||||
if !ok {
|
||||
panic("srp: bad hex constant for N")
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// srpClient implements the client side of SRP-6a for Apple's iCloud auth.
|
||||
type srpClient struct {
|
||||
a *big.Int // client secret
|
||||
A *big.Int // client public value
|
||||
k *big.Int // multiplier
|
||||
M1 []byte
|
||||
M2 []byte
|
||||
K []byte
|
||||
}
|
||||
|
||||
// newSRPClient generates a random 32-byte secret and computes the public value A.
|
||||
func newSRPClient() (*srpClient, error) {
|
||||
secret := make([]byte, 32)
|
||||
if _, err := rand.Read(secret); err != nil {
|
||||
return nil, fmt.Errorf("srp: rand.Read failed: %w", err)
|
||||
}
|
||||
|
||||
a := new(big.Int).SetBytes(secret)
|
||||
A := new(big.Int).Exp(srpG, a, srpN)
|
||||
k := getMultiplier()
|
||||
|
||||
return &srpClient{
|
||||
a: a,
|
||||
A: A,
|
||||
k: k,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getABytes returns the padded public value A.
|
||||
func (c *srpClient) getABytes() []byte {
|
||||
return padToN(c.A)
|
||||
}
|
||||
|
||||
// processChallenge computes the session key and proof values from the server challenge.
|
||||
// username is the Apple ID, derivedKey is the output of derivePassword, salt and B come
|
||||
// from the server's init response (raw bytes, already base64-decoded).
|
||||
func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) error {
|
||||
B := new(big.Int).SetBytes(serverB)
|
||||
|
||||
// Validate B (RFC 5054)
|
||||
if B.Cmp(big.NewInt(0)) <= 0 || B.Cmp(srpN) >= 0 {
|
||||
return fmt.Errorf("srp: invalid server-supplied B, must be 1..N-1")
|
||||
}
|
||||
|
||||
x := calculateX(salt, derivedKey)
|
||||
u := calculateU(c.A, B)
|
||||
|
||||
// Validate u (RFC 5054)
|
||||
if u.Sign() == 0 {
|
||||
return fmt.Errorf("srp: calculated u is zero, aborting")
|
||||
}
|
||||
S := calculateS(c.k, x, c.a, B, u)
|
||||
c.K = calculateK(S)
|
||||
|
||||
aBytes := padToN(c.A)
|
||||
bBytes := padToN(B)
|
||||
c.M1 = calculateM1(username, salt, aBytes, bBytes, c.K)
|
||||
c.M2 = calculateM2(aBytes, c.M1, c.K)
|
||||
return nil
|
||||
}
|
||||
|
||||
// derivePassword performs Apple's password key derivation.
|
||||
// For "s2k": PBKDF2(SHA256(password), salt, iterations, 32, SHA256)
|
||||
// For "s2k_fo": PBKDF2(hex(SHA256(password)), salt, iterations, 32, SHA256)
|
||||
func derivePassword(password string, salt []byte, iterations int, protocol string) ([]byte, error) {
|
||||
passHash := sha256.Sum256([]byte(password))
|
||||
|
||||
var passInput string
|
||||
switch protocol {
|
||||
case "s2k":
|
||||
passInput = string(passHash[:])
|
||||
case "s2k_fo":
|
||||
passInput = hex.EncodeToString(passHash[:])
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported SRP protocol: %q", protocol)
|
||||
}
|
||||
return pbkdf2.Key(sha256.New, passInput, salt, iterations, 32)
|
||||
}
|
||||
|
||||
// padToN pads a big.Int to the SRP group size (256 bytes for 2048-bit).
|
||||
func padToN(n *big.Int) []byte {
|
||||
b := n.Bytes()
|
||||
if len(b) >= srpNLenBytes {
|
||||
return b
|
||||
}
|
||||
padded := make([]byte, srpNLenBytes)
|
||||
copy(padded[srpNLenBytes-len(b):], b)
|
||||
return padded
|
||||
}
|
||||
|
||||
func srpHash(data []byte) []byte {
|
||||
h := srpHashFunc()
|
||||
h.Write(data)
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
func hashToInt(h hash.Hash) *big.Int {
|
||||
return new(big.Int).SetBytes(h.Sum(nil))
|
||||
}
|
||||
|
||||
// getMultiplier computes k = H(N | pad(g))
|
||||
func getMultiplier() *big.Int {
|
||||
h := srpHashFunc()
|
||||
nBytes := srpN.Bytes()
|
||||
gBytes := srpG.Bytes()
|
||||
// pad g to same length as N
|
||||
for len(gBytes) < len(nBytes) {
|
||||
gBytes = append([]byte{0}, gBytes...)
|
||||
}
|
||||
h.Write(nBytes)
|
||||
h.Write(gBytes)
|
||||
return hashToInt(h)
|
||||
}
|
||||
|
||||
// calculateX computes x = H(salt | H(":" | password))
|
||||
// Apple's variant: NoUserNameInX - username is omitted from the inner hash.
|
||||
// The "password" here is actually the derived key from derivePassword.
|
||||
func calculateX(salt, derivedKey []byte) *big.Int {
|
||||
h := srpHashFunc()
|
||||
// Apple omits username (NoUserNameInX), only writes ":" + derivedKey
|
||||
h.Write([]byte(":"))
|
||||
h.Write(derivedKey)
|
||||
digest := h.Sum(nil)
|
||||
|
||||
h2 := srpHashFunc()
|
||||
h2.Write(salt)
|
||||
h2.Write(digest)
|
||||
return new(big.Int).SetBytes(h2.Sum(nil))
|
||||
}
|
||||
|
||||
// calculateU computes u = H(pad(A) | pad(B))
|
||||
func calculateU(A, B *big.Int) *big.Int {
|
||||
h := srpHashFunc()
|
||||
h.Write(padToN(A))
|
||||
h.Write(padToN(B))
|
||||
return hashToInt(h)
|
||||
}
|
||||
|
||||
// calculateS computes the client-side shared secret:
|
||||
// S = (B - k * g^x) ^ (a + u*x) mod N
|
||||
func calculateS(k, x, a, B, u *big.Int) []byte {
|
||||
// g^x mod N
|
||||
gx := new(big.Int).Exp(srpG, x, srpN)
|
||||
|
||||
// k * g^x
|
||||
kgx := new(big.Int).Mul(k, gx)
|
||||
|
||||
// B - k*g^x
|
||||
diff := new(big.Int).Sub(B, kgx)
|
||||
|
||||
// u*x
|
||||
ux := new(big.Int).Mul(u, x)
|
||||
|
||||
// a + u*x
|
||||
exp := new(big.Int).Add(a, ux)
|
||||
|
||||
// (B - k*g^x) ^ (a + u*x) mod N
|
||||
S := new(big.Int).Exp(diff, exp, srpN)
|
||||
|
||||
// Ensure positive
|
||||
S.Mod(S, srpN)
|
||||
|
||||
return padToN(S)
|
||||
}
|
||||
|
||||
// calculateK computes K = H(S)
|
||||
func calculateK(S []byte) []byte {
|
||||
return srpHash(S)
|
||||
}
|
||||
|
||||
// calculateM1 computes the client proof (Apple/iCloud variant):
|
||||
// M1 = H(H(g) XOR H(N) | H(username) | salt | A | B | K)
|
||||
func calculateM1(username, salt, A, B, K []byte) []byte {
|
||||
hg := srpHash(padToN(srpG))
|
||||
hn := srpHash(srpN.Bytes())
|
||||
hi := srpHash(username)
|
||||
|
||||
hxor := make([]byte, len(hg))
|
||||
for i := range hg {
|
||||
hxor[i] = hg[i] ^ hn[i]
|
||||
}
|
||||
|
||||
h := srpHashFunc()
|
||||
h.Write(hxor)
|
||||
h.Write(hi)
|
||||
h.Write(salt)
|
||||
h.Write(A)
|
||||
h.Write(B)
|
||||
h.Write(K)
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
// calculateM2 computes the server proof: M2 = H(A | M1 | K)
|
||||
func calculateM2(A, M1, K []byte) []byte {
|
||||
h := srpHashFunc()
|
||||
h.Write(A)
|
||||
h.Write(M1)
|
||||
h.Write(K)
|
||||
return h.Sum(nil)
|
||||
}
|
||||
@@ -0,0 +1,411 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"math/big"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPadToN(t *testing.T) {
|
||||
t.Run("pads small value to 256 bytes", func(t *testing.T) {
|
||||
result := padToN(big.NewInt(1))
|
||||
assert.Equal(t, srpNLenBytes, len(result))
|
||||
assert.Equal(t, byte(0), result[0])
|
||||
assert.Equal(t, byte(1), result[srpNLenBytes-1])
|
||||
})
|
||||
|
||||
t.Run("pads zero to 256 bytes", func(t *testing.T) {
|
||||
result := padToN(big.NewInt(0))
|
||||
assert.Equal(t, srpNLenBytes, len(result))
|
||||
for _, b := range result {
|
||||
assert.Equal(t, byte(0), b)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("preserves value that fills N", func(t *testing.T) {
|
||||
// Use N itself - already 256 bytes
|
||||
result := padToN(srpN)
|
||||
assert.Equal(t, srpNLenBytes, len(result))
|
||||
assert.Equal(t, srpN.Bytes(), result)
|
||||
})
|
||||
|
||||
t.Run("round-trips through big.Int", func(t *testing.T) {
|
||||
original := big.NewInt(123456789)
|
||||
padded := padToN(original)
|
||||
recovered := new(big.Int).SetBytes(padded)
|
||||
assert.Equal(t, 0, original.Cmp(recovered))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetMultiplier(t *testing.T) {
|
||||
k := getMultiplier()
|
||||
|
||||
// k must be non-zero and less than N
|
||||
assert.True(t, k.Sign() > 0, "multiplier must be positive")
|
||||
assert.True(t, k.Cmp(srpN) < 0, "multiplier must be less than N")
|
||||
|
||||
// k is deterministic
|
||||
k2 := getMultiplier()
|
||||
assert.Equal(t, 0, k.Cmp(k2), "multiplier must be deterministic")
|
||||
|
||||
// k = H(N | pad(g)) - verify by manual computation
|
||||
h := srpHashFunc()
|
||||
nBytes := srpN.Bytes()
|
||||
gBytes := srpG.Bytes()
|
||||
for len(gBytes) < len(nBytes) {
|
||||
gBytes = append([]byte{0}, gBytes...)
|
||||
}
|
||||
h.Write(nBytes)
|
||||
h.Write(gBytes)
|
||||
expected := new(big.Int).SetBytes(h.Sum(nil))
|
||||
assert.Equal(t, 0, expected.Cmp(k), "multiplier must equal H(N|pad(g))")
|
||||
}
|
||||
|
||||
func TestDerivePassword(t *testing.T) {
|
||||
salt := []byte("test_salt_value!")
|
||||
password := "test_password"
|
||||
|
||||
t.Run("s2k produces 32-byte key", func(t *testing.T) {
|
||||
key, err := derivePassword(password, salt, 1000, "s2k")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 32, len(key))
|
||||
})
|
||||
|
||||
t.Run("s2k_fo produces 32-byte key", func(t *testing.T) {
|
||||
key, err := derivePassword(password, salt, 1000, "s2k_fo")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 32, len(key))
|
||||
})
|
||||
|
||||
t.Run("s2k and s2k_fo produce different keys", func(t *testing.T) {
|
||||
key1, err := derivePassword(password, salt, 1000, "s2k")
|
||||
require.NoError(t, err)
|
||||
key2, err := derivePassword(password, salt, 1000, "s2k_fo")
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, key1, key2, "s2k and s2k_fo must produce different keys")
|
||||
})
|
||||
|
||||
t.Run("deterministic output", func(t *testing.T) {
|
||||
key1, err := derivePassword(password, salt, 1000, "s2k")
|
||||
require.NoError(t, err)
|
||||
key2, err := derivePassword(password, salt, 1000, "s2k")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, key1, key2)
|
||||
})
|
||||
|
||||
t.Run("different passwords produce different keys", func(t *testing.T) {
|
||||
key1, err := derivePassword("password1", salt, 1000, "s2k")
|
||||
require.NoError(t, err)
|
||||
key2, err := derivePassword("password2", salt, 1000, "s2k")
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, key1, key2)
|
||||
})
|
||||
|
||||
t.Run("s2k uses raw SHA256 bytes", func(t *testing.T) {
|
||||
// Verify that s2k uses SHA256(password) as the PBKDF2 input.
|
||||
// Two passwords with the same SHA256 hash would produce the same key,
|
||||
// while two passwords with different SHA256 hashes must produce different keys.
|
||||
key1, err := derivePassword("hello", salt, 10, "s2k")
|
||||
require.NoError(t, err)
|
||||
key2, err := derivePassword("world", salt, 10, "s2k")
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, key1, key2)
|
||||
})
|
||||
|
||||
t.Run("s2k_fo uses hex-encoded SHA256", func(t *testing.T) {
|
||||
// For s2k_fo, the PBKDF2 input is hex(SHA256(password)), which is
|
||||
// a 64-character ASCII string, not raw 32-byte hash.
|
||||
key, err := derivePassword("hello", salt, 10, "s2k_fo")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 32, len(key))
|
||||
})
|
||||
|
||||
t.Run("unknown protocol returns error", func(t *testing.T) {
|
||||
_, err := derivePassword(password, salt, 1000, "unknown")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unsupported SRP protocol")
|
||||
})
|
||||
}
|
||||
|
||||
func TestCalculateX(t *testing.T) {
|
||||
salt := []byte("some_salt")
|
||||
derivedKey := []byte("some_derived_key_value_32_bytes!")
|
||||
|
||||
x := calculateX(salt, derivedKey)
|
||||
|
||||
// x must be positive and less than the hash output space
|
||||
assert.True(t, x.Sign() > 0, "x must be positive")
|
||||
|
||||
// x is deterministic
|
||||
x2 := calculateX(salt, derivedKey)
|
||||
assert.Equal(t, 0, x.Cmp(x2), "x must be deterministic")
|
||||
|
||||
// Verify manually: x = H(salt | H(":" | derivedKey))
|
||||
h1 := srpHashFunc()
|
||||
h1.Write([]byte(":"))
|
||||
h1.Write(derivedKey)
|
||||
inner := h1.Sum(nil)
|
||||
|
||||
h2 := srpHashFunc()
|
||||
h2.Write(salt)
|
||||
h2.Write(inner)
|
||||
expected := new(big.Int).SetBytes(h2.Sum(nil))
|
||||
assert.Equal(t, 0, expected.Cmp(x), "x must equal H(salt | H(\":\" | derivedKey))")
|
||||
}
|
||||
|
||||
func TestCalculateU(t *testing.T) {
|
||||
A := big.NewInt(12345)
|
||||
B := big.NewInt(67890)
|
||||
|
||||
u := calculateU(A, B)
|
||||
assert.True(t, u.Sign() > 0, "u must be positive")
|
||||
|
||||
// u is deterministic
|
||||
u2 := calculateU(A, B)
|
||||
assert.Equal(t, 0, u.Cmp(u2))
|
||||
|
||||
// u changes when inputs change
|
||||
u3 := calculateU(A, big.NewInt(99999))
|
||||
assert.NotEqual(t, 0, u.Cmp(u3), "u must change when B changes")
|
||||
|
||||
// Verify: u = H(pad(A) | pad(B))
|
||||
h := srpHashFunc()
|
||||
h.Write(padToN(A))
|
||||
h.Write(padToN(B))
|
||||
expected := new(big.Int).SetBytes(h.Sum(nil))
|
||||
assert.Equal(t, 0, expected.Cmp(u))
|
||||
}
|
||||
|
||||
func TestCalculateK(t *testing.T) {
|
||||
S := []byte("some_shared_secret_value_32bytes")
|
||||
|
||||
K := calculateK(S)
|
||||
|
||||
// K is a SHA-256 hash, so 32 bytes
|
||||
assert.Equal(t, sha256.Size, len(K))
|
||||
|
||||
// K is deterministic
|
||||
K2 := calculateK(S)
|
||||
assert.Equal(t, K, K2)
|
||||
|
||||
// Verify: K = H(S)
|
||||
expected := srpHash(S)
|
||||
assert.Equal(t, expected, K)
|
||||
}
|
||||
|
||||
func TestCalculateM1(t *testing.T) {
|
||||
username := []byte("test@example.com")
|
||||
salt := []byte("test_salt")
|
||||
A := make([]byte, srpNLenBytes)
|
||||
B := make([]byte, srpNLenBytes)
|
||||
K := make([]byte, 32)
|
||||
|
||||
// Fill with non-zero values
|
||||
A[srpNLenBytes-1] = 0x42
|
||||
B[srpNLenBytes-1] = 0x43
|
||||
K[0] = 0x44
|
||||
|
||||
m1 := calculateM1(username, salt, A, B, K)
|
||||
|
||||
// M1 is a SHA-256 hash
|
||||
assert.Equal(t, sha256.Size, len(m1))
|
||||
|
||||
// M1 is deterministic
|
||||
m1b := calculateM1(username, salt, A, B, K)
|
||||
assert.Equal(t, m1, m1b)
|
||||
|
||||
// M1 changes when username changes
|
||||
m1c := calculateM1([]byte("other@example.com"), salt, A, B, K)
|
||||
assert.NotEqual(t, m1, m1c)
|
||||
|
||||
// Verify: M1 = H(H(g) XOR H(N) | H(username) | salt | A | B | K)
|
||||
hg := srpHash(padToN(srpG))
|
||||
hn := srpHash(srpN.Bytes())
|
||||
hi := srpHash(username)
|
||||
hxor := make([]byte, len(hg))
|
||||
for i := range hg {
|
||||
hxor[i] = hg[i] ^ hn[i]
|
||||
}
|
||||
h := srpHashFunc()
|
||||
h.Write(hxor)
|
||||
h.Write(hi)
|
||||
h.Write(salt)
|
||||
h.Write(A)
|
||||
h.Write(B)
|
||||
h.Write(K)
|
||||
expected := h.Sum(nil)
|
||||
assert.Equal(t, expected, m1)
|
||||
}
|
||||
|
||||
func TestCalculateM2(t *testing.T) {
|
||||
A := make([]byte, srpNLenBytes)
|
||||
M1 := make([]byte, 32)
|
||||
K := make([]byte, 32)
|
||||
|
||||
A[srpNLenBytes-1] = 0x01
|
||||
M1[0] = 0x02
|
||||
K[0] = 0x03
|
||||
|
||||
m2 := calculateM2(A, M1, K)
|
||||
assert.Equal(t, sha256.Size, len(m2))
|
||||
|
||||
// Verify: M2 = H(A | M1 | K)
|
||||
h := srpHashFunc()
|
||||
h.Write(A)
|
||||
h.Write(M1)
|
||||
h.Write(K)
|
||||
assert.Equal(t, h.Sum(nil), m2)
|
||||
}
|
||||
|
||||
func TestNewSRPClient(t *testing.T) {
|
||||
client, err := newSRPClient()
|
||||
require.NoError(t, err)
|
||||
|
||||
// A must be non-zero and in range [1, N-1]
|
||||
assert.True(t, client.A.Sign() > 0, "A must be positive")
|
||||
assert.True(t, client.A.Cmp(srpN) < 0, "A must be less than N")
|
||||
|
||||
// A = g^a mod N
|
||||
expectedA := new(big.Int).Exp(srpG, client.a, srpN)
|
||||
assert.Equal(t, 0, expectedA.Cmp(client.A), "A must equal g^a mod N")
|
||||
|
||||
// getABytes returns padded A
|
||||
aBytes := client.getABytes()
|
||||
assert.Equal(t, srpNLenBytes, len(aBytes))
|
||||
recoveredA := new(big.Int).SetBytes(aBytes)
|
||||
assert.Equal(t, 0, client.A.Cmp(recoveredA))
|
||||
|
||||
// Two clients should have different secrets (probabilistic, but 2^256 collision chance)
|
||||
client2, err := newSRPClient()
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, 0, client.a.Cmp(client2.a), "two clients must have different secrets")
|
||||
}
|
||||
|
||||
func TestProcessChallenge(t *testing.T) {
|
||||
// Use a fixed secret for deterministic testing
|
||||
fixedSecret := make([]byte, 32)
|
||||
fixedSecret[31] = 0x42 // small non-zero value
|
||||
|
||||
a := new(big.Int).SetBytes(fixedSecret)
|
||||
A := new(big.Int).Exp(srpG, a, srpN)
|
||||
k := getMultiplier()
|
||||
|
||||
client := &srpClient{
|
||||
a: a,
|
||||
A: A,
|
||||
k: k,
|
||||
}
|
||||
|
||||
username := []byte("testuser@apple.com")
|
||||
derivedKey := make([]byte, 32)
|
||||
for i := range derivedKey {
|
||||
derivedKey[i] = byte(i)
|
||||
}
|
||||
salt := []byte("fixed_salt_for_test!")
|
||||
|
||||
// Compute a valid B: B = (k*v + g^b) mod N where v = g^x mod N
|
||||
x := calculateX(salt, derivedKey)
|
||||
v := new(big.Int).Exp(srpG, x, srpN)
|
||||
|
||||
bSecret := make([]byte, 32)
|
||||
bSecret[31] = 0x99
|
||||
b := new(big.Int).SetBytes(bSecret)
|
||||
|
||||
// B = (k*v + g^b) mod N
|
||||
kv := new(big.Int).Mul(k, v)
|
||||
gb := new(big.Int).Exp(srpG, b, srpN)
|
||||
B := new(big.Int).Add(kv, gb)
|
||||
B.Mod(B, srpN)
|
||||
|
||||
serverB := padToN(B)
|
||||
|
||||
err := client.processChallenge(username, derivedKey, salt, serverB)
|
||||
require.NoError(t, err)
|
||||
|
||||
// M1, M2, K must be populated
|
||||
assert.NotNil(t, client.M1)
|
||||
assert.NotNil(t, client.M2)
|
||||
assert.NotNil(t, client.K)
|
||||
assert.Equal(t, sha256.Size, len(client.M1))
|
||||
assert.Equal(t, sha256.Size, len(client.M2))
|
||||
assert.Equal(t, sha256.Size, len(client.K))
|
||||
|
||||
// M1 and M2 must be different
|
||||
assert.NotEqual(t, client.M1, client.M2)
|
||||
|
||||
// Results must be deterministic - run again with same inputs
|
||||
client2 := &srpClient{
|
||||
a: new(big.Int).Set(a),
|
||||
A: new(big.Int).Set(A),
|
||||
k: new(big.Int).Set(k),
|
||||
}
|
||||
err = client2.processChallenge(username, derivedKey, salt, serverB)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, client.M1, client2.M1, "M1 must be deterministic")
|
||||
assert.Equal(t, client.M2, client2.M2, "M2 must be deterministic")
|
||||
assert.Equal(t, client.K, client2.K, "K must be deterministic")
|
||||
|
||||
// Verify the server could also compute S and get the same K.
|
||||
// Server computes: S_server = (A * v^u)^b mod N
|
||||
u := calculateU(A, B)
|
||||
vu := new(big.Int).Exp(v, u, srpN)
|
||||
avu := new(big.Int).Mul(A, vu)
|
||||
avu.Mod(avu, srpN)
|
||||
sServer := new(big.Int).Exp(avu, b, srpN)
|
||||
sServer.Mod(sServer, srpN)
|
||||
kServer := calculateK(padToN(sServer))
|
||||
assert.Equal(t, client.K, kServer, "client and server must derive the same session key")
|
||||
}
|
||||
|
||||
func TestProcessChallenge_InvalidB(t *testing.T) {
|
||||
k := getMultiplier()
|
||||
client := &srpClient{
|
||||
a: big.NewInt(42),
|
||||
A: new(big.Int).Exp(srpG, big.NewInt(42), srpN),
|
||||
k: k,
|
||||
}
|
||||
username := []byte("test@apple.com")
|
||||
derivedKey := make([]byte, 32)
|
||||
salt := []byte("salt")
|
||||
|
||||
t.Run("B=0 rejected", func(t *testing.T) {
|
||||
err := client.processChallenge(username, derivedKey, salt, padToN(big.NewInt(0)))
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("B=N rejected", func(t *testing.T) {
|
||||
err := client.processChallenge(username, derivedKey, salt, padToN(srpN))
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSRPGroupParameters(t *testing.T) {
|
||||
// Verify the 2048-bit prime starts with known hex prefix
|
||||
nHex := hex.EncodeToString(srpN.Bytes())
|
||||
assert.True(t, len(nHex) > 0)
|
||||
assert.Equal(t, "ac6bdb41", nHex[:8], "N must start with known prefix")
|
||||
|
||||
// g must be 2
|
||||
assert.Equal(t, int64(2), srpG.Int64())
|
||||
|
||||
// N byte length must match expected
|
||||
assert.Equal(t, 256, srpNLenBytes)
|
||||
assert.Equal(t, 256, len(srpN.Bytes()))
|
||||
}
|
||||
|
||||
func TestSRPHash(t *testing.T) {
|
||||
data := []byte("hello world")
|
||||
hash := srpHash(data)
|
||||
|
||||
// Must be SHA-256 output
|
||||
assert.Equal(t, sha256.Size, len(hash))
|
||||
|
||||
// Must equal standard sha256
|
||||
expected := sha256.Sum256(data)
|
||||
assert.Equal(t, expected[:], hash)
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
//go:build !plan9 && !solaris
|
||||
|
||||
// Package iclouddrive provides access to iCloud Drive and Photos
|
||||
package iclouddrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/backend/iclouddrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
)
|
||||
|
||||
const configAuthSession = "_auth_session"
|
||||
|
||||
// configAuthState holds session fields that must be preserved across
|
||||
// Config() state machine steps (push 2FA and SMS both need it)
|
||||
type configAuthState struct {
|
||||
Scnt string `json:"s"`
|
||||
SessionID string `json:"i"`
|
||||
AuthAttributes string `json:"a"`
|
||||
FrameID string `json:"f"`
|
||||
SessionToken string `json:"t"`
|
||||
ClientID string `json:"c"`
|
||||
AccountCountry string `json:"ac"`
|
||||
Phones []smsPhone `json:"p,omitempty"`
|
||||
}
|
||||
|
||||
type smsPhone struct {
|
||||
ID int `json:"id"`
|
||||
Num string `json:"n"`
|
||||
Mode string `json:"m"`
|
||||
}
|
||||
|
||||
func saveAuthSession(m configmap.Mapper, s *api.Session, phones []smsPhone) {
|
||||
st := configAuthState{
|
||||
Scnt: s.Scnt,
|
||||
SessionID: s.SessionID,
|
||||
AuthAttributes: s.AuthAttributes,
|
||||
FrameID: s.FrameID,
|
||||
SessionToken: s.SessionToken,
|
||||
ClientID: s.ClientID,
|
||||
AccountCountry: s.AccountCountry,
|
||||
Phones: phones,
|
||||
}
|
||||
data, err := json.Marshal(st)
|
||||
if err != nil {
|
||||
fs.Debugf(nil, "iclouddrive: failed to marshal auth session: %v", err)
|
||||
return
|
||||
}
|
||||
m.Set(configAuthSession, base64.StdEncoding.EncodeToString(data))
|
||||
}
|
||||
|
||||
func loadAuthSession(m configmap.Mapper) (*configAuthState, error) {
|
||||
raw, _ := m.Get(configAuthSession)
|
||||
if raw == "" {
|
||||
return nil, errors.New("auth session state lost, please reconfigure")
|
||||
}
|
||||
data, err := base64.StdEncoding.DecodeString(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("corrupt auth session state: %w", err)
|
||||
}
|
||||
var st configAuthState
|
||||
if err := json.Unmarshal(data, &st); err != nil {
|
||||
return nil, fmt.Errorf("invalid auth session state: %w", err)
|
||||
}
|
||||
return &st, nil
|
||||
}
|
||||
|
||||
func restoreAuthSession(icloud *api.Client, st *configAuthState) {
|
||||
icloud.Session.Scnt = st.Scnt
|
||||
icloud.Session.SessionID = st.SessionID
|
||||
icloud.Session.AuthAttributes = st.AuthAttributes
|
||||
icloud.Session.FrameID = st.FrameID
|
||||
icloud.Session.SessionToken = st.SessionToken
|
||||
icloud.Session.ClientID = st.ClientID
|
||||
icloud.Session.AccountCountry = st.AccountCountry
|
||||
}
|
||||
|
||||
// resumeConfigClient recreates an API client and restores session state saved
|
||||
// from a previous Config() step. Used across 2FA states to avoid re-authenticating
|
||||
func resumeConfigClient(m configmap.Mapper, appleid, password, trustToken, clientID string, cookies []*http.Cookie) (*api.Client, *configAuthState, error) {
|
||||
st, err := loadAuthSession(m)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
restoreAuthSession(icloud, st)
|
||||
return icloud, st, nil
|
||||
}
|
||||
|
||||
// saveAuthCredentials persists trust token, cookies, and clears session state
|
||||
// after successful 2FA validation
|
||||
func saveAuthCredentials(m configmap.Mapper, icloud *api.Client, name string) {
|
||||
m.Set(configTrustToken, icloud.Session.TrustToken)
|
||||
m.Set(configCookies, icloud.Session.GetCookieString())
|
||||
m.Set(configAuthSession, "")
|
||||
api.ClearCacheDir(name)
|
||||
}
|
||||
|
||||
// triggerSMSFlow handles phone selection and SMS triggering
|
||||
// Single phone: triggers SMS immediately, returns code prompt
|
||||
// Multiple phones: saves session + phone list, returns phone picker
|
||||
func triggerSMSFlow(ctx context.Context, icloud *api.Client, phones []api.TrustedPhoneNumber, m configmap.Mapper) (*fs.ConfigOut, error) {
|
||||
smsPhones := make([]smsPhone, len(phones))
|
||||
for i, p := range phones {
|
||||
mode := p.PushMode
|
||||
if mode == "" {
|
||||
mode = "sms"
|
||||
}
|
||||
smsPhones[i] = smsPhone{ID: p.ID, Num: p.NumberWithDialCode, Mode: mode}
|
||||
}
|
||||
|
||||
if len(phones) == 1 {
|
||||
p := smsPhones[0]
|
||||
if err := icloud.Session.RequestSMSCode(ctx, p.ID, p.Mode); err != nil {
|
||||
return nil, fmt.Errorf("failed to send SMS code: %w", err)
|
||||
}
|
||||
saveAuthSession(m, icloud.Session, nil)
|
||||
nextState := fmt.Sprintf("2fa_sms_%d_%s", p.ID, p.Mode)
|
||||
return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", p.Num))
|
||||
}
|
||||
|
||||
// Multiple phones - save session and present picker
|
||||
saveAuthSession(m, icloud.Session, smsPhones)
|
||||
items := make([]fs.OptionExample, len(smsPhones))
|
||||
for i, p := range smsPhones {
|
||||
items[i] = fs.OptionExample{
|
||||
Value: fmt.Sprintf("%d_%s", p.ID, p.Mode),
|
||||
Help: p.Num,
|
||||
}
|
||||
}
|
||||
return fs.ConfigChooseExclusiveFixed("2fa_sms_select", "config_2fa_phone", "Select phone number for SMS verification", items)
|
||||
}
|
||||
|
||||
const (
|
||||
configService = "service"
|
||||
|
||||
// Service types
|
||||
serviceDrive = "drive"
|
||||
servicePhotos = "photos"
|
||||
)
|
||||
|
||||
// ServiceOptions defines the configuration for service selection
|
||||
type ServiceOptions struct {
|
||||
Service string `config:"service"`
|
||||
}
|
||||
|
||||
// Register with rclone
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "iclouddrive",
|
||||
Description: "iCloud Drive and Photos",
|
||||
Config: Config,
|
||||
NewFs: NewServiceFs,
|
||||
MetadataInfo: &fs.MetadataInfo{
|
||||
System: map[string]fs.MetadataHelp{
|
||||
"width": {
|
||||
Help: "Image width in pixels",
|
||||
Type: "int",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"height": {
|
||||
Help: "Image height in pixels",
|
||||
Type: "int",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"added-time": {
|
||||
Help: "Time the item was added to the iCloud library",
|
||||
Type: "RFC 3339",
|
||||
Example: "2006-01-02T15:04:05Z",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"favorite": {
|
||||
Help: "Whether the item is marked as favorite",
|
||||
Type: "bool",
|
||||
ReadOnly: true,
|
||||
},
|
||||
"hidden": {
|
||||
Help: "Whether the item is hidden",
|
||||
Type: "bool",
|
||||
ReadOnly: true,
|
||||
},
|
||||
},
|
||||
Help: "Metadata is read-only and available for the Photos service only.",
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: configService,
|
||||
Help: "iCloud service to use.",
|
||||
Required: true,
|
||||
Default: serviceDrive,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: serviceDrive,
|
||||
Help: "iCloud Drive",
|
||||
}, {
|
||||
Value: servicePhotos,
|
||||
Help: "iCloud Photos",
|
||||
}},
|
||||
}, {
|
||||
Name: configAppleID,
|
||||
Help: "Apple ID.",
|
||||
Required: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: configPassword,
|
||||
Help: "Password.",
|
||||
Required: true,
|
||||
IsPassword: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: configTrustToken,
|
||||
Help: "Trust token for session authentication.",
|
||||
IsPassword: false,
|
||||
Required: false,
|
||||
Sensitive: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: configCookies,
|
||||
Help: "Session cookies.",
|
||||
Required: false,
|
||||
Advanced: false,
|
||||
Sensitive: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: configClientID,
|
||||
Help: "Client ID for iCloud API access.",
|
||||
Required: false,
|
||||
Advanced: true,
|
||||
Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeSlash |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Config handles the authentication and configuration flow
|
||||
func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
var err error
|
||||
appleid, _ := m.Get(configAppleID)
|
||||
if appleid == "" {
|
||||
return nil, errors.New("an Apple ID is required")
|
||||
}
|
||||
|
||||
password, _ := m.Get(configPassword)
|
||||
if password != "" {
|
||||
password, err = obscure.Reveal(password)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
trustToken, _ := m.Get(configTrustToken)
|
||||
cookieRaw, _ := m.Get(configCookies)
|
||||
clientID, _ := m.Get(configClientID)
|
||||
cookies := ReadCookies(cookieRaw)
|
||||
|
||||
switch {
|
||||
case config.State == "":
|
||||
// Force fresh SRP authentication - ignore stale trust token and cookies
|
||||
// so that reconnect always prompts for 2FA
|
||||
m.Set(configAuthSession, "")
|
||||
icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := icloud.Authenticate(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Set(configCookies, icloud.Session.GetCookieString())
|
||||
if icloud.Session.Requires2FA() {
|
||||
// Check if user has no trusted devices - auto-trigger SMS if so
|
||||
authState, err := icloud.Session.GetAuthState(ctx)
|
||||
if err == nil && authState.NoTrustedDevices && len(authState.TrustedPhoneNumbers) > 0 {
|
||||
return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m)
|
||||
}
|
||||
// Explicitly request push to trusted devices - required for iOS 26.4+
|
||||
// where the SRP 409 no longer auto-pushes. GET /appleauth/auth above
|
||||
// may also trigger a push (cosmetic double on pre-26.4, harmless)
|
||||
if err := icloud.Session.RequestPushNotification(ctx); err != nil {
|
||||
fs.Debugf(nil, "iclouddrive: push notification request failed (SMS fallback available): %v", err)
|
||||
} else {
|
||||
fs.Debugf(nil, "iclouddrive: push notification requested to trusted devices")
|
||||
}
|
||||
// Save session state so 2fa_do can validate without re-authenticating
|
||||
// Push codes are account-scoped so session reuse is not strictly required,
|
||||
// but it avoids a redundant SRP roundtrip and a second push on pre-26.4
|
||||
saveAuthSession(m, icloud.Session, nil)
|
||||
return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: enter your 2FA code or type 'sms' for a text message")
|
||||
}
|
||||
// Auth succeeded without 2FA - save updated credentials and clear stale cache
|
||||
m.Set(configTrustToken, icloud.Session.TrustToken)
|
||||
api.ClearCacheDir(name)
|
||||
return nil, nil
|
||||
|
||||
case config.State == "2fa_do":
|
||||
code := config.Result
|
||||
if code == "" {
|
||||
return fs.ConfigError("authenticate", "2FA codes can't be blank")
|
||||
}
|
||||
|
||||
// Restore session from initial sign-in instead of re-authenticating
|
||||
// This avoids a redundant SRP roundtrip and extra push on pre-26.4
|
||||
icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.EqualFold(code, "sms") {
|
||||
authState, err := icloud.Session.GetAuthState(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get trusted phone numbers: %w", err)
|
||||
}
|
||||
if len(authState.TrustedPhoneNumbers) == 0 {
|
||||
return nil, errors.New("no trusted phone numbers on this account")
|
||||
}
|
||||
return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m)
|
||||
}
|
||||
|
||||
if err := icloud.Session.Validate2FACode(ctx, code); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
saveAuthCredentials(m, icloud, name)
|
||||
return nil, nil
|
||||
|
||||
case config.State == "2fa_sms_select":
|
||||
// User selected a phone from ConfigChooseExclusiveFixed (value: "ID_mode")
|
||||
idStr, mode, _ := strings.Cut(config.Result, "_")
|
||||
phoneID, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
m.Set(configAuthSession, "")
|
||||
return nil, fmt.Errorf("invalid phone selection %q", config.Result)
|
||||
}
|
||||
if mode == "" {
|
||||
mode = "sms"
|
||||
}
|
||||
icloud, smsState, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Find the selected phone's display number
|
||||
var phoneNum string
|
||||
for _, p := range smsState.Phones {
|
||||
if p.ID == phoneID {
|
||||
phoneNum = p.Num
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := icloud.Session.RequestSMSCode(ctx, phoneID, mode); err != nil {
|
||||
m.Set(configAuthSession, "")
|
||||
return nil, fmt.Errorf("failed to send SMS code: %w", err)
|
||||
}
|
||||
saveAuthSession(m, icloud.Session, nil)
|
||||
nextState := fmt.Sprintf("2fa_sms_%d_%s", phoneID, mode)
|
||||
return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", phoneNum))
|
||||
|
||||
case strings.HasPrefix(config.State, "2fa_sms_"):
|
||||
code := config.Result
|
||||
if code == "" {
|
||||
return fs.ConfigError("authenticate", "SMS code can't be blank")
|
||||
}
|
||||
// State encodes phone ID and mode: "2fa_sms_<ID>_<mode>"
|
||||
suffix := strings.TrimPrefix(config.State, "2fa_sms_")
|
||||
idStr, mode, _ := strings.Cut(suffix, "_")
|
||||
phoneID, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid phone ID in state %q: %w", config.State, err)
|
||||
}
|
||||
if mode == "" {
|
||||
mode = "sms"
|
||||
}
|
||||
|
||||
icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := icloud.Session.ValidateSMSCode(ctx, code, phoneID, mode); err != nil {
|
||||
m.Set(configAuthSession, "")
|
||||
return nil, err
|
||||
}
|
||||
saveAuthCredentials(m, icloud, name)
|
||||
return nil, nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown state %q", config.State)
|
||||
}
|
||||
}
|
||||
|
||||
// newICloudClient parses options, authenticates, and returns a ready client
|
||||
// Shared between NewFs (Drive) and NewFsPhotos (Photos) to avoid duplication
|
||||
func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*api.Client, *Options, error) {
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if opt.Password != "" {
|
||||
opt.Password, err = obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("couldn't decrypt user password: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if opt.TrustToken == "" {
|
||||
return nil, nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name)
|
||||
}
|
||||
|
||||
cookies := ReadCookies(opt.Cookies)
|
||||
|
||||
callback := func(session *api.Session) {
|
||||
m.Set(configCookies, session.GetCookieString())
|
||||
}
|
||||
|
||||
icloud, err := api.New(
|
||||
opt.AppleID,
|
||||
opt.Password,
|
||||
opt.TrustToken,
|
||||
opt.ClientID,
|
||||
cookies,
|
||||
callback,
|
||||
name,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err := icloud.Authenticate(ctx); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if icloud.Session.Requires2FA() {
|
||||
return nil, nil, errors.New("trust token expired, please reauth")
|
||||
}
|
||||
|
||||
return icloud, opt, nil
|
||||
}
|
||||
|
||||
// disconnectClient clears authentication state and removes disk caches
|
||||
// Shared between Drive Fs and Photos Fs Disconnect() implementations
|
||||
func disconnectClient(m configmap.Mapper, icloud *api.Client) error {
|
||||
m.Set(configTrustToken, "")
|
||||
m.Set(configCookies, "")
|
||||
m.Set(configAuthSession, "")
|
||||
return os.RemoveAll(icloud.CacheDir())
|
||||
}
|
||||
|
||||
// NewServiceFs creates a filesystem instance for the selected service
|
||||
func NewServiceFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse the service selection
|
||||
opt := new(ServiceOptions)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set default service if not specified
|
||||
if opt.Service == "" {
|
||||
opt.Service = serviceDrive
|
||||
}
|
||||
|
||||
// Route to the appropriate backend
|
||||
switch opt.Service {
|
||||
case serviceDrive:
|
||||
// Create Drive filesystem
|
||||
return NewFs(ctx, name, root, m)
|
||||
case servicePhotos:
|
||||
// Create Photos filesystem
|
||||
return NewFsPhotos(ctx, name, root, m)
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid service selection: %s (must be 'drive' or 'photos')", opt.Service)
|
||||
}
|
||||
}
|
||||
@@ -9,17 +9,13 @@ import (
|
||||
"path"
|
||||
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
|
||||
"github.com/rclone/rclone/backend/iclouddrive/api"
|
||||
@@ -27,6 +23,8 @@ import (
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
/*
|
||||
@@ -49,61 +47,10 @@ const (
|
||||
decayConstant = 2
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "iclouddrive",
|
||||
Description: "iCloud Drive",
|
||||
Config: Config,
|
||||
NewFs: NewFs,
|
||||
Options: []fs.Option{{
|
||||
Name: configAppleID,
|
||||
Help: "Apple ID.",
|
||||
Required: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: configPassword,
|
||||
Help: "Password.",
|
||||
Required: true,
|
||||
IsPassword: true,
|
||||
Sensitive: true,
|
||||
}, {
|
||||
Name: configTrustToken,
|
||||
Help: "Trust token (internal use)",
|
||||
IsPassword: false,
|
||||
Required: false,
|
||||
Sensitive: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: configCookies,
|
||||
Help: "cookies (internal use only)",
|
||||
Required: false,
|
||||
Advanced: false,
|
||||
Sensitive: true,
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: configClientID,
|
||||
Help: "Client id",
|
||||
Required: false,
|
||||
Advanced: true,
|
||||
Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d",
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
Advanced: true,
|
||||
Default: (encoder.Display |
|
||||
//encoder.EncodeDot |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
AppleID string `config:"apple_id"`
|
||||
Password string `config:"password"`
|
||||
Photos bool `config:"photos"`
|
||||
TrustToken string `config:"trust_token"`
|
||||
Cookies string `config:"cookies"`
|
||||
ClientID string `config:"client_id"`
|
||||
@@ -116,6 +63,7 @@ type Fs struct {
|
||||
root string // the path we are working on.
|
||||
rootID string
|
||||
opt Options // parsed config options
|
||||
m configmap.Mapper // config map for persisting auth state
|
||||
features *fs.Features // optional features
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
icloud *api.Client
|
||||
@@ -137,72 +85,6 @@ type Object struct {
|
||||
downloadURL string
|
||||
}
|
||||
|
||||
// Config configures the iCloud remote.
|
||||
func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
|
||||
var err error
|
||||
appleid, _ := m.Get(configAppleID)
|
||||
if appleid == "" {
|
||||
return nil, errors.New("a apple ID is required")
|
||||
}
|
||||
|
||||
password, _ := m.Get(configPassword)
|
||||
if password != "" {
|
||||
password, err = obscure.Reveal(password)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
trustToken, _ := m.Get(configTrustToken)
|
||||
cookieRaw, _ := m.Get(configCookies)
|
||||
clientID, _ := m.Get(configClientID)
|
||||
cookies := ReadCookies(cookieRaw)
|
||||
|
||||
switch config.State {
|
||||
case "":
|
||||
icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := icloud.Authenticate(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Set(configCookies, icloud.Session.GetCookieString())
|
||||
if icloud.Session.Requires2FA() {
|
||||
return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: please enter your 2FA code")
|
||||
}
|
||||
return nil, nil
|
||||
case "2fa_do":
|
||||
code := config.Result
|
||||
if code == "" {
|
||||
return fs.ConfigError("authenticate", "2FA codes can't be blank")
|
||||
}
|
||||
|
||||
icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := icloud.SignIn(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := icloud.Session.Validate2FACode(ctx, code); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m.Set(configTrustToken, icloud.Session.TrustToken)
|
||||
m.Set(configCookies, icloud.Session.GetCookieString())
|
||||
return nil, nil
|
||||
|
||||
case "2fa_error":
|
||||
if config.Result == "true" {
|
||||
return fs.ConfigGoto("2fa")
|
||||
}
|
||||
return nil, errors.New("2fa authentication failed")
|
||||
}
|
||||
return nil, fmt.Errorf("unknown state %q", config.State)
|
||||
}
|
||||
|
||||
// find item by path. Will not return any children for the item
|
||||
func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, found bool, err error) {
|
||||
var resp *http.Response
|
||||
@@ -225,7 +107,8 @@ func (f *Fs) findLeafItem(ctx context.Context, pathID string, leaf string) (item
|
||||
return nil, false, err
|
||||
}
|
||||
for _, item := range items {
|
||||
if strings.EqualFold(item.FullName(), leaf) {
|
||||
// iCloud returns file names in NFD Unicode normalization, so normalized to NFC for consistent comparison
|
||||
if strings.EqualFold(norm.NFC.String(item.FullName()), leaf) {
|
||||
return item, true, nil
|
||||
}
|
||||
}
|
||||
@@ -794,51 +677,11 @@ func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bo
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
icloud, opt, err := newICloudClient(ctx, name, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if opt.Password != "" {
|
||||
var err error
|
||||
opt.Password, err = obscure.Reveal(opt.Password)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't decrypt user password: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if opt.TrustToken == "" {
|
||||
return nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name)
|
||||
}
|
||||
|
||||
cookies := ReadCookies(opt.Cookies)
|
||||
|
||||
callback := func(session *api.Session) {
|
||||
m.Set(configCookies, session.GetCookieString())
|
||||
}
|
||||
|
||||
icloud, err := api.New(
|
||||
opt.AppleID,
|
||||
opt.Password,
|
||||
opt.TrustToken,
|
||||
opt.ClientID,
|
||||
cookies,
|
||||
callback,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := icloud.Authenticate(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if icloud.Session.Requires2FA() {
|
||||
return nil, errors.New("trust token expired, please reauth")
|
||||
}
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &Fs{
|
||||
@@ -847,6 +690,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
icloud: icloud,
|
||||
rootID: "FOLDER::com.apple.CloudDocs::root",
|
||||
opt: *opt,
|
||||
m: m,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
@@ -1161,9 +1005,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
return nil
|
||||
}
|
||||
|
||||
// Disconnect clears authentication state and removes disk caches
|
||||
func (f *Fs) Disconnect(ctx context.Context) error {
|
||||
return disconnectClient(f.m, f.icloud)
|
||||
}
|
||||
|
||||
// Check interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Disconnecter = (*Fs)(nil)
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.Purger = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
|
||||
@@ -4,4 +4,4 @@
|
||||
//go:build plan9 || solaris
|
||||
|
||||
// Package iclouddrive implements the iCloud Drive backend
|
||||
package iclouddrive
|
||||
package iclouddrive
|
||||
@@ -0,0 +1,892 @@
|
||||
//go:build !plan9 && !solaris
|
||||
|
||||
package iclouddrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/iclouddrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
)
|
||||
|
||||
const rootID = "photos-root"
|
||||
|
||||
// PhotosFs represents a remote iCloud Photos server
|
||||
type PhotosFs struct {
|
||||
name string
|
||||
root string
|
||||
opt Options
|
||||
features *fs.Features
|
||||
icloud *api.Client
|
||||
m configmap.Mapper
|
||||
pacer *fs.Pacer
|
||||
dirCache *dircache.DirCache
|
||||
httpClient *http.Client
|
||||
startTime time.Time
|
||||
mu sync.Mutex
|
||||
photos *api.PhotosService
|
||||
}
|
||||
|
||||
// PhotosObject describes an iCloud Photos object
|
||||
type PhotosObject struct {
|
||||
fs *PhotosFs
|
||||
remote string
|
||||
size int64
|
||||
modTime time.Time
|
||||
masterID string // CloudKit recordName for fresh URL lookup (master or asset depending on resource)
|
||||
zone string // CloudKit zone (e.g. "PrimarySync")
|
||||
resourceKey string // CloudKit resource field (resOriginalRes or resOriginalVidComplRes)
|
||||
width int
|
||||
height int
|
||||
addedDate int64
|
||||
isFavorite bool
|
||||
isHidden bool
|
||||
}
|
||||
|
||||
// NewFsPhotos constructs an Fs for Photos from the path, container:path
|
||||
func NewFsPhotos(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
icloud, opt, err := newICloudClient(ctx, name, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
root = strings.Trim(root, "/")
|
||||
|
||||
f := &PhotosFs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
icloud: icloud,
|
||||
m: m,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
httpClient: fshttp.NewClient(ctx),
|
||||
startTime: time.Now(),
|
||||
}
|
||||
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
PartialUploads: false,
|
||||
ReadMimeType: false,
|
||||
ReadMetadata: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
f.dirCache = dircache.New(root, rootID, f)
|
||||
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// Check if root points to a file (e.g. PrimarySync/Videos/file.mp4)
|
||||
newRoot, remote := dircache.SplitPath(root)
|
||||
tempF := &PhotosFs{
|
||||
name: f.name,
|
||||
root: newRoot,
|
||||
opt: f.opt,
|
||||
features: f.features,
|
||||
icloud: f.icloud,
|
||||
pacer: f.pacer,
|
||||
httpClient: f.httpClient,
|
||||
startTime: f.startTime,
|
||||
photos: f.photos, // nil here; photosService() creates lazily on first use
|
||||
}
|
||||
tempF.dirCache = dircache.New(newRoot, rootID, tempF)
|
||||
if err2 := tempF.dirCache.FindRoot(ctx, false); err2 != nil {
|
||||
return f, nil
|
||||
}
|
||||
_, err2 := tempF.NewObject(ctx, remote)
|
||||
if err2 != nil {
|
||||
return f, nil
|
||||
}
|
||||
// Root is a file - adjust f to parent dir, signal ErrorIsFile
|
||||
f.root = newRoot
|
||||
f.dirCache = tempF.dirCache
|
||||
return f, fs.ErrorIsFile
|
||||
}
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// photosService returns the PhotosService, creating it on first call
|
||||
func (f *PhotosFs) photosService(ctx context.Context) (*api.PhotosService, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.photos == nil {
|
||||
var err error
|
||||
f.photos, err = api.NewPhotosService(ctx, f.icloud, f.pacer, shouldRetry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return f.photos, nil
|
||||
}
|
||||
|
||||
// Name of the remote (as passed into NewFs)
|
||||
func (f *PhotosFs) Name() string {
|
||||
return f.name
|
||||
}
|
||||
|
||||
// Root of the remote (as passed into NewFs)
|
||||
func (f *PhotosFs) Root() string {
|
||||
return f.opt.Enc.ToStandardPath(f.root)
|
||||
}
|
||||
|
||||
// String converts this Fs to a string
|
||||
func (f *PhotosFs) String() string {
|
||||
return fmt.Sprintf("iCloud Photos root '%s'", f.root)
|
||||
}
|
||||
|
||||
// Precision of the object storage system
|
||||
func (f *PhotosFs) Precision() time.Duration {
|
||||
return time.Second
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets
|
||||
func (f *PhotosFs) Hashes() hash.Set {
|
||||
return hash.Set(hash.None)
|
||||
}
|
||||
|
||||
// Features returns the optional features of this Fs
|
||||
func (f *PhotosFs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// List the objects and directories in dir into entries
|
||||
func (f *PhotosFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
photosService, err := f.photosService(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get photos service: %w", err)
|
||||
}
|
||||
|
||||
dirID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch {
|
||||
case dirID == rootID:
|
||||
// List libraries
|
||||
libraries, err := photosService.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
albumCounts, err := photosService.GetLibraryAlbumCounts(ctx)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to get library album counts: %v", err)
|
||||
}
|
||||
|
||||
for libraryName := range libraries {
|
||||
count := int64(-1)
|
||||
if albumCounts != nil {
|
||||
if c, ok := albumCounts[libraryName]; ok {
|
||||
count = c
|
||||
}
|
||||
}
|
||||
d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(libraryName)), f.startTime).SetItems(count)
|
||||
entries = append(entries, d)
|
||||
}
|
||||
|
||||
case strings.HasPrefix(dirID, "lib:"):
|
||||
// List albums in a library
|
||||
libraryName := strings.TrimPrefix(dirID, "lib:")
|
||||
libraries, err := photosService.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
library, exists := libraries[libraryName]
|
||||
if !exists {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
albums, err := library.GetAlbums(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
counts, err := library.GetAlbumCounts(ctx)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to get album counts for library %q: %v", libraryName, err)
|
||||
}
|
||||
|
||||
for albumName := range albums {
|
||||
count := int64(-1)
|
||||
if counts != nil {
|
||||
if c, ok := counts[albumName]; ok {
|
||||
count = c
|
||||
}
|
||||
}
|
||||
d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(albumName)), f.startTime).SetItems(count)
|
||||
entries = append(entries, d)
|
||||
}
|
||||
|
||||
case strings.HasPrefix(dirID, "album:"):
|
||||
// List contents of an album or folder
|
||||
libraryName, albumPath, ok := parseAlbumDirID(dirID)
|
||||
if !ok {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Folders list their child albums as subdirectories
|
||||
if album.IsFolder {
|
||||
for childName, child := range album.Children {
|
||||
d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(childName)), f.startTime)
|
||||
if child.IsFolder {
|
||||
d.SetItems(int64(len(child.Children)))
|
||||
}
|
||||
entries = append(entries, d)
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
photos, err := album.GetPhotos(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, photo := range photos {
|
||||
if photo.Filename == "" {
|
||||
continue
|
||||
}
|
||||
encodedName := f.opt.Enc.ToStandardName(photo.Filename)
|
||||
remotePath := encodedName
|
||||
if dir != "" {
|
||||
remotePath = path.Join(dir, encodedName)
|
||||
}
|
||||
o := f.newPhotosObject(remotePath, photo, libraryName)
|
||||
entries = append(entries, o)
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error fs.ErrorObjectNotFound
|
||||
func (f *PhotosFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
dir, leaf := dircache.SplitPath(remote)
|
||||
filename := f.opt.Enc.FromStandardName(leaf)
|
||||
|
||||
dirID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "NewObject(%s): FindDir: %v", remote, err)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(dirID, "album:") {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
libraryName, albumPath, ok := parseAlbumDirID(dirID)
|
||||
if !ok {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
photosService, err := f.photosService(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get photos service: %w", err)
|
||||
}
|
||||
|
||||
album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)
|
||||
if err != nil || album.IsFolder {
|
||||
fs.Debugf(f, "NewObject(%s): album resolve failed or is folder", remote)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
photo, err := album.GetPhotoByName(ctx, filename)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "NewObject(%s): %v", remote, err)
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
|
||||
return f.newPhotosObject(remote, photo, libraryName), nil
|
||||
}
|
||||
|
||||
// newPhotosObject creates a PhotosObject from a Photo and zone
|
||||
func (f *PhotosFs) newPhotosObject(remote string, photo *api.Photo, zone string) *PhotosObject {
|
||||
return &PhotosObject{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
size: photo.Size,
|
||||
modTime: time.UnixMilli(photo.AssetDate),
|
||||
masterID: photo.ID,
|
||||
zone: zone,
|
||||
resourceKey: photo.ResourceKey,
|
||||
width: photo.Width,
|
||||
height: photo.Height,
|
||||
addedDate: photo.AddedDate,
|
||||
isFavorite: photo.IsFavorite,
|
||||
isHidden: photo.IsHidden,
|
||||
}
|
||||
}
|
||||
|
||||
// Put is not supported for this read-only backend
|
||||
func (f *PhotosFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
return nil, fs.ErrorNotImplemented
|
||||
}
|
||||
|
||||
// Mkdir is not supported for this read-only backend
|
||||
func (f *PhotosFs) Mkdir(ctx context.Context, dir string) error {
|
||||
return fs.ErrorNotImplemented
|
||||
}
|
||||
|
||||
// Rmdir is not supported for this read-only backend
|
||||
func (f *PhotosFs) Rmdir(ctx context.Context, dir string) error {
|
||||
return fs.ErrorNotImplemented
|
||||
}
|
||||
|
||||
// FindLeaf finds a directory of name leaf in the folder with ID pathID
|
||||
func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
|
||||
decodedLeaf := f.opt.Enc.FromStandardName(leaf)
|
||||
|
||||
photosService, err := f.photosService(ctx)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("failed to get photos service: %w", err)
|
||||
}
|
||||
|
||||
if pathID == rootID {
|
||||
libraries, err := photosService.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
if _, exists := libraries[decodedLeaf]; exists {
|
||||
return "lib:" + decodedLeaf, true, nil
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(pathID, "lib:") {
|
||||
libraryName := strings.TrimPrefix(pathID, "lib:")
|
||||
libraries, err := photosService.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
library, exists := libraries[libraryName]
|
||||
if !exists {
|
||||
return "", false, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
albums, err := library.GetAlbums(ctx)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
if _, exists := albums[decodedLeaf]; exists {
|
||||
albumID := "album:" + libraryName + ":" + decodedLeaf
|
||||
return albumID, true, nil
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(pathID, "album:") {
|
||||
// Check if this is a folder containing child albums
|
||||
libraryName, albumPath, ok := parseAlbumDirID(pathID)
|
||||
if !ok {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath)
|
||||
if err != nil {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
if album.IsFolder {
|
||||
if _, exists := album.Children[decodedLeaf]; exists {
|
||||
childID := "album:" + libraryName + ":" + albumPath + "/" + decodedLeaf
|
||||
return childID, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// CreateDir makes a directory with pathID as parent and name leaf
|
||||
func (f *PhotosFs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
|
||||
return "", fs.ErrorNotImplemented
|
||||
}
|
||||
|
||||
// Fs returns the parent Fs
|
||||
func (o *PhotosObject) Fs() fs.Info {
|
||||
return o.fs
|
||||
}
|
||||
|
||||
// Return a string version
|
||||
func (o *PhotosObject) String() string {
|
||||
if o == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// Remote returns the remote path
|
||||
func (o *PhotosObject) Remote() string {
|
||||
return o.remote
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
func (o *PhotosObject) ModTime(ctx context.Context) time.Time {
|
||||
return o.modTime
|
||||
}
|
||||
|
||||
// Size returns the size of an object in bytes
|
||||
func (o *PhotosObject) Size() int64 {
|
||||
return o.size
|
||||
}
|
||||
|
||||
// Storable returns a boolean as to whether this object is storable
|
||||
func (o *PhotosObject) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Hash returns the hash of an object returning a lowercase hex string
|
||||
func (o *PhotosObject) Hash(ctx context.Context, ty hash.Type) (string, error) {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
|
||||
// SetModTime sets the modification time of the object
|
||||
func (o *PhotosObject) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return fs.ErrorCantSetModTime
|
||||
}
|
||||
|
||||
// Metadata returns metadata for the photo object
|
||||
func (o *PhotosObject) Metadata(ctx context.Context) (fs.Metadata, error) {
|
||||
metadata := make(fs.Metadata, 5)
|
||||
if o.width > 0 {
|
||||
metadata["width"] = strconv.Itoa(o.width)
|
||||
}
|
||||
if o.height > 0 {
|
||||
metadata["height"] = strconv.Itoa(o.height)
|
||||
}
|
||||
if o.addedDate > 0 {
|
||||
metadata["added-time"] = time.UnixMilli(o.addedDate).UTC().Format(time.RFC3339)
|
||||
}
|
||||
metadata["favorite"] = strconv.FormatBool(o.isFavorite)
|
||||
metadata["hidden"] = strconv.FormatBool(o.isHidden)
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// Open an object for read, fetching a fresh download URL via records/lookup
|
||||
func (o *PhotosObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
photosService, err := o.fs.photosService(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get photos service: %w", err)
|
||||
}
|
||||
|
||||
downloadURL, err := photosService.LookupDownloadURL(ctx, o.masterID, o.zone, o.resourceKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get download URL for %q: %w", o.remote, err)
|
||||
}
|
||||
|
||||
fs.FixRangeOption(options, o.size)
|
||||
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
if resp != nil {
|
||||
_ = resp.Body.Close()
|
||||
resp = nil
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
||||
|
||||
resp, err = o.fs.httpClient.Do(req)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
_ = resp.Body.Close()
|
||||
return nil, fmt.Errorf("download %q failed: %s", o.remote, resp.Status)
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Update is not supported for this read-only backend
|
||||
func (o *PhotosObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
return fs.ErrorNotImplemented
|
||||
}
|
||||
|
||||
// Remove is not supported for this read-only backend
|
||||
func (o *PhotosObject) Remove(ctx context.Context) error {
|
||||
return fs.ErrorNotImplemented
|
||||
}
|
||||
|
||||
// resolveAlbum finds an album by path, traversing folder hierarchy
|
||||
// albumPath can be "AlbumName" or "FolderName/AlbumName" for nested albums
|
||||
func (f *PhotosFs) resolveAlbum(ctx context.Context, ps *api.PhotosService, libraryName, albumPath string) (*api.Album, error) {
|
||||
libraries, err := ps.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
library, exists := libraries[libraryName]
|
||||
if !exists {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
albums, err := library.GetAlbums(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return resolveAlbumPath(albums, albumPath)
|
||||
}
|
||||
|
||||
// resolveAlbumPath finds an album inside an album tree by slash-separated path
|
||||
func resolveAlbumPath(albums map[string]*api.Album, albumPath string) (*api.Album, error) {
|
||||
parts := strings.Split(albumPath, "/")
|
||||
current := albums
|
||||
for i, part := range parts {
|
||||
album, exists := current[part]
|
||||
if !exists {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
if i == len(parts)-1 {
|
||||
return album, nil
|
||||
}
|
||||
if !album.IsFolder || album.Children == nil {
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
current = album.Children
|
||||
}
|
||||
return nil, fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
// parseAlbumDirID extracts library and album names from "album:lib:album" dirID
|
||||
func parseAlbumDirID(dirID string) (libraryName, albumName string, ok bool) {
|
||||
parts := strings.SplitN(strings.TrimPrefix(dirID, "album:"), ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing as an
|
||||
// optional interface
|
||||
func (f *PhotosFs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
// Also flush API-layer caches so albums/photos are re-fetched
|
||||
// DirCacheFlusher interface has no ctx param; use a bounded timeout
|
||||
// to avoid hanging indefinitely if photosService triggers first init
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if ps, err := f.photosService(ctx); err == nil {
|
||||
ps.FlushCaches()
|
||||
}
|
||||
}
|
||||
|
||||
// listRAlbumJob represents an album to be listed by a ListR worker
|
||||
type listRAlbumJob struct {
|
||||
album *api.Album
|
||||
zone string
|
||||
dirPath string // path relative to Fs root (e.g. "Videos" when f.root="PrimarySync")
|
||||
}
|
||||
|
||||
// ListR lists the objects and directories of the Fs starting from dir
|
||||
// recursively into out, calling callback for each batch of entries
|
||||
// Albums are listed in parallel using a goroutine pool
|
||||
func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
||||
ps, err := f.photosService(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
helper := list.NewHelper(callback)
|
||||
var mu sync.Mutex // protects helper (not thread-safe)
|
||||
|
||||
addEntry := func(entry fs.DirEntry) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return helper.Add(entry)
|
||||
}
|
||||
|
||||
// Resolve dir through dirCache - this respects f.root
|
||||
dirID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Collect album jobs, scoped to the resolved directory
|
||||
var jobs []listRAlbumJob
|
||||
|
||||
// collectAlbumJobs emits directory entries for albums/folders under basePath
|
||||
// and collects leaf albums as jobs for parallel photo listing
|
||||
// Recurses into folders to handle arbitrary nesting depth
|
||||
var collectAlbumJobs func(albums map[string]*api.Album, zone, basePath string) error
|
||||
collectAlbumJobs = func(albums map[string]*api.Album, zone, basePath string) error {
|
||||
for albumName, album := range albums {
|
||||
albumPath := path.Join(basePath, f.opt.Enc.ToStandardName(albumName))
|
||||
if err := addEntry(fs.NewDir(albumPath, f.startTime)); err != nil {
|
||||
return err
|
||||
}
|
||||
if album.IsFolder {
|
||||
if err := collectAlbumJobs(album.Children, zone, albumPath); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
jobs = append(jobs, listRAlbumJob{album: album, zone: zone, dirPath: albumPath})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case dirID == rootID:
|
||||
// Root level - emit libraries, then all albums
|
||||
libraries, err := ps.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for libName, lib := range libraries {
|
||||
libPath := path.Join(dir, f.opt.Enc.ToStandardName(libName))
|
||||
if err := addEntry(fs.NewDir(libPath, f.startTime)); err != nil {
|
||||
return err
|
||||
}
|
||||
albums, err := lib.GetAlbums(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := collectAlbumJobs(albums, libName, libPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
case strings.HasPrefix(dirID, "lib:"):
|
||||
// Library level - emit and collect all albums
|
||||
libraryName := strings.TrimPrefix(dirID, "lib:")
|
||||
libraries, err := ps.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
library, exists := libraries[libraryName]
|
||||
if !exists {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
albums, err := library.GetAlbums(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := collectAlbumJobs(albums, libraryName, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
case strings.HasPrefix(dirID, "album:"):
|
||||
// Album or folder level
|
||||
libraryName, albumPath, ok := parseAlbumDirID(dirID)
|
||||
if !ok {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
album, err := f.resolveAlbum(ctx, ps, libraryName, albumPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if album.IsFolder {
|
||||
if err := collectAlbumJobs(album.Children, libraryName, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Single album - just list its photos
|
||||
jobs = append(jobs, listRAlbumJob{album: album, zone: libraryName, dirPath: dir})
|
||||
}
|
||||
|
||||
default:
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
|
||||
if len(jobs) == 0 {
|
||||
return helper.Flush()
|
||||
}
|
||||
|
||||
// Fan out album photo listing across goroutines
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
jobCh := make(chan listRAlbumJob, len(jobs))
|
||||
for _, job := range jobs {
|
||||
jobCh <- job
|
||||
}
|
||||
close(jobCh)
|
||||
|
||||
workers := fs.GetConfig(ctx).Checkers
|
||||
if len(jobs) < workers {
|
||||
workers = len(jobs)
|
||||
}
|
||||
errs := make(chan error, workers)
|
||||
|
||||
for range workers {
|
||||
go func() {
|
||||
for job := range jobCh {
|
||||
photos, err := job.album.GetPhotos(ctx)
|
||||
if err != nil {
|
||||
errs <- err
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
for _, photo := range photos {
|
||||
if photo.Filename == "" {
|
||||
continue
|
||||
}
|
||||
remotePath := path.Join(job.dirPath, f.opt.Enc.ToStandardName(photo.Filename))
|
||||
o := f.newPhotosObject(remotePath, photo, job.zone)
|
||||
if err := addEntry(o); err != nil {
|
||||
errs <- err
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
errs <- nil
|
||||
}()
|
||||
}
|
||||
|
||||
for range workers {
|
||||
if e := <-errs; e != nil && err == nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return helper.Flush()
|
||||
}
|
||||
|
||||
func notifyAlbumTree(albums map[string]*api.Album, base string, notifyFunc func(string, fs.EntryType)) {
|
||||
for name, album := range albums {
|
||||
albumPath := name
|
||||
if base != "" {
|
||||
albumPath = path.Join(base, name)
|
||||
}
|
||||
notifyFunc(albumPath, fs.EntryDirectory)
|
||||
if album.IsFolder {
|
||||
notifyAlbumTree(album.Children, albumPath, notifyFunc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *PhotosFs) notifyZoneChange(ctx context.Context, ps *api.PhotosService, zone string, notifyFunc func(string, fs.EntryType)) {
|
||||
if f.root == "" {
|
||||
notifyFunc(zone, fs.EntryDirectory)
|
||||
return
|
||||
}
|
||||
|
||||
libraryName, albumPath, _ := strings.Cut(f.root, "/")
|
||||
if libraryName != zone {
|
||||
return
|
||||
}
|
||||
|
||||
// Always invalidate the mounted root when its backing zone changes
|
||||
notifyFunc("", fs.EntryDirectory)
|
||||
|
||||
libraries, err := ps.GetLibraries(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
library, ok := libraries[zone]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
albums, err := library.GetAlbums(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if albumPath == "" {
|
||||
notifyAlbumTree(albums, "", notifyFunc)
|
||||
return
|
||||
}
|
||||
|
||||
album, err := resolveAlbumPath(albums, albumPath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if album.IsFolder {
|
||||
notifyAlbumTree(album.Children, "", notifyFunc)
|
||||
}
|
||||
}
|
||||
|
||||
// ChangeNotify polls for changes and notifies the VFS when directories are modified
|
||||
func (f *PhotosFs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
|
||||
go func() {
|
||||
var ticker *time.Ticker
|
||||
var tickerC <-chan time.Time
|
||||
for {
|
||||
select {
|
||||
case pollInterval, ok := <-pollIntervalChan:
|
||||
if !ok {
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
}
|
||||
return
|
||||
}
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
tickerC = nil
|
||||
}
|
||||
if pollInterval > 0 {
|
||||
ticker = time.NewTicker(pollInterval)
|
||||
tickerC = ticker.C
|
||||
}
|
||||
case <-tickerC:
|
||||
ps, err := f.photosService(ctx)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "ChangeNotify: failed to get photos service: %v", err)
|
||||
continue
|
||||
}
|
||||
changedZones := ps.PollForChanges(ctx)
|
||||
for _, zone := range changedZones {
|
||||
f.notifyZoneChange(ctx, ps, zone, notifyFunc)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Disconnect clears authentication state and removes disk caches
|
||||
func (f *PhotosFs) Disconnect(ctx context.Context) error {
|
||||
return disconnectClient(f.m, f.icloud)
|
||||
}
|
||||
|
||||
// Check interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = (*PhotosFs)(nil)
|
||||
_ fs.Disconnecter = (*PhotosFs)(nil)
|
||||
_ fs.ListRer = (*PhotosFs)(nil)
|
||||
_ fs.ChangeNotifier = (*PhotosFs)(nil)
|
||||
_ fs.DirCacheFlusher = (*PhotosFs)(nil)
|
||||
_ fs.Object = (*PhotosObject)(nil)
|
||||
_ fs.Metadataer = (*PhotosObject)(nil)
|
||||
)
|
||||
@@ -0,0 +1,516 @@
|
||||
//go:build !plan9 && !solaris
|
||||
|
||||
package iclouddrive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/backend/iclouddrive/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestPhotosFs() *PhotosFs {
|
||||
return &PhotosFs{
|
||||
name: "test-icp",
|
||||
root: "",
|
||||
opt: Options{},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmartAlbumDefinitions(t *testing.T) {
|
||||
expected := map[string]struct {
|
||||
ObjectType string
|
||||
ListType string
|
||||
}{
|
||||
"All Photos": {"CPLAssetByAssetDateWithoutHiddenOrDeleted", "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted"},
|
||||
"Time-lapse": {"CPLAssetInSmartAlbumByAssetDate:Timelapse", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Videos": {"CPLAssetInSmartAlbumByAssetDate:Video", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Slo-mo": {"CPLAssetInSmartAlbumByAssetDate:Slomo", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Bursts": {"CPLAssetBurstStackAssetByAssetDate", "CPLBurstStackAssetAndMasterByAssetDate"},
|
||||
"Favorites": {"CPLAssetInSmartAlbumByAssetDate:Favorite", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Panoramas": {"CPLAssetInSmartAlbumByAssetDate:Panorama", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Screenshots": {"CPLAssetInSmartAlbumByAssetDate:Screenshot", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Live": {"CPLAssetInSmartAlbumByAssetDate:Live", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Recently Deleted": {"CPLAssetDeletedByExpungedDate", "CPLAssetAndMasterDeletedByExpungedDate"},
|
||||
"Hidden": {"CPLAssetHiddenByAssetDate", "CPLAssetAndMasterHiddenByAssetDate"},
|
||||
"Portrait": {"CPLAssetInSmartAlbumByAssetDate:Depth", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Long Exposure": {"CPLAssetInSmartAlbumByAssetDate:Exposure", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
"Animated": {"CPLAssetInSmartAlbumByAssetDate:Animated", "CPLAssetAndMasterInSmartAlbumByAssetDate"},
|
||||
}
|
||||
|
||||
for name, exp := range expected {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
album, exists := api.SmartAlbums[name]
|
||||
require.True(t, exists, "smart album %q must be defined", name)
|
||||
assert.Equal(t, exp.ObjectType, album.ObjectType)
|
||||
assert.Equal(t, exp.ListType, album.ListType)
|
||||
})
|
||||
}
|
||||
|
||||
assert.Equal(t, len(expected), len(api.SmartAlbums))
|
||||
}
|
||||
|
||||
func TestSmartAlbumFilters(t *testing.T) {
|
||||
filtered := map[string]string{
|
||||
"Time-lapse": "TIMELAPSE",
|
||||
"Videos": "VIDEO",
|
||||
"Slo-mo": "SLOMO",
|
||||
"Favorites": "FAVORITE",
|
||||
"Panoramas": "PANORAMA",
|
||||
"Screenshots": "SCREENSHOT",
|
||||
"Live": "LIVE",
|
||||
"Portrait": "DEPTH",
|
||||
"Long Exposure": "EXPOSURE",
|
||||
"Animated": "ANIMATED",
|
||||
}
|
||||
|
||||
for name, filterVal := range filtered {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
album := api.SmartAlbums[name]
|
||||
require.NotEmpty(t, album.Filters, "album %q must have filters", name)
|
||||
assert.Equal(t, "smartAlbum", album.Filters[0].FieldName)
|
||||
assert.Equal(t, "EQUALS", album.Filters[0].Comparator)
|
||||
fv, ok := album.Filters[0].FieldValue.(map[string]string)
|
||||
require.True(t, ok, "filter value must be map[string]string")
|
||||
assert.Equal(t, filterVal, fv["value"])
|
||||
})
|
||||
}
|
||||
|
||||
unfiltered := []string{"All Photos", "Bursts", "Recently Deleted", "Hidden"}
|
||||
for _, name := range unfiltered {
|
||||
t.Run(name+"_no_filter", func(t *testing.T) {
|
||||
album := api.SmartAlbums[name]
|
||||
assert.Empty(t, album.Filters, "album %q must not have filters", name)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAlbumDirID(t *testing.T) {
|
||||
lib, album, ok := parseAlbumDirID("album:PrimarySync:Favorites")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "PrimarySync", lib)
|
||||
assert.Equal(t, "Favorites", album)
|
||||
|
||||
// Colon in album name
|
||||
lib, album, ok = parseAlbumDirID("album:PrimarySync:My:Album")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "PrimarySync", lib)
|
||||
assert.Equal(t, "My:Album", album)
|
||||
|
||||
// Missing album
|
||||
_, _, ok = parseAlbumDirID("album:PrimarySync")
|
||||
assert.False(t, ok)
|
||||
|
||||
// Empty
|
||||
_, _, ok = parseAlbumDirID("album:")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestPhotosObject_Metadata(t *testing.T) {
|
||||
o := &PhotosObject{
|
||||
fs: &PhotosFs{name: "test"},
|
||||
remote: "PrimarySync/All Photos/IMG_0001.HEIC",
|
||||
size: 4200000,
|
||||
modTime: time.Date(2025, 6, 15, 14, 30, 0, 0, time.UTC),
|
||||
masterID: "master-001",
|
||||
zone: "PrimarySync",
|
||||
width: 4032,
|
||||
height: 3024,
|
||||
addedDate: 1718459400000, // 2024-06-15T13:50:00Z in millis
|
||||
isFavorite: true,
|
||||
isHidden: false,
|
||||
}
|
||||
|
||||
metadata, err := o.Metadata(context.Background())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "4032", metadata["width"])
|
||||
assert.Equal(t, "3024", metadata["height"])
|
||||
assert.Equal(t, "2024-06-15T13:50:00Z", metadata["added-time"])
|
||||
assert.Equal(t, "true", metadata["favorite"])
|
||||
assert.Equal(t, "false", metadata["hidden"])
|
||||
}
|
||||
|
||||
func TestPhotosObject_MetadataZeroDimensions(t *testing.T) {
|
||||
// Live Photo .MOV companion - no width/height, not favorite
|
||||
o := &PhotosObject{
|
||||
fs: &PhotosFs{name: "test"},
|
||||
remote: "PrimarySync/Live/IMG_3031.MOV",
|
||||
}
|
||||
|
||||
metadata, err := o.Metadata(context.Background())
|
||||
require.NoError(t, err)
|
||||
_, hasWidth := metadata["width"]
|
||||
_, hasHeight := metadata["height"]
|
||||
assert.False(t, hasWidth, "zero width should be omitted")
|
||||
assert.False(t, hasHeight, "zero height should be omitted")
|
||||
assert.Equal(t, "false", metadata["favorite"])
|
||||
assert.Equal(t, "false", metadata["hidden"])
|
||||
}
|
||||
|
||||
// newTestPhotosService builds a PhotosService with pre-populated libraries and albums
|
||||
// for testing resolveAlbum and FindLeaf without HTTP calls
|
||||
func newTestPhotosService() *api.PhotosService {
|
||||
return api.NewTestPhotosService(map[string]map[string]*api.Album{
|
||||
"PrimarySync": {
|
||||
"All Photos": {Name: "All Photos", ObjectType: "CPLAssetByAssetDateWithoutHiddenOrDeleted"},
|
||||
"Videos": {Name: "Videos", ObjectType: "CPLAssetInSmartAlbumByAssetDate:Video"},
|
||||
"UserAlbum": {Name: "UserAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec1", RecordName: "rec1"},
|
||||
"Folder": {
|
||||
Name: "Folder", RecordName: "folder1", IsFolder: true,
|
||||
Children: map[string]*api.Album{
|
||||
"ChildAlbum": {Name: "ChildAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec2", RecordName: "rec2"},
|
||||
"NestedFolder": {
|
||||
Name: "NestedFolder", RecordName: "folder2", IsFolder: true,
|
||||
Children: map[string]*api.Album{
|
||||
"LeafAlbum": {Name: "LeafAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec3", RecordName: "rec3"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolveAlbum(t *testing.T) {
|
||||
f := newTestPhotosFs()
|
||||
f.photos = newTestPhotosService()
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("simple album", func(t *testing.T) {
|
||||
album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "Videos", album.Name)
|
||||
})
|
||||
|
||||
t.Run("nested folder child", func(t *testing.T) {
|
||||
album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/ChildAlbum")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ChildAlbum", album.Name)
|
||||
})
|
||||
|
||||
t.Run("two-level nesting", func(t *testing.T) {
|
||||
album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/NestedFolder/LeafAlbum")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "LeafAlbum", album.Name)
|
||||
})
|
||||
|
||||
t.Run("folder itself", func(t *testing.T) {
|
||||
album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, album.IsFolder)
|
||||
})
|
||||
|
||||
t.Run("missing album", func(t *testing.T) {
|
||||
_, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "NoSuchAlbum")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
})
|
||||
|
||||
t.Run("missing library", func(t *testing.T) {
|
||||
_, err := f.resolveAlbum(ctx, f.photos, "NoSuchLib", "Videos")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
})
|
||||
|
||||
t.Run("traverse into non-folder", func(t *testing.T) {
|
||||
_, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos/SubAlbum")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFindLeaf(t *testing.T) {
|
||||
f := newTestPhotosFs()
|
||||
f.photos = newTestPhotosService()
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("root to library", func(t *testing.T) {
|
||||
id, found, err := f.FindLeaf(ctx, rootID, "PrimarySync")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, "lib:PrimarySync", id)
|
||||
})
|
||||
|
||||
t.Run("root to missing library", func(t *testing.T) {
|
||||
_, found, err := f.FindLeaf(ctx, rootID, "NoSuchLib")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
})
|
||||
|
||||
t.Run("library to album", func(t *testing.T) {
|
||||
id, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "Videos")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, "album:PrimarySync:Videos", id)
|
||||
})
|
||||
|
||||
t.Run("library to missing album", func(t *testing.T) {
|
||||
_, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "NoSuchAlbum")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
})
|
||||
|
||||
t.Run("missing library returns ErrorDirNotFound", func(t *testing.T) {
|
||||
_, _, err := f.FindLeaf(ctx, "lib:NoSuchLib", "Videos")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound)
|
||||
})
|
||||
|
||||
t.Run("folder to child album", func(t *testing.T) {
|
||||
id, found, err := f.FindLeaf(ctx, "album:PrimarySync:Folder", "ChildAlbum")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, "album:PrimarySync:Folder/ChildAlbum", id)
|
||||
})
|
||||
|
||||
t.Run("non-folder album returns not found", func(t *testing.T) {
|
||||
_, found, err := f.FindLeaf(ctx, "album:PrimarySync:Videos", "SubAlbum")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
})
|
||||
|
||||
t.Run("unknown prefix returns not found", func(t *testing.T) {
|
||||
_, found, err := f.FindLeaf(ctx, "unknown:prefix", "leaf")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolveAlbum_EmptyPath(t *testing.T) {
|
||||
f := newTestPhotosFs()
|
||||
f.photos = newTestPhotosService()
|
||||
ctx := context.Background()
|
||||
|
||||
// parseAlbumDirID("album:PrimarySync:") returns ("PrimarySync", "", true)
|
||||
// This empty album path feeds into resolveAlbum
|
||||
_, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "")
|
||||
assert.ErrorIs(t, err, fs.ErrorDirNotFound, "empty album path should return ErrorDirNotFound")
|
||||
}
|
||||
|
||||
func TestNewObject_ErrorPaths(t *testing.T) {
|
||||
f := newTestPhotosFs()
|
||||
f.photos = newTestPhotosService()
|
||||
ctx := context.Background()
|
||||
|
||||
// Pre-populate photoCache on the "Videos" album so GetPhotoByName works
|
||||
// without HTTP. Access via the test PhotosService's internal structure
|
||||
libs, _ := f.photos.GetLibraries(ctx)
|
||||
albums, _ := libs["PrimarySync"].GetAlbums(ctx)
|
||||
videos := albums["Videos"]
|
||||
videos.SetTestPhotoCache(map[string]*api.Photo{
|
||||
"existing.mp4": {ID: "m1", Filename: "existing.mp4", Size: 1024, ResourceKey: "resOriginalRes"},
|
||||
})
|
||||
|
||||
// NewObject depends on dircache.FindDir which requires FindRoot + HTTP
|
||||
// Test the reachable paths through resolveAlbum and GetPhotoByName
|
||||
|
||||
// Test resolveAlbum → folder → GetPhotoByName on folder (should fail)
|
||||
album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, album.IsFolder, "Folder should be a folder")
|
||||
// NewObject would return ErrorObjectNotFound for a folder path
|
||||
|
||||
// Test GetPhotoByName on album with populated cache
|
||||
photo, err := videos.GetPhotoByName(ctx, "existing.mp4")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "m1", photo.ID)
|
||||
|
||||
// Test GetPhotoByName cache miss
|
||||
_, err = videos.GetPhotoByName(ctx, "nonexistent.mp4")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "nonexistent.mp4")
|
||||
}
|
||||
|
||||
func TestParseAlbumDirID_Exhaustive(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
lib, album string
|
||||
ok bool
|
||||
}{
|
||||
{"album:PrimarySync:Videos", "PrimarySync", "Videos", true},
|
||||
{"album:PrimarySync:Folder/Child", "PrimarySync", "Folder/Child", true},
|
||||
{"album:PrimarySync:Name:With:Colons", "PrimarySync", "Name:With:Colons", true},
|
||||
{"album:PrimarySync:", "PrimarySync", "", true},
|
||||
{"album:", "", "", false},
|
||||
{"notalbum:foo:bar", "notalbum", "foo:bar", true}, // strips "album:" prefix literally, "notalbum:" stays
|
||||
{"", "", "", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
lib, album, ok := parseAlbumDirID(tt.input)
|
||||
if tt.ok {
|
||||
assert.True(t, ok, "input=%q", tt.input)
|
||||
assert.Equal(t, tt.lib, lib, "input=%q", tt.input)
|
||||
assert.Equal(t, tt.album, album, "input=%q", tt.input)
|
||||
} else {
|
||||
assert.False(t, ok, "input=%q", tt.input)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setEmptyPhotoCaches recursively sets empty photo caches on all leaf albums
|
||||
// so GetPhotos returns without HTTP (service=nil test fast path)
|
||||
func setEmptyPhotoCaches(albums map[string]*api.Album) {
|
||||
for _, album := range albums {
|
||||
if album.IsFolder {
|
||||
setEmptyPhotoCaches(album.Children)
|
||||
} else {
|
||||
album.SetTestPhotoCache(map[string]*api.Photo{})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListR_NestedFolderRecursion(t *testing.T) {
|
||||
f := newTestPhotosFs()
|
||||
f.photos = newTestPhotosService()
|
||||
f.dirCache = dircache.New("", rootID, f)
|
||||
f.features = (&fs.Features{}).Fill(context.Background(), f)
|
||||
f.startTime = time.Now()
|
||||
ctx := context.Background()
|
||||
|
||||
err := f.dirCache.FindRoot(ctx, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Pre-populate empty photo caches on all leaf albums
|
||||
libs, err := f.photos.GetLibraries(ctx)
|
||||
require.NoError(t, err)
|
||||
for _, lib := range libs {
|
||||
albums, err := lib.GetAlbums(ctx)
|
||||
require.NoError(t, err)
|
||||
setEmptyPhotoCaches(albums)
|
||||
}
|
||||
|
||||
var dirs []string
|
||||
err = f.ListR(ctx, "", func(entries fs.DirEntries) error {
|
||||
for _, entry := range entries {
|
||||
if _, ok := entry.(fs.Directory); ok {
|
||||
dirs = append(dirs, entry.Remote())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
sort.Strings(dirs)
|
||||
|
||||
// All directories must be present, including deeply nested ones
|
||||
assert.Contains(t, dirs, "PrimarySync")
|
||||
assert.Contains(t, dirs, "PrimarySync/All Photos")
|
||||
assert.Contains(t, dirs, "PrimarySync/Videos")
|
||||
assert.Contains(t, dirs, "PrimarySync/Folder")
|
||||
assert.Contains(t, dirs, "PrimarySync/Folder/ChildAlbum")
|
||||
assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder")
|
||||
assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder/LeafAlbum")
|
||||
}
|
||||
|
||||
func TestListR_FolderLevel(t *testing.T) {
|
||||
f := &PhotosFs{
|
||||
name: "test",
|
||||
root: "PrimarySync/Folder",
|
||||
opt: Options{},
|
||||
startTime: time.Now(),
|
||||
}
|
||||
f.features = (&fs.Features{}).Fill(context.Background(), f)
|
||||
f.photos = newTestPhotosService()
|
||||
f.dirCache = dircache.New("PrimarySync/Folder", rootID, f)
|
||||
ctx := context.Background()
|
||||
|
||||
err := f.dirCache.FindRoot(ctx, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
libs, _ := f.photos.GetLibraries(ctx)
|
||||
for _, lib := range libs {
|
||||
albums, _ := lib.GetAlbums(ctx)
|
||||
setEmptyPhotoCaches(albums)
|
||||
}
|
||||
|
||||
var dirs []string
|
||||
err = f.ListR(ctx, "", func(entries fs.DirEntries) error {
|
||||
for _, entry := range entries {
|
||||
if _, ok := entry.(fs.Directory); ok {
|
||||
dirs = append(dirs, entry.Remote())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// When rooted at a folder, should see children recursively
|
||||
assert.Contains(t, dirs, "ChildAlbum")
|
||||
assert.Contains(t, dirs, "NestedFolder")
|
||||
assert.Contains(t, dirs, "NestedFolder/LeafAlbum")
|
||||
}
|
||||
|
||||
func TestNotifyZoneChange(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ps := newTestPhotosService()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
root string
|
||||
zone string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "top level root",
|
||||
root: "",
|
||||
zone: "PrimarySync",
|
||||
want: []string{"PrimarySync"},
|
||||
},
|
||||
{
|
||||
name: "library root",
|
||||
root: "PrimarySync",
|
||||
zone: "PrimarySync",
|
||||
want: []string{"", "All Photos", "Folder", "Folder/ChildAlbum", "Folder/NestedFolder", "Folder/NestedFolder/LeafAlbum", "UserAlbum", "Videos"},
|
||||
},
|
||||
{
|
||||
name: "folder root",
|
||||
root: "PrimarySync/Folder",
|
||||
zone: "PrimarySync",
|
||||
want: []string{"", "ChildAlbum", "NestedFolder", "NestedFolder/LeafAlbum"},
|
||||
},
|
||||
{
|
||||
name: "nested folder root",
|
||||
root: "PrimarySync/Folder/NestedFolder",
|
||||
zone: "PrimarySync",
|
||||
want: []string{"", "LeafAlbum"},
|
||||
},
|
||||
{
|
||||
name: "leaf album root",
|
||||
root: "PrimarySync/Folder/NestedFolder/LeafAlbum",
|
||||
zone: "PrimarySync",
|
||||
want: []string{""},
|
||||
},
|
||||
{
|
||||
name: "missing nested album still invalidates root",
|
||||
root: "PrimarySync/Folder/NoSuchAlbum",
|
||||
zone: "PrimarySync",
|
||||
want: []string{""},
|
||||
},
|
||||
{
|
||||
name: "different zone ignored",
|
||||
root: "PrimarySync/Folder/NestedFolder",
|
||||
zone: "SharedSync",
|
||||
want: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f := newTestPhotosFs()
|
||||
f.root = tt.root
|
||||
|
||||
var got []string
|
||||
f.notifyZoneChange(ctx, ps, tt.zone, func(remote string, entryType fs.EntryType) {
|
||||
assert.Equal(t, fs.EntryDirectory, entryType)
|
||||
got = append(got, remote)
|
||||
})
|
||||
|
||||
sort.Strings(got)
|
||||
want := append([]string(nil), tt.want...)
|
||||
sort.Strings(want)
|
||||
assert.Equal(t, want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -30,15 +30,21 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
)
|
||||
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2
|
||||
maxUploadParts = 10000
|
||||
minChunkSize = fs.SizeSuffix(5 * 1024 * 1024)
|
||||
minUploadCutoff = fs.SizeSuffix(100 * 1024 * 1024)
|
||||
defaultUploadCutoff = fs.SizeSuffix(100 * 1024 * 1024)
|
||||
maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
|
||||
)
|
||||
|
||||
// shouldRetry determines if an error should be retried.
|
||||
@@ -101,6 +107,21 @@ func init() {
|
||||
Default: true,
|
||||
Advanced: true,
|
||||
Help: "Skip hash validation when downloading files.\n\nBy default, hash validation is disabled. Set this to false to enable validation.",
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
Help: "Concurrency for multipart uploads.\n\nThis is the number of chunks of the same file that are uploaded concurrently.\n\nNote that each chunk is buffered in memory.",
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: "Cutoff for switching to multipart upload.\n\nAny files larger than this will be uploaded in chunks of chunk_size.\nThe minimum is 100 MiB and the maximum is 5 GiB.",
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: "Chunk size for multipart uploads.\n\nFiles larger than upload_cutoff will be uploaded in chunks of this size.\n\nMemory usage is approximately chunk_size * upload_concurrency.",
|
||||
Default: fs.SizeSuffix(30 * 1024 * 1024),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: rclone_config.ConfigEncoding,
|
||||
Help: rclone_config.ConfigEncodingHelp,
|
||||
@@ -194,6 +215,9 @@ type Options struct {
|
||||
TwoFA string `config:"2fa"`
|
||||
Mnemonic string `config:"mnemonic"`
|
||||
SkipHashValidation bool `config:"skip_hash_validation"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Encoding encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
@@ -238,6 +262,11 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
}
|
||||
|
||||
// Hashes returns type of hashes supported by Internxt
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
return hash.NewHashSet()
|
||||
@@ -255,6 +284,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := checkUploadChunkSize(opt.ChunkSize); err != nil {
|
||||
return nil, fmt.Errorf("internxt: chunk size: %w", err)
|
||||
}
|
||||
if err := checkUploadCutoff(opt.UploadCutoff); err != nil {
|
||||
return nil, fmt.Errorf("internxt: upload cutoff: %w", err)
|
||||
}
|
||||
|
||||
if opt.Mnemonic == "" {
|
||||
return nil, errors.New("mnemonic is required - please run: rclone config reconnect " + name + ":")
|
||||
}
|
||||
@@ -884,32 +920,53 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
fs.Debugf(o.f, "Renamed existing file %s to backup %s.%s (UUID: %s)", remote, backupName, backupType, backupUUID)
|
||||
}
|
||||
|
||||
size := src.Size()
|
||||
|
||||
var meta *buckets.CreateMetaResponse
|
||||
err = o.f.pacer.CallNoRetry(func() (bool, error) {
|
||||
var err error
|
||||
meta, err = buckets.UploadFileStreamAuto(ctx,
|
||||
o.f.cfg,
|
||||
dirID,
|
||||
o.f.opt.Encoding.FromStandardName(path.Base(remote)),
|
||||
in,
|
||||
src.Size(),
|
||||
src.ModTime(ctx),
|
||||
)
|
||||
return o.f.shouldRetry(ctx, err)
|
||||
})
|
||||
if size < 0 || size >= int64(o.f.opt.UploadCutoff) {
|
||||
chunkWriter, uploadErr := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
|
||||
Open: o.f,
|
||||
OpenOptions: options,
|
||||
})
|
||||
|
||||
if err != nil && isEmptyFileLimitError(err) {
|
||||
o.restoreBackupFile(ctx, backupUUID, origName, origType)
|
||||
return fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
if uploadErr != nil {
|
||||
if isEmptyFileLimitError(uploadErr) {
|
||||
o.restoreBackupFile(ctx, backupUUID, origName, origType)
|
||||
return fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
o.restoreBackupFile(ctx, backupUUID, origName, origType)
|
||||
return uploadErr
|
||||
}
|
||||
w := chunkWriter.(*internxtChunkWriter)
|
||||
meta = w.meta
|
||||
} else {
|
||||
// Use single-part upload for small files
|
||||
err = o.f.pacer.CallNoRetry(func() (bool, error) {
|
||||
var err error
|
||||
meta, err = buckets.UploadFileStreamAuto(ctx,
|
||||
o.f.cfg,
|
||||
dirID,
|
||||
o.f.opt.Encoding.FromStandardName(path.Base(remote)),
|
||||
in,
|
||||
size,
|
||||
src.ModTime(ctx),
|
||||
)
|
||||
return o.f.shouldRetry(ctx, err)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
|
||||
}
|
||||
if err != nil && isEmptyFileLimitError(err) {
|
||||
o.restoreBackupFile(ctx, backupUUID, origName, origType)
|
||||
return fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
o.restoreBackupFile(ctx, backupUUID, origName, origType)
|
||||
return err
|
||||
if err != nil {
|
||||
meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
o.restoreBackupFile(ctx, backupUUID, origName, origType)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Update object metadata
|
||||
|
||||
@@ -3,6 +3,7 @@ package internxt_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
@@ -10,5 +11,9 @@ import (
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestInternxt:",
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: 100 * fs.Mebi,
|
||||
NeedMultipleChunks: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package internxt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/cipher"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/internxt/rclone-adapter/buckets"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/lib/multipart"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
)
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
if cs < minChunkSize {
|
||||
return fmt.Errorf("%s is less than %s", cs, minChunkSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetUploadChunkSize sets the chunk size used for multipart uploads
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
err := checkUploadChunkSize(cs)
|
||||
if err == nil {
|
||||
old := f.opt.ChunkSize
|
||||
f.opt.ChunkSize = cs
|
||||
return old, nil
|
||||
}
|
||||
return f.opt.ChunkSize, err
|
||||
}
|
||||
|
||||
func checkUploadCutoff(cs fs.SizeSuffix) error {
|
||||
if cs < minUploadCutoff {
|
||||
return fmt.Errorf("%s is less than %s (Internxt requires minimum %s for multipart uploads)", cs, minUploadCutoff, minUploadCutoff)
|
||||
}
|
||||
if cs > maxUploadCutoff {
|
||||
return fmt.Errorf("%s is greater than %s", cs, maxUploadCutoff)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetUploadCutoff sets the cutoff for switching to multipart upload
|
||||
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
err := checkUploadCutoff(cs)
|
||||
if err == nil {
|
||||
old := f.opt.UploadCutoff
|
||||
f.opt.UploadCutoff = cs
|
||||
return old, nil
|
||||
}
|
||||
return f.opt.UploadCutoff, err
|
||||
}
|
||||
|
||||
// internxtChunkWriter implements fs.ChunkWriter for Internxt multipart uploads.
|
||||
type internxtChunkWriter struct {
|
||||
f *Fs
|
||||
remote string
|
||||
src fs.ObjectInfo
|
||||
session *buckets.ChunkUploadSession
|
||||
completedParts []buckets.CompletedPart
|
||||
partsMu sync.Mutex
|
||||
size int64
|
||||
dirID string
|
||||
meta *buckets.CreateMetaResponse
|
||||
chunkSize int64
|
||||
hashMu sync.Mutex
|
||||
nextHashChunk int
|
||||
pendingChunks map[int]*pool.RW
|
||||
}
|
||||
|
||||
// OpenChunkWriter returns the chunk size and a ChunkWriter for multipart uploads.
|
||||
func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
|
||||
size := src.Size()
|
||||
|
||||
info = fs.ChunkWriterInfo{
|
||||
ChunkSize: int64(f.opt.ChunkSize),
|
||||
Concurrency: f.opt.UploadConcurrency,
|
||||
LeavePartsOnError: false,
|
||||
}
|
||||
|
||||
// Reject files below the upload cutoff
|
||||
if size >= 0 && size < int64(f.opt.UploadCutoff) {
|
||||
return info, nil, &fs.FileTooSmallError{MinSize: int64(f.opt.UploadCutoff)}
|
||||
}
|
||||
|
||||
chunkSize := f.opt.ChunkSize
|
||||
if size < 0 {
|
||||
warnStreamUpload.Do(func() {
|
||||
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
|
||||
chunkSize, fs.SizeSuffix(int64(chunkSize)*int64(maxUploadParts)))
|
||||
})
|
||||
} else {
|
||||
chunkSize = chunksize.Calculator(src, size, maxUploadParts, chunkSize)
|
||||
info.ChunkSize = int64(chunkSize)
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
_, dirID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to find parent directory: %w", err)
|
||||
}
|
||||
|
||||
var session *buckets.ChunkUploadSession
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
session, err = buckets.NewChunkUploadSession(ctx, f.cfg, size, int64(chunkSize))
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return info, nil, fmt.Errorf("failed to create upload session: %w", err)
|
||||
}
|
||||
|
||||
w := &internxtChunkWriter{
|
||||
f: f,
|
||||
remote: remote,
|
||||
src: src,
|
||||
session: session,
|
||||
size: size,
|
||||
dirID: dirID,
|
||||
chunkSize: int64(chunkSize),
|
||||
pendingChunks: make(map[int]*pool.RW),
|
||||
}
|
||||
|
||||
return info, w, nil
|
||||
}
|
||||
|
||||
// WriteChunk encrypts plaintext per-chunk using AES-256-CTR at the correct
|
||||
// byte offset, uploads the encrypted data, then feeds it into the ordered
|
||||
// hash accumulator
|
||||
func (w *internxtChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (int64, error) {
|
||||
byteOffset := int64(chunkNumber) * w.chunkSize
|
||||
cipherStream, err := w.session.NewCipherAtOffset(byteOffset)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
encRW := multipart.NewRW().Reserve(w.chunkSize)
|
||||
cipherReader := &cipher.StreamReader{S: cipherStream, R: reader}
|
||||
size, err := io.Copy(encRW, cipherReader)
|
||||
if err != nil {
|
||||
_ = encRW.Close()
|
||||
return 0, err
|
||||
}
|
||||
if size == 0 {
|
||||
_ = encRW.Close()
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var etag string
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
if _, err := encRW.Seek(0, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
var uploadErr error
|
||||
etag, uploadErr = w.session.UploadChunk(ctx, chunkNumber, encRW, size)
|
||||
return w.f.shouldRetry(ctx, uploadErr)
|
||||
})
|
||||
if err != nil {
|
||||
_ = encRW.Close()
|
||||
return 0, err
|
||||
}
|
||||
|
||||
w.recordCompletedPart(chunkNumber, etag)
|
||||
|
||||
if _, err := encRW.Seek(0, io.SeekStart); err != nil {
|
||||
_ = encRW.Close()
|
||||
return 0, err
|
||||
}
|
||||
w.submitForHashing(chunkNumber, encRW)
|
||||
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// recordCompletedPart appends a completed part to the list (thread-safe).
|
||||
func (w *internxtChunkWriter) recordCompletedPart(chunkNumber int, etag string) {
|
||||
w.partsMu.Lock()
|
||||
w.completedParts = append(w.completedParts, buckets.CompletedPart{
|
||||
PartNumber: chunkNumber + 1,
|
||||
ETag: etag,
|
||||
})
|
||||
w.partsMu.Unlock()
|
||||
}
|
||||
|
||||
// hashWriter is an io.Writer that feeds data into the session's SHA-256 hash.
|
||||
type hashWriter struct {
|
||||
session *buckets.ChunkUploadSession
|
||||
}
|
||||
|
||||
func (hw hashWriter) Write(p []byte) (int, error) {
|
||||
hw.session.HashEncryptedData(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// submitForHashing feeds encrypted chunk data into the session's hash in order.
|
||||
func (w *internxtChunkWriter) submitForHashing(chunkNumber int, encRW *pool.RW) {
|
||||
w.hashMu.Lock()
|
||||
defer w.hashMu.Unlock()
|
||||
|
||||
hw := hashWriter{w.session}
|
||||
|
||||
if chunkNumber == w.nextHashChunk {
|
||||
_, _ = encRW.WriteTo(hw)
|
||||
_ = encRW.Close()
|
||||
w.nextHashChunk++
|
||||
for {
|
||||
next, ok := w.pendingChunks[w.nextHashChunk]
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
_, _ = next.WriteTo(hw)
|
||||
_ = next.Close()
|
||||
delete(w.pendingChunks, w.nextHashChunk)
|
||||
w.nextHashChunk++
|
||||
}
|
||||
} else {
|
||||
w.pendingChunks[chunkNumber] = encRW
|
||||
}
|
||||
}
|
||||
|
||||
// Close completes the multipart upload and registers the file in Internxt Drive.
|
||||
func (w *internxtChunkWriter) Close(ctx context.Context) error {
|
||||
w.hashMu.Lock()
|
||||
pending := len(w.pendingChunks)
|
||||
if pending != 0 {
|
||||
for _, rw := range w.pendingChunks {
|
||||
_ = rw.Close()
|
||||
}
|
||||
w.pendingChunks = nil
|
||||
}
|
||||
w.hashMu.Unlock()
|
||||
if pending != 0 {
|
||||
return fmt.Errorf("internal error: %d chunks still pending hash", pending)
|
||||
}
|
||||
|
||||
// Sort parts by part number
|
||||
w.partsMu.Lock()
|
||||
sort.Slice(w.completedParts, func(i, j int) bool {
|
||||
return w.completedParts[i].PartNumber < w.completedParts[j].PartNumber
|
||||
})
|
||||
parts := make([]buckets.CompletedPart, len(w.completedParts))
|
||||
copy(parts, w.completedParts)
|
||||
w.partsMu.Unlock()
|
||||
|
||||
// Finish multipart upload (SDK computes hash + calls FinishMultipartUpload)
|
||||
var finishResp *buckets.FinishUploadResp
|
||||
err := w.f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
finishResp, err = w.session.Finish(ctx, parts)
|
||||
return w.f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to finish multipart upload: %w", err)
|
||||
}
|
||||
|
||||
// Create file metadata in Internxt Drive
|
||||
baseName := w.f.opt.Encoding.FromStandardName(path.Base(w.remote))
|
||||
name := strings.TrimSuffix(baseName, path.Ext(baseName))
|
||||
ext := strings.TrimPrefix(path.Ext(baseName), ".")
|
||||
|
||||
var meta *buckets.CreateMetaResponse
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
meta, err = buckets.CreateMetaFile(ctx, w.f.cfg,
|
||||
name, w.f.cfg.Bucket, &finishResp.ID, "03-aes",
|
||||
w.dirID, name, ext, w.size, w.src.ModTime(ctx))
|
||||
return w.f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file metadata: %w", err)
|
||||
}
|
||||
w.meta = meta
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Abort cleans up after a failed upload.
|
||||
func (w *internxtChunkWriter) Abort(ctx context.Context) error {
|
||||
w.hashMu.Lock()
|
||||
for _, rw := range w.pendingChunks {
|
||||
_ = rw.Close()
|
||||
}
|
||||
w.pendingChunks = nil
|
||||
w.hashMu.Unlock()
|
||||
fs.Logf(w.f, "Multipart upload aborted for %s", w.remote)
|
||||
return nil
|
||||
}
|
||||
@@ -91,6 +91,7 @@ func getServices() []service {
|
||||
{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
|
||||
{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"onlime", "Onlime (Denmark)", "cloud-auth.onlime.dk", "onlime_wl", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"phonero", "Phonero Sky", "sky.phonero.no", "phonero", "desktop-win", []string{"openid", "offline_access"}},
|
||||
{"tele2", "Tele2 Cloud (Sweden)", "mittcloud-auth.tele2.se", "comhem", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"telia_no", "Telia Sky (Norway)", "sky-auth.telia.no", "get", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
{"telia_se", "Telia Cloud (Sweden)", "cloud-auth.telia.se", "telia_se", "desktop", []string{"openid", "jotta-default", "offline_access"}},
|
||||
@@ -166,7 +167,8 @@ func init() {
|
||||
// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
|
||||
Default: (encoder.Display |
|
||||
encoder.EncodeWin | // :?"*<>|
|
||||
encoder.EncodeInvalidUtf8),
|
||||
encoder.EncodeInvalidUtf8 |
|
||||
encoder.EncodePercent),
|
||||
}}...),
|
||||
})
|
||||
}
|
||||
|
||||
+162
-43
@@ -14,19 +14,21 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
@@ -41,6 +43,7 @@ const (
|
||||
maxSleep = 2 * time.Second
|
||||
pacerBurst = 1
|
||||
linkboxAPIURL = "https://www.linkbox.to/api/open/"
|
||||
linkboxWebAPIURL = "https://www.linkbox.to/api/"
|
||||
rootID = "0" // ID of root directory
|
||||
)
|
||||
|
||||
@@ -54,6 +57,20 @@ func init() {
|
||||
Help: "Token from https://www.linkbox.to/admin/account",
|
||||
Sensitive: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "email",
|
||||
Help: "Email for login",
|
||||
Sensitive: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "password",
|
||||
Help: "Password for login",
|
||||
IsPassword: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "web_token",
|
||||
Help: "Web API login token - set automatically.",
|
||||
Hide: fs.OptionHideBoth,
|
||||
}},
|
||||
}
|
||||
fs.Register(fsi)
|
||||
@@ -61,19 +78,26 @@ func init() {
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Token string `config:"token"`
|
||||
Token string `config:"token"`
|
||||
Email string `config:"email"`
|
||||
Password string `config:"password"`
|
||||
WebToken string `config:"web_token"`
|
||||
}
|
||||
|
||||
// Fs stores the interface to the remote Linkbox files
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
opt Options // options for this backend
|
||||
features *fs.Features // optional features
|
||||
ci *fs.ConfigInfo // global config
|
||||
srv *rest.Client // the connection to the server
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
pacer *fs.Pacer
|
||||
name string
|
||||
root string
|
||||
opt Options // options for this backend
|
||||
features *fs.Features // optional features
|
||||
ci *fs.ConfigInfo // global config
|
||||
srv *rest.Client // the connection to the server
|
||||
cdnSrv *rest.Client // HTTP client for CDN downloads (HTTP/2 disabled)
|
||||
dirCache *dircache.DirCache // Map of directory path to directory id
|
||||
pacer *fs.Pacer
|
||||
m configmap.Mapper // config mapper for saving tokens
|
||||
webToken string // token from web API login
|
||||
webTokenMu sync.Mutex // protects webToken
|
||||
}
|
||||
|
||||
// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
|
||||
@@ -104,14 +128,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
root: root,
|
||||
ci: ci,
|
||||
srv: rest.NewClient(fshttp.NewClient(ctx)),
|
||||
// Create a separate HTTP client for CDN downloads with HTTP/2 disabled.
|
||||
// The Linkbox CDN intermittently blocks Go's HTTP/2 fingerprint and
|
||||
// non-browser User-Agents.
|
||||
cdnClient := fshttp.NewClientCustom(ctx, func(t *http.Transport) {
|
||||
t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
|
||||
})
|
||||
if tr, ok := cdnClient.Transport.(*fshttp.Transport); ok {
|
||||
tr.SetRequestFilter(func(req *http.Request) {
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
||||
})
|
||||
}
|
||||
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep))),
|
||||
f := &Fs{
|
||||
name: name,
|
||||
opt: *opt,
|
||||
root: root,
|
||||
ci: ci,
|
||||
srv: rest.NewClient(fshttp.NewClient(ctx)),
|
||||
cdnSrv: rest.NewClient(cdnClient),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep))),
|
||||
m: m,
|
||||
}
|
||||
f.dirCache = dircache.New(root, rootID, f)
|
||||
|
||||
@@ -120,12 +157,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
CaseInsensitive: true,
|
||||
}).Fill(ctx, f)
|
||||
|
||||
if f.opt.Email == "" || f.opt.Password == "" {
|
||||
return nil, fmt.Errorf("email and password are required - run `rclone config`")
|
||||
}
|
||||
|
||||
// Load cached web token or login to get a new one
|
||||
if opt.WebToken != "" {
|
||||
f.webToken = opt.WebToken
|
||||
fs.Debugf(f, "Using cached web token")
|
||||
} else {
|
||||
err = f.login(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Find the current root
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
// Assume it is a file
|
||||
newRoot, remote := dircache.SplitPath(root)
|
||||
tempF := *f
|
||||
tempF := *f //nolint:govet // copying mutex is OK here as it is a new Fs
|
||||
tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
|
||||
tempF.root = newRoot
|
||||
// Make new Fs which is the parent
|
||||
@@ -154,6 +206,78 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// loginResponse is the response from the web API login
|
||||
type loginResponse struct {
|
||||
response
|
||||
Data struct {
|
||||
Token string `json:"token"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// login authenticates with the web API using email and password
|
||||
// and saves the token in the config for reuse.
|
||||
func (f *Fs) login(ctx context.Context) error {
|
||||
password, err := obscure.Reveal(f.opt.Password)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt password: %w", err)
|
||||
}
|
||||
opts := &rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: linkboxWebAPIURL,
|
||||
Path: "user/login_email",
|
||||
Parameters: url.Values{
|
||||
"email": {f.opt.Email},
|
||||
"pwd": {password},
|
||||
"platform": {"web"},
|
||||
"pf": {"web"},
|
||||
"lan": {"en"},
|
||||
},
|
||||
}
|
||||
var result loginResponse
|
||||
err = getUnmarshaledResponse(ctx, f, opts, &result)
|
||||
if err != nil {
|
||||
return fmt.Errorf("login failed: %w", err)
|
||||
}
|
||||
if result.Data.Token == "" {
|
||||
return fmt.Errorf("login returned empty token")
|
||||
}
|
||||
f.webTokenMu.Lock()
|
||||
f.webToken = result.Data.Token
|
||||
f.webTokenMu.Unlock()
|
||||
// Save token to config for reuse across invocations
|
||||
f.m.Set("web_token", f.webToken)
|
||||
fs.Debugf(f, "Login successful")
|
||||
return nil
|
||||
}
|
||||
|
||||
// shouldRetryWeb determines whether a web API error should be retried.
|
||||
//
|
||||
// If the API returns an error it may be because the token has expired,
|
||||
// so this refreshes the token and updates the request parameters for
|
||||
// the retry.
|
||||
func (f *Fs) shouldRetryWeb(ctx context.Context, resp *http.Response, err error, result responser, opts *rest.Opts) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes) {
|
||||
return true, err
|
||||
}
|
||||
// If the web API returned an error, it may be due to an expired token.
|
||||
// Refresh the token and retry.
|
||||
if result.IsError() {
|
||||
fs.Debugf(f, "Web API returned error %q - refreshing token", result.Error())
|
||||
loginErr := f.login(ctx)
|
||||
if loginErr != nil {
|
||||
return false, loginErr
|
||||
}
|
||||
f.webTokenMu.Lock()
|
||||
opts.Parameters.Set("token", f.webToken)
|
||||
f.webTokenMu.Unlock()
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type entity struct {
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
@@ -216,40 +340,28 @@ func getUnmarshaledResponse(ctx context.Context, f *Fs, opts *rest.Opts, result
|
||||
// Should return true to finish processing
|
||||
type listAllFn func(*entity) bool
|
||||
|
||||
// Search is a bit fussy about which characters match
|
||||
//
|
||||
// If the name doesn't match this then do an dir list instead
|
||||
// N.B.: Linkbox doesn't support search by name that is longer than 50 chars
|
||||
var searchOK = regexp.MustCompile(`^[a-zA-Z0-9_ -.]{1,50}$`)
|
||||
|
||||
// Lists the directory required calling the user function on each item found
|
||||
//
|
||||
// If the user fn ever returns true then it early exits with found = true
|
||||
//
|
||||
// If you set name then search ignores dirID. name is a substring
|
||||
// search also so name="dir" matches "sub dir" also. This filters it
|
||||
// down so it only returns items in dirID
|
||||
func (f *Fs) listAll(ctx context.Context, dirID string, name string, fn listAllFn) (found bool, err error) {
|
||||
_ = name // name parameter kept for call-site compatibility but unused with web API
|
||||
var (
|
||||
pageNumber = 0
|
||||
numberOfEntities = maxEntitiesPerPage
|
||||
)
|
||||
name = strings.TrimSpace(name) // search doesn't like spaces
|
||||
if !searchOK.MatchString(name) {
|
||||
// If name isn't good then do an unbounded search
|
||||
name = ""
|
||||
}
|
||||
|
||||
OUTER:
|
||||
for numberOfEntities == maxEntitiesPerPage {
|
||||
pageNumber++
|
||||
f.webTokenMu.Lock()
|
||||
webToken := f.webToken
|
||||
f.webTokenMu.Unlock()
|
||||
opts := &rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: linkboxAPIURL,
|
||||
Path: "file_search",
|
||||
RootURL: linkboxWebAPIURL,
|
||||
Path: "file/my_file_list/web",
|
||||
Parameters: url.Values{
|
||||
"token": {f.opt.Token},
|
||||
"name": {name},
|
||||
"token": {webToken},
|
||||
"pid": {dirID},
|
||||
"pageNo": {itoa(pageNumber)},
|
||||
"pageSize": {itoa64(maxEntitiesPerPage)},
|
||||
@@ -257,7 +369,10 @@ OUTER:
|
||||
}
|
||||
|
||||
var responseResult fileSearchRes
|
||||
err = getUnmarshaledResponse(ctx, f, opts, &responseResult)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, opts, nil, &responseResult)
|
||||
return f.shouldRetryWeb(ctx, resp, err, &responseResult, opts)
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("getting files failed: %w", err)
|
||||
}
|
||||
@@ -403,7 +518,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
}
|
||||
|
||||
// get an entity with leaf from dirID
|
||||
func getEntity(ctx context.Context, f *Fs, leaf string, directoryID string, token string) (*entity, error) {
|
||||
func getEntity(ctx context.Context, f *Fs, leaf string, directoryID string) (*entity, error) {
|
||||
var result *entity
|
||||
var resultErr = fs.ErrorObjectNotFound
|
||||
_, err := f.listAll(ctx, directoryID, leaf, func(entity *entity) bool {
|
||||
@@ -441,7 +556,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entity, err := getEntity(ctx, f, leaf, dirID, f.opt.Token)
|
||||
entity, err := getEntity(ctx, f, leaf, dirID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -518,7 +633,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
downloadURL := o.fullURL
|
||||
if downloadURL == "" {
|
||||
_, name := splitDirAndName(o.Remote())
|
||||
newObject, err := getEntity(ctx, o.fs, name, itoa64(o.dirID), o.fs.opt.Token)
|
||||
newObject, err := getEntity(ctx, o.fs, name, itoa64(o.dirID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -530,6 +645,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
downloadURL = newObject.URL
|
||||
}
|
||||
|
||||
if downloadURL == "" {
|
||||
return nil, fmt.Errorf("no download URL for %q - try re-running `rclone config reconnect`", o.Remote())
|
||||
}
|
||||
|
||||
opts := &rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: downloadURL,
|
||||
@@ -538,7 +657,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
|
||||
err := o.fs.pacer.Call(func() (bool, error) {
|
||||
var err error
|
||||
res, err = o.fs.srv.Call(ctx, opts)
|
||||
res, err = o.fs.cdnSrv.Call(ctx, opts)
|
||||
return o.fs.shouldRetry(ctx, res, err)
|
||||
})
|
||||
|
||||
@@ -697,7 +816,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
var sleepTime = 100 * time.Millisecond
|
||||
var entity *entity
|
||||
for try := 1; try <= maxTries; try++ {
|
||||
entity, err = getEntity(ctx, o.fs, leaf, dirID, o.fs.opt.Token)
|
||||
entity, err = getEntity(ctx, o.fs, leaf, dirID)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
//go:build !linux
|
||||
|
||||
package local
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
func newFadviseReadCloser(o *Object, f *os.File, offset, limit int64) io.ReadCloser {
|
||||
return f
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
//go:build linux
|
||||
|
||||
package local
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// fadvise provides means to automate freeing pages in kernel page cache for
|
||||
// a given file descriptor as the file is sequentially processed (read or
|
||||
// written).
|
||||
//
|
||||
// When copying a file to a remote backend all the file content is read by
|
||||
// kernel and put to page cache to make future reads faster.
|
||||
// This causes memory pressure visible in both memory usage and CPU consumption
|
||||
// and can even cause OOM errors in applications consuming large amounts memory.
|
||||
//
|
||||
// In case of an upload to a remote backend, there is no benefits from caching.
|
||||
//
|
||||
// fadvise would orchestrate calling POSIX_FADV_DONTNEED
|
||||
//
|
||||
// POSIX_FADV_DONTNEED attempts to free cached pages associated
|
||||
// with the specified region. This is useful, for example, while
|
||||
// streaming large files. A program may periodically request the
|
||||
// kernel to free cached data that has already been used, so that
|
||||
// more useful cached pages are not discarded instead.
|
||||
//
|
||||
// Requests to discard partial pages are ignored. It is
|
||||
// preferable to preserve needed data than discard unneeded data.
|
||||
// If the application requires that data be considered for
|
||||
// discarding, then offset and len must be page-aligned.
|
||||
//
|
||||
// The implementation may attempt to write back dirty pages in
|
||||
// the specified region, but this is not guaranteed. Any
|
||||
// unwritten dirty pages will not be freed. If the application
|
||||
// wishes to ensure that dirty pages will be released, it should
|
||||
// call fsync(2) or fdatasync(2) first.
|
||||
type fadvise struct {
|
||||
o *Object
|
||||
fd int
|
||||
lastPos int64
|
||||
curPos int64
|
||||
windowSize int64
|
||||
|
||||
freePagesCh chan offsetLength
|
||||
doneCh chan struct{}
|
||||
}
|
||||
|
||||
type offsetLength struct {
|
||||
offset int64
|
||||
length int64
|
||||
}
|
||||
|
||||
const (
|
||||
defaultAllowPages = 32
|
||||
defaultWorkerQueueSize = 64
|
||||
)
|
||||
|
||||
func newFadvise(o *Object, fd int, offset int64) *fadvise {
|
||||
f := &fadvise{
|
||||
o: o,
|
||||
fd: fd,
|
||||
lastPos: offset,
|
||||
curPos: offset,
|
||||
windowSize: int64(os.Getpagesize()) * defaultAllowPages,
|
||||
|
||||
freePagesCh: make(chan offsetLength, defaultWorkerQueueSize),
|
||||
doneCh: make(chan struct{}),
|
||||
}
|
||||
go f.worker()
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
// sequential configures readahead strategy in Linux kernel.
|
||||
//
|
||||
// Under Linux, POSIX_FADV_NORMAL sets the readahead window to the
|
||||
// default size for the backing device; POSIX_FADV_SEQUENTIAL doubles
|
||||
// this size, and POSIX_FADV_RANDOM disables file readahead entirely.
|
||||
func (f *fadvise) sequential(limit int64) bool {
|
||||
l := int64(0)
|
||||
if limit > 0 {
|
||||
l = limit
|
||||
}
|
||||
if err := unix.Fadvise(f.fd, f.curPos, l, unix.FADV_SEQUENTIAL); err != nil {
|
||||
fs.Debugf(f.o, "fadvise sequential failed on file descriptor %d: %s", f.fd, err)
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (f *fadvise) next(n int) {
|
||||
f.curPos += int64(n)
|
||||
f.freePagesIfNeeded()
|
||||
}
|
||||
|
||||
func (f *fadvise) freePagesIfNeeded() {
|
||||
if f.curPos >= f.lastPos+f.windowSize {
|
||||
f.freePages()
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fadvise) freePages() {
|
||||
f.freePagesCh <- offsetLength{f.lastPos, f.curPos - f.lastPos}
|
||||
f.lastPos = f.curPos
|
||||
}
|
||||
|
||||
func (f *fadvise) worker() {
|
||||
for p := range f.freePagesCh {
|
||||
if err := unix.Fadvise(f.fd, p.offset, p.length, unix.FADV_DONTNEED); err != nil {
|
||||
fs.Debugf(f.o, "fadvise dontneed failed on file descriptor %d: %s", f.fd, err)
|
||||
}
|
||||
}
|
||||
|
||||
close(f.doneCh)
|
||||
}
|
||||
|
||||
func (f *fadvise) wait() {
|
||||
close(f.freePagesCh)
|
||||
<-f.doneCh
|
||||
}
|
||||
|
||||
type fadviseReadCloser struct {
|
||||
*fadvise
|
||||
inner io.ReadCloser
|
||||
}
|
||||
|
||||
// newFadviseReadCloser wraps os.File so that reading from that file would
|
||||
// remove already consumed pages from kernel page cache.
|
||||
// In addition to that it instructs kernel to double the readahead window to
|
||||
// make sequential reads faster.
|
||||
// See also fadvise.
|
||||
func newFadviseReadCloser(o *Object, f *os.File, offset, limit int64) io.ReadCloser {
|
||||
r := fadviseReadCloser{
|
||||
fadvise: newFadvise(o, int(f.Fd()), offset),
|
||||
inner: f,
|
||||
}
|
||||
|
||||
// If syscall failed it's likely that the subsequent syscalls to that
|
||||
// file descriptor would also fail. In that case return the provided os.File
|
||||
// pointer.
|
||||
if !r.sequential(limit) {
|
||||
r.wait()
|
||||
return f
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
func (f fadviseReadCloser) Read(p []byte) (n int, err error) {
|
||||
n, err = f.inner.Read(p)
|
||||
f.next(n)
|
||||
return
|
||||
}
|
||||
|
||||
func (f fadviseReadCloser) Close() error {
|
||||
f.freePages()
|
||||
f.wait()
|
||||
return f.inner.Close()
|
||||
}
|
||||
@@ -1168,7 +1168,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
var fd *os.File
|
||||
fd, err = file.Open(o.path)
|
||||
if fd != nil {
|
||||
in = newFadviseReadCloser(o, fd, 0, 0)
|
||||
in = fd
|
||||
}
|
||||
} else {
|
||||
in, err = o.openTranslatedLink(0, -1)
|
||||
@@ -1374,7 +1374,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
wrappedFd := readers.NewLimitedReadCloser(newFadviseReadCloser(o, fd, offset, limit), limit)
|
||||
wrappedFd := readers.NewLimitedReadCloser(fd, limit)
|
||||
if offset != 0 {
|
||||
// seek the object
|
||||
_, err = fd.Seek(offset, io.SeekStart)
|
||||
|
||||
@@ -284,7 +284,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
}
|
||||
err = srv.LoginWithKeys(opt.SessionID, decodedMasterKey)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "login with previous auth keys failed: %v", err)
|
||||
return nil, fmt.Errorf("login with previous auth keys failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,6 +238,10 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
|
||||
|
||||
// Check if it is an api.Error
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
// Handle error 1101 specifically - don't retry as it's a permanent API restriction
|
||||
if apiErr.Result == 1101 {
|
||||
return false, err
|
||||
}
|
||||
// See https://docs.pcloud.com/errors/ for error treatment
|
||||
// Errors are classified as 1xxx, 2xxx, etc.
|
||||
switch apiErr.Result / 1000 {
|
||||
@@ -531,10 +535,63 @@ func fileIDtoNumber(fileID string) string {
|
||||
// Should return true to finish processing
|
||||
type listAllFn func(*api.Item) bool
|
||||
|
||||
// listAllRootRecursive handles recursive listing for root directory (folderid=0)
|
||||
// using a two-step approach to work around pCloud API limitation
|
||||
func (f *Fs) listAllRootRecursive(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
|
||||
// List root directory non-recursively to get immediate contents
|
||||
var rootSubdirs []*api.Item
|
||||
found, err = f.listAll(ctx, dirID, false, false, false, func(item *api.Item) bool {
|
||||
if item.IsFolder {
|
||||
// Store subdirectories for recursive list later
|
||||
rootSubdirs = append(rootSubdirs, item)
|
||||
// Handle directories immediately if not filesOnly
|
||||
if !filesOnly {
|
||||
if fn(item) {
|
||||
return true // stop iteration
|
||||
}
|
||||
}
|
||||
} else if !directoriesOnly {
|
||||
// Handle files immediately if not directoriesOnly
|
||||
if fn(item) {
|
||||
return true // stop iteration
|
||||
}
|
||||
}
|
||||
return false // continue iteration
|
||||
})
|
||||
if err != nil || found {
|
||||
return found, err
|
||||
}
|
||||
|
||||
// Recursively list each subdirectory
|
||||
for _, subdir := range rootSubdirs {
|
||||
subdirName := f.opt.Enc.ToStandardName(subdir.Name)
|
||||
found, err = f.listAll(ctx, subdir.ID, directoriesOnly, filesOnly, true, func(item *api.Item) bool {
|
||||
// Prepend the subdirectory name to create correct relative path
|
||||
origItemName := item.Name
|
||||
item.Name = subdirName + "/" + item.Name
|
||||
result := fn(item)
|
||||
item.Name = origItemName
|
||||
return result
|
||||
})
|
||||
if err != nil || found {
|
||||
return found, err
|
||||
}
|
||||
}
|
||||
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// Lists the directory required calling the user function on each item found
|
||||
//
|
||||
// If the user fn ever returns true then it early exits with found = true
|
||||
func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, recursive bool, fn listAllFn) (found bool, err error) {
|
||||
// Special case: root directory with recursive listing
|
||||
// pCloud API rejects folderid=0 with recursive=1 (error 1101)
|
||||
// See: https://github.com/rclone/rclone/issues/9315
|
||||
if recursive && dirIDtoNumber(dirID) == "0" {
|
||||
return f.listAllRootRecursive(ctx, dirID, directoriesOnly, filesOnly, fn)
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/listfolder",
|
||||
|
||||
@@ -7,9 +7,11 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-semver/semver"
|
||||
protonDriveAPI "github.com/rclone/Proton-API-Bridge"
|
||||
"github.com/rclone/go-proton-api"
|
||||
|
||||
@@ -20,6 +22,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
@@ -49,6 +52,7 @@ const (
|
||||
var (
|
||||
errCanNotUploadFileWithUnknownSize = errors.New("proton Drive can't upload files with unknown size")
|
||||
errCanNotPurgeRootDirectory = errors.New("can't purge root directory")
|
||||
protonDriveInvalidVersionChars = regexp.MustCompile(`[^0-9A-Za-z.+-]+`)
|
||||
|
||||
// for the auth/deauth handler
|
||||
_mapper configmap.Mapper
|
||||
@@ -151,11 +155,12 @@ size, will fail to operate properly`,
|
||||
Name: "app_version",
|
||||
Help: `The app version string
|
||||
|
||||
The app version string indicates the client that is currently performing
|
||||
the API request. This information is required and will be sent with every
|
||||
API request.`,
|
||||
The app version string identifies the client that is currently performing
|
||||
the API request. Third-party Proton Drive integrations should use the form
|
||||
external-drive-<project>@<version>. If this option is left empty, rclone
|
||||
derives a compliant value from its own version. This value is sent with
|
||||
every API request; the option itself is optional.`,
|
||||
Advanced: true,
|
||||
Default: "macos-drive@1.0.0-alpha.1+rclone",
|
||||
}, {
|
||||
Name: "replace_existing_draft",
|
||||
Help: `Create a new revision when filename conflict is detected
|
||||
@@ -328,11 +333,121 @@ func deAuthHandler() {
|
||||
clearConfigMap(_mapper)
|
||||
}
|
||||
|
||||
func protonDriveAppVersionFromRcloneVersion(version string) string {
|
||||
const fallback = "external-drive-rclone@1.0.0-stable"
|
||||
|
||||
version = strings.TrimSpace(strings.TrimPrefix(version, "v"))
|
||||
version = protonDriveInvalidVersionChars.ReplaceAllString(version, "-")
|
||||
if version == "" {
|
||||
return fallback
|
||||
}
|
||||
|
||||
parsedVersion, err := semver.NewVersion(version)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
|
||||
appVersion := fmt.Sprintf(
|
||||
"external-drive-rclone@%d.%d.%d",
|
||||
parsedVersion.Major,
|
||||
parsedVersion.Minor,
|
||||
parsedVersion.Patch,
|
||||
)
|
||||
|
||||
metadataParts := protonDriveMetadataParts(parsedVersion.Metadata)
|
||||
preRelease := strings.ToLower(string(parsedVersion.PreRelease))
|
||||
|
||||
switch {
|
||||
case preRelease == "":
|
||||
return protonDriveJoinAppVersion(appVersion, "-stable", metadataParts)
|
||||
case preRelease == "dev":
|
||||
return protonDriveJoinAppVersion(appVersion, "-dev", metadataParts)
|
||||
case preRelease == "beta" || strings.HasPrefix(preRelease, "beta."):
|
||||
betaSuffix, betaMetadataParts := protonDriveBetaSuffixAndMetadata(preRelease)
|
||||
return protonDriveJoinAppVersion(appVersion, betaSuffix, append(betaMetadataParts, metadataParts...))
|
||||
default:
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
func protonDriveMetadataParts(metadata string) []string {
|
||||
if metadata == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
parts := strings.Split(metadata, ".")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func protonDriveBetaSuffixAndMetadata(preRelease string) (suffix string, metadata []string) {
|
||||
suffix = "-beta"
|
||||
remainder := strings.TrimPrefix(strings.TrimPrefix(preRelease, "beta"), ".")
|
||||
if remainder == "" {
|
||||
return suffix, nil
|
||||
}
|
||||
|
||||
parts := protonDriveMetadataParts(remainder)
|
||||
i := 0
|
||||
for i < len(parts) && isDecimalString(parts[i]) {
|
||||
suffix += "." + parts[i]
|
||||
i++
|
||||
}
|
||||
return suffix, parts[i:]
|
||||
}
|
||||
|
||||
func protonDriveJoinAppVersion(appVersion, suffix string, metadata []string) string {
|
||||
version := appVersion + suffix
|
||||
if len(metadata) != 0 {
|
||||
version += "+" + strings.Join(metadata, ".")
|
||||
}
|
||||
return version
|
||||
}
|
||||
|
||||
func isDecimalString(value string) bool {
|
||||
if value == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// protonLogger adapts rclone's fs.Debugf/Logf/Errorf into the
|
||||
// resty.Logger / common.Logger shape expected by Proton-API-Bridge and
|
||||
// go-proton-api so that library output participates in -v / -vv levels
|
||||
// and is captured by --log-file.
|
||||
type protonLogger struct{ f *Fs }
|
||||
|
||||
func (l protonLogger) Errorf(format string, v ...interface{}) { fs.Errorf(l.f, format, v...) }
|
||||
func (l protonLogger) Warnf(format string, v ...interface{}) { fs.Logf(l.f, format, v...) }
|
||||
func (l protonLogger) Debugf(format string, v ...interface{}) { fs.Debugf(l.f, format, v...) }
|
||||
|
||||
func newProtonDrive(ctx context.Context, f *Fs, opt *Options, m configmap.Mapper) (*protonDriveAPI.ProtonDrive, error) {
|
||||
config := protonDriveAPI.NewDefaultConfig()
|
||||
config.AppVersion = opt.AppVersion
|
||||
if config.AppVersion == "" {
|
||||
config.AppVersion = protonDriveAppVersionFromRcloneVersion(fs.Version)
|
||||
}
|
||||
config.UserAgent = f.ci.UserAgent // opt.UserAgent
|
||||
|
||||
// Route HTTP through rclone's transport so global flags such as
|
||||
// --dump headers, --no-check-certificate, --user-agent, --bind,
|
||||
// --ca-cert and --header all take effect against Proton Drive.
|
||||
config.Transport = fshttp.NewTransport(ctx)
|
||||
|
||||
// Route bridge and go-proton-api log output through rclone's
|
||||
// logging system so it honours -v / -vv and --log-file.
|
||||
config.Logger = protonLogger{f: f}
|
||||
|
||||
config.ReplaceExistingDraft = opt.ReplaceExistingDraft
|
||||
config.EnableCaching = opt.EnableCaching
|
||||
|
||||
@@ -854,7 +969,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
|
||||
}
|
||||
|
||||
if o.digests != nil {
|
||||
return *o.digests, nil
|
||||
return strings.ToLower(*o.digests), nil
|
||||
}
|
||||
|
||||
// sha1 not cached: we fetch and try to obtain the sha1 of the link
|
||||
@@ -904,7 +1019,7 @@ func (o *Object) Storable() bool {
|
||||
|
||||
// Open opens the file for read. Call Close() on the returned io.ReadCloser
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
|
||||
fs.FixRangeOption(options, *o.originalSize)
|
||||
fs.FixRangeOption(options, o.Size())
|
||||
var offset, limit int64 = 0, -1
|
||||
for _, option := range options { // if the caller passes in nil for options, it will become array of nil
|
||||
switch x := option.(type) {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package protondrive
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var protonDriveAppVersionPattern = regexp.MustCompile(`(?i)^external-drive(-[a-z_]+)+@[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?-((stable|beta|RC|alpha)(([.-]?\d+)*)?)?([.-]?dev)?(\+.*)?$`)
|
||||
|
||||
func TestProtonDriveAppVersionFromRcloneVersion(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
rcloneVersion string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "release",
|
||||
rcloneVersion: "v1.73.5",
|
||||
want: "external-drive-rclone@1.73.5-stable",
|
||||
},
|
||||
{
|
||||
name: "dev build",
|
||||
rcloneVersion: "v1.74.0-DEV",
|
||||
want: "external-drive-rclone@1.74.0-dev",
|
||||
},
|
||||
{
|
||||
name: "beta build with extra metadata",
|
||||
rcloneVersion: "v1.74.0-beta.9519.990f33f2a.fix-protondrive-sdk-2026",
|
||||
want: "external-drive-rclone@1.74.0-beta.9519+990f33f2a.fix-protondrive-sdk-2026",
|
||||
},
|
||||
{
|
||||
name: "beta build with unsanitized branch name",
|
||||
rcloneVersion: "v1.74.0-beta.9519.990f33f2a.fix/protondrive-sdk-2026",
|
||||
want: "external-drive-rclone@1.74.0-beta.9519+990f33f2a.fix-protondrive-sdk-2026",
|
||||
},
|
||||
{
|
||||
name: "invalid version falls back to stable",
|
||||
rcloneVersion: "not-a-version",
|
||||
want: "external-drive-rclone@1.0.0-stable",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
got := protonDriveAppVersionFromRcloneVersion(testCase.rcloneVersion)
|
||||
|
||||
if got != testCase.want {
|
||||
t.Fatalf("unexpected app version: got %q, want %q", got, testCase.want)
|
||||
}
|
||||
if !protonDriveAppVersionPattern.MatchString(got) {
|
||||
t.Fatalf("app version %q does not match Proton pattern", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -16,11 +16,12 @@ type Authenticator interface {
|
||||
}
|
||||
|
||||
// IbmIamSigner is a structure for signing requests using IBM IAM.
|
||||
// Requeres APIKey and Resource InstanceID
|
||||
// Requires APIKey and Resource InstanceID
|
||||
type IbmIamSigner struct {
|
||||
APIKey string
|
||||
InstanceID string
|
||||
Auth Authenticator
|
||||
APIKey string
|
||||
InstanceID string
|
||||
IAMEndpoint string
|
||||
Auth Authenticator
|
||||
}
|
||||
|
||||
// SignHTTP signs requests using IBM IAM token.
|
||||
@@ -29,7 +30,7 @@ func (signer *IbmIamSigner) SignHTTP(ctx context.Context, credentials aws.Creden
|
||||
if signer.Auth != nil {
|
||||
authenticator = signer.Auth
|
||||
} else {
|
||||
authenticator = &core.IamAuthenticator{ApiKey: signer.APIKey}
|
||||
authenticator = &core.IamAuthenticator{ApiKey: signer.APIKey, URL: signer.IAMEndpoint}
|
||||
}
|
||||
token, err := authenticator.GetToken()
|
||||
if err != nil {
|
||||
|
||||
@@ -23,14 +23,16 @@ func TestSignHTTP(t *testing.T) {
|
||||
apiKey := "mock-api-key"
|
||||
instanceID := "mock-instance-id"
|
||||
token := "mock-iam-token"
|
||||
iamEndpoint := "https://iamendpoint.com"
|
||||
mockAuth := &MockAuthenticator{
|
||||
Token: token,
|
||||
Error: nil,
|
||||
}
|
||||
signer := &IbmIamSigner{
|
||||
APIKey: apiKey,
|
||||
InstanceID: instanceID,
|
||||
Auth: mockAuth,
|
||||
APIKey: apiKey,
|
||||
InstanceID: instanceID,
|
||||
IAMEndpoint: iamEndpoint,
|
||||
Auth: mockAuth,
|
||||
}
|
||||
req, err := http.NewRequest("GET", "https://example.com", nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -18,3 +18,4 @@ quirks:
|
||||
list_version: 1
|
||||
force_path_style: true
|
||||
list_url_encode: false
|
||||
use_already_exists: false
|
||||
@@ -4,6 +4,7 @@ region:
|
||||
eu-west-1: Europe West
|
||||
endpoint:
|
||||
s3.cubbit.eu: Cubbit DS3 Object Storage endpoint
|
||||
s3.{tenant_name}.cubbit.eu: Multi-tenant endpoint - replace {tenant_name} with your actual tenant name. Do not select this directly
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
|
||||
@@ -4,20 +4,26 @@ region:
|
||||
au-east-1: AU East 1
|
||||
eu-central: EU Central
|
||||
eu-south-1: EU South 1
|
||||
eu-west-1: EU West 1
|
||||
jp-central-1: JP Central 1
|
||||
uk-east-1: UK East 1
|
||||
us-central-1: US Central 1
|
||||
us-east: US East
|
||||
us-east-1: US East 1
|
||||
us-west: US West
|
||||
us-west-1: US West 1
|
||||
endpoint:
|
||||
au-east-1.object.fastlystorage.app: AU East 1
|
||||
eu-central.object.fastlystorage.app: EU Central
|
||||
eu-south-1.object.fastlystorage.app: EU South 1
|
||||
eu-west-1.object.fastlystorage.app: EU West 1
|
||||
jp-central-1.object.fastlystorage.app: JP Central 1
|
||||
uk-east-1.object.fastlystorage.app: UK East 1
|
||||
us-central-1.object.fastlystorage.app: US Central 1
|
||||
us-east.object.fastlystorage.app: US East
|
||||
us-east-1.object.fastlystorage.app: US East 1
|
||||
us-west.object.fastlystorage.app: US West
|
||||
us-west-1.object.fastlystorage.app: US West 1
|
||||
quirks:
|
||||
force_path_style: true
|
||||
use_already_exists: false
|
||||
|
||||
@@ -18,3 +18,7 @@ quirks:
|
||||
# See: https://issuetracker.google.com/issues/323465186
|
||||
# So make cutoff very large which it does seem to support
|
||||
copy_cutoff: 9223372036854775807
|
||||
# GCS S3 API doesn't fully support Object Lock:
|
||||
# - Uses x-goog-bypass-governance-retention instead of x-amz-bypass-governance-retention
|
||||
# - PutObjectLegalHold / GetObjectLegalHold not implemented
|
||||
object_lock_supported: false
|
||||
@@ -0,0 +1,18 @@
|
||||
name: HCP
|
||||
description: Hitachi Content Platform (HCP)
|
||||
region: {}
|
||||
endpoint: {}
|
||||
location_constraint: {}
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
force_path_style: true
|
||||
list_url_encode: true
|
||||
# HCP returns a SHA256-based ETag for multipart uploads instead of the
|
||||
# standard MD5-based ETag, so ETag verification after multipart upload
|
||||
# must be disabled:
|
||||
use_multipart_etag: false
|
||||
# HCP returns object versions oldest first (ascending chronological order)
|
||||
# instead of the S3 standard newest first:
|
||||
# https://docs.hitachivantara.com/r/en-us/content-platform/9.6.x/mk-95hcph002/using-the-hitachi-api-for-amazon-s3/working-with-buckets/listing-bucket-contents-version-2
|
||||
list_versions_oldest_first: true
|
||||
@@ -117,6 +117,7 @@ acl:
|
||||
This acl is available on IBM Cloud (Infra) and On-Premise IBM COS.
|
||||
ibm_api_key: true
|
||||
ibm_resource_instance_id: true
|
||||
ibm_iam_endpoint: true
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
list_version: 1
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
name: ImpossibleCloud
|
||||
description: Impossible Cloud Object Storage
|
||||
region:
|
||||
eu-central-2: Frankfurt, Germany
|
||||
eu-west-1: Amsterdam, Netherlands
|
||||
eu-west-2: London, UK
|
||||
eu-west-3: Paris, France
|
||||
eu-east-1: Poznań, Poland
|
||||
eu-north-1: Copenhagen, Denmark
|
||||
us-east-1: New York, USA
|
||||
endpoint:
|
||||
eu-central-2.storage.impossibleapi.net: Frankfurt, Germany
|
||||
eu-west-1.storage.impossibleapi.net: Amsterdam, Netherlands
|
||||
eu-west-2.storage.impossibleapi.net: London, UK
|
||||
eu-west-3.storage.impossibleapi.net: Paris, France
|
||||
eu-east-1.storage.impossibleapi.net: Poznań, Poland
|
||||
eu-north-1.storage.impossibleapi.net: Copenhagen, Denmark
|
||||
us-east-1.storage.impossibleapi.net: New York, USA
|
||||
location_constraint: {}
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
@@ -1,10 +1,17 @@
|
||||
name: Mega
|
||||
description: MEGA S4 Object Storage
|
||||
endpoint:
|
||||
s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam)
|
||||
s3.eu-central-2.s4.mega.io: Mega S4 eu-central-2 (Bettembourg)
|
||||
s3.ca-central-1.s4.mega.io: Mega S4 ca-central-1 (Montreal)
|
||||
s3.ca-west-1.s4.mega.io: Mega S4 ca-west-1 (Vancouver)
|
||||
s3.eu-amsterdam.megas4.com: Mega S4 Amsterdam
|
||||
s3.eu-luxembourg.megas4.com: Mega S4 Luxembourg
|
||||
s3.eu-paris.megas4.com: Mega S4 Paris
|
||||
s3.eu-barcelona.megas4.com: Mega S4 Barcelona
|
||||
s3.ca-montreal.megas4.com: Mega S4 Montreal
|
||||
s3.ca-vancouver.megas4.com: Mega S4 Vancouver
|
||||
s3.ap-tokyo.megas4.com: Mega S4 Tokyo
|
||||
s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam, legacy)
|
||||
s3.eu-central-2.s4.mega.io: Mega S4 eu-central-2 (Bettembourg, legacy)
|
||||
s3.ca-central-1.s4.mega.io: Mega S4 ca-central-1 (Montreal, legacy)
|
||||
s3.ca-west-1.s4.mega.io: Mega S4 ca-west-1 (Vancouver, legacy)
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
list_version: 2
|
||||
|
||||
@@ -32,5 +32,15 @@ endpoint:
|
||||
s3.us-east-va.io.cloud.ovh.us: OVHcloud Vint Hill, Virginia, USA
|
||||
s3.us-west-or.io.cloud.ovh.us: OVHcloud Hillsboro, Oregon, USA
|
||||
s3.rbx-archive.io.cloud.ovh.net: OVHcloud Roubaix, France (Cold Archive)
|
||||
storage_class:
|
||||
'': Default
|
||||
EXPRESS_ONEZONE: High Performance storage class
|
||||
STANDARD: Standard storage class
|
||||
STANDARD_IA: Standard Infrequent Access storage class
|
||||
ONEZONE_IA: Standard Infrequent Access storage class
|
||||
GLACIER: Active Archive storage class
|
||||
GLACIER_IR: Active Archive storage class
|
||||
DEEP_ARCHIVE: Cold Archive storage class
|
||||
INTELLIGENT_TIERING: Standard storage class
|
||||
acl: {}
|
||||
bucket_acl: true
|
||||
@@ -0,0 +1,44 @@
|
||||
name: US3
|
||||
description: US3 Object Storage
|
||||
endpoint:
|
||||
s3-cn-bj.ufileos.com: North China (Beijing 2)
|
||||
s3-cn-wlcb.ufileos.com: North China (Ulanqab)
|
||||
s3-cn-sh2.ufileos.com: Shanghai
|
||||
s3-cn-gd.ufileos.com: South China (Guangzhou)
|
||||
s3-hk.ufileos.com: Hongkong
|
||||
s3-us-ca.ufileos.com: United States (Los Angeles)
|
||||
s3-sg.ufileos.com: Singapore
|
||||
s3-idn-jakarta.ufileos.com: Indonesia (Jakarta)
|
||||
s3-tw-tp.ufileos.com: Taiwan (Taipei)
|
||||
s3-afr-nigeria.ufileos.com: Nigeria (Lagos)
|
||||
s3-bra-saopaulo.ufileos.com: Brazil (São Paulo)
|
||||
s3-uae-dubai.ufileos.com: United Arab Emirates (Dubai)
|
||||
s3-ge-fra.ufileos.com: Germany (Frankfurt)
|
||||
s3-vn-sng.ufileos.com: Vietnam (Ho Chi Minh City)
|
||||
s3-us-ws.ufileos.com: United States (Washington)
|
||||
s3-ind-mumbai.ufileos.com: India (Mumbai)
|
||||
s3-kr-seoul.ufileos.com: Seoul (South Korea)
|
||||
s3-jpn-tky.ufileos.com: Japan (Tokyo)
|
||||
s3-th-bkk.ufileos.com: Bangkok (Thailand)
|
||||
s3-uk-london.ufileos.com: United Kingdom (London)
|
||||
s3-rus-mosc.ufileos.com: Russia (Moscow)
|
||||
s3-cn-guiyang1.ufileos.com: Guiyang 1
|
||||
s3-pk-khi.ufileos.com: Pakistan
|
||||
acl:
|
||||
private: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
No one else has access rights (default).
|
||||
public-read: |-
|
||||
Owner gets FULL_CONTROL.
|
||||
The AllUsers group gets READ access.
|
||||
storage_class:
|
||||
'': Default
|
||||
STANDARD: Standard storage class
|
||||
ARCHIVE: Archive storage mode
|
||||
STANDARD_IA: Infrequent access storage mode
|
||||
bucket_acl: true
|
||||
quirks:
|
||||
list_version: 1
|
||||
use_multipart_etag: false
|
||||
use_already_exists: false
|
||||
list_url_encode: true
|
||||
@@ -30,9 +30,11 @@ type Quirks struct {
|
||||
MightGzip *bool `yaml:"might_gzip,omitempty"`
|
||||
UseMultipartUploads *bool `yaml:"use_multipart_uploads,omitempty"`
|
||||
UseUnsignedPayload *bool `yaml:"use_unsigned_payload,omitempty"`
|
||||
ListVersionsOldestFirst *bool `yaml:"list_versions_oldest_first,omitempty"`
|
||||
UseXID *bool `yaml:"use_x_id,omitempty"`
|
||||
SignAcceptEncoding *bool `yaml:"sign_accept_encoding,omitempty"`
|
||||
EtagIsNotMD5 *bool `yaml:"etag_is_not_md5,omitempty"`
|
||||
ObjectLockSupported *bool `yaml:"object_lock_supported,omitempty"`
|
||||
CopyCutoff *int64 `yaml:"copy_cutoff,omitempty"`
|
||||
MaxUploadParts *int `yaml:"max_upload_parts,omitempty"`
|
||||
MinChunkSize *int64 `yaml:"min_chunk_size,omitempty"`
|
||||
@@ -52,6 +54,7 @@ type Provider struct {
|
||||
// other
|
||||
IBMApiKey bool `yaml:"ibm_api_key,omitempty"`
|
||||
IBMResourceInstanceID bool `yaml:"ibm_resource_instance_id,omitempty"`
|
||||
IBMIAMEndpoint bool `yaml:"ibm_iam_endpoint,omitempty"`
|
||||
|
||||
// advanced
|
||||
BucketACL bool `yaml:"bucket_acl,omitempty"`
|
||||
@@ -224,6 +227,8 @@ func constructProviders(options fs.Options, providerMap map[string]*Provider) st
|
||||
addBool(opt, p, p.IBMApiKey)
|
||||
case "ibm_resource_instance_id":
|
||||
addBool(opt, p, p.IBMResourceInstanceID)
|
||||
case "ibm_iam_endpoint":
|
||||
addBool(opt, p, p.IBMIAMEndpoint)
|
||||
case "leave_parts_on_error":
|
||||
addBool(opt, p, p.LeavePartsOnError)
|
||||
case "sts_endpoint":
|
||||
|
||||
+90
-8
@@ -4,6 +4,7 @@ package s3
|
||||
//go:generate go run gen_setfrom.go -o setfrom.go
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/tls"
|
||||
@@ -370,7 +371,7 @@ this may help to speed up the transfers.`,
|
||||
|
||||
If this is true (the default) then rclone will use path style access,
|
||||
if false then rclone will use virtual path style. See [the AWS S3
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
|
||||
docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html)
|
||||
for more info.
|
||||
|
||||
Some providers (e.g. AWS, Aliyun OSS, Netease COS, or Tencent COS) require this set to
|
||||
@@ -741,6 +742,21 @@ knows about - please make a bug report if not.
|
||||
You can change this if you want to disable the use of multipart uploads.
|
||||
This shouldn't be necessary in normal operation.
|
||||
|
||||
This should be automatically set correctly for all providers rclone
|
||||
knows about - please make a bug report if not.
|
||||
`,
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "list_versions_oldest_first",
|
||||
Help: `Set if the backend returns object versions oldest first.
|
||||
|
||||
The S3 standard returns object versions newest first. Some backends
|
||||
(e.g. Hitachi HCP) return them oldest first instead.
|
||||
|
||||
Set this quirk if --s3-version-at or --s3-versions produce incorrect
|
||||
results with your backend.
|
||||
|
||||
This should be automatically set correctly for all providers rclone
|
||||
knows about - please make a bug report if not.
|
||||
`,
|
||||
@@ -829,6 +845,10 @@ use |-vv| to see the debug level logs.
|
||||
}, {
|
||||
Name: "ibm_resource_instance_id",
|
||||
Help: "IBM service instance id",
|
||||
}, {
|
||||
Name: "ibm_iam_endpoint",
|
||||
Help: "IBM IAM Endpoint to use for authentication.\n\nLeave blank to use the default public endpoint.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "object_lock_mode",
|
||||
Help: `Object Lock mode to apply when uploading or copying objects.
|
||||
@@ -930,6 +950,18 @@ PutObjectRetention and PutObjectLegalHold API calls after the upload completes.
|
||||
This adds extra API calls per object, so only enable if your provider requires it.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "object_lock_supported",
|
||||
Help: `Whether the provider supports S3 Object Lock.
|
||||
|
||||
This should be true, false or left unset to use the default for the provider.
|
||||
|
||||
Set to false for providers that don't fully support the S3 Object Lock API
|
||||
(e.g. GCS which uses non-standard headers for bypass governance retention
|
||||
and doesn't implement Legal Hold via the S3 API).
|
||||
`,
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
},
|
||||
}}))
|
||||
}
|
||||
@@ -1107,6 +1139,8 @@ type Options struct {
|
||||
DirectoryBucket bool `config:"directory_bucket"`
|
||||
IBMAPIKey string `config:"ibm_api_key"`
|
||||
IBMInstanceID string `config:"ibm_resource_instance_id"`
|
||||
IBMIAMEndpoint string `config:"ibm_iam_endpoint"`
|
||||
ListVersionsOldestFirst fs.Tristate `config:"list_versions_oldest_first"`
|
||||
UseXID fs.Tristate `config:"use_x_id"`
|
||||
SignAcceptEncoding fs.Tristate `config:"sign_accept_encoding"`
|
||||
ObjectLockMode string `config:"object_lock_mode"`
|
||||
@@ -1115,6 +1149,7 @@ type Options struct {
|
||||
BypassGovernanceRetention bool `config:"bypass_governance_retention"`
|
||||
BucketObjectLockEnabled bool `config:"bucket_object_lock_enabled"`
|
||||
ObjectLockSetAfterUpload bool `config:"object_lock_set_after_upload"`
|
||||
ObjectLockSupported fs.Tristate `config:"object_lock_supported"`
|
||||
}
|
||||
|
||||
// Fs represents a remote s3 server
|
||||
@@ -1487,8 +1522,10 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
|
||||
}
|
||||
}
|
||||
|
||||
// Create AssumeRole credentials provider
|
||||
awsConfig.Credentials = stscreds.NewAssumeRoleProvider(stsClient, opt.RoleARN, assumeRoleOptions)
|
||||
// Create AssumeRole credentials provider, wrapped in a
|
||||
// CredentialsCache so we don't call AssumeRole on every
|
||||
// request.
|
||||
awsConfig.Credentials = aws.NewCredentialsCache(stscreds.NewAssumeRoleProvider(stsClient, opt.RoleARN, assumeRoleOptions))
|
||||
}
|
||||
|
||||
provider = loadProvider(opt.Provider)
|
||||
@@ -1541,7 +1578,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (s3Cli
|
||||
fs.Debugf(nil, "Using v2 auth")
|
||||
if opt.Provider == "IBMCOS" && opt.IBMAPIKey != "" && opt.IBMInstanceID != "" {
|
||||
options = append(options, func(s3Opt *s3.Options) {
|
||||
s3Opt.HTTPSignerV4 = &IbmIamSigner{APIKey: opt.IBMAPIKey, InstanceID: opt.IBMInstanceID}
|
||||
s3Opt.HTTPSignerV4 = &IbmIamSigner{APIKey: opt.IBMAPIKey, InstanceID: opt.IBMInstanceID, IAMEndpoint: opt.IBMIAMEndpoint}
|
||||
})
|
||||
} else {
|
||||
options = append(options, func(s3Opt *s3.Options) {
|
||||
@@ -1721,6 +1758,8 @@ func setQuirks(opt *Options, provider *Provider) {
|
||||
set(&opt.UseUnsignedPayload, true, provider.Quirks.UseUnsignedPayload)
|
||||
set(&opt.UseXID, true, provider.Quirks.UseXID)
|
||||
set(&opt.SignAcceptEncoding, true, provider.Quirks.SignAcceptEncoding)
|
||||
set(&opt.ObjectLockSupported, true, provider.Quirks.ObjectLockSupported)
|
||||
set(&opt.ListVersionsOldestFirst, false, provider.Quirks.ListVersionsOldestFirst)
|
||||
}
|
||||
|
||||
// setRoot changes the root of the Fs
|
||||
@@ -1841,6 +1880,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if opt.Provider == "Rabata" {
|
||||
f.features.Copy = nil
|
||||
}
|
||||
if opt.Provider == "TencentCOS" && strings.Contains(opt.Endpoint, "cos.accelerate.myqcloud.com") {
|
||||
// Global Acceleration endpoint does not support bucket creation.
|
||||
f.opt.NoCheckBucket = true
|
||||
}
|
||||
if opt.DirectoryMarkers {
|
||||
f.features.CanHaveEmptyDirectories = true
|
||||
}
|
||||
@@ -2234,6 +2277,13 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
|
||||
//structs.SetFrom(resp, respVersions)
|
||||
setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(resp, respVersions)
|
||||
|
||||
// Some backends (e.g. Hitachi HCP) return versions oldest first instead
|
||||
// of newest first. Reverse both lists so mergeDeleteMarkers works correctly.
|
||||
if ls.f.opt.ListVersionsOldestFirst.Value {
|
||||
slices.Reverse(respVersions.Versions)
|
||||
slices.Reverse(respVersions.DeleteMarkers)
|
||||
}
|
||||
|
||||
// Merge in delete Markers as types.ObjectVersion if we need them
|
||||
if ls.hidden || ls.usingVersionAt {
|
||||
respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers)
|
||||
@@ -2319,9 +2369,11 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
|
||||
opt.directory += "/"
|
||||
}
|
||||
}
|
||||
delimiter := ""
|
||||
// Use nil delimiter for recursive listings to omit the parameter
|
||||
// entirely. Some S3-compatible servers reject an empty delimiter.
|
||||
var delimiter *string
|
||||
if !opt.recurse {
|
||||
delimiter = "/"
|
||||
delimiter = aws.String("/")
|
||||
}
|
||||
// URL encode the listings so we can use control characters in object names
|
||||
// See: https://github.com/aws/aws-sdk-go/issues/1914
|
||||
@@ -2341,7 +2393,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
|
||||
urlEncodeListings := f.opt.ListURLEncode.Value
|
||||
req := s3.ListObjectsV2Input{
|
||||
Bucket: &opt.bucket,
|
||||
Delimiter: &delimiter,
|
||||
Delimiter: delimiter,
|
||||
Prefix: &opt.directory,
|
||||
MaxKeys: &f.opt.ListChunk,
|
||||
}
|
||||
@@ -3763,7 +3815,9 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
|
||||
if bucket == "" {
|
||||
return errors.New("can't purge from root")
|
||||
}
|
||||
versioned := f.isVersioned(ctx)
|
||||
// If the user explicitly set --s3-versions, trust that the bucket is
|
||||
// versioned even if GetBucketVersioning fails (e.g. missing permission).
|
||||
versioned := f.opt.Versions || f.isVersioned(ctx)
|
||||
if !versioned && oldOnly {
|
||||
fs.Infof(f, "bucket is not versioned so not removing old versions")
|
||||
return nil
|
||||
@@ -4603,8 +4657,31 @@ func (o *Object) uploadMultipart(ctx context.Context, src fs.ObjectInfo, in io.R
|
||||
return wantETag, gotETag, versionID, s3cw.ui, nil
|
||||
}
|
||||
|
||||
// bufferForObjectLockMD5 buffers the body and computes Content-MD5 when
|
||||
// Object Lock parameters are set on the request. AWS S3 requires Content-MD5
|
||||
// for PutObject with Object Lock params and cannot compute it automatically
|
||||
// from a non-seekable io.Reader.
|
||||
// See: https://github.com/aws/aws-sdk-go-v2/discussions/2960
|
||||
func bufferForObjectLockMD5(req *s3.PutObjectInput, in io.Reader) (io.Reader, error) {
|
||||
if req.ObjectLockMode == "" && req.ObjectLockRetainUntilDate == nil && req.ObjectLockLegalHoldStatus == "" {
|
||||
return in, nil
|
||||
}
|
||||
buf, err := io.ReadAll(in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read body for Content-MD5: %w", err)
|
||||
}
|
||||
md5sum := md5.Sum(buf)
|
||||
md5base64 := base64.StdEncoding.EncodeToString(md5sum[:])
|
||||
req.ContentMD5 = &md5base64
|
||||
return bytes.NewReader(buf), nil
|
||||
}
|
||||
|
||||
// Upload a single part using PutObject
|
||||
func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, versionID *string, err error) {
|
||||
in, err = bufferForObjectLockMD5(req, in)
|
||||
if err != nil {
|
||||
return etag, lastModified, nil, err
|
||||
}
|
||||
req.Body = io.NopCloser(in)
|
||||
var options = []func(*s3.Options){}
|
||||
if o.fs.opt.UseUnsignedPayload.Value {
|
||||
@@ -4635,6 +4712,11 @@ func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjec
|
||||
|
||||
// Upload a single part using a presigned request
|
||||
func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, versionID *string, err error) {
|
||||
// Content-MD5 must be set before signing so it's included in the presigned URL.
|
||||
in, err = bufferForObjectLockMD5(req, in)
|
||||
if err != nil {
|
||||
return etag, lastModified, nil, err
|
||||
}
|
||||
// Create the presigned request
|
||||
putReq, err := s3.NewPresignClient(o.fs.c).PresignPutObject(ctx, req, s3.WithPresignExpires(15*time.Minute))
|
||||
if err != nil {
|
||||
|
||||
@@ -44,6 +44,13 @@ func md5sum(t *testing.T, s string) string {
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestMetadata(t *testing.T) {
|
||||
// Impossible Cloud silently drops cache-control,
|
||||
// content-disposition, content-encoding and content-language
|
||||
// system metadata on retrieval (and thus also breaks the
|
||||
// Content-Encoding response header that --s3-decompress relies on).
|
||||
if f.opt.Provider == "ImpossibleCloud" {
|
||||
t.Skip("Impossible Cloud does not preserve all S3 system metadata")
|
||||
}
|
||||
ctx := context.Background()
|
||||
original := random.String(1000)
|
||||
contents := gz(t, original)
|
||||
@@ -499,6 +506,9 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTestObjectLock(t *testing.T) {
|
||||
if !f.opt.ObjectLockSupported.Value {
|
||||
t.Skip("Object Lock not supported by this provider (quirk object_lock_supported = false)")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a temporary bucket with Object Lock enabled to test on.
|
||||
@@ -721,6 +731,55 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
|
||||
assert.WithinDuration(t, retainUntilDate, gotRetainDate, time.Second)
|
||||
assert.Equal(t, "ON", gotMetadata["object-lock-legal-hold-status"])
|
||||
})
|
||||
|
||||
t.Run("Multipart", func(t *testing.T) {
|
||||
// Force multipart upload by setting a very low cutoff
|
||||
oldCutoff := f.opt.UploadCutoff
|
||||
f.opt.UploadCutoff = fs.SizeSuffix(1)
|
||||
f.opt.ObjectLockMode = "GOVERNANCE"
|
||||
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
|
||||
defer func() {
|
||||
f.opt.UploadCutoff = oldCutoff
|
||||
f.opt.ObjectLockMode = ""
|
||||
f.opt.ObjectLockRetainUntilDate = ""
|
||||
}()
|
||||
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-multipart", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
|
||||
})
|
||||
|
||||
t.Run("Presigned", func(t *testing.T) {
|
||||
// Use presigned request upload path
|
||||
f.opt.UsePresignedRequest = true
|
||||
f.opt.ObjectLockMode = "GOVERNANCE"
|
||||
f.opt.ObjectLockRetainUntilDate = retainUntilDate.Format(time.RFC3339)
|
||||
defer func() {
|
||||
f.opt.UsePresignedRequest = false
|
||||
f.opt.ObjectLockMode = ""
|
||||
f.opt.ObjectLockRetainUntilDate = ""
|
||||
}()
|
||||
|
||||
contents := random.String(100)
|
||||
item := fstest.NewItem("test-object-lock-presigned", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
|
||||
obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
|
||||
defer func() {
|
||||
removeLocked(t, obj)
|
||||
}()
|
||||
|
||||
o := obj.(*Object)
|
||||
gotMetadata, err := o.Metadata(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "GOVERNANCE", gotMetadata["object-lock-mode"])
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) InternalTest(t *testing.T) {
|
||||
|
||||
+12
-5
@@ -928,11 +928,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
// Set up sshConfig here from opt
|
||||
// **NB** everything else should be setup in NewFsWithConnection
|
||||
sshConfig := &ssh.ClientConfig{
|
||||
User: opt.User,
|
||||
Auth: []ssh.AuthMethod{},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: time.Duration(f.ci.ConnectTimeout),
|
||||
ClientVersion: "SSH-2.0-" + f.ci.UserAgent,
|
||||
User: opt.User,
|
||||
Auth: []ssh.AuthMethod{},
|
||||
Timeout: time.Duration(f.ci.ConnectTimeout),
|
||||
ClientVersion: "SSH-2.0-" + f.ci.UserAgent,
|
||||
}
|
||||
|
||||
if len(opt.HostKeyAlgorithms) != 0 {
|
||||
@@ -945,6 +944,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
return nil, fmt.Errorf("couldn't parse known_hosts_file: %w", err)
|
||||
}
|
||||
sshConfig.HostKeyCallback = hostcallback
|
||||
} else {
|
||||
// Set insecure HostKeyCallback if no known_hosts_file is
|
||||
// configured. Rclone has no mechanism to manage
|
||||
// known_hosts files so we can't enable host key
|
||||
// validation by default. Users can enable it by setting
|
||||
// known_hosts_file. See: https://rclone.org/sftp/#host-key-validation
|
||||
sshConfig.HostKeyCallback = ssh.InsecureIgnoreHostKey()
|
||||
fs.Logf(name, "No host key validation is being performed. Set known_hosts_file to enable it. See: https://rclone.org/sftp/#host-key-validation")
|
||||
}
|
||||
|
||||
if opt.UseInsecureCipher && (opt.Ciphers != nil || opt.KeyExchange != nil) {
|
||||
|
||||
+24
-12
@@ -225,6 +225,7 @@ type Fs struct {
|
||||
hasOCMD5 bool // set if can use owncloud style checksums for MD5
|
||||
hasOCSHA1 bool // set if can use owncloud style checksums for SHA1
|
||||
hasMESHA1 bool // set if can use fastmail style checksums for SHA1
|
||||
useStandardProps bool // set if should use standard props for PROPFIND
|
||||
ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips
|
||||
chunksUploadURL string // upload URL for nextcloud chunked
|
||||
canChunk bool // set if nextcloud and nextcloud_chunk_size is set
|
||||
@@ -340,19 +341,22 @@ func itemIsDir(item *api.Response) bool {
|
||||
}
|
||||
|
||||
// readMetaDataForPath reads the metadata from the path
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string) (info *api.Prop, err error) {
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Prop, err error) {
|
||||
// FIXME how do we read back additional properties?
|
||||
opts := rest.Opts{
|
||||
Method: "PROPFIND",
|
||||
Path: f.filePath(path),
|
||||
ExtraHeaders: map[string]string{
|
||||
"Depth": depth,
|
||||
"Depth": "0",
|
||||
},
|
||||
CheckRedirect: rest.PreserveMethodRedirectFn,
|
||||
}
|
||||
if f.hasOCMD5 || f.hasOCSHA1 {
|
||||
opts.Body = bytes.NewBuffer(owncloudProps)
|
||||
} else if f.useStandardProps {
|
||||
opts.Body = bytes.NewBuffer(standardProps)
|
||||
}
|
||||
// Note: According to WebDAV RFC 4918, empty PROPFIND body defaults to allprop
|
||||
var result api.Multistatus
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -363,9 +367,6 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
|
||||
// does not exist
|
||||
switch apiErr.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
if f.retryWithZeroDepth && depth != "0" {
|
||||
return f.readMetaDataForPath(ctx, path, "0")
|
||||
}
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther:
|
||||
// Some sort of redirect - go doesn't deal with these properly (it resets
|
||||
@@ -711,6 +712,7 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
|
||||
f.precision = time.Second
|
||||
f.useOCMtime = true
|
||||
case "other":
|
||||
f.useStandardProps = true
|
||||
default:
|
||||
fs.Debugf(f, "Unknown vendor %q", vendor)
|
||||
}
|
||||
@@ -759,13 +761,23 @@ var owncloudProps = []byte(`<?xml version="1.0"?>
|
||||
<d:getlastmodified />
|
||||
<d:getcontentlength />
|
||||
<d:resourcetype />
|
||||
<d:getcontenttype />
|
||||
<oc:checksums />
|
||||
<oc:permissions />
|
||||
</d:prop>
|
||||
</d:propfind>
|
||||
`)
|
||||
|
||||
var standardProps = []byte(`<?xml version="1.0"?>
|
||||
<d:propfind xmlns:d="DAV:">
|
||||
<d:prop>
|
||||
<d:displayname/>
|
||||
<d:getlastmodified/>
|
||||
<d:getcontentlength/>
|
||||
<d:resourcetype/>
|
||||
</d:prop>
|
||||
</d:propfind>
|
||||
`)
|
||||
|
||||
// list the objects into the function supplied
|
||||
//
|
||||
// If directories is set it only sends directories
|
||||
@@ -784,10 +796,14 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
||||
ExtraHeaders: map[string]string{
|
||||
"Depth": depth,
|
||||
},
|
||||
AuthRedirect: f.opt.AuthRedirect, // allow redirects to preserve Auth
|
||||
}
|
||||
if f.hasOCMD5 || f.hasOCSHA1 {
|
||||
opts.Body = bytes.NewBuffer(owncloudProps)
|
||||
} else if f.useStandardProps {
|
||||
opts.Body = bytes.NewBuffer(standardProps)
|
||||
}
|
||||
// Note: According to WebDAV RFC 4918, empty PROPFIND body defaults to `allprop`
|
||||
var result api.Multistatus
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -1091,11 +1107,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
// does not really exist so we perform an extra check here.
|
||||
// Only the existence is checked, all other errors must be
|
||||
// ignored here to make the rclone test suite pass.
|
||||
depth := defaultDepth
|
||||
if f.retryWithZeroDepth {
|
||||
depth = "0"
|
||||
}
|
||||
_, err := f.readMetaDataForPath(ctx, dir, depth)
|
||||
_, err := f.readMetaDataForPath(ctx, dir)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
return fs.ErrorDirNotFound
|
||||
}
|
||||
@@ -1404,7 +1416,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
if o.hasMetaData {
|
||||
return nil
|
||||
}
|
||||
info, err := o.fs.readMetaDataForPath(ctx, o.remote, defaultDepth)
|
||||
info, err := o.fs.readMetaDataForPath(ctx, o.remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/configfile"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -81,6 +82,44 @@ func TestHeaders(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestListAllAuthRedirect checks auth_redirect is honoured on listAll PROPFIND.
|
||||
func TestListAllAuthRedirect(t *testing.T) {
|
||||
var targetAuth string
|
||||
var targetHits int
|
||||
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
targetHits++
|
||||
targetAuth = r.Header.Get("Authorization")
|
||||
_, err := fmt.Fprint(w, `<d:multistatus xmlns:d="DAV:"></d:multistatus>`)
|
||||
require.NoError(t, err)
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
// Redirect via a different hostname so net/http strips Authorization on cross-host redirect.
|
||||
targetURL := strings.Replace(target.URL, "127.0.0.1", "localhost", 1)
|
||||
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, targetURL+r.URL.Path, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer source.Close()
|
||||
|
||||
configfile.Install()
|
||||
m := configmap.Simple{
|
||||
"type": "webdav",
|
||||
"url": source.URL,
|
||||
"user": "alice",
|
||||
"pass": obscure.MustObscure("secret"),
|
||||
"auth_redirect": "true",
|
||||
}
|
||||
|
||||
f, err := webdav.NewFs(context.Background(), remoteName, "", m)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _ = f.List(context.Background(), "")
|
||||
|
||||
assert.GreaterOrEqual(t, targetHits, 1, "redirect target should receive the request")
|
||||
assert.NotEmpty(t, targetAuth, "Authorization header should be preserved across redirect")
|
||||
}
|
||||
|
||||
// TestReservedCharactersInPathAreEscaped verifies that reserved characters
|
||||
// like semicolons and equals signs in file paths are percent-encoded in
|
||||
// HTTP requests to the WebDAV server (RFC 3986 compliance).
|
||||
|
||||
Executable
+97
@@ -0,0 +1,97 @@
|
||||
#!/bin/bash
|
||||
# Fetch the latest GUI dist.zip from rclone/rclone-web GitHub releases.
|
||||
#
|
||||
# Downloads dist.zip from the latest release to cmd/gui/dist.zip and
|
||||
# records the tag in cmd/gui/dist.tag. Both files are committed to the
|
||||
# repo so that builds are reproducible and `go build` works on a fresh
|
||||
# clone without needing to fetch anything.
|
||||
#
|
||||
# Skips the download when both dist.zip and dist.tag exist and the tag
|
||||
# matches the latest release.
|
||||
#
|
||||
# Requires: curl
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO="rclone/rclone-web"
|
||||
DEST_DIR="cmd/gui"
|
||||
ZIP_FILE="${DEST_DIR}/dist.zip"
|
||||
TAG_FILE="${DEST_DIR}/dist.tag"
|
||||
|
||||
COMMIT=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--commit) COMMIT=1 ;;
|
||||
*) echo "Unknown argument: $arg" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
CURL_OPTS=(-fSs --retry 5 --retry-delay 2 --retry-all-errors)
|
||||
|
||||
# Use GITHUB_TOKEN (or GH_TOKEN) if present so that GitHub API calls are
|
||||
# authenticated. Unauthenticated calls are limited to 60/hour per source IP,
|
||||
# which is regularly exhausted on shared GitHub Actions runners.
|
||||
AUTH_HEADER=()
|
||||
TOKEN="${GITHUB_TOKEN:-${GH_TOKEN:-}}"
|
||||
if [ -n "${TOKEN}" ]; then
|
||||
AUTH_HEADER=(-H "Authorization: Bearer ${TOKEN}")
|
||||
fi
|
||||
|
||||
# Get the latest release info
|
||||
echo "Checking latest release of ${REPO}..."
|
||||
RELEASE_JSON=$(curl "${CURL_OPTS[@]}" "${AUTH_HEADER[@]}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"https://api.github.com/repos/${REPO}/releases/latest") || {
|
||||
echo "Error: failed to fetch release info from GitHub API" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
TAG=$(echo "${RELEASE_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'])")
|
||||
ASSET_URL=$(echo "${RELEASE_JSON}" | python3 -c "
|
||||
import sys, json
|
||||
r = json.load(sys.stdin)
|
||||
for a in r['assets']:
|
||||
if a['name'] == 'dist.zip':
|
||||
print(a['browser_download_url'])
|
||||
sys.exit(0)
|
||||
print('', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
") || {
|
||||
echo "Error: dist.zip asset not found in release ${TAG}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "Latest release: ${TAG}"
|
||||
|
||||
# Skip only when both the zip and the tag are present and the tag matches.
|
||||
# If only the tag exists (e.g. someone deleted the zip), force a re-download.
|
||||
if [ -f "${ZIP_FILE}" ] && [ -f "${TAG_FILE}" ] && [ "$(cat "${TAG_FILE}")" = "${TAG}" ]; then
|
||||
echo "Already up to date (${TAG})"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Download dist.zip directly to its final location, via a temp file so a
|
||||
# failed download doesn't leave a partial file behind.
|
||||
TMPFILE=$(mktemp "${DEST_DIR}/.dist.zip.XXXXXX")
|
||||
trap 'rm -f "${TMPFILE}"' EXIT
|
||||
|
||||
echo "Downloading dist.zip from ${TAG}..."
|
||||
curl -L "${CURL_OPTS[@]}" "${AUTH_HEADER[@]}" -o "${TMPFILE}" "${ASSET_URL}" || {
|
||||
echo "Error: failed to download dist.zip" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
mv "${TMPFILE}" "${ZIP_FILE}"
|
||||
chmod 644 "${ZIP_FILE}"
|
||||
echo -n "${TAG}" > "${TAG_FILE}"
|
||||
|
||||
echo "Done. ${ZIP_FILE} updated to ${TAG}"
|
||||
|
||||
if [ "${COMMIT}" -eq 1 ]; then
|
||||
git add "${ZIP_FILE}" "${TAG_FILE}"
|
||||
if git diff --cached --quiet -- "${ZIP_FILE}" "${TAG_FILE}"; then
|
||||
echo "No changes to commit (zip and tag are identical to HEAD)"
|
||||
else
|
||||
git commit -m "gui: update embedded release to ${TAG}" -- "${ZIP_FILE}" "${TAG_FILE}"
|
||||
fi
|
||||
fi
|
||||
@@ -56,6 +56,7 @@ docs = [
|
||||
"drive.md",
|
||||
"googlephotos.md",
|
||||
"hasher.md",
|
||||
"huaweidrive.md",
|
||||
"hdfs.md",
|
||||
"hidrive.md",
|
||||
"http.md",
|
||||
@@ -135,6 +136,7 @@ ignore_docs = [
|
||||
"privacy.md",
|
||||
"sponsor.md",
|
||||
"amazonclouddrive.md",
|
||||
"backends.md", # Makes JSON confusingly
|
||||
]
|
||||
|
||||
def read_doc(doc):
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
_ "github.com/rclone/rclone/cmd/genautocomplete"
|
||||
_ "github.com/rclone/rclone/cmd/gendocs"
|
||||
_ "github.com/rclone/rclone/cmd/gitannex"
|
||||
_ "github.com/rclone/rclone/cmd/gui"
|
||||
_ "github.com/rclone/rclone/cmd/hashsum"
|
||||
_ "github.com/rclone/rclone/cmd/link"
|
||||
_ "github.com/rclone/rclone/cmd/listremotes"
|
||||
|
||||
@@ -131,7 +131,11 @@ func ArchiveExtract(ctx context.Context, dst fs.Fs, dstDir string, src fs.Fs, sr
|
||||
}
|
||||
// account and buffer the transfer
|
||||
// in = tr.Account(ctx, in).WithBuffer()
|
||||
in := tr.Account(ctx, in0)
|
||||
acc := tr.Account(ctx, in0)
|
||||
in, err := acc.WithReadAtSeeker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// identify format
|
||||
format, _, err := archives.Identify(ctx, "", in)
|
||||
if err != nil {
|
||||
|
||||
@@ -183,7 +183,12 @@ func newFile(ctx context.Context, obj fs.Object, fi stdfs.FileInfo) (stdfs.File,
|
||||
return nil, f.err
|
||||
}
|
||||
// Account the transfer
|
||||
f.reader = f.transfer.Account(ctx, f.reader)
|
||||
acc := f.transfer.Account(ctx, f.reader)
|
||||
h, err := acc.WithReadAtSeeker()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.reader = h
|
||||
|
||||
return f, f.err
|
||||
}
|
||||
|
||||
@@ -167,7 +167,11 @@ func ArchiveList(ctx context.Context, src fs.Fs, srcFile string, listFn archives
|
||||
}
|
||||
// account and buffer the transfer
|
||||
// in = tr.Account(ctx, in).WithBuffer()
|
||||
in := tr.Account(ctx, in0)
|
||||
acc := tr.Account(ctx, in0)
|
||||
in, err := acc.WithReadAtSeeker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// identify format
|
||||
format, _, err := archives.Identify(ctx, "", in)
|
||||
|
||||
|
||||
@@ -85,6 +85,8 @@ var logReplacements = []string{
|
||||
`^NOTICE: .*?: Replacing invalid UTF-8 characters in "[^"]*"$`, dropMe,
|
||||
// ignore rclone debug messages
|
||||
`^DEBUG : .*$`, dropMe,
|
||||
// ignore SFTP host key messages
|
||||
`^NOTICE: .*?No host key validation is being performed.*$`, dropMe,
|
||||
// ignore dropbox info messages
|
||||
`^NOTICE: too_many_(requests|write_operations)/\.*: Too many requests or write operations.*$`, dropMe,
|
||||
`^NOTICE: .*?: Forced to upload files to set modification times on this backend.$`, dropMe,
|
||||
@@ -329,7 +331,7 @@ func testBisync(ctx context.Context, t *testing.T, path1, path2 string) {
|
||||
|
||||
baseDir, err := os.Getwd()
|
||||
require.NoError(t, err, "get current directory")
|
||||
randName := time.Now().Format("150405") + random.String(2) // some bucket backends don't like dots, keep this short to avoid linux errors
|
||||
randName := time.Now().Format("150405") + random.String(8) // some bucket backends don't like dots, keep this short to avoid linux errors
|
||||
tempDir := filepath.Join(os.TempDir(), randName)
|
||||
workDir := filepath.Join(tempDir, "workdir")
|
||||
|
||||
|
||||
+4
-3
@@ -127,14 +127,14 @@ func init() {
|
||||
// and the Command line syntax section of docs/content/bisync.md (it doesn't update automatically)
|
||||
flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.", "")
|
||||
flags.FVarP(cmdFlags, &Opt.ResyncMode, "resync-mode", "", "During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.)", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, makeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")
|
||||
flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, MakeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")
|
||||
flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, MakeHelp("Filename for --check-access (default: {CHECKFILE})"), "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "")
|
||||
flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")
|
||||
flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "")
|
||||
flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")
|
||||
flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, MakeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")
|
||||
flags.StringVarP(cmdFlags, &Opt.BackupDir1, "backup-dir1", "", Opt.BackupDir1, "--backup-dir for Path1. Must be a non-overlapping path on the same remote.", "")
|
||||
flags.StringVarP(cmdFlags, &Opt.BackupDir2, "backup-dir2", "", Opt.BackupDir2, "--backup-dir for Path2. Must be a non-overlapping path on the same remote.", "")
|
||||
flags.StringVarP(cmdFlags, &Opt.DebugName, "debugname", "", Opt.DebugName, "Debug by tracking one file at various points throughout a bisync run (when -v or -vv)", "")
|
||||
@@ -153,6 +153,7 @@ func init() {
|
||||
flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")
|
||||
_ = cmdFlags.MarkHidden("debugname")
|
||||
_ = cmdFlags.MarkHidden("localtime")
|
||||
addRC()
|
||||
}
|
||||
|
||||
// bisync command definition
|
||||
|
||||
+65
-53
@@ -1,65 +1,77 @@
|
||||
package bisync
|
||||
//go:build none
|
||||
|
||||
// Create the help text for the rc
|
||||
//
|
||||
// Run with go generate ./cmd/bisync (defined in rc.go)
|
||||
package main
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/muesli/reflow/wordwrap"
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/bisync"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
)
|
||||
|
||||
func makeHelp(help string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"|", "`",
|
||||
"{MAXDELETE}", strconv.Itoa(DefaultMaxDelete),
|
||||
"{CHECKFILE}", DefaultCheckFilename,
|
||||
// "{WORKDIR}", DefaultWorkdir,
|
||||
)
|
||||
return replacer.Replace(help)
|
||||
// Output the help to stdout
|
||||
func main() {
|
||||
out := os.Stdout
|
||||
if len(os.Args) > 1 {
|
||||
var err error
|
||||
out, err = os.Create(os.Args[1])
|
||||
if err != nil {
|
||||
fs.Fatalf(nil, "Open output failed: %v", err)
|
||||
}
|
||||
defer out.Close()
|
||||
}
|
||||
fmt.Fprintf(out, "<!--- Docs generated by help.go - use go generate to rebuild - DO NOT EDIT --->\n\n")
|
||||
fmt.Fprint(out, RcHelp())
|
||||
}
|
||||
|
||||
var shortHelp = `Perform bidirectional synchronization between two paths.`
|
||||
|
||||
var rcHelp = makeHelp(`This takes the following parameters
|
||||
|
||||
- path1 - a remote directory string e.g. |drive:path1|
|
||||
- path2 - a remote directory string e.g. |drive:path2|
|
||||
- dryRun - dry-run mode
|
||||
- resync - performs the resync run
|
||||
- checkAccess - abort if {CHECKFILE} files are not found on both filesystems
|
||||
- checkFilename - file name for checkAccess (default: {CHECKFILE})
|
||||
- maxDelete - abort sync if percentage of deleted files is above
|
||||
this threshold (default: {MAXDELETE})
|
||||
- force - Bypass maxDelete safety check and run the sync
|
||||
- checkSync - |true| by default, |false| disables comparison of final listings,
|
||||
|only| will skip sync, only compare listings from the last run
|
||||
- createEmptySrcDirs - Sync creation and deletion of empty directories.
|
||||
(Not compatible with --remove-empty-dirs)
|
||||
- removeEmptyDirs - remove empty directories at the final cleanup step
|
||||
- filtersFile - read filtering patterns from a file
|
||||
- ignoreListingChecksum - Do not use checksums for listings
|
||||
- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync.
|
||||
- workdir - server directory for history files (default: |~/.cache/rclone/bisync|)
|
||||
- backupdir1 - --backup-dir for Path1. Must be a non-overlapping path on the same remote.
|
||||
- backupdir2 - --backup-dir for Path2. Must be a non-overlapping path on the same remote.
|
||||
- noCleanup - retain working files
|
||||
// RcHelp returns the rc help
|
||||
func RcHelp() string {
|
||||
return wordwrap.String(bisync.MakeHelp(`This takes the following parameters:
|
||||
|
||||
- path1 (required) - (string) a remote directory string e.g. ||drive:path1||
|
||||
- path2 (required) - (string) a remote directory string e.g. ||drive:path2||
|
||||
- dryRun - (bool) dry-run mode
|
||||
`+GenerateParams()+`
|
||||
See [bisync command help](https://rclone.org/commands/rclone_bisync/)
|
||||
and [full bisync description](https://rclone.org/bisync/)
|
||||
for more information.`)
|
||||
for more information.
|
||||
`), 76)
|
||||
}
|
||||
|
||||
var longHelp = shortHelp + makeHelp(`
|
||||
// example: "create-empty-src-dirs" -> "createEmptySrcDirs"
|
||||
func toCamel(s string) string {
|
||||
split := strings.Split(s, "-")
|
||||
builder := strings.Builder{}
|
||||
for i, word := range split {
|
||||
if i == 0 { // first word always all lowercase
|
||||
builder.WriteString(strings.ToLower(word))
|
||||
continue
|
||||
}
|
||||
builder.WriteString(cases.Title(language.AmericanEnglish).String(word))
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
[Bisync](https://rclone.org/bisync/) provides a
|
||||
bidirectional cloud sync solution in rclone.
|
||||
It retains the Path1 and Path2 filesystem listings from the prior run.
|
||||
On each successive run it will:
|
||||
|
||||
- list files on Path1 and Path2, and check for changes on each side.
|
||||
Changes include |New|, |Newer|, |Older|, and |Deleted| files.
|
||||
- Propagate changes on Path1 to Path2, and vice-versa.
|
||||
|
||||
Bisync is considered an **advanced command**, so use with care.
|
||||
Make sure you have read and understood the entire [manual](https://rclone.org/bisync)
|
||||
(especially the [Limitations](https://rclone.org/bisync/#limitations) section)
|
||||
before using, or data loss can result. Questions can be asked in the
|
||||
[Rclone Forum](https://forum.rclone.org/).
|
||||
|
||||
See [full bisync description](https://rclone.org/bisync/) for details.`)
|
||||
// GenerateParams automatically generates the param list from commandDefinition.Flags
|
||||
func GenerateParams() string {
|
||||
builder := strings.Builder{}
|
||||
fn := func(flag *pflag.Flag) {
|
||||
if flag.Hidden {
|
||||
return
|
||||
}
|
||||
builder.WriteString(fmt.Sprintf("- %s - (%s) %s \n", toCamel(flag.Name), flag.Value.Type(), flag.Usage))
|
||||
}
|
||||
commandDefinition, _, _ := cmd.Root.Find([]string{"bisync"})
|
||||
commandDefinition.Flags().VisitAll(fn)
|
||||
return builder.String()
|
||||
}
|
||||
+12
-1
@@ -98,15 +98,26 @@ func (b *bisyncRun) lockFileIsExpired() bool {
|
||||
rdf, err := os.Open(b.lockFile)
|
||||
b.handleErr(b.lockFile, "error reading lock file", err, true, true)
|
||||
dec := json.NewDecoder(rdf)
|
||||
var decodeErr error
|
||||
for {
|
||||
if err := dec.Decode(&b.lockFileOpt.data); err != nil {
|
||||
if err != io.EOF {
|
||||
fs.Errorf(b.lockFile, "err: %v", err)
|
||||
decodeErr = err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
b.handleErr(b.lockFile, "error closing file", rdf.Close(), true, true)
|
||||
if decodeErr != nil {
|
||||
if b.opt.MaxLock < basicallyforever {
|
||||
fs.Infof(b.lockFile, Color(terminal.YellowFg, "Lock file is unreadable (decode error: %v) and --max-lock is set. Treating as expired."), decodeErr)
|
||||
markFailed(b.listing1)
|
||||
markFailed(b.listing2)
|
||||
return true
|
||||
}
|
||||
fs.Errorf(b.lockFile, Color(terminal.RedFg, "Lock file exists, but contents are unreadable. (decode error: %v)"), decodeErr)
|
||||
return false
|
||||
}
|
||||
if !b.lockFileOpt.data.TimeExpires.IsZero() && b.lockFileOpt.data.TimeExpires.Before(time.Now()) {
|
||||
fs.Infof(b.lockFile, Color(terminal.GreenFg, "Lock file found, but it expired at %v. Will delete it and proceed."), b.lockFileOpt.data.TimeExpires)
|
||||
markFailed(b.listing1) // listing is untrusted so force revert to prior (if --recover) or create new ones (if --resync)
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package bisync
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestLockfileBisyncRun(t *testing.T, lockContent string, maxLock fs.Duration) *bisyncRun {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
lockPath := filepath.Join(dir, "test.lck")
|
||||
require.NoError(t, os.WriteFile(lockPath, []byte(lockContent), 0600))
|
||||
|
||||
listing1 := filepath.Join(dir, "listing1")
|
||||
listing2 := filepath.Join(dir, "listing2")
|
||||
require.NoError(t, os.WriteFile(listing1, []byte(""), 0600))
|
||||
require.NoError(t, os.WriteFile(listing2, []byte(""), 0600))
|
||||
|
||||
return &bisyncRun{
|
||||
lockFile: lockPath,
|
||||
opt: &Options{MaxLock: maxLock},
|
||||
listing1: listing1,
|
||||
listing2: listing2,
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockfileIsExpired_UnreadableWithMaxLock(t *testing.T) {
|
||||
b := newTestLockfileBisyncRun(t, "not json!!!", fs.Duration(5*time.Minute))
|
||||
assert.True(t, b.lockFileIsExpired(), "unreadable lockfile with --max-lock set should be treated as expired")
|
||||
}
|
||||
|
||||
func TestLockfileIsExpired_UnreadableWithoutMaxLock(t *testing.T) {
|
||||
b := newTestLockfileBisyncRun(t, "not json!!!", basicallyforever)
|
||||
assert.False(t, b.lockFileIsExpired(), "unreadable lockfile without --max-lock should not be treated as expired")
|
||||
}
|
||||
|
||||
func TestLockfileIsExpired_ValidExpired(t *testing.T) {
|
||||
data := struct {
|
||||
Session string
|
||||
PID string
|
||||
TimeRenewed time.Time
|
||||
TimeExpires time.Time
|
||||
}{
|
||||
Session: "test",
|
||||
PID: "12345",
|
||||
TimeRenewed: time.Now().Add(-10 * time.Minute),
|
||||
TimeExpires: time.Now().Add(-5 * time.Minute),
|
||||
}
|
||||
content, err := json.Marshal(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := newTestLockfileBisyncRun(t, string(content), fs.Duration(5*time.Minute))
|
||||
assert.True(t, b.lockFileIsExpired(), "valid lockfile with past expiry should be expired")
|
||||
}
|
||||
|
||||
func TestLockfileIsExpired_ValidNotExpired(t *testing.T) {
|
||||
data := struct {
|
||||
Session string
|
||||
PID string
|
||||
TimeRenewed time.Time
|
||||
TimeExpires time.Time
|
||||
}{
|
||||
Session: "test",
|
||||
PID: "12345",
|
||||
TimeRenewed: time.Now(),
|
||||
TimeExpires: time.Now().Add(10 * time.Minute),
|
||||
}
|
||||
content, err := json.Marshal(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := newTestLockfileBisyncRun(t, string(content), fs.Duration(5*time.Minute))
|
||||
assert.False(t, b.lockFileIsExpired(), "valid lockfile with future expiry should not be expired")
|
||||
}
|
||||
@@ -196,7 +196,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
|
||||
}
|
||||
|
||||
if b.critical {
|
||||
if b.retryable && b.opt.Resilient {
|
||||
if b.retryable && b.opt.Resilient && !b.opt.Resync {
|
||||
fs.Errorf(nil, Color(terminal.RedFg, "Bisync critical error: %v"), err)
|
||||
fs.Error(nil, Color(terminal.YellowFg, "Bisync aborted. Error is retryable without --resync due to --resilient mode."))
|
||||
} else {
|
||||
|
||||
+128
-30
@@ -1,25 +1,66 @@
|
||||
//go:generate go run help.go rc.md
|
||||
package bisync
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"errors"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd/bisync/bilib"
|
||||
"github.com/rclone/rclone/fs"
|
||||
fslog "github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
)
|
||||
|
||||
func init() {
|
||||
func addRC() {
|
||||
rc.Add(rc.Call{
|
||||
Path: "sync/bisync",
|
||||
AuthRequired: true,
|
||||
Fn: rcBisync,
|
||||
Title: shortHelp,
|
||||
Help: rcHelp,
|
||||
Path: "sync/bisync",
|
||||
Fn: rcBisync,
|
||||
Title: shortHelp,
|
||||
Help: rcHelp,
|
||||
})
|
||||
}
|
||||
|
||||
//go:embed rc.md
|
||||
var rcHelp string
|
||||
|
||||
var shortHelp = `Perform bidirectional synchronization between two paths.`
|
||||
|
||||
var longHelp = shortHelp + MakeHelp(`
|
||||
|
||||
[Bisync](https://rclone.org/bisync/) provides a
|
||||
bidirectional cloud sync solution in rclone.
|
||||
It retains the Path1 and Path2 filesystem listings from the prior run.
|
||||
On each successive run it will:
|
||||
|
||||
- list files on Path1 and Path2, and check for changes on each side.
|
||||
Changes include ||New||, ||Newer||, ||Older||, and ||Deleted|| files.
|
||||
- Propagate changes on Path1 to Path2, and vice-versa.
|
||||
|
||||
Bisync is considered an **advanced command**, so use with care.
|
||||
Make sure you have read and understood the entire [manual](https://rclone.org/bisync)
|
||||
(especially the [Limitations](https://rclone.org/bisync/#limitations) section)
|
||||
before using, or data loss can result. Questions can be asked in the
|
||||
[Rclone Forum](https://forum.rclone.org/).
|
||||
|
||||
See [full bisync description](https://rclone.org/bisync/) for details.
|
||||
`)
|
||||
|
||||
// MakeHelp replaces some dynamic variables for the help docs
|
||||
func MakeHelp(help string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"||", "`",
|
||||
"{MAXDELETE}", strconv.Itoa(DefaultMaxDelete),
|
||||
"{CHECKFILE}", DefaultCheckFilename,
|
||||
"{WORKDIR}", DefaultWorkdir,
|
||||
)
|
||||
return replacer.Replace(help)
|
||||
}
|
||||
|
||||
func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
opt := &Options{}
|
||||
octx, ci := fs.AddConfig(ctx)
|
||||
@@ -41,56 +82,86 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
}
|
||||
|
||||
if opt.Resync, err = in.GetBool("resync"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("resync", "optional parameter is missing. using default value: %v", opt.Resync)
|
||||
}
|
||||
if opt.CheckAccess, err = in.GetBool("checkAccess"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("checkAccess", "optional parameter is missing. using default value: %v", opt.CheckAccess)
|
||||
}
|
||||
if opt.Force, err = in.GetBool("force"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("force", "optional parameter is missing. using default value: %v", opt.Force)
|
||||
}
|
||||
if opt.CreateEmptySrcDirs, err = in.GetBool("createEmptySrcDirs"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("createEmptySrcDirs", "optional parameter is missing. using default value: %v", opt.CreateEmptySrcDirs)
|
||||
}
|
||||
if opt.RemoveEmptyDirs, err = in.GetBool("removeEmptyDirs"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("removeEmptyDirs", "optional parameter is missing. using default value: %v", opt.RemoveEmptyDirs)
|
||||
}
|
||||
if opt.NoCleanup, err = in.GetBool("noCleanup"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("noCleanup", "optional parameter is missing. using default value: %v", opt.NoCleanup)
|
||||
}
|
||||
if opt.IgnoreListingChecksum, err = in.GetBool("ignoreListingChecksum"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("ignoreListingChecksum", "optional parameter is missing. using default value: %v", opt.IgnoreListingChecksum)
|
||||
}
|
||||
if opt.Resilient, err = in.GetBool("resilient"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("resilient", "optional parameter is missing. using default value: %v", opt.Resilient)
|
||||
}
|
||||
|
||||
if opt.CheckFilename, err = in.GetString("checkFilename"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
opt.CheckFilename = DefaultCheckFilename
|
||||
fs.Debugf("checkFilename", "optional parameter is missing. using default value: %v", opt.CheckFilename)
|
||||
}
|
||||
if opt.FiltersFile, err = in.GetString("filtersFile"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
fs.Debugf("filtersFile", "optional parameter is missing. using default value: %v", opt.FiltersFile)
|
||||
}
|
||||
if opt.Workdir, err = in.GetString("workdir"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
// "" sets correct default later
|
||||
fs.Debugf("workdir", "optional parameter is missing. using default value: %v", opt.Workdir)
|
||||
}
|
||||
if opt.BackupDir1, err = in.GetString("backupdir1"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
if opt.BackupDir1, err = in.GetString("backupDir1"); rc.NotErrParamNotFound(err) {
|
||||
// we accept an alternate capitalization here for backward compatibility.
|
||||
if opt.BackupDir1, err = in.GetString("backupdir1"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("backupDir1", "optional parameter is missing. using default value: %v", opt.BackupDir1)
|
||||
}
|
||||
}
|
||||
if opt.BackupDir2, err = in.GetString("backupdir2"); rc.NotErrParamNotFound(err) {
|
||||
return
|
||||
if opt.BackupDir2, err = in.GetString("backupDir2"); rc.NotErrParamNotFound(err) {
|
||||
// we accept an alternate capitalization here for backward compatibility.
|
||||
if opt.BackupDir2, err = in.GetString("backupdir2"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("backupDir2", "optional parameter is missing. using default value: %v", opt.BackupDir2)
|
||||
}
|
||||
}
|
||||
|
||||
checkSync, err := in.GetString("checkSync")
|
||||
if rc.NotErrParamNotFound(err) {
|
||||
if err = setEnum(in, "checkSync", "true", opt.CheckSync.Set); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if checkSync == "" {
|
||||
checkSync = "true"
|
||||
}
|
||||
if err := opt.CheckSync.Set(checkSync); err != nil {
|
||||
if err = setEnum(in, "resyncMode", opt.ResyncMode.String(), opt.ResyncMode.Set); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = setEnum(in, "conflictResolve", opt.ConflictResolve.String(), opt.ConflictResolve.Set); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = setEnum(in, "conflictLoser", opt.ConflictLoser.String(), opt.ConflictLoser.Set); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.ConflictSuffixFlag, err = in.GetString("conflictSuffix"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("conflictSuffix", "optional parameter is missing. using default value: %v", opt.ConflictSuffixFlag)
|
||||
}
|
||||
if opt.Recover, err = in.GetBool("recover"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("recover", "optional parameter is missing. using default value: %v", opt.Recover)
|
||||
}
|
||||
if opt.CompareFlag, err = in.GetString("compare"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("compare", "optional parameter is missing. using default value: %v", opt.CompareFlag)
|
||||
}
|
||||
if opt.Compare.NoSlowHash, err = in.GetBool("noSlowHash"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("noSlowHash", "optional parameter is missing. using default value: %v", opt.Compare.NoSlowHash)
|
||||
}
|
||||
if opt.Compare.SlowHashSyncOnly, err = in.GetBool("slowHashSyncOnly"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("slowHashSyncOnly", "optional parameter is missing. using default value: %v", opt.Compare.SlowHashSyncOnly)
|
||||
}
|
||||
if opt.Compare.DownloadHash, err = in.GetBool("downloadHash"); rc.NotErrParamNotFound(err) {
|
||||
fs.Debugf("downloadHash", "optional parameter is missing. using default value: %v", opt.Compare.DownloadHash)
|
||||
}
|
||||
if opt.MaxLock, err = in.GetFsDuration("maxLock"); rc.NotErrParamNotFound(err) {
|
||||
opt.MaxLock = 0
|
||||
fs.Debugf("maxLock", "optional parameter is missing. using default value: %v", opt.MaxLock)
|
||||
}
|
||||
|
||||
fs1, err := rc.GetFsNamed(octx, in, "path1")
|
||||
if err != nil {
|
||||
@@ -105,6 +176,33 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
output := bilib.CaptureOutput(func() {
|
||||
err = Bisync(octx, fs1, fs2, opt)
|
||||
})
|
||||
|
||||
workDir, _ := filepath.Abs(DefaultWorkdir)
|
||||
if opt.Workdir != "" {
|
||||
workDir, _ = filepath.Abs(opt.Workdir)
|
||||
}
|
||||
basePath := bilib.BasePath(ctx, workDir, fs1, fs2)
|
||||
|
||||
_, _ = log.Writer().Write(output)
|
||||
return rc.Params{"output": string(output)}, err
|
||||
return rc.Params{
|
||||
"output": string(output),
|
||||
"session": bilib.SessionName(fs1, fs2),
|
||||
"workDir": workDir,
|
||||
"basePath": basePath,
|
||||
"listing1": basePath + ".path1.lst",
|
||||
"listing2": basePath + ".path2.lst",
|
||||
"logFile": fslog.Opt.File,
|
||||
}, err
|
||||
}
|
||||
|
||||
func setEnum(in rc.Params, name string, defaultVal string, set func(s string) error) error {
|
||||
v, err := in.GetString(name)
|
||||
if rc.NotErrParamNotFound(err) || v == "" {
|
||||
v = defaultVal
|
||||
fs.Debugf(name, "optional parameter is missing. using default value: %v", v)
|
||||
}
|
||||
if err := set(v); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
<!--- Docs generated by help.go - use go generate to rebuild - DO NOT EDIT --->
|
||||
|
||||
This takes the following parameters:
|
||||
|
||||
- path1 (required) - (string) a remote directory string e.g. `drive:path1`
|
||||
- path2 (required) - (string) a remote directory string e.g. `drive:path2`
|
||||
- dryRun - (bool) dry-run mode
|
||||
- backupDir1 - (string) --backup-dir for Path1. Must be a non-overlapping path on
|
||||
the same remote.
|
||||
- backupDir2 - (string) --backup-dir for Path2. Must be a non-overlapping path on
|
||||
the same remote.
|
||||
- checkAccess - (bool) Ensure expected RCLONE_TEST files are found on both
|
||||
Path1 and Path2 filesystems, else abort.
|
||||
- checkFilename - (string) Filename for --check-access (default: RCLONE_TEST)
|
||||
- checkSync - (string) Controls comparison of final listings: true|false|only
|
||||
(default: true)
|
||||
- compare - (string) Comma-separated list of bisync-specific compare options ex.
|
||||
'size,modtime,checksum' (default: 'size,modtime')
|
||||
- conflictLoser - (ConflictLoserAction) Action to take on the loser of a sync
|
||||
conflict (when there is a winner) or on both files (when there is no
|
||||
winner): , num, pathname, delete (default: num)
|
||||
- conflictResolve - (string) Automatically resolve conflicts by preferring the
|
||||
version that is: none, path1, path2, newer, older, larger, smaller (default:
|
||||
none)
|
||||
- conflictSuffix - (string) Suffix to use when renaming a --conflict-loser. Can
|
||||
be either one string or two comma-separated strings to assign different
|
||||
suffixes to Path1/Path2. (default: 'conflict')
|
||||
- createEmptySrcDirs - (bool) Sync creation and deletion of empty directories.
|
||||
(Not compatible with --remove-empty-dirs)
|
||||
- downloadHash - (bool) Compute hash by downloading when otherwise
|
||||
unavailable. (warning: may be slow and use lots of data!)
|
||||
- filtersFile - (string) Read filtering patterns from a file
|
||||
- force - (bool) Bypass --max-delete safety check and run the sync. Consider
|
||||
using with --verbose
|
||||
- ignoreListingChecksum - (bool) Do not use checksums for listings (add --ignore-
|
||||
checksum to additionally skip post-copy checksum checks)
|
||||
- maxLock - (Duration) Consider lock files older than this to be expired
|
||||
(default: 0 (never expire)) (minimum: 2m)
|
||||
- noCleanup - (bool) Retain working files (useful for troubleshooting and
|
||||
testing).
|
||||
- noSlowHash - (bool) Ignore listing checksums only on backends where they are
|
||||
slow
|
||||
- recover - (bool) Automatically recover from interruptions without requiring --
|
||||
resync.
|
||||
- removeEmptyDirs - (bool) Remove ALL empty directories at the final cleanup
|
||||
step.
|
||||
- resilient - (bool) Allow future runs to retry after certain less-serious
|
||||
errors, instead of requiring --resync.
|
||||
- resync - (bool) Performs the resync run. Equivalent to --resync-mode path1.
|
||||
Consider using --verbose or --dry-run first.
|
||||
- resyncMode - (string) During resync, prefer the version that is: path1,
|
||||
path2, newer, older, larger, smaller (default: path1 if --resync, otherwise
|
||||
none for no resync.)
|
||||
- slowHashSyncOnly - (bool) Ignore slow checksums for listings and deltas, but
|
||||
still consider them during sync calls.
|
||||
- workdir - (string) Use custom working dir - useful for testing. (default:
|
||||
/home/ncw/.cache/rclone/bisync)
|
||||
|
||||
See [bisync command help](https://rclone.org/commands/rclone_bisync/)
|
||||
and [full bisync description](https://rclone.org/bisync/)
|
||||
for more information.
|
||||
@@ -165,6 +165,16 @@ func (b *bisyncRun) resolve(ctxMove context.Context, path1, path2, file, alias s
|
||||
},
|
||||
}
|
||||
|
||||
if b.opt.ConflictLoser == ConflictLoserPathname {
|
||||
switch winningPath {
|
||||
case 1: // keep path1, rename path2
|
||||
r.path1.newName = r.path1.oldName
|
||||
case 2: // keep path2, rename path1
|
||||
r.path2.newName = r.path2.oldName
|
||||
}
|
||||
// case 0 keeps default
|
||||
}
|
||||
|
||||
// handle auto-numbering
|
||||
// note that we still queue copies for both files, whether or not we renamed
|
||||
// we also set these for ConflictLoserDelete in case there is no winner.
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
"file2.txt"
|
||||
"file1.txt.path2"
|
||||
+3
-1
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-25T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -8,3 +8,5 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt.path2"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file2.txt"
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-25T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -8,4 +8,4 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 33 - - 2005-01-02T00:00:00.000000000+0000 "file2.txt"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file2.txt"
|
||||
+1
-1
@@ -1,5 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -7,4 +8,3 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
|
||||
+3
-1
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-25T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -8,3 +8,5 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt.path2"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file2.txt"
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2005-01-02T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
|
||||
+1
-1
@@ -1,5 +1,6 @@
|
||||
# bisync listing v1 from test
|
||||
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
|
||||
- 33 - - 2006-03-04T00:00:00.000000000+0000 "file1.txt"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path1"
|
||||
- 33 - - 2004-07-23T00:00:00.000000000+0000 "file1.txt..path2"
|
||||
- 33 - - 2002-08-26T00:00:00.000000000+0000 "file1.txt.apple1"
|
||||
@@ -7,4 +8,3 @@
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.cloud1"
|
||||
- 33 - - 2001-08-26T00:00:00.000000000+0000 "file1.txt.dinosaur1"
|
||||
- 33 - - 2003-07-23T00:00:00.000000000+0000 "file1.txt.local1"
|
||||
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt"
|
||||
@@ -305,3 +305,56 @@ INFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mP
|
||||
INFO : Updating listings
|
||||
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
|
||||
INFO : [32mBisync successful[0m
|
||||
|
||||
[36m(48) :[0m [34mtest --conflict-resolve newer --conflict-loser pathname --conflict-suffix path[0m
|
||||
[36m(49) :[0m [34mtouch-glob 2004-07-23 {datadir/} file1R.txt[0m
|
||||
[36m(50) :[0m [34mcopy-as {datadir/}file1R.txt {path2/} file1.txt[0m
|
||||
[36m(51) :[0m [34mtouch-glob 2004-07-25 {datadir/} file1L.txt[0m
|
||||
[36m(52) :[0m [34mcopy-as {datadir/}file1L.txt {path1/} file1.txt[0m
|
||||
|
||||
[36m(53) :[0m [34mtest bisync run with --conflict-resolve=newer --conflict-loser=pathname --conflict-suffix=path[0m
|
||||
|
||||
[36m(54) :[0m [34mbisync conflict-resolve=newer conflict-loser=pathname conflict-suffix=path[0m
|
||||
INFO : [2mSetting --ignore-listing-checksum as neither --checksum nor --compare checksum are set.[0m
|
||||
INFO : Bisyncing with Comparison Settings:
|
||||
{
|
||||
"Modtime": true,
|
||||
"Size": true,
|
||||
"Checksum": false,
|
||||
"NoSlowHash": false,
|
||||
"SlowHashSyncOnly": false,
|
||||
"DownloadHash": false
|
||||
}
|
||||
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
|
||||
INFO : Building Path1 and Path2 listings
|
||||
INFO : Path1 checking for diffs
|
||||
INFO : - [36mPath1[0m [35m[33mFile changed: [35mtime (older)[0m[0m[0m - [36mfile1.txt[0m
|
||||
INFO : - [36mPath1[0m [35m[32mFile is new[0m[0m - [36mfile2.txt[0m
|
||||
INFO : Path1: 2 changes: [32m 1 new[0m, [33m 1 modified[0m, [31m 0 deleted[0m
|
||||
INFO : ([33mModified[0m: [36m 0 newer[0m, [34m 1 older[0m)
|
||||
INFO : Path2 checking for diffs
|
||||
INFO : - [34mPath2[0m [35m[33mFile changed: [35mtime (older)[0m[0m[0m - [36mfile1.txt[0m
|
||||
INFO : - [34mPath2[0m [35m[32mFile is new[0m[0m - [36mfile2.txt[0m
|
||||
INFO : Path2: 2 changes: [32m 1 new[0m, [33m 1 modified[0m, [31m 0 deleted[0m
|
||||
INFO : ([33mModified[0m: [36m 0 newer[0m, [34m 1 older[0m)
|
||||
INFO : Applying changes
|
||||
INFO : Checking potential conflicts...
|
||||
ERROR : file1.txt: {hashtype} differ
|
||||
NOTICE: {path2String}: 1 differences found
|
||||
NOTICE: {path2String}: 1 errors while checking
|
||||
NOTICE: {path2String}: 1 matching files
|
||||
INFO : Finished checking the potential conflicts. 1 differences found
|
||||
NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfile1.txt[0m
|
||||
INFO : file1.txt: Path1 is newer. Path1: 2004-07-25 00:00:00 +0000 UTC, Path2: 2004-07-23 00:00:00 +0000 UTC, Difference: 48h0m0s
|
||||
INFO : file1.txt: [32mThe winner is: Path1[0m
|
||||
NOTICE: - [36mPath1[0m [35mNot renaming Path1 copy, as it was determined the winner[0m - [36m{path1/}file1.txt[0m
|
||||
NOTICE: - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{path2/}file1.txt[0m
|
||||
NOTICE: - [34mPath2[0m [35mRenaming Path2 copy[0m - [36m{path2/}file1.txt.path2[0m
|
||||
NOTICE: - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file1.txt.path2[0m
|
||||
NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfile2.txt[0m
|
||||
INFO : Files are equal! Skipping: file2.txt
|
||||
INFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m
|
||||
INFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m
|
||||
INFO : Updating listings
|
||||
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
|
||||
INFO : [32mBisync successful[0m
|
||||
+11
-1
@@ -64,4 +64,14 @@ copy-as {datadir/}file1L.txt {path1/} file1.txt
|
||||
copy-as {datadir/}file1L.txt {path2/} file2.txt
|
||||
|
||||
test bisync run with --conflict-resolve=newer --conflict-loser=delete
|
||||
bisync conflict-resolve=newer conflict-loser=delete
|
||||
bisync conflict-resolve=newer conflict-loser=delete
|
||||
|
||||
test --conflict-resolve newer --conflict-loser pathname --conflict-suffix path
|
||||
touch-glob 2004-07-23 {datadir/} file1R.txt
|
||||
copy-as {datadir/}file1R.txt {path2/} file1.txt
|
||||
touch-glob 2004-07-25 {datadir/} file1L.txt
|
||||
copy-as {datadir/}file1L.txt {path1/} file1.txt
|
||||
|
||||
test bisync run with --conflict-resolve=newer --conflict-loser=pathname --conflict-suffix=path
|
||||
# result should be -- "file1.txt", "file1.txt.path2"
|
||||
bisync conflict-resolve=newer conflict-loser=pathname conflict-suffix=path
|
||||
+4
-4
@@ -124,12 +124,12 @@ func waitFor(fn func() bool) (ok bool) {
|
||||
//
|
||||
// returns an error, and an error channel for the serve process to
|
||||
// report an error when fusermount is called.
|
||||
func mount(VFS *vfs.VFS, mountPath string, opt *mountlib.Options) (<-chan error, func() error, error) {
|
||||
func mount(VFS *vfs.VFS, mountPath string, opt *mountlib.Options) (<-chan error, func() error, string, error) {
|
||||
// Get mountpoint using OS specific logic
|
||||
f := VFS.Fs()
|
||||
mountpoint, err := getMountpoint(f, mountPath, opt)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, "", err
|
||||
}
|
||||
fs.Debugf(nil, "Mounting on %q (%q)", mountpoint, opt.VolumeName)
|
||||
|
||||
@@ -205,7 +205,7 @@ func mount(VFS *vfs.VFS, mountPath string, opt *mountlib.Options) (<-chan error,
|
||||
select {
|
||||
case err := <-errChan:
|
||||
err = fmt.Errorf("mount stopped before calling Init: %w", err)
|
||||
return nil, nil, err
|
||||
return nil, nil, "", err
|
||||
case <-fsys.ready:
|
||||
}
|
||||
|
||||
@@ -220,5 +220,5 @@ func mount(VFS *vfs.VFS, mountPath string, opt *mountlib.Options) (<-chan error,
|
||||
}
|
||||
}
|
||||
|
||||
return errChan, unmount, nil
|
||||
return errChan, unmount, mountpoint, nil
|
||||
}
|
||||
@@ -26,8 +26,8 @@ func init() {
|
||||
//
|
||||
// returns an error, and an error channel for the serve process to
|
||||
// report an error when fusermount is called.
|
||||
func mount(_ *vfs.VFS, _ string, _ *mountlib.Options) (<-chan error, func() error, error) {
|
||||
return nil, nil, errors.New("rclone mount is not supported on MacOS when rclone is installed via Homebrew. " +
|
||||
func mount(_ *vfs.VFS, _ string, _ *mountlib.Options) (<-chan error, func() error, string, error) {
|
||||
return nil, nil, "", errors.New("rclone mount is not supported on MacOS when rclone is installed via Homebrew. " +
|
||||
"Please install the rclone binaries available at https://rclone.org/downloads/ " +
|
||||
"instead if you want to use the rclone mount command")
|
||||
}
|
||||
Loaded 100 of 346 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user