Compare commits
146
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dfc5b0460b | ||
|
|
26db80c270 | ||
|
|
9eb3470c9c | ||
|
|
a449dd7d1c | ||
|
|
fc4fe33703 | ||
|
|
e11bfacfcf | ||
|
|
a9c49c50a0 | ||
|
|
8979337313 | ||
|
|
7ffab5d998 | ||
|
|
3ccf222acb | ||
|
|
2781f8e2f1 | ||
|
|
3d55f69338 | ||
|
|
cc9bc2cb80 | ||
|
|
80ac59ee5b | ||
|
|
5d6a6dd6c0 | ||
|
|
c676e2139d | ||
|
|
7361c98b2d | ||
|
|
5cc47de912 | ||
|
|
6d342a3c5b | ||
|
|
336051870e | ||
|
|
38c6d022bd | ||
|
|
c138367df6 | ||
|
|
da404dc0f2 | ||
|
|
28e43fe7af | ||
|
|
3ec25f437b | ||
|
|
a34276e9b3 | ||
|
|
c2baacc0a4 | ||
|
|
fcec4bedbe | ||
|
|
813a5e0931 | ||
|
|
bd4abb15a3 | ||
|
|
7f84283539 | ||
|
|
47b1a0d6fa | ||
|
|
ce168ecac2 | ||
|
|
4f0ddb60e7 | ||
|
|
b929a56f46 | ||
|
|
74af6409d4 | ||
|
|
0e77072dcc | ||
|
|
2437eb3cce | ||
|
|
a12c94caff | ||
|
|
542c1616b8 | ||
|
|
8697f0bd26 | ||
|
|
a9f18f8093 | ||
|
|
8e5e230b81 | ||
|
|
c0985e93b7 | ||
|
|
fb4f7555c7 | ||
|
|
f2e7a2e794 | ||
|
|
9e4854955c | ||
|
|
319ac225e4 | ||
|
|
a9d3283d97 | ||
|
|
edf0412464 | ||
|
|
e6194a4b83 | ||
|
|
7f05990623 | ||
|
|
e16f2a566f | ||
|
|
a36fef8a66 | ||
|
|
6500e1d205 | ||
|
|
9f7484e4e9 | ||
|
|
0ba702ccf4 | ||
|
|
6f91198b57 | ||
|
|
cf0a72aecd | ||
|
|
f6fd6ee777 | ||
|
|
1e66d052fd | ||
|
|
e5974ac4b0 | ||
|
|
50a0c3482d | ||
|
|
389a29b017 | ||
|
|
9dcf9375e8 | ||
|
|
1d6d41fb91 | ||
|
|
a3d4307892 | ||
|
|
a446106041 | ||
|
|
607172b6ec | ||
|
|
94757277bc | ||
|
|
deab86867c | ||
|
|
c0c5b3bc6b | ||
|
|
a947f298e6 | ||
|
|
1b0128ecb2 | ||
|
|
c5395db1f1 | ||
|
|
6e5382fc99 | ||
|
|
134592adaa | ||
|
|
36e614f550 | ||
|
|
7bfed98b48 | ||
|
|
f471096fd0 | ||
|
|
4cebade95d | ||
|
|
a8cd18faf3 | ||
|
|
e34c543660 | ||
|
|
598364ad0f | ||
|
|
211dbe9aee | ||
|
|
4829527dac | ||
|
|
cc8dde402f | ||
|
|
2b67ad17aa | ||
|
|
6da3522499 | ||
|
|
97606bbdef | ||
|
|
a15885dd74 | ||
|
|
87c201c92a | ||
|
|
d77736c21a | ||
|
|
86bd5f6922 | ||
|
|
fe271a4e35 | ||
|
|
75455d4000 | ||
|
|
82e24f521f | ||
|
|
5605e34f7b | ||
|
|
06598531e0 | ||
|
|
b1d43f8d41 | ||
|
|
b53c38c9fd | ||
|
|
03715f6c6b | ||
|
|
07481396e0 | ||
|
|
bab91e4402 | ||
|
|
fde40319ef | ||
|
|
94e330d4fa | ||
|
|
087543d723 | ||
|
|
6a759d936a | ||
|
|
7c31240bb8 | ||
|
|
25146b4306 | ||
|
|
240561850b | ||
|
|
39a1e37441 | ||
|
|
4c02f50ef5 | ||
|
|
f583b86334 | ||
|
|
118e8e1470 | ||
|
|
afcea9c72b | ||
|
|
27176cc6bb | ||
|
|
f1e4b7da7b | ||
|
|
f065a267f6 | ||
|
|
17f8014909 | ||
|
|
8ba04562c3 | ||
|
|
285747b1d1 | ||
|
|
7bb8b8f4ba | ||
|
|
59c242bbf6 | ||
|
|
a2bacd7d3f | ||
|
|
9babcc4811 | ||
|
|
a0f665ec3c | ||
|
|
ecdf42c17f | ||
|
|
be9ee1d138 | ||
|
|
9e9ead2ac4 | ||
|
|
4f78226f8b | ||
|
|
54c9c3156c | ||
|
|
6ecbbf796e | ||
|
|
603e51c43f | ||
|
|
ca4671126e | ||
|
|
6ea26b508a | ||
|
|
887cccb2c1 | ||
|
|
d975196cfa | ||
|
|
1f39b28f49 | ||
|
|
2738db22fb | ||
|
|
1978ddde73 | ||
|
|
c2bfda22ab | ||
|
|
d4da9b98d6 | ||
|
|
e4f5912294 | ||
|
|
750fffdf71 | ||
|
|
388e74af52 |
No files matched your search
+30
-36
@@ -25,12 +25,12 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.15', 'go1.16']
|
||||
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '1.17.x'
|
||||
go: '1.18.x'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
|
||||
- job_name: mac_amd64
|
||||
os: macos-11
|
||||
go: '1.17.x'
|
||||
go: '1.18.x'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -50,51 +50,40 @@ jobs:
|
||||
|
||||
- job_name: mac_arm64
|
||||
os: macos-11
|
||||
go: '1.17.x'
|
||||
go: '1.18.x'
|
||||
gotags: 'cmount'
|
||||
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
|
||||
deploy: true
|
||||
|
||||
- job_name: windows_amd64
|
||||
- job_name: windows
|
||||
os: windows-latest
|
||||
go: '1.17.x'
|
||||
go: '1.18.x'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^windows/amd64" -cgo'
|
||||
build_args: '-buildmode exe'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
deploy: true
|
||||
|
||||
- job_name: windows_386
|
||||
os: windows-latest
|
||||
go: '1.17.x'
|
||||
gotags: cmount
|
||||
goarch: '386'
|
||||
cgo: '1'
|
||||
build_flags: '-include "^windows/386" -cgo'
|
||||
cgo: '0'
|
||||
build_flags: '-include "^windows/"'
|
||||
build_args: '-buildmode exe'
|
||||
quicktest: true
|
||||
deploy: true
|
||||
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '1.17.x'
|
||||
build_flags: '-exclude "^(windows/(386|amd64)|darwin/|linux/)"'
|
||||
go: '1.18.x'
|
||||
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: go1.15
|
||||
os: ubuntu-latest
|
||||
go: '1.15.x'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
- job_name: go1.16
|
||||
os: ubuntu-latest
|
||||
go: '1.16.x'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
- job_name: go1.17
|
||||
os: ubuntu-latest
|
||||
go: '1.17.x'
|
||||
quicktest: true
|
||||
racequicktest: true
|
||||
|
||||
name: ${{ matrix.job_name }}
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
@@ -178,6 +167,11 @@ jobs:
|
||||
run: |
|
||||
make
|
||||
|
||||
- name: Rclone version
|
||||
shell: bash
|
||||
run: |
|
||||
rclone version
|
||||
|
||||
- name: Run tests
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -246,14 +240,14 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
# Upgrade together with NDK version
|
||||
- name: Set up Go 1.16
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.16
|
||||
go-version: 1.18.x
|
||||
|
||||
# Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes.
|
||||
- name: Force NDK version
|
||||
run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;22.1.7171670" | grep -v = || true
|
||||
run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;23.1.7779620" | grep -v = || true
|
||||
|
||||
- name: Go module cache
|
||||
uses: actions/cache@v2
|
||||
@@ -274,8 +268,8 @@ jobs:
|
||||
|
||||
- name: install gomobile
|
||||
run: |
|
||||
go get golang.org/x/mobile/cmd/gobind
|
||||
go get golang.org/x/mobile/cmd/gomobile
|
||||
go install golang.org/x/mobile/cmd/gobind@latest
|
||||
go install golang.org/x/mobile/cmd/gomobile@latest
|
||||
env PATH=$PATH:~/go/bin gomobile init
|
||||
|
||||
- name: arm-v7a gomobile build
|
||||
@@ -284,7 +278,7 @@ jobs:
|
||||
- name: arm-v7a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=arm' >> $GITHUB_ENV
|
||||
@@ -297,7 +291,7 @@ jobs:
|
||||
- name: arm64-v8a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=arm64' >> $GITHUB_ENV
|
||||
@@ -310,7 +304,7 @@ jobs:
|
||||
- name: x86 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=386' >> $GITHUB_ENV
|
||||
@@ -323,7 +317,7 @@ jobs:
|
||||
- name: x64 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/22.1.7171670/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
|
||||
echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
echo 'GOOS=android' >> $GITHUB_ENV
|
||||
echo 'GOARCH=amd64' >> $GITHUB_ENV
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
with:
|
||||
tag: beta
|
||||
imageName: rclone/rclone
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
|
||||
publish: true
|
||||
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
|
||||
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
with:
|
||||
tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
|
||||
imageName: rclone/rclone
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
|
||||
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
|
||||
publish: true
|
||||
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
|
||||
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
PLUGIN_USER=rclone
|
||||
docker login --username ${{ secrets.DOCKER_HUB_USER }} \
|
||||
--password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
|
||||
for PLUGIN_ARCH in amd64 arm64 arm/v7 ;do
|
||||
for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
|
||||
export PLUGIN_USER PLUGIN_ARCH
|
||||
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
|
||||
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
|
||||
|
||||
Generated
+2
-2
@@ -19720,7 +19720,7 @@ y/e/d> y</code></pre>
|
||||
<p>Here are the standard options specific to netstorage (Akamai NetStorage).</p>
|
||||
<h4 id="netstorage-host">--netstorage-host</h4>
|
||||
<p>Domain+path of NetStorage host to connect to.</p>
|
||||
<p>Format should be <domain>/<internal folders></p>
|
||||
<p>Format should be <code><domain>/<internal folders></code></p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: host</li>
|
||||
@@ -19785,7 +19785,7 @@ y/e/d> y</code></pre>
|
||||
<h3 id="symlink">symlink</h3>
|
||||
<p>You can create a symbolic link in ObjectStore with the symlink action.</p>
|
||||
<pre><code>rclone backend symlink remote: [options] [<arguments>+]</code></pre>
|
||||
<p>The desired path location (including applicable sub-directories) ending in the object that will be the target of the symlink (for example, /links/mylink). Include the file extension for the object, if applicable. rclone backend symlink <src> <path></p>
|
||||
<p>The desired path location (including applicable sub-directories) ending in the object that will be the target of the symlink (for example, /links/mylink). Include the file extension for the object, if applicable. <code>rclone backend symlink <src> <path></code></p>
|
||||
<h1 id="microsoft-azure-blob-storage">Microsoft Azure Blob Storage</h1>
|
||||
<p>Paths are specified as <code>remote:container</code> (or <code>remote:</code> for the <code>lsd</code> command.) You may put subdirectories in too, e.g. <code>remote:container/path/to/dir</code>.</p>
|
||||
<h2 id="configuration-25">Configuration</h2>
|
||||
|
||||
@@ -29340,7 +29340,7 @@ Here are the standard options specific to netstorage (Akamai NetStorage).
|
||||
|
||||
Domain+path of NetStorage host to connect to.
|
||||
|
||||
Format should be <domain>/<internal folders>
|
||||
Format should be `<domain>/<internal folders>`
|
||||
|
||||
Properties:
|
||||
|
||||
@@ -29432,7 +29432,7 @@ You can create a symbolic link in ObjectStore with the symlink action.
|
||||
The desired path location (including applicable sub-directories) ending in
|
||||
the object that will be the target of the symlink (for example, /links/mylink).
|
||||
Include the file extension for the object, if applicable.
|
||||
rclone backend symlink <src> <path>
|
||||
`rclone backend symlink <src> <path>`
|
||||
|
||||
|
||||
|
||||
|
||||
Generated
+2
-2
@@ -28719,7 +28719,7 @@ NetStorage).
|
||||
|
||||
Domain+path of NetStorage host to connect to.
|
||||
|
||||
Format should be /
|
||||
Format should be <domain>/<internal folders>
|
||||
|
||||
Properties:
|
||||
|
||||
@@ -28813,7 +28813,7 @@ You can create a symbolic link in ObjectStore with the symlink action.
|
||||
The desired path location (including applicable sub-directories) ending
|
||||
in the object that will be the target of the symlink (for example,
|
||||
/links/mylink). Include the file extension for the object, if
|
||||
applicable. rclone backend symlink
|
||||
applicable. rclone backend symlink <src> <path>
|
||||
|
||||
Microsoft Azure Blob Storage
|
||||
|
||||
|
||||
@@ -97,7 +97,7 @@ release_dep_linux:
|
||||
|
||||
# Get the release dependencies we only install on Windows
|
||||
release_dep_windows:
|
||||
GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
|
||||
GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
|
||||
|
||||
# Update dependencies
|
||||
showupdates:
|
||||
@@ -245,18 +245,18 @@ retag:
|
||||
startdev:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next version is $(NEXT_VERSION)"
|
||||
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
|
||||
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
startstable:
|
||||
@echo "Version is $(VERSION)"
|
||||
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
|
||||
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
|
||||
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
|
||||
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
|
||||
echo "$(NEXT_PATCH_VERSION)" > VERSION
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
|
||||
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
|
||||
|
||||
winzip:
|
||||
zip -9 rclone-$(TAG).zip rclone.exe
|
||||
|
||||
@@ -28,6 +28,9 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
|
||||
* Box [:page_facing_up:](https://rclone.org/box/)
|
||||
* Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
|
||||
* China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
|
||||
* Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
|
||||
* Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
|
||||
* Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
|
||||
* DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
|
||||
* Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
|
||||
@@ -41,6 +44,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
|
||||
* HTTP [:page_facing_up:](https://rclone.org/http/)
|
||||
* Hubic [:page_facing_up:](https://rclone.org/hubic/)
|
||||
* Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
|
||||
* Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
|
||||
* IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
|
||||
* Koofr [:page_facing_up:](https://rclone.org/koofr/)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/hdfs"
|
||||
_ "github.com/rclone/rclone/backend/http"
|
||||
_ "github.com/rclone/rclone/backend/hubic"
|
||||
_ "github.com/rclone/rclone/backend/internetarchive"
|
||||
_ "github.com/rclone/rclone/backend/jottacloud"
|
||||
_ "github.com/rclone/rclone/backend/koofr"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
"github.com/Azure/go-autorest/autorest/adal"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
@@ -43,8 +44,9 @@ import (
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 10 * time.Second
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
maxListChunkSize = 5000 // number of items to read at once
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
maxListChunkSize = 5000 // number of items to read at once
|
||||
maxUploadParts = 50000 // maximum allowed number of parts/blocks in a multi-part upload
|
||||
modTimeKey = "mtime"
|
||||
timeFormatIn = time.RFC3339
|
||||
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
@@ -371,15 +373,9 @@ func (o *Object) split() (container, containerPath string) {
|
||||
|
||||
// validateAccessTier checks if azureblob supports user supplied tier
|
||||
func validateAccessTier(tier string) bool {
|
||||
switch tier {
|
||||
case string(azblob.AccessTierHot),
|
||||
string(azblob.AccessTierCool),
|
||||
string(azblob.AccessTierArchive):
|
||||
// valid cases
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(tier, string(azblob.AccessTierHot)) ||
|
||||
strings.EqualFold(tier, string(azblob.AccessTierCool)) ||
|
||||
strings.EqualFold(tier, string(azblob.AccessTierArchive))
|
||||
}
|
||||
|
||||
// validatePublicAccess checks if azureblob supports use supplied public access level
|
||||
@@ -612,7 +608,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
serviceURL = azblob.NewServiceURL(*u, pipeline)
|
||||
case opt.UseMSI:
|
||||
var token adal.Token
|
||||
var userMSI *userMSI = &userMSI{}
|
||||
var userMSI = &userMSI{}
|
||||
if len(opt.MSIClientID) > 0 || len(opt.MSIObjectID) > 0 || len(opt.MSIResourceID) > 0 {
|
||||
// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
|
||||
// Validate and ensure exactly one is set. (To do: better validation.)
|
||||
@@ -1689,8 +1685,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
uploadParts := int64(maxUploadParts)
|
||||
if uploadParts < 1 {
|
||||
uploadParts = 1
|
||||
} else if uploadParts > maxUploadParts {
|
||||
uploadParts = maxUploadParts
|
||||
}
|
||||
// calculate size of parts/blocks
|
||||
partSize := chunksize.Calculator(o, int(uploadParts), o.fs.opt.ChunkSize)
|
||||
|
||||
putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
|
||||
BufferSize: int(o.fs.opt.ChunkSize),
|
||||
BufferSize: int(partSize),
|
||||
MaxBuffers: o.fs.opt.UploadConcurrency,
|
||||
Metadata: o.meta,
|
||||
BlobHTTPHeaders: httpHeaders,
|
||||
|
||||
@@ -61,3 +61,25 @@ func TestServicePrincipalFileFailure(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
|
||||
}
|
||||
|
||||
func TestValidateAccessTier(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
accessTier string
|
||||
want bool
|
||||
}{
|
||||
"hot": {"hot", true},
|
||||
"HOT": {"HOT", true},
|
||||
"Hot": {"Hot", true},
|
||||
"cool": {"cool", true},
|
||||
"archive": {"archive", true},
|
||||
"empty": {"", false},
|
||||
"unknown": {"unknown", false},
|
||||
}
|
||||
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got := validateAccessTier(test.accessTier)
|
||||
assert.Equal(t, test.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
+32
-2
@@ -64,7 +64,8 @@ const (
|
||||
|
||||
// Globals
|
||||
var (
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
|
||||
errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -106,6 +107,11 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
|
||||
Help: "Include old versions in directory listings.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "version_at",
|
||||
Help: "Show file versions as they were at the specified time.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
|
||||
Default: fs.Time{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "hard_delete",
|
||||
Help: "Permanently delete files on remote removal, otherwise hide files.",
|
||||
@@ -211,6 +217,7 @@ type Options struct {
|
||||
Endpoint string `config:"endpoint"`
|
||||
TestMode string `config:"test_mode"`
|
||||
Versions bool `config:"versions"`
|
||||
VersionAt fs.Time `config:"version_at"`
|
||||
HardDelete bool `config:"hard_delete"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
|
||||
@@ -696,9 +703,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
Method: "POST",
|
||||
Path: "/b2_list_file_names",
|
||||
}
|
||||
if hidden {
|
||||
if hidden || f.opt.VersionAt.IsSet() {
|
||||
opts.Path = "/b2_list_file_versions"
|
||||
}
|
||||
|
||||
lastFileName := ""
|
||||
|
||||
for {
|
||||
var response api.ListFileNamesResponse
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
@@ -728,7 +738,21 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
if addBucket {
|
||||
remote = path.Join(bucket, remote)
|
||||
}
|
||||
|
||||
if f.opt.VersionAt.IsSet() {
|
||||
if time.Time(file.UploadTimestamp).After(time.Time(f.opt.VersionAt)) {
|
||||
// Ignore versions that were created after the specified time
|
||||
continue
|
||||
}
|
||||
|
||||
if file.Name == lastFileName {
|
||||
// Ignore versions before the already returned version
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Send object
|
||||
lastFileName = file.Name
|
||||
err = fn(remote, file, isDirectory)
|
||||
if err != nil {
|
||||
if err == errEndList {
|
||||
@@ -1828,6 +1852,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
if o.fs.opt.Versions {
|
||||
return errNotWithVersions
|
||||
}
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
return errNotWithVersionAt
|
||||
}
|
||||
size := src.Size()
|
||||
|
||||
bucket, bucketPath := o.split()
|
||||
@@ -1983,6 +2010,9 @@ func (o *Object) Remove(ctx context.Context) error {
|
||||
if o.fs.opt.Versions {
|
||||
return errNotWithVersions
|
||||
}
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
return errNotWithVersionAt
|
||||
}
|
||||
if o.fs.opt.HardDelete {
|
||||
return o.fs.deleteByID(ctx, o.id, bucketPath)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/rclone/rclone/backend/b2/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
@@ -88,21 +89,19 @@ type largeUpload struct {
|
||||
// newLargeUpload starts an upload of object o from in with metadata in src
|
||||
//
|
||||
// If newInfo is set then metadata from that will be used instead of reading it from src
|
||||
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
|
||||
remote := o.remote
|
||||
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
|
||||
size := src.Size()
|
||||
parts := int64(0)
|
||||
sha1SliceSize := int64(maxParts)
|
||||
chunkSize := defaultChunkSize
|
||||
if size == -1 {
|
||||
fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
|
||||
} else {
|
||||
chunkSize = chunksize.Calculator(src, maxParts, defaultChunkSize)
|
||||
parts = size / int64(chunkSize)
|
||||
if size%int64(chunkSize) != 0 {
|
||||
parts++
|
||||
}
|
||||
if parts > maxParts {
|
||||
return nil, fmt.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
|
||||
}
|
||||
sha1SliceSize = parts
|
||||
}
|
||||
|
||||
|
||||
Vendored
+6
-2
@@ -394,7 +394,11 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
|
||||
notifiedRemotes: make(map[string]bool),
|
||||
}
|
||||
cache.PinUntilFinalized(f.Fs, f)
|
||||
f.rateLimiter = rate.NewLimiter(rate.Limit(float64(opt.Rps)), opt.TotalWorkers)
|
||||
rps := rate.Inf
|
||||
if opt.Rps > 0 {
|
||||
rps = rate.Limit(float64(opt.Rps))
|
||||
}
|
||||
f.rateLimiter = rate.NewLimiter(rps, opt.TotalWorkers)
|
||||
|
||||
f.plexConnector = &plexConnector{}
|
||||
if opt.PlexURL != "" {
|
||||
@@ -1743,7 +1747,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.Fs.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
@@ -515,7 +515,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
|
||||
|
||||
strRegex := regexp.QuoteMeta(pattern)
|
||||
strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl)
|
||||
strRegex = strings.Replace(strRegex, "\\*", mainNameRegStr, -1)
|
||||
strRegex = strings.ReplaceAll(strRegex, "\\*", mainNameRegStr)
|
||||
strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld)
|
||||
f.nameRegexp = regexp.MustCompile(strRegex)
|
||||
|
||||
@@ -524,7 +524,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
|
||||
if numDigits > 1 {
|
||||
fmtDigits = fmt.Sprintf("%%0%dd", numDigits)
|
||||
}
|
||||
strFmt := strings.Replace(pattern, "%", "%%", -1)
|
||||
strFmt := strings.ReplaceAll(pattern, "%", "%%")
|
||||
strFmt = strings.Replace(strFmt, "*", "%s", 1)
|
||||
f.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits)
|
||||
f.ctrlNameFmt = reHashes.ReplaceAllLiteralString(strFmt, "_%s")
|
||||
@@ -1895,7 +1895,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.base.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("can't CleanUp")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
@@ -1904,7 +1904,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.base.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
@@ -906,7 +906,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.Fs.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("can't CleanUp: not supported by underlying remote")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
@@ -915,7 +915,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.Fs.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("can't About: not supported by underlying remote")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
@@ -597,7 +597,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
do := f.Fs.Features().CleanUp
|
||||
if do == nil {
|
||||
return errors.New("can't CleanUp")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
@@ -606,7 +606,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
do := f.Fs.Features().About
|
||||
if do == nil {
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
return do(ctx)
|
||||
}
|
||||
|
||||
+22
-7
@@ -70,7 +70,7 @@ const (
|
||||
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
|
||||
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
|
||||
defaultChunkSize = 8 * fs.Mebi
|
||||
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
|
||||
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
|
||||
listRGrouping = 50 // number of IDs to search at once when using ListR
|
||||
listRInputBuffer = 1000 // size of input buffer when using ListR
|
||||
defaultXDGIcon = "text-html"
|
||||
@@ -660,6 +660,7 @@ type baseObject struct {
|
||||
mimeType string // The object MIME type
|
||||
bytes int64 // size of the object
|
||||
parents []string // IDs of the parent directories
|
||||
resourceKey *string // resourceKey is needed for link shared objects
|
||||
}
|
||||
type documentObject struct {
|
||||
baseObject
|
||||
@@ -829,8 +830,8 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
|
||||
if title != "" {
|
||||
searchTitle := f.opt.Enc.FromStandardName(title)
|
||||
// Escaping the backslash isn't documented but seems to work
|
||||
searchTitle = strings.Replace(searchTitle, `\`, `\\`, -1)
|
||||
searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
|
||||
searchTitle = strings.ReplaceAll(searchTitle, `\`, `\\`)
|
||||
searchTitle = strings.ReplaceAll(searchTitle, `'`, `\'`)
|
||||
|
||||
var titleQuery bytes.Buffer
|
||||
_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
|
||||
@@ -1319,12 +1320,16 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
|
||||
}
|
||||
}
|
||||
}
|
||||
return &Object{
|
||||
o := &Object{
|
||||
baseObject: f.newBaseObject(remote, info),
|
||||
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
|
||||
md5sum: strings.ToLower(info.Md5Checksum),
|
||||
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
|
||||
}
|
||||
if info.ResourceKey != "" {
|
||||
o.resourceKey = &info.ResourceKey
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// newDocumentObject creates an fs.Object for a google docs drive.File
|
||||
@@ -2429,11 +2434,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
|
||||
var info *drive.File
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
info, err = f.svc.Files.Copy(id, createInfo).
|
||||
copy := f.svc.Files.Copy(id, createInfo).
|
||||
Fields(partialFields).
|
||||
SupportsAllDrives(true).
|
||||
KeepRevisionForever(f.opt.KeepRevisionForever).
|
||||
Context(ctx).Do()
|
||||
KeepRevisionForever(f.opt.KeepRevisionForever)
|
||||
srcObj.addResourceKey(copy.Header())
|
||||
info, err = copy.Context(ctx).Do()
|
||||
return f.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
@@ -3530,6 +3536,14 @@ func (o *baseObject) Storable() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// addResourceKey adds a X-Goog-Drive-Resource-Keys header for this
|
||||
// object if required.
|
||||
func (o *baseObject) addResourceKey(header http.Header) {
|
||||
if o.resourceKey != nil {
|
||||
header.Add("X-Goog-Drive-Resource-Keys", fmt.Sprintf("%s/%s", o.id, *o.resourceKey))
|
||||
}
|
||||
}
|
||||
|
||||
// httpResponse gets an http.Response object for the object
|
||||
// using the url and method passed in
|
||||
func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
|
||||
@@ -3545,6 +3559,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
|
||||
// Don't supply range requests for 0 length objects as they always fail
|
||||
delete(req.Header, "Range")
|
||||
}
|
||||
o.addResourceKey(req.Header)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
res, err = o.fs.client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
@@ -118,12 +118,12 @@ func (b *batcher) Batching() bool {
|
||||
}
|
||||
|
||||
// finishBatch commits the batch, returning a batch status to poll or maybe complete
|
||||
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (batchStatus *files.UploadSessionFinishBatchLaunch, err error) {
|
||||
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
|
||||
var arg = &files.UploadSessionFinishBatchArg{
|
||||
Entries: items,
|
||||
}
|
||||
err = b.f.pacer.Call(func() (bool, error) {
|
||||
batchStatus, err = b.f.srv.UploadSessionFinishBatch(arg)
|
||||
complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
|
||||
// If error is insufficient space then don't retry
|
||||
if e, ok := err.(files.UploadSessionFinishAPIError); ok {
|
||||
if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
|
||||
@@ -137,7 +137,7 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("batch commit failed: %w", err)
|
||||
}
|
||||
return batchStatus, nil
|
||||
return complete, nil
|
||||
}
|
||||
|
||||
// finishBatchJobStatus waits for the batch to complete returning completed entries
|
||||
@@ -199,26 +199,11 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
|
||||
fs.Debugf(b.f, "Committing %s", desc)
|
||||
|
||||
// finalise the batch getting either a result or a job id to poll
|
||||
batchStatus, err := b.finishBatch(ctx, items)
|
||||
complete, err := b.finishBatch(ctx, items)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// check whether batch is complete
|
||||
var complete *files.UploadSessionFinishBatchResult
|
||||
switch batchStatus.Tag {
|
||||
case "async_job_id":
|
||||
// wait for batch to complete
|
||||
complete, err = b.finishBatchJobStatus(ctx, batchStatus)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case "complete":
|
||||
complete = batchStatus.Complete
|
||||
default:
|
||||
return fmt.Errorf("batch returned unknown status %q", batchStatus.Tag)
|
||||
}
|
||||
|
||||
// Check we got the right number of entries
|
||||
entries := complete.Entries
|
||||
if len(entries) != len(results) {
|
||||
|
||||
@@ -1269,7 +1269,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
var total uint64
|
||||
if q.Allocation != nil {
|
||||
@@ -1370,10 +1370,12 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
|
||||
|
||||
if timeout < 30 {
|
||||
timeout = 30
|
||||
fs.Debugf(f, "Increasing poll interval to minimum 30s")
|
||||
}
|
||||
|
||||
if timeout > 480 {
|
||||
timeout = 480
|
||||
fs.Debugf(f, "Decreasing poll interval to maximum 480s")
|
||||
}
|
||||
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -1650,13 +1652,37 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
|
||||
}
|
||||
|
||||
chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
|
||||
skip := int64(0)
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
// seek to the start in case this is a retry
|
||||
if _, err = chunk.Seek(0, io.SeekStart); err != nil {
|
||||
return false, nil
|
||||
if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
|
||||
// after session is started, we retry everything
|
||||
if err != nil {
|
||||
// Check for incorrect offset error and retry with new offset
|
||||
if uErr, ok := err.(files.UploadSessionAppendV2APIError); ok {
|
||||
if uErr.EndpointError != nil && uErr.EndpointError.IncorrectOffset != nil {
|
||||
correctOffset := uErr.EndpointError.IncorrectOffset.CorrectOffset
|
||||
delta := int64(correctOffset) - int64(cursor.Offset)
|
||||
skip += delta
|
||||
what := fmt.Sprintf("incorrect offset error receved: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
|
||||
if skip < 0 {
|
||||
return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
|
||||
} else if skip == chunkSize {
|
||||
fs.Debugf(o, "%s: chunk received OK - continuing", what)
|
||||
return false, nil
|
||||
} else if skip > chunkSize {
|
||||
// This error should never happen
|
||||
return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
|
||||
}
|
||||
// Skip the sent data on next retry
|
||||
cursor.Offset = uint64(int64(cursor.Offset) + delta)
|
||||
fs.Debugf(o, "%s: skipping bytes on retry to fix offset", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
return err != nil, err
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1760,7 +1786,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
entry, err = o.uploadChunked(ctx, in, commitInfo, size)
|
||||
} else {
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
entry, err = o.fs.srv.Upload(commitInfo, in)
|
||||
entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
}
|
||||
|
||||
+10
-1
@@ -15,7 +15,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/ftp"
|
||||
"github.com/jlaffaye/ftp"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
@@ -100,6 +100,11 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
|
||||
Help: "Disable using MLSD even if server advertises support.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "disable_utf8",
|
||||
Help: "Disable using UTF-8 even if server advertises support.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "writing_mdtm",
|
||||
Help: "Use MDTM to set modification time (VsFtpd quirk)",
|
||||
@@ -184,6 +189,7 @@ type Options struct {
|
||||
SkipVerifyTLSCert bool `config:"no_check_certificate"`
|
||||
DisableEPSV bool `config:"disable_epsv"`
|
||||
DisableMLSD bool `config:"disable_mlsd"`
|
||||
DisableUTF8 bool `config:"disable_utf8"`
|
||||
WritingMDTM bool `config:"writing_mdtm"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
CloseTimeout fs.Duration `config:"close_timeout"`
|
||||
@@ -338,6 +344,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
|
||||
if f.opt.DisableMLSD {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
|
||||
}
|
||||
if f.opt.DisableUTF8 {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithDisabledUTF8(true))
|
||||
}
|
||||
if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -295,6 +296,30 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
|
||||
Value: "DURABLE_REDUCED_AVAILABILITY",
|
||||
Help: "Durable reduced availability storage class",
|
||||
}},
|
||||
}, {
|
||||
Name: "no_check_bucket",
|
||||
Help: `If set, don't attempt to check the bucket exists or create it.
|
||||
|
||||
This can be useful when trying to minimise the number of transactions
|
||||
rclone does if you know the bucket exists already.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "download_compressed",
|
||||
Help: `If set this will download compressed objects as-is.
|
||||
|
||||
It is possible to upload objects to GCS with "Content-Encoding: gzip"
|
||||
set. Normally rclone will transparently decompress these files on
|
||||
download. This means that rclone can't check the hash or the size of
|
||||
the file as both of these refer to the compressed object.
|
||||
|
||||
If this flag is set then rclone will download files with
|
||||
"Content-Encoding: gzip" as they are received. This means that rclone
|
||||
can check the size and hash but the file contents will be compressed.
|
||||
`,
|
||||
Advanced: true,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -317,21 +342,24 @@ type Options struct {
|
||||
BucketPolicyOnly bool `config:"bucket_policy_only"`
|
||||
Location string `config:"location"`
|
||||
StorageClass string `config:"storage_class"`
|
||||
NoCheckBucket bool `config:"no_check_bucket"`
|
||||
DownloadCompressed bool `config:"download_compressed"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote storage server
|
||||
type Fs struct {
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
svc *storage.Service // the connection to the storage server
|
||||
client *http.Client // authorized client
|
||||
rootBucket string // bucket part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
cache *bucket.Cache // cache of bucket status
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
name string // name of this remote
|
||||
root string // the path we are working on if any
|
||||
opt Options // parsed options
|
||||
features *fs.Features // optional features
|
||||
svc *storage.Service // the connection to the storage server
|
||||
client *http.Client // authorized client
|
||||
rootBucket string // bucket part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
cache *bucket.Cache // cache of bucket status
|
||||
pacer *fs.Pacer // To pace the API calls
|
||||
warnCompressed sync.Once // warn once about compressed files
|
||||
}
|
||||
|
||||
// Object describes a storage object
|
||||
@@ -345,6 +373,7 @@ type Object struct {
|
||||
bytes int64 // Bytes in the object
|
||||
modTime time.Time // Modified time of the object
|
||||
mimeType string
|
||||
gzipped bool // set if object has Content-Encoding: gzip
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -482,7 +511,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
|
||||
pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
|
||||
cache: bucket.NewCache(),
|
||||
}
|
||||
f.setRoot(root)
|
||||
@@ -840,6 +869,14 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// checkBucket creates the bucket if it doesn't exist unless NoCheckBucket is true
|
||||
func (f *Fs) checkBucket(ctx context.Context, bucket string) error {
|
||||
if f.opt.NoCheckBucket {
|
||||
return nil
|
||||
}
|
||||
return f.makeBucket(ctx, bucket)
|
||||
}
|
||||
|
||||
// Rmdir deletes the bucket if the fs is at the root
|
||||
//
|
||||
// Returns an error if it isn't empty: Error 409: The bucket you tried
|
||||
@@ -873,7 +910,7 @@ func (f *Fs) Precision() time.Duration {
|
||||
// If it isn't possible then return fs.ErrorCantCopy
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
dstBucket, dstPath := f.split(remote)
|
||||
err := f.makeBucket(ctx, dstBucket)
|
||||
err := f.checkBucket(ctx, dstBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -957,6 +994,7 @@ func (o *Object) setMetaData(info *storage.Object) {
|
||||
o.url = info.MediaLink
|
||||
o.bytes = int64(info.Size)
|
||||
o.mimeType = info.ContentType
|
||||
o.gzipped = info.ContentEncoding == "gzip"
|
||||
|
||||
// Read md5sum
|
||||
md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
|
||||
@@ -995,6 +1033,15 @@ func (o *Object) setMetaData(info *storage.Object) {
|
||||
} else {
|
||||
o.modTime = modTime
|
||||
}
|
||||
|
||||
// If gunzipping then size and md5sum are unknown
|
||||
if o.gzipped && !o.fs.opt.DownloadCompressed {
|
||||
o.bytes = -1
|
||||
o.md5sum = ""
|
||||
o.fs.warnCompressed.Do(func() {
|
||||
fs.Logf(o.fs, "Decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-download-compressed to override")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// readObjectInfo reads the definition for an object
|
||||
@@ -1095,6 +1142,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
return nil, err
|
||||
}
|
||||
fs.FixRangeOption(options, o.bytes)
|
||||
if o.gzipped && o.fs.opt.DownloadCompressed {
|
||||
// Allow files which are stored on the cloud storage system
|
||||
// compressed to be downloaded without being decompressed. Note
|
||||
// that setting this here overrides the automatic decompression
|
||||
// in the Transport.
|
||||
//
|
||||
// See: https://cloud.google.com/storage/docs/transcoding
|
||||
req.Header.Set("Accept-Encoding", "gzip")
|
||||
}
|
||||
fs.OpenOptionAddHTTPHeaders(req.Header, options)
|
||||
var res *http.Response
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
@@ -1123,7 +1179,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
// The new object may have been created if an error is returned
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
bucket, bucketPath := o.split()
|
||||
err := o.fs.makeBucket(ctx, bucket)
|
||||
err := o.fs.checkBucket(ctx, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -562,7 +562,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
|
||||
for i := range items {
|
||||
item := &result.MediaItems[i]
|
||||
remote := item.Filename
|
||||
remote = strings.Replace(remote, "/", "/", -1)
|
||||
remote = strings.ReplaceAll(remote, "/", "/")
|
||||
err = fn(remote, item, false)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -282,7 +282,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
if do := f.Fs.Features().CleanUp; do != nil {
|
||||
return do(ctx)
|
||||
}
|
||||
return errors.New("CleanUp not supported")
|
||||
return errors.New("not supported by underlying remote")
|
||||
}
|
||||
|
||||
// About gets quota information from the Fs
|
||||
@@ -290,7 +290,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
if do := f.Fs.Features().About; do != nil {
|
||||
return do(ctx)
|
||||
}
|
||||
return nil, errors.New("About not supported")
|
||||
return nil, errors.New("not supported by underlying remote")
|
||||
}
|
||||
|
||||
// ChangeNotify calls the passed function with a path that has had changes.
|
||||
|
||||
@@ -119,7 +119,7 @@ func (f *Fs) getCredentials(ctx context.Context) (err error) {
|
||||
defer fs.CheckClose(resp.Body, &err)
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
body, _ := ioutil.ReadAll(resp.Body)
|
||||
bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
|
||||
bodyStr := strings.TrimSpace(strings.ReplaceAll(string(body), "\n", " "))
|
||||
return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
|
||||
}
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,17 @@
|
||||
// Test internetarchive filesystem interface
|
||||
package internetarchive_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/internetarchive"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestIA:lesmi-rclone-test/",
|
||||
NilObject: (*internetarchive.Object)(nil),
|
||||
})
|
||||
}
|
||||
@@ -8,42 +8,69 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// default time format for almost all request and responses
|
||||
timeFormat = "2006-01-02-T15:04:05Z0700"
|
||||
// the API server seems to use a different format
|
||||
apiTimeFormat = "2006-01-02T15:04:05Z07:00"
|
||||
// default time format historically used for all request and responses.
|
||||
// Similar to time.RFC3339, but with an extra '-' in front of 'T',
|
||||
// and no ':' separator in timezone offset. Some newer endpoints have
|
||||
// moved to proper time.RFC3339 conformant format instead.
|
||||
jottaTimeFormat = "2006-01-02-T15:04:05Z0700"
|
||||
)
|
||||
|
||||
// Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
|
||||
type Time time.Time
|
||||
|
||||
// UnmarshalXML turns XML into a Time
|
||||
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
// unmarshalXML turns XML into a Time
|
||||
func unmarshalXMLTime(d *xml.Decoder, start xml.StartElement, timeFormat string) (time.Time, error) {
|
||||
var v string
|
||||
if err := d.DecodeElement(&v, &start); err != nil {
|
||||
return err
|
||||
return time.Time{}, err
|
||||
}
|
||||
if v == "" {
|
||||
*t = Time(time.Time{})
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
}
|
||||
newTime, err := time.Parse(timeFormat, v)
|
||||
if err == nil {
|
||||
*t = Time(newTime)
|
||||
return newTime, nil
|
||||
}
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
// JottaTime represents time values in the classic API using a custom RFC3339 like format
|
||||
type JottaTime time.Time
|
||||
|
||||
// String returns JottaTime string in Jottacloud classic format
|
||||
func (t JottaTime) String() string { return time.Time(t).Format(jottaTimeFormat) }
|
||||
|
||||
// UnmarshalXML turns XML into a JottaTime
|
||||
func (t *JottaTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
tm, err := unmarshalXMLTime(d, start, jottaTimeFormat)
|
||||
*t = JottaTime(tm)
|
||||
return err
|
||||
}
|
||||
|
||||
// MarshalXML turns a Time into XML
|
||||
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
// MarshalXML turns a JottaTime into XML
|
||||
func (t *JottaTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return e.EncodeElement(t.String(), start)
|
||||
}
|
||||
|
||||
// Return Time string in Jottacloud format
|
||||
func (t Time) String() string { return time.Time(t).Format(timeFormat) }
|
||||
// Rfc3339Time represents time values in the newer APIs using standard RFC3339 format
|
||||
type Rfc3339Time time.Time
|
||||
|
||||
// APIString returns Time string in Jottacloud API format
|
||||
func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
|
||||
// String returns Rfc3339Time string in Jottacloud RFC3339 format
|
||||
func (t Rfc3339Time) String() string { return time.Time(t).Format(time.RFC3339) }
|
||||
|
||||
// UnmarshalXML turns XML into a Rfc3339Time
|
||||
func (t *Rfc3339Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
tm, err := unmarshalXMLTime(d, start, time.RFC3339)
|
||||
*t = Rfc3339Time(tm)
|
||||
return err
|
||||
}
|
||||
|
||||
// MarshalXML turns a Rfc3339Time into XML
|
||||
func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
return e.EncodeElement(t.String(), start)
|
||||
}
|
||||
|
||||
// MarshalJSON turns a Rfc3339Time into JSON
|
||||
func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
|
||||
return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
|
||||
}
|
||||
|
||||
// LoginToken is struct representing the login token generated in the WebUI
|
||||
type LoginToken struct {
|
||||
@@ -122,16 +149,11 @@ type AllocateFileResponse struct {
|
||||
|
||||
// UploadResponse after an upload
|
||||
type UploadResponse struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Kind string `json:"kind"`
|
||||
ContentID string `json:"content_id"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Md5 string `json:"md5"`
|
||||
Created int64 `json:"created"`
|
||||
Modified int64 `json:"modified"`
|
||||
Deleted interface{} `json:"deleted"`
|
||||
Mime string `json:"mime"`
|
||||
Path string `json:"path"`
|
||||
ContentID string `json:"content_id"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Md5 string `json:"md5"`
|
||||
Modified int64 `json:"modified"`
|
||||
}
|
||||
|
||||
// DeviceRegistrationResponse is the response to registering a device
|
||||
@@ -338,9 +360,9 @@ type JottaFolder struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Deleted Flag `xml:"deleted,attr"`
|
||||
Path string `xml:"path"`
|
||||
CreatedAt Time `xml:"created"`
|
||||
ModifiedAt Time `xml:"modified"`
|
||||
Updated Time `xml:"updated"`
|
||||
CreatedAt JottaTime `xml:"created"`
|
||||
ModifiedAt JottaTime `xml:"modified"`
|
||||
Updated JottaTime `xml:"updated"`
|
||||
Folders []JottaFolder `xml:"folders>folder"`
|
||||
Files []JottaFile `xml:"files>file"`
|
||||
}
|
||||
@@ -365,17 +387,17 @@ GET http://www.jottacloud.com/JFS/<account>/<device>/<mountpoint>/.../<file>
|
||||
// JottaFile represents a Jottacloud file
|
||||
type JottaFile struct {
|
||||
XMLName xml.Name
|
||||
Name string `xml:"name,attr"`
|
||||
Deleted Flag `xml:"deleted,attr"`
|
||||
PublicURI string `xml:"publicURI"`
|
||||
PublicSharePath string `xml:"publicSharePath"`
|
||||
State string `xml:"currentRevision>state"`
|
||||
CreatedAt Time `xml:"currentRevision>created"`
|
||||
ModifiedAt Time `xml:"currentRevision>modified"`
|
||||
Updated Time `xml:"currentRevision>updated"`
|
||||
Size int64 `xml:"currentRevision>size"`
|
||||
MimeType string `xml:"currentRevision>mime"`
|
||||
MD5 string `xml:"currentRevision>md5"`
|
||||
Name string `xml:"name,attr"`
|
||||
Deleted Flag `xml:"deleted,attr"`
|
||||
PublicURI string `xml:"publicURI"`
|
||||
PublicSharePath string `xml:"publicSharePath"`
|
||||
State string `xml:"currentRevision>state"`
|
||||
CreatedAt JottaTime `xml:"currentRevision>created"`
|
||||
ModifiedAt JottaTime `xml:"currentRevision>modified"`
|
||||
Updated JottaTime `xml:"currentRevision>updated"`
|
||||
Size int64 `xml:"currentRevision>size"`
|
||||
MimeType string `xml:"currentRevision>mime"`
|
||||
MD5 string `xml:"currentRevision>md5"`
|
||||
}
|
||||
|
||||
// Error is a custom Error for wrapping Jottacloud error responses
|
||||
|
||||
@@ -191,7 +191,7 @@ machines.`)
|
||||
m.Set("auth_code", "")
|
||||
return fs.ConfigGoto("legacy_do_auth")
|
||||
case "legacy_auth_code":
|
||||
authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
|
||||
authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
|
||||
m.Set("auth_code", authCode)
|
||||
return fs.ConfigGoto("legacy_do_auth")
|
||||
case "legacy_do_auth":
|
||||
@@ -519,7 +519,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
|
||||
values.Set("client_id", defaultClientID)
|
||||
values.Set("grant_type", "password")
|
||||
values.Set("password", loginToken.AuthToken)
|
||||
values.Set("scope", "offline_access+openid")
|
||||
values.Set("scope", "openid offline_access")
|
||||
values.Set("username", loginToken.Username)
|
||||
values.Encode()
|
||||
opts = rest.Opts{
|
||||
@@ -649,7 +649,7 @@ func errorHandler(resp *http.Response) error {
|
||||
|
||||
// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
|
||||
func urlPathEscape(in string) string {
|
||||
return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
|
||||
return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
|
||||
}
|
||||
|
||||
// filePathRaw returns an unescaped file path (f.root, file)
|
||||
@@ -932,26 +932,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
type listStreamTime time.Time
|
||||
|
||||
func (c *listStreamTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||
var v string
|
||||
if err := d.DecodeElement(&v, &start); err != nil {
|
||||
return err
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*c = listStreamTime(t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c listStreamTime) MarshalJSON() ([]byte, error) {
|
||||
return []byte(fmt.Sprintf("\"%s\"", time.Time(c).Format(time.RFC3339))), nil
|
||||
}
|
||||
|
||||
func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, filesystem *Fs, callback func(fs.DirEntry) error) error {
|
||||
func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
|
||||
|
||||
type stats struct {
|
||||
Folders int `xml:"folders"`
|
||||
@@ -960,12 +941,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
|
||||
var expected, actual stats
|
||||
|
||||
type xmlFile struct {
|
||||
Path string `xml:"path"`
|
||||
Name string `xml:"filename"`
|
||||
Checksum string `xml:"md5"`
|
||||
Size int64 `xml:"size"`
|
||||
Modified listStreamTime `xml:"modified"`
|
||||
Created listStreamTime `xml:"created"`
|
||||
Path string `xml:"path"`
|
||||
Name string `xml:"filename"`
|
||||
Checksum string `xml:"md5"`
|
||||
Size int64 `xml:"size"`
|
||||
Modified api.Rfc3339Time `xml:"modified"` // Note: Liststream response includes 3 decimal milliseconds, but we ignore them since there is second precision everywhere else
|
||||
Created api.Rfc3339Time `xml:"created"`
|
||||
}
|
||||
|
||||
type xmlFolder struct {
|
||||
@@ -987,8 +968,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
|
||||
})
|
||||
}
|
||||
|
||||
// liststream paths are /mountpoint/root/path
|
||||
// so the returned paths should have /mountpoint/root/ trimmed
|
||||
// as the caller is expecting path.
|
||||
pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
|
||||
trimPathPrefix := func(p string) string {
|
||||
p = strings.TrimPrefix(p, trimPrefix)
|
||||
p = strings.TrimPrefix(p, pathPrefix)
|
||||
p = strings.TrimPrefix(p, "/")
|
||||
return p
|
||||
}
|
||||
@@ -1071,11 +1056,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
return shouldRetry(ctx, resp, err)
|
||||
}
|
||||
|
||||
// liststream paths are /mountpoint/root/path
|
||||
// so the returned paths should have /mountpoint/root/ trimmed
|
||||
// as the caller is expecting path.
|
||||
trimPrefix := path.Join("/", f.opt.Mountpoint, f.root)
|
||||
err = parseListRStream(ctx, resp.Body, trimPrefix, f, func(d fs.DirEntry) error {
|
||||
err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
|
||||
if d.Remote() == dir {
|
||||
return nil
|
||||
}
|
||||
@@ -1210,6 +1191,45 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
return f.purgeCheck(ctx, dir, false)
|
||||
}
|
||||
|
||||
// createOrUpdate tries to make remote file match without uploading.
|
||||
// If the remote file exists, and has matching size and md5, only
|
||||
// timestamps are updated. If the file does not exist or does does
|
||||
// not match size and md5, but matching content can be constructed
|
||||
// from deduplication, the file will be updated/created. If the file
|
||||
// is currently in trash, but can be made to match, it will be
|
||||
// restored. Returns ErrorObjectNotFound if upload will be necessary
|
||||
// to get a matching remote file.
|
||||
func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: f.filePath(file),
|
||||
Parameters: url.Values{},
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
|
||||
opts.Parameters.Set("cphash", "true")
|
||||
|
||||
fileDate := api.JottaTime(modTime).String()
|
||||
opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
|
||||
opts.ExtraHeaders["JMd5"] = md5
|
||||
opts.ExtraHeaders["JCreated"] = fileDate
|
||||
opts.ExtraHeaders["JModified"] = fileDate
|
||||
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
// does not exist, i.e. not matching size and md5, and not possible to make it by deduplication
|
||||
if apiErr.StatusCode == http.StatusNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// copyOrMoves copies or moves directories or files depending on the method parameter
|
||||
func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
|
||||
opts := rest.Opts{
|
||||
@@ -1253,6 +1273,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
|
||||
|
||||
// if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?)
|
||||
if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
|
||||
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
|
||||
info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't copy file: %w", err)
|
||||
}
|
||||
@@ -1554,40 +1580,19 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// prepare allocate request with existing metadata but changed timestamps
|
||||
var resp *http.Response
|
||||
var options []fs.OpenOption
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "files/v1/allocate",
|
||||
Options: options,
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
fileDate := api.Time(modTime).APIString()
|
||||
var request = api.AllocateFileRequest{
|
||||
Bytes: o.size,
|
||||
Created: fileDate,
|
||||
Modified: fileDate,
|
||||
Md5: o.md5,
|
||||
Path: path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
|
||||
}
|
||||
|
||||
// send it
|
||||
var response api.AllocateFileResponse
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
// request check/update with existing metadata and new modtime
|
||||
// (note that if size/md5 does not match, the file content will
|
||||
// also be modified if deduplication is possible, i.e. it is
|
||||
// important to use correct/latest values)
|
||||
_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
|
||||
if err != nil {
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
|
||||
return errors.New("metadata did not match")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// check response
|
||||
if response.State != "COMPLETED" {
|
||||
// could be the file was modified (size/md5 changed) between readMetaData and the allocate request
|
||||
return errors.New("metadata did not match")
|
||||
}
|
||||
|
||||
// update local metadata
|
||||
o.modTime = modTime
|
||||
return nil
|
||||
@@ -1725,7 +1730,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
Options: options,
|
||||
ExtraHeaders: make(map[string]string),
|
||||
}
|
||||
fileDate := api.Time(src.ModTime(ctx)).APIString()
|
||||
fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
|
||||
|
||||
// the allocate request
|
||||
var request = api.AllocateFileRequest{
|
||||
|
||||
@@ -65,7 +65,7 @@ HTTP is provided primarily for debugging purposes.`,
|
||||
Name: "host",
|
||||
Help: `Domain+path of NetStorage host to connect to.
|
||||
|
||||
Format should be <domain>/<internal folders>`,
|
||||
Format should be ` + "`<domain>/<internal folders>`",
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "account",
|
||||
@@ -94,7 +94,7 @@ files stored in any sub-directories that may exist.`,
|
||||
Long: `The desired path location (including applicable sub-directories) ending in
|
||||
the object that will be the target of the symlink (for example, /links/mylink).
|
||||
Include the file extension for the object, if applicable.
|
||||
rclone backend symlink <src> <path>`,
|
||||
` + "`rclone backend symlink <src> <path>`",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -301,6 +301,10 @@ type siteResource struct {
|
||||
type siteResponse struct {
|
||||
Sites []siteResource `json:"value"`
|
||||
}
|
||||
type deltaResponse struct {
|
||||
DeltaLink string `json:"@odata.deltaLink"`
|
||||
Value []api.Item `json:"value"`
|
||||
}
|
||||
|
||||
// Get the region and graphURL from the config
|
||||
func getRegionURL(m configmap.Mapper) (region, graphURL string) {
|
||||
@@ -1502,7 +1506,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
q := drive.Quota
|
||||
// On (some?) Onedrive sharepoints these are all 0 so return unknown in that case
|
||||
@@ -2302,6 +2306,142 @@ func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
|
||||
return canonicalDriveID
|
||||
}
|
||||
|
||||
// ChangeNotify calls the passed function with a path that has had changes.
|
||||
// If the implementation uses polling, it should adhere to the given interval.
|
||||
//
|
||||
// Automatically restarts itself in case of unexpected behavior of the remote.
|
||||
//
|
||||
// Close the returned channel to stop being notified.
|
||||
//
|
||||
// The Onedrive implementation gives the whole hierarchy up to the top when
|
||||
// an object is changed. For instance, if a/b/c is changed, this function
|
||||
// will call notifyFunc with a, a/b and a/b/c.
|
||||
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
|
||||
go func() {
|
||||
// get the StartPageToken early so all changes from now on get processed
|
||||
nextDeltaToken, err := f.changeNotifyStartPageToken(ctx)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get first deltaLink: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
fs.Debugf(f, "Next delta token is: %s", nextDeltaToken)
|
||||
|
||||
var ticker *time.Ticker
|
||||
var tickerC <-chan time.Time
|
||||
for {
|
||||
select {
|
||||
case pollInterval, ok := <-pollIntervalChan:
|
||||
if !ok {
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
}
|
||||
return
|
||||
}
|
||||
if ticker != nil {
|
||||
ticker.Stop()
|
||||
ticker, tickerC = nil, nil
|
||||
}
|
||||
if pollInterval != 0 {
|
||||
ticker = time.NewTicker(pollInterval)
|
||||
tickerC = ticker.C
|
||||
}
|
||||
case <-tickerC:
|
||||
fs.Debugf(f, "Checking for changes on remote")
|
||||
nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
|
||||
if err != nil {
|
||||
fs.Infof(f, "Change notify listener failure: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, "latest")
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
nextDeltaToken = parsedURL.Query().Get("token")
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta deltaResponse, err error) {
|
||||
opts := f.buildDriveDeltaOpts(token)
|
||||
|
||||
_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
|
||||
rootURL := graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
|
||||
|
||||
return rest.Opts{
|
||||
Method: "GET",
|
||||
RootURL: rootURL,
|
||||
Path: "/" + f.driveID + "/root/delta",
|
||||
Parameters: map[string][]string{"token": {token}},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
|
||||
delta, err := f.changeNotifyNextChange(ctx, deltaToken)
|
||||
parsedURL, err := url.Parse(delta.DeltaLink)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
nextDeltaToken = parsedURL.Query().Get("token")
|
||||
|
||||
for _, item := range delta.Value {
|
||||
isDriveRootFolder := item.GetParentReference().ID == ""
|
||||
if isDriveRootFolder {
|
||||
continue
|
||||
}
|
||||
|
||||
fullPath, err := getItemFullPath(&item)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Could not get item full path: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if fullPath == f.root {
|
||||
continue
|
||||
}
|
||||
|
||||
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
|
||||
if !insideRoot {
|
||||
continue
|
||||
}
|
||||
|
||||
if item.GetFile() != nil {
|
||||
notifyFunc(relName, fs.EntryObject)
|
||||
} else if item.GetFolder() != nil {
|
||||
notifyFunc(relName, fs.EntryDirectory)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func getItemFullPath(item *api.Item) (fullPath string, err error) {
|
||||
err = nil
|
||||
fullPath = item.GetName()
|
||||
if parent := item.GetParentReference(); parent != nil && parent.Path != "" {
|
||||
pathParts := strings.SplitN(parent.Path, ":", 2)
|
||||
if len(pathParts) != 2 {
|
||||
err = fmt.Errorf("invalid parent path: %s", parent.Path)
|
||||
return
|
||||
}
|
||||
|
||||
if pathParts[1] != "" {
|
||||
fullPath = strings.TrimPrefix(pathParts[1], "/") + "/" + fullPath
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// getRelativePathInsideBase checks if `target` is inside `base`. If so, it
|
||||
// returns a relative path for `target` based on `base` and a boolean `true`.
|
||||
// Otherwise returns "", false.
|
||||
|
||||
@@ -906,7 +906,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
usage = &fs.Usage{
|
||||
Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used
|
||||
|
||||
@@ -783,10 +783,10 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("CreateDir http: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
if err = info.AsErr(); err != nil {
|
||||
return nil, fmt.Errorf("CreateDir: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
usage = &fs.Usage{
|
||||
Used: fs.NewUsageValue(int64(info.SpaceUsed)),
|
||||
|
||||
+22
-5
@@ -4,16 +4,21 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/putdotio/go-putio/putio"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
)
|
||||
|
||||
func checkStatusCode(resp *http.Response, expected int) error {
|
||||
if resp.StatusCode != expected {
|
||||
return &statusCodeError{response: resp}
|
||||
func checkStatusCode(resp *http.Response, expected ...int) error {
|
||||
for _, code := range expected {
|
||||
if resp.StatusCode == code {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return &statusCodeError{response: resp}
|
||||
}
|
||||
|
||||
type statusCodeError struct {
|
||||
@@ -24,8 +29,10 @@ func (e *statusCodeError) Error() string {
|
||||
return fmt.Sprintf("unexpected status code (%d) response while doing %s to %s", e.response.StatusCode, e.response.Request.Method, e.response.Request.URL.String())
|
||||
}
|
||||
|
||||
// This method is called from fserrors.ShouldRetry() to determine if an error should be retried.
|
||||
// Some errors (e.g. 429 Too Many Requests) are handled before this step, so they are not included here.
|
||||
func (e *statusCodeError) Temporary() bool {
|
||||
return e.response.StatusCode == 429 || e.response.StatusCode >= 500
|
||||
return e.response.StatusCode >= 500
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err deserves to be
|
||||
@@ -40,6 +47,16 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if perr, ok := err.(*putio.ErrorResponse); ok {
|
||||
err = &statusCodeError{response: perr.Response}
|
||||
}
|
||||
if scerr, ok := err.(*statusCodeError); ok && scerr.response.StatusCode == 429 {
|
||||
delay := defaultRateLimitSleep
|
||||
header := scerr.response.Header.Get("x-ratelimit-reset")
|
||||
if header != "" {
|
||||
if resetTime, cerr := strconv.ParseInt(header, 10, 64); cerr == nil {
|
||||
delay = time.Until(time.Unix(resetTime+1, 0))
|
||||
}
|
||||
}
|
||||
return true, pacer.RetryAfterError(scerr, delay)
|
||||
}
|
||||
if fserrors.ShouldRetry(err) {
|
||||
return true, err
|
||||
}
|
||||
|
||||
+3
-3
@@ -302,8 +302,8 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if resp.StatusCode != 201 {
|
||||
return false, fmt.Errorf("unexpected status code from upload create: %d", resp.StatusCode)
|
||||
if err := checkStatusCode(resp, 201); err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
location = resp.Header.Get("location")
|
||||
if location == "" {
|
||||
@@ -647,7 +647,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
return shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return &fs.Usage{
|
||||
Total: fs.NewUsageValue(ai.Disk.Size), // quota of bytes that can be used
|
||||
|
||||
@@ -241,7 +241,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
// fs.Debugf(o, "opening file: id=%d", o.file.ID)
|
||||
resp, err = o.fs.httpClient.Do(req)
|
||||
return shouldRetry(ctx, err)
|
||||
if err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
if err := checkStatusCode(resp, 200, 206); err != nil {
|
||||
return shouldRetry(ctx, err)
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 {
|
||||
_ = resp.Body.Close()
|
||||
|
||||
@@ -33,8 +33,9 @@ const (
|
||||
rcloneObscuredClientSecret = "cMwrjWVmrHZp3gf1ZpCrlyGAmPpB-YY5BbVnO1fj-G9evcd8"
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
decayConstant = 1 // bigger for slower decay, exponential
|
||||
defaultChunkSize = 48 * fs.Mebi
|
||||
defaultRateLimitSleep = 60 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
+483
-99
@@ -32,9 +32,11 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/endpoints"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/ncw/swift/v2"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
@@ -58,7 +60,7 @@ import (
|
||||
func init() {
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "s3",
|
||||
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS",
|
||||
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Options: []fs.Option{{
|
||||
@@ -75,6 +77,15 @@ func init() {
|
||||
}, {
|
||||
Value: "Ceph",
|
||||
Help: "Ceph Object Storage",
|
||||
}, {
|
||||
Value: "ChinaMobile",
|
||||
Help: "China Mobile Ecloud Elastic Object Storage (EOS)",
|
||||
}, {
|
||||
Value: "Cloudflare",
|
||||
Help: "Cloudflare R2 Storage",
|
||||
}, {
|
||||
Value: "ArvanCloud",
|
||||
Help: "Arvan Cloud Object Storage (AOS)",
|
||||
}, {
|
||||
Value: "DigitalOcean",
|
||||
Help: "Digital Ocean Spaces",
|
||||
@@ -290,11 +301,22 @@ func init() {
|
||||
}, {
|
||||
Value: "fr-par",
|
||||
Help: "Paris, France",
|
||||
}, {
|
||||
Value: "pl-waw",
|
||||
Help: "Warsaw, Poland",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.",
|
||||
Provider: "Cloudflare",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "auto",
|
||||
Help: "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
|
||||
Provider: "!AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS",
|
||||
Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
|
||||
@@ -306,6 +328,114 @@ func init() {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\n\nLeave blank if using AWS to use the default endpoint for the region.",
|
||||
Provider: "AWS",
|
||||
}, {
|
||||
// ChinaMobile endpoints: https://ecloud.10086.cn/op-help-center/doc/article/24534
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.",
|
||||
Provider: "ChinaMobile",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "eos-wuxi-1.cmecloud.cn",
|
||||
Help: "The default endpoint - a good choice if you are unsure.\nEast China (Suzhou)",
|
||||
}, {
|
||||
Value: "eos-jinan-1.cmecloud.cn",
|
||||
Help: "East China (Jinan)",
|
||||
}, {
|
||||
Value: "eos-ningbo-1.cmecloud.cn",
|
||||
Help: "East China (Hangzhou)",
|
||||
}, {
|
||||
Value: "eos-shanghai-1.cmecloud.cn",
|
||||
Help: "East China (Shanghai-1)",
|
||||
}, {
|
||||
Value: "eos-zhengzhou-1.cmecloud.cn",
|
||||
Help: "Central China (Zhengzhou)",
|
||||
}, {
|
||||
Value: "eos-hunan-1.cmecloud.cn",
|
||||
Help: "Central China (Changsha-1)",
|
||||
}, {
|
||||
Value: "eos-zhuzhou-1.cmecloud.cn",
|
||||
Help: "Central China (Changsha-2)",
|
||||
}, {
|
||||
Value: "eos-guangzhou-1.cmecloud.cn",
|
||||
Help: "South China (Guangzhou-2)",
|
||||
}, {
|
||||
Value: "eos-dongguan-1.cmecloud.cn",
|
||||
Help: "South China (Guangzhou-3)",
|
||||
}, {
|
||||
Value: "eos-beijing-1.cmecloud.cn",
|
||||
Help: "North China (Beijing-1)",
|
||||
}, {
|
||||
Value: "eos-beijing-2.cmecloud.cn",
|
||||
Help: "North China (Beijing-2)",
|
||||
}, {
|
||||
Value: "eos-beijing-4.cmecloud.cn",
|
||||
Help: "North China (Beijing-3)",
|
||||
}, {
|
||||
Value: "eos-huhehaote-1.cmecloud.cn",
|
||||
Help: "North China (Huhehaote)",
|
||||
}, {
|
||||
Value: "eos-chengdu-1.cmecloud.cn",
|
||||
Help: "Southwest China (Chengdu)",
|
||||
}, {
|
||||
Value: "eos-chongqing-1.cmecloud.cn",
|
||||
Help: "Southwest China (Chongqing)",
|
||||
}, {
|
||||
Value: "eos-guiyang-1.cmecloud.cn",
|
||||
Help: "Southwest China (Guiyang)",
|
||||
}, {
|
||||
Value: "eos-xian-1.cmecloud.cn",
|
||||
Help: "Nouthwest China (Xian)",
|
||||
}, {
|
||||
Value: "eos-yunnan.cmecloud.cn",
|
||||
Help: "Yunnan China (Kunming)",
|
||||
}, {
|
||||
Value: "eos-yunnan-2.cmecloud.cn",
|
||||
Help: "Yunnan China (Kunming-2)",
|
||||
}, {
|
||||
Value: "eos-tianjin-1.cmecloud.cn",
|
||||
Help: "Tianjin China (Tianjin)",
|
||||
}, {
|
||||
Value: "eos-jilin-1.cmecloud.cn",
|
||||
Help: "Jilin China (Changchun)",
|
||||
}, {
|
||||
Value: "eos-hubei-1.cmecloud.cn",
|
||||
Help: "Hubei China (Xiangyan)",
|
||||
}, {
|
||||
Value: "eos-jiangxi-1.cmecloud.cn",
|
||||
Help: "Jiangxi China (Nanchang)",
|
||||
}, {
|
||||
Value: "eos-gansu-1.cmecloud.cn",
|
||||
Help: "Gansu China (Lanzhou)",
|
||||
}, {
|
||||
Value: "eos-shanxi-1.cmecloud.cn",
|
||||
Help: "Shanxi China (Taiyuan)",
|
||||
}, {
|
||||
Value: "eos-liaoning-1.cmecloud.cn",
|
||||
Help: "Liaoning China (Shenyang)",
|
||||
}, {
|
||||
Value: "eos-hebei-1.cmecloud.cn",
|
||||
Help: "Hebei China (Shijiazhuang)",
|
||||
}, {
|
||||
Value: "eos-fujian-1.cmecloud.cn",
|
||||
Help: "Fujian China (Xiamen)",
|
||||
}, {
|
||||
Value: "eos-guangxi-1.cmecloud.cn",
|
||||
Help: "Guangxi China (Nanning)",
|
||||
}, {
|
||||
Value: "eos-anhui-1.cmecloud.cn",
|
||||
Help: "Anhui China (Huainan)",
|
||||
}},
|
||||
}, {
|
||||
// ArvanCloud endpoints: https://www.arvancloud.com/en/products/cloud-storage
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for Arvan Cloud Object Storage (AOS) API.",
|
||||
Provider: "ArvanCloud",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "s3.ir-thr-at1.arvanstorage.com",
|
||||
Help: "The default endpoint - a good choice if you are unsure.\nTehran Iran (Asiatech)",
|
||||
}, {
|
||||
Value: "s3.ir-tbz-sh1.arvanstorage.com",
|
||||
Help: "Tabriz Iran (Shahriar)",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for IBM COS S3 API.\n\nSpecify if using an IBM COS On Premise.",
|
||||
@@ -588,6 +718,9 @@ func init() {
|
||||
}, {
|
||||
Value: "s3.fr-par.scw.cloud",
|
||||
Help: "Paris Endpoint",
|
||||
}, {
|
||||
Value: "s3.pl-waw.scw.cloud",
|
||||
Help: "Warsaw Endpoint",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
@@ -746,7 +879,7 @@ func init() {
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
|
||||
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp",
|
||||
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "objects-us-east-1.dream.io",
|
||||
Help: "Dream Objects endpoint",
|
||||
@@ -799,6 +932,10 @@ func init() {
|
||||
Value: "s3.ap-northeast-2.wasabisys.com",
|
||||
Help: "Wasabi AP Northeast 2 (Osaka) endpoint",
|
||||
Provider: "Wasabi",
|
||||
}, {
|
||||
Value: "s3.ir-thr-at1.arvanstorage.com",
|
||||
Help: "ArvanCloud Tehran Iran (Asiatech) endpoint",
|
||||
Provider: "ArvanCloud",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
@@ -880,6 +1017,112 @@ func init() {
|
||||
Value: "us-gov-west-1",
|
||||
Help: "AWS GovCloud (US) Region",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must match endpoint.\n\nUsed when creating buckets only.",
|
||||
Provider: "ChinaMobile",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "wuxi1",
|
||||
Help: "East China (Suzhou)",
|
||||
}, {
|
||||
Value: "jinan1",
|
||||
Help: "East China (Jinan)",
|
||||
}, {
|
||||
Value: "ningbo1",
|
||||
Help: "East China (Hangzhou)",
|
||||
}, {
|
||||
Value: "shanghai1",
|
||||
Help: "East China (Shanghai-1)",
|
||||
}, {
|
||||
Value: "zhengzhou1",
|
||||
Help: "Central China (Zhengzhou)",
|
||||
}, {
|
||||
Value: "hunan1",
|
||||
Help: "Central China (Changsha-1)",
|
||||
}, {
|
||||
Value: "zhuzhou1",
|
||||
Help: "Central China (Changsha-2)",
|
||||
}, {
|
||||
Value: "guangzhou1",
|
||||
Help: "South China (Guangzhou-2)",
|
||||
}, {
|
||||
Value: "dongguan1",
|
||||
Help: "South China (Guangzhou-3)",
|
||||
}, {
|
||||
Value: "beijing1",
|
||||
Help: "North China (Beijing-1)",
|
||||
}, {
|
||||
Value: "beijing2",
|
||||
Help: "North China (Beijing-2)",
|
||||
}, {
|
||||
Value: "beijing4",
|
||||
Help: "North China (Beijing-3)",
|
||||
}, {
|
||||
Value: "huhehaote1",
|
||||
Help: "North China (Huhehaote)",
|
||||
}, {
|
||||
Value: "chengdu1",
|
||||
Help: "Southwest China (Chengdu)",
|
||||
}, {
|
||||
Value: "chongqing1",
|
||||
Help: "Southwest China (Chongqing)",
|
||||
}, {
|
||||
Value: "guiyang1",
|
||||
Help: "Southwest China (Guiyang)",
|
||||
}, {
|
||||
Value: "xian1",
|
||||
Help: "Nouthwest China (Xian)",
|
||||
}, {
|
||||
Value: "yunnan",
|
||||
Help: "Yunnan China (Kunming)",
|
||||
}, {
|
||||
Value: "yunnan2",
|
||||
Help: "Yunnan China (Kunming-2)",
|
||||
}, {
|
||||
Value: "tianjin1",
|
||||
Help: "Tianjin China (Tianjin)",
|
||||
}, {
|
||||
Value: "jilin1",
|
||||
Help: "Jilin China (Changchun)",
|
||||
}, {
|
||||
Value: "hubei1",
|
||||
Help: "Hubei China (Xiangyan)",
|
||||
}, {
|
||||
Value: "jiangxi1",
|
||||
Help: "Jiangxi China (Nanchang)",
|
||||
}, {
|
||||
Value: "gansu1",
|
||||
Help: "Gansu China (Lanzhou)",
|
||||
}, {
|
||||
Value: "shanxi1",
|
||||
Help: "Shanxi China (Taiyuan)",
|
||||
}, {
|
||||
Value: "liaoning1",
|
||||
Help: "Liaoning China (Shenyang)",
|
||||
}, {
|
||||
Value: "hebei1",
|
||||
Help: "Hebei China (Shijiazhuang)",
|
||||
}, {
|
||||
Value: "fujian1",
|
||||
Help: "Fujian China (Xiamen)",
|
||||
}, {
|
||||
Value: "guangxi1",
|
||||
Help: "Guangxi China (Nanning)",
|
||||
}, {
|
||||
Value: "anhui1",
|
||||
Help: "Anhui China (Huainan)",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must match endpoint.\n\nUsed when creating buckets only.",
|
||||
Provider: "ArvanCloud",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "ir-thr-at1",
|
||||
Help: "Tehran Iran (Asiatech)",
|
||||
}, {
|
||||
Value: "ir-tbz-sh1",
|
||||
Help: "Tabriz Iran (Shahriar)",
|
||||
}},
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must match endpoint when using IBM Cloud Public.\n\nFor on-prem COS, do not make a selection from this list, hit enter.",
|
||||
@@ -1046,7 +1289,7 @@ func init() {
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
|
||||
}, {
|
||||
Name: "acl",
|
||||
Help: `Canned ACL used when creating buckets and storing or copying objects.
|
||||
@@ -1057,7 +1300,7 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
|
||||
|
||||
Note that this ACL is applied when server-side copying objects as S3
|
||||
doesn't copy the ACL from the source but rather writes a fresh one.`,
|
||||
Provider: "!Storj",
|
||||
Provider: "!Storj,Cloudflare",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "default",
|
||||
Help: "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
|
||||
@@ -1081,11 +1324,11 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
|
||||
}, {
|
||||
Value: "bucket-owner-read",
|
||||
Help: "Object owner gets FULL_CONTROL.\nBucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.",
|
||||
Provider: "!IBMCOS",
|
||||
Provider: "!IBMCOS,ChinaMobile",
|
||||
}, {
|
||||
Value: "bucket-owner-full-control",
|
||||
Help: "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.",
|
||||
Provider: "!IBMCOS",
|
||||
Provider: "!IBMCOS,ChinaMobile",
|
||||
}, {
|
||||
Value: "private",
|
||||
Help: "Owner gets FULL_CONTROL.\nNo one else has access rights (default).\nThis acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS.",
|
||||
@@ -1134,7 +1377,7 @@ isn't set then "acl" is used instead.`,
|
||||
}, {
|
||||
Name: "server_side_encryption",
|
||||
Help: "The server-side encryption algorithm used when storing this object in S3.",
|
||||
Provider: "AWS,Ceph,Minio",
|
||||
Provider: "AWS,Ceph,ChinaMobile,Minio",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "None",
|
||||
@@ -1142,13 +1385,14 @@ isn't set then "acl" is used instead.`,
|
||||
Value: "AES256",
|
||||
Help: "AES256",
|
||||
}, {
|
||||
Value: "aws:kms",
|
||||
Help: "aws:kms",
|
||||
Value: "aws:kms",
|
||||
Help: "aws:kms",
|
||||
Provider: "!ChinaMobile",
|
||||
}},
|
||||
}, {
|
||||
Name: "sse_customer_algorithm",
|
||||
Help: "If using SSE-C, the server-side encryption algorithm used when storing this object in S3.",
|
||||
Provider: "AWS,Ceph,Minio",
|
||||
Provider: "AWS,Ceph,ChinaMobile,Minio",
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
@@ -1171,7 +1415,7 @@ isn't set then "acl" is used instead.`,
|
||||
}, {
|
||||
Name: "sse_customer_key",
|
||||
Help: "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
|
||||
Provider: "AWS,Ceph,Minio",
|
||||
Provider: "AWS,Ceph,ChinaMobile,Minio",
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
@@ -1183,7 +1427,7 @@ isn't set then "acl" is used instead.`,
|
||||
|
||||
If you leave it blank, this is calculated automatically from the sse_customer_key provided.
|
||||
`,
|
||||
Provider: "AWS,Ceph,Minio",
|
||||
Provider: "AWS,Ceph,ChinaMobile,Minio",
|
||||
Advanced: true,
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
@@ -1239,6 +1483,33 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
|
||||
Value: "STANDARD_IA",
|
||||
Help: "Infrequent access storage mode",
|
||||
}},
|
||||
}, {
|
||||
// Mapping from here: https://ecloud.10086.cn/op-help-center/doc/article/24495
|
||||
Name: "storage_class",
|
||||
Help: "The storage class to use when storing new objects in ChinaMobile.",
|
||||
Provider: "ChinaMobile",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Default",
|
||||
}, {
|
||||
Value: "STANDARD",
|
||||
Help: "Standard storage class",
|
||||
}, {
|
||||
Value: "GLACIER",
|
||||
Help: "Archive storage mode",
|
||||
}, {
|
||||
Value: "STANDARD_IA",
|
||||
Help: "Infrequent access storage mode",
|
||||
}},
|
||||
}, {
|
||||
// Mapping from here: https://www.arvancloud.com/en/products/cloud-storage
|
||||
Name: "storage_class",
|
||||
Help: "The storage class to use when storing new objects in ArvanCloud.",
|
||||
Provider: "ArvanCloud",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "STANDARD",
|
||||
Help: "Standard storage class",
|
||||
}},
|
||||
}, {
|
||||
// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
|
||||
Name: "storage_class",
|
||||
@@ -1571,6 +1842,20 @@ This should be true, false or left unset to use the default for the provider.
|
||||
`,
|
||||
Default: fs.Tristate{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "use_presigned_request",
|
||||
Help: `Whether to use a presigned request or PutObject for single part uploads
|
||||
|
||||
If this is false rclone will use PutObject from the AWS SDK to upload
|
||||
an object.
|
||||
|
||||
Versions of rclone < 1.59 use presigned requests to upload a single
|
||||
part object and setting this flag to true will re-enable that
|
||||
functionality. This shouldn't be necessary except in exceptional
|
||||
circumstances or for testing.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
},
|
||||
}})
|
||||
}
|
||||
@@ -1636,6 +1921,7 @@ type Options struct {
|
||||
DisableHTTP2 bool `config:"disable_http2"`
|
||||
DownloadURL string `config:"download_url"`
|
||||
UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`
|
||||
UsePresignedRequest bool `config:"use_presigned_request"`
|
||||
}
|
||||
|
||||
// Fs represents a remote s3 server
|
||||
@@ -1647,6 +1933,7 @@ type Fs struct {
|
||||
ctx context.Context // global context for reading config
|
||||
features *fs.Features // optional features
|
||||
c *s3.S3 // the connection to the s3 server
|
||||
cu *s3.S3 // unsigned connection to the s3 server for PutObject
|
||||
ses *session.Session // the s3 session
|
||||
rootBucket string // bucket part of root (if any)
|
||||
rootDirectory string // directory part of root (if any)
|
||||
@@ -1779,7 +2066,11 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
|
||||
}
|
||||
|
||||
// s3Connection makes a connection to s3
|
||||
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
|
||||
//
|
||||
// If unsignedBody is set then the connection is configured for
|
||||
// unsigned bodies which is necessary for PutObject if we don't want
|
||||
// it to seek
|
||||
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
// Make the auth
|
||||
v := credentials.Value{
|
||||
@@ -1796,7 +2087,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
// start a new AWS session
|
||||
awsSession, err := session.NewSession()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("NewSession: %w", err)
|
||||
return nil, nil, nil, fmt.Errorf("NewSession: %w", err)
|
||||
}
|
||||
|
||||
// first provider to supply a credential set "wins"
|
||||
@@ -1836,9 +2127,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
|
||||
cred = credentials.AnonymousCredentials
|
||||
case v.AccessKeyID == "":
|
||||
return nil, nil, errors.New("access_key_id not found")
|
||||
return nil, nil, nil, errors.New("access_key_id not found")
|
||||
case v.SecretAccessKey == "":
|
||||
return nil, nil, errors.New("secret_access_key not found")
|
||||
return nil, nil, nil, errors.New("secret_access_key not found")
|
||||
}
|
||||
|
||||
if opt.Region == "" {
|
||||
@@ -1877,25 +2168,36 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
|
||||
// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
|
||||
awsSessionOpts.Config.Credentials = nil
|
||||
}
|
||||
// Setting this stops PutObject reading the body twice and seeking
|
||||
// We add our own Content-MD5 for data protection
|
||||
awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true)
|
||||
ses, err := session.NewSessionWithOptions(awsSessionOpts)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
c := s3.New(ses)
|
||||
if opt.V2Auth || opt.Region == "other-v2-signature" {
|
||||
fs.Debugf(nil, "Using v2 auth")
|
||||
signer := func(req *request.Request) {
|
||||
// Ignore AnonymousCredentials object
|
||||
if req.Config.Credentials == credentials.AnonymousCredentials {
|
||||
return
|
||||
newC := func(unsignedBody bool) *s3.S3 {
|
||||
c := s3.New(ses)
|
||||
if opt.V2Auth || opt.Region == "other-v2-signature" {
|
||||
fs.Debugf(nil, "Using v2 auth")
|
||||
signer := func(req *request.Request) {
|
||||
// Ignore AnonymousCredentials object
|
||||
if req.Config.Credentials == credentials.AnonymousCredentials {
|
||||
return
|
||||
}
|
||||
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
|
||||
}
|
||||
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
|
||||
c.Handlers.Sign.Clear()
|
||||
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
|
||||
c.Handlers.Sign.PushBack(signer)
|
||||
} else if unsignedBody {
|
||||
// If the body is unsigned then tell the signer that we aren't signing the payload
|
||||
c.Handlers.Sign.Clear()
|
||||
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
|
||||
c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload))
|
||||
}
|
||||
c.Handlers.Sign.Clear()
|
||||
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
|
||||
c.Handlers.Sign.PushBack(signer)
|
||||
return c
|
||||
}
|
||||
return c, ses, nil
|
||||
return newC(false), newC(true), ses, nil
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
@@ -1950,6 +2252,17 @@ func setQuirks(opt *Options) {
|
||||
listObjectsV2 = false
|
||||
virtualHostStyle = false
|
||||
urlEncodeListings = false
|
||||
case "ChinaMobile":
|
||||
listObjectsV2 = false
|
||||
virtualHostStyle = false
|
||||
urlEncodeListings = false
|
||||
case "Cloudflare":
|
||||
virtualHostStyle = false
|
||||
useMultipartEtag = false // currently multipart Etags are random
|
||||
case "ArvanCloud":
|
||||
listObjectsV2 = false
|
||||
virtualHostStyle = false
|
||||
urlEncodeListings = false
|
||||
case "DigitalOcean":
|
||||
urlEncodeListings = false
|
||||
case "Dreamhost":
|
||||
@@ -2069,7 +2382,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
|
||||
}
|
||||
srv := getClient(ctx, opt)
|
||||
c, ses, err := s3Connection(ctx, opt, srv)
|
||||
c, cu, ses, err := s3Connection(ctx, opt, srv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -2087,6 +2400,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
ci: ci,
|
||||
ctx: ctx,
|
||||
c: c,
|
||||
cu: cu,
|
||||
ses: ses,
|
||||
pacer: pc,
|
||||
cache: bucket.NewCache(),
|
||||
@@ -2210,11 +2524,12 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
|
||||
// Make a new session with the new region
|
||||
oldRegion := f.opt.Region
|
||||
f.opt.Region = region
|
||||
c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
|
||||
c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating new session failed: %w", err)
|
||||
}
|
||||
f.c = c
|
||||
f.cu = cu
|
||||
f.ses = ses
|
||||
|
||||
fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
|
||||
@@ -2657,7 +2972,7 @@ func (f *Fs) Precision() time.Duration {
|
||||
// pathEscape escapes s as for a URL path. It uses rest.URLPathEscape
|
||||
// but also escapes '+' for S3 and Digital Ocean spaces compatibility
|
||||
func pathEscape(s string) string {
|
||||
return strings.Replace(rest.URLPathEscape(s), "+", "%2B", -1)
|
||||
return strings.ReplaceAll(rest.URLPathEscape(s), "+", "%2B")
|
||||
}
|
||||
|
||||
// copy does a server-side copy
|
||||
@@ -3030,6 +3345,10 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
|
||||
st.Status = "Not an S3 object"
|
||||
return
|
||||
}
|
||||
if o.storageClass != "GLACIER" && o.storageClass != "DEEP_ARCHIVE" {
|
||||
st.Status = "Not GLACIER or DEEP_ARCHIVE storage class"
|
||||
return
|
||||
}
|
||||
bucket, bucketPath := o.split()
|
||||
reqCopy := req
|
||||
reqCopy.Bucket = &bucket
|
||||
@@ -3526,7 +3845,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
}
|
||||
|
||||
// calculate size of parts
|
||||
partSize := int(f.opt.ChunkSize)
|
||||
partSize := f.opt.ChunkSize
|
||||
|
||||
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
|
||||
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
|
||||
@@ -3537,11 +3856,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
|
||||
})
|
||||
} else {
|
||||
// Adjust partSize until the number of parts is small enough.
|
||||
if size/int64(partSize) >= uploadParts {
|
||||
// Calculate partition size rounded up to the nearest MiB
|
||||
partSize = int((((size / uploadParts) >> 20) + 1) << 20)
|
||||
}
|
||||
partSize = chunksize.Calculator(o, int(uploadParts), f.opt.ChunkSize)
|
||||
}
|
||||
|
||||
memPool := f.getMemoryPool(int64(partSize))
|
||||
@@ -3711,6 +4026,117 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
return etag, nil
|
||||
}
|
||||
|
||||
// unWrapAwsError unwraps AWS errors, looking for a non AWS error
|
||||
//
|
||||
// It returns true if one was found and the error, or false and the
|
||||
// error passed in.
|
||||
func unWrapAwsError(err error) (found bool, outErr error) {
|
||||
if awsErr, ok := err.(awserr.Error); ok {
|
||||
var origErrs []error
|
||||
if batchErr, ok := awsErr.(awserr.BatchError); ok {
|
||||
origErrs = batchErr.OrigErrs()
|
||||
} else {
|
||||
origErrs = []error{awsErr.OrigErr()}
|
||||
}
|
||||
for _, origErr := range origErrs {
|
||||
found, newErr := unWrapAwsError(origErr)
|
||||
if found {
|
||||
return found, newErr
|
||||
}
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, err
|
||||
}
|
||||
|
||||
// Upload a single part using PutObject
|
||||
func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
|
||||
req.Body = readers.NewFakeSeeker(in, size)
|
||||
var resp *s3.PutObjectOutput
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
resp, err = o.fs.cu.PutObject(req)
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
// Return the underlying error if we have a Serialization error if possible
|
||||
//
|
||||
// Serialization errors are synthesized locally in the SDK (not returned from the
|
||||
// server). We'll get one if the SDK attempts a retry, however the FakeSeeker will
|
||||
// remember the previous error from Read and return that.
|
||||
if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization {
|
||||
if found, newErr := unWrapAwsError(err); found {
|
||||
err = newErr
|
||||
}
|
||||
}
|
||||
return etag, lastModified, err
|
||||
}
|
||||
lastModified = time.Now()
|
||||
etag = aws.StringValue(resp.ETag)
|
||||
return etag, lastModified, nil
|
||||
}
|
||||
|
||||
// Upload a single part using a presigned request
|
||||
func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
|
||||
// Create the request
|
||||
putObj, _ := o.fs.c.PutObjectRequest(req)
|
||||
|
||||
// Sign it so we can upload using a presigned request.
|
||||
//
|
||||
// Note the SDK didn't used to support streaming to
|
||||
// PutObject so we used this work-around.
|
||||
url, headers, err := putObj.PresignRequest(15 * time.Minute)
|
||||
if err != nil {
|
||||
return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err)
|
||||
}
|
||||
|
||||
if o.fs.opt.V2Auth && headers == nil {
|
||||
headers = putObj.HTTPRequest.Header
|
||||
}
|
||||
|
||||
// Set request to nil if empty so as not to make chunked encoding
|
||||
if size == 0 {
|
||||
in = nil
|
||||
}
|
||||
|
||||
// create the vanilla http request
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
|
||||
if err != nil {
|
||||
return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err)
|
||||
}
|
||||
|
||||
// set the headers we signed and the length
|
||||
httpReq.Header = headers
|
||||
httpReq.ContentLength = size
|
||||
|
||||
var resp *http.Response
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = o.fs.srv.Do(httpReq)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
|
||||
return false, nil
|
||||
}
|
||||
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
|
||||
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
})
|
||||
if err != nil {
|
||||
return etag, lastModified, err
|
||||
}
|
||||
if resp != nil {
|
||||
if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil {
|
||||
lastModified = date
|
||||
}
|
||||
etag = resp.Header.Get("Etag")
|
||||
}
|
||||
return etag, lastModified, nil
|
||||
}
|
||||
|
||||
// Update the Object from in with modTime and size
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
|
||||
bucket, bucketPath := o.split()
|
||||
@@ -3762,6 +4188,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
ContentType: &mimeType,
|
||||
Metadata: metadata,
|
||||
}
|
||||
if size >= 0 {
|
||||
req.ContentLength = &size
|
||||
}
|
||||
if md5sumBase64 != "" {
|
||||
req.ContentMD5 = &md5sumBase64
|
||||
}
|
||||
@@ -3816,66 +4245,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
}
|
||||
|
||||
var resp *http.Response // response from PUT
|
||||
var wantETag string // Multipart upload Etag to check
|
||||
var wantETag string // Multipart upload Etag to check
|
||||
var gotEtag string // Etag we got from the upload
|
||||
var lastModified time.Time // Time we got from the upload
|
||||
if multipart {
|
||||
wantETag, err = o.uploadMultipart(ctx, &req, size, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
|
||||
// Create the request
|
||||
putObj, _ := o.fs.c.PutObjectRequest(&req)
|
||||
|
||||
// Sign it so we can upload using a presigned request.
|
||||
//
|
||||
// Note the SDK doesn't currently support streaming to
|
||||
// PutObject so we'll use this work-around.
|
||||
url, headers, err := putObj.PresignRequest(15 * time.Minute)
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3 upload: sign request: %w", err)
|
||||
}
|
||||
|
||||
if o.fs.opt.V2Auth && headers == nil {
|
||||
headers = putObj.HTTPRequest.Header
|
||||
}
|
||||
|
||||
// Set request to nil if empty so as not to make chunked encoding
|
||||
if size == 0 {
|
||||
in = nil
|
||||
}
|
||||
|
||||
// create the vanilla http request
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("s3 upload: new request: %w", err)
|
||||
}
|
||||
|
||||
// set the headers we signed and the length
|
||||
httpReq.Header = headers
|
||||
httpReq.ContentLength = size
|
||||
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
var err error
|
||||
resp, err = o.fs.srv.Do(httpReq)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
body, err := rest.ReadBody(resp)
|
||||
if err != nil {
|
||||
return o.fs.shouldRetry(ctx, err)
|
||||
}
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
|
||||
return false, nil
|
||||
}
|
||||
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
|
||||
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
if o.fs.opt.UsePresignedRequest {
|
||||
gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
|
||||
} else {
|
||||
gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// User requested we don't HEAD the object after uploading it
|
||||
// so make up the object as best we can assuming it got
|
||||
@@ -3888,11 +4272,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
o.mimeType = aws.StringValue(req.ContentType)
|
||||
o.storageClass = aws.StringValue(req.StorageClass)
|
||||
// If we have done a single part PUT request then we can read these
|
||||
if resp != nil {
|
||||
if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
|
||||
o.lastModified = date
|
||||
}
|
||||
o.setMD5FromEtag(resp.Header.Get("Etag"))
|
||||
if gotEtag != "" {
|
||||
o.setMD5FromEtag(gotEtag)
|
||||
}
|
||||
if !o.lastModified.IsZero() {
|
||||
o.lastModified = lastModified
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -109,7 +109,7 @@ when the ssh-agent contains many keys.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "use_insecure_cipher",
|
||||
Help: `Enable the use of insecure ciphers and key exchange methods.
|
||||
Help: `Enable the use of insecure ciphers and key exchange methods.
|
||||
|
||||
This enables the use of the following insecure ciphers and key exchange methods:
|
||||
|
||||
@@ -1220,7 +1220,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
}
|
||||
stdout, err := f.run(ctx, "df -k "+escapedPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("your remote may not support About: %w", err)
|
||||
return nil, fmt.Errorf("your remote may not have the required df utility: %w", err)
|
||||
}
|
||||
|
||||
usageTotal, usageUsed, usageAvail := parseUsage(stdout)
|
||||
@@ -1311,7 +1311,7 @@ var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\
|
||||
// when sending it to a shell.
|
||||
func shellEscape(str string) string {
|
||||
safe := shellEscapeRegex.ReplaceAllString(str, `\$0`)
|
||||
return strings.Replace(safe, "\n", "'\n'", -1)
|
||||
return strings.ReplaceAll(safe, "\n", "'\n'")
|
||||
}
|
||||
|
||||
// Converts a byte array from the SSH session returned by
|
||||
|
||||
+11
-2
@@ -579,7 +579,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return newObjectFromUplink(f, "", upload.Info()), nil
|
||||
return newObjectFromUplink(f, src.Remote(), upload.Info()), nil
|
||||
}
|
||||
|
||||
// PutStream uploads to the remote path with the modTime given of indeterminate
|
||||
@@ -705,7 +705,16 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
// Do the move
|
||||
err := f.project.MoveObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rename object failed: %w", err)
|
||||
// Make sure destination bucket exists
|
||||
_, err := f.project.EnsureBucket(ctx, dstBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rename object failed to create destination bucket: %w", err)
|
||||
}
|
||||
// And try again
|
||||
err = f.project.MoveObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rename object failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Read the new object
|
||||
|
||||
+18
-13
@@ -141,22 +141,20 @@ func (f *Fs) Hashes() hash.Set {
|
||||
return f.hashSet
|
||||
}
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
// mkdir makes the directory passed in and returns the upstreams used
|
||||
func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
|
||||
upstreams, err := f.create(ctx, dir)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
if dir != parentDir(dir) {
|
||||
if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
|
||||
return err
|
||||
}
|
||||
upstreams, err = f.create(ctx, dir)
|
||||
parent := parentDir(dir)
|
||||
if dir != parent {
|
||||
upstreams, err = f.mkdir(ctx, parent)
|
||||
} else if dir == "" {
|
||||
// If root dirs not created then create them
|
||||
upstreams, err = f.upstreams, nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
errs := Errors(make([]error, len(upstreams)))
|
||||
multithread(len(upstreams), func(i int) {
|
||||
@@ -165,7 +163,17 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
|
||||
}
|
||||
})
|
||||
return errs.Err()
|
||||
err = errs.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return upstreams, nil
|
||||
}
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.mkdir(ctx, dir)
|
||||
return err
|
||||
}
|
||||
|
||||
// Purge all files in the directory
|
||||
@@ -449,10 +457,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
|
||||
srcPath := src.Remote()
|
||||
upstreams, err := f.create(ctx, srcPath)
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
if err := f.Mkdir(ctx, parentDir(srcPath)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upstreams, err = f.create(ctx, srcPath)
|
||||
upstreams, err = f.mkdir(ctx, parentDir(srcPath))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -124,6 +124,22 @@ You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'
|
||||
`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "base_path",
|
||||
Help: `Base path of expected replies
|
||||
|
||||
Normally WebDAV servers return the files they are listing under the
|
||||
url path as specified above. However some WebDAV servers return files
|
||||
with URLs that are not under the endpoint URL. This causes rclone to
|
||||
get confused and return errors like
|
||||
|
||||
Item with unknown path received: "/remote.php/webdav/folder1/", "/elsewhere/remote.php/webdav/folder1/"
|
||||
|
||||
errors. If that is the case, then set "base_path" to the path
|
||||
specified in the error message up to the first item, in the above
|
||||
example "/elsewhere/remote.php/webdav/".
|
||||
`,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -138,6 +154,7 @@ type Options struct {
|
||||
BearerTokenCommand string `config:"bearer_token_command"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
Headers fs.CommaSepList `config:"headers"`
|
||||
BasePath string `config:"base_path"`
|
||||
}
|
||||
|
||||
// Fs represents a remote webdav
|
||||
@@ -454,7 +471,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.srv.SetHeader("Referer", u.String())
|
||||
if !f.findHeader(opt.Headers, "Referer") {
|
||||
f.srv.SetHeader("Referer", u.String())
|
||||
}
|
||||
|
||||
if root != "" && !rootIsDir {
|
||||
// Check to see if the root actually an existing file
|
||||
@@ -517,6 +536,17 @@ func (f *Fs) addHeaders(headers fs.CommaSepList) {
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if the header was configured
|
||||
func (f *Fs) findHeader(headers fs.CommaSepList, find string) bool {
|
||||
for i := 0; i < len(headers); i += 2 {
|
||||
key := f.opt.Headers[i]
|
||||
if strings.EqualFold(key, find) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fetch the bearer token and set it if successful
|
||||
func (f *Fs) fetchAndSetBearerToken() error {
|
||||
if f.opt.BearerTokenCommand == "" {
|
||||
@@ -680,6 +710,10 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("couldn't join URL: %w", err)
|
||||
}
|
||||
basePath := baseURL.Path
|
||||
if f.opt.BasePath != "" {
|
||||
basePath = f.opt.BasePath
|
||||
}
|
||||
for i := range result.Responses {
|
||||
item := &result.Responses[i]
|
||||
isDir := itemIsDir(item)
|
||||
@@ -694,11 +728,11 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
|
||||
if isDir {
|
||||
u.Path = addSlash(u.Path)
|
||||
}
|
||||
if !strings.HasPrefix(u.Path, baseURL.Path) {
|
||||
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path)
|
||||
if !strings.HasPrefix(u.Path, basePath) {
|
||||
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, basePath)
|
||||
continue
|
||||
}
|
||||
subPath := u.Path[len(baseURL.Path):]
|
||||
subPath := u.Path[len(basePath):]
|
||||
if f.opt.Enc != encoder.EncodeZero {
|
||||
subPath = f.opt.Enc.ToStandardPath(subPath)
|
||||
}
|
||||
@@ -1148,7 +1182,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("about call failed: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
usage := &fs.Usage{}
|
||||
if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
docker build -t rclone/xgo-cgofuse https://github.com/billziss-gh/cgofuse.git
|
||||
docker build -t rclone/xgo-cgofuse https://github.com/winfsp/cgofuse.git
|
||||
docker images
|
||||
docker push rclone/xgo-cgofuse
|
||||
@@ -173,8 +173,8 @@ func buildZip(dir string) string {
|
||||
func buildDebAndRpm(dir, version, goarch string) []string {
|
||||
// Make internal version number acceptable to .deb and .rpm
|
||||
pkgVersion := version[1:]
|
||||
pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1)
|
||||
pkgVersion = strings.Replace(pkgVersion, "-", ".", -1)
|
||||
pkgVersion = strings.ReplaceAll(pkgVersion, "β", "-beta")
|
||||
pkgVersion = strings.ReplaceAll(pkgVersion, "-", ".")
|
||||
nfpmArch, ok := goarchToNfpm[goarch]
|
||||
if !ok {
|
||||
nfpmArch = goarch
|
||||
@@ -265,9 +265,12 @@ func buildWindowsResourceSyso(goarch string, versionTag string) string {
|
||||
"-o",
|
||||
sysoPath,
|
||||
}
|
||||
if goarch == "amd64" {
|
||||
if strings.Contains(goarch, "64") {
|
||||
args = append(args, "-64") // Make the syso a 64-bit coff file
|
||||
}
|
||||
if strings.Contains(goarch, "arm") {
|
||||
args = append(args, "-arm") // Make the syso an arm binary
|
||||
}
|
||||
args = append(args, jsonPath)
|
||||
err = runEnv(args, nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -48,6 +48,7 @@ docs = [
|
||||
"hdfs.md",
|
||||
"http.md",
|
||||
"hubic.md",
|
||||
"internetarchive.md",
|
||||
"jottacloud.md",
|
||||
"koofr.md",
|
||||
"mailru.md",
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@ see complete list in [documentation](https://rclone.org/overview/#optional-featu
|
||||
}
|
||||
u, err := doAbout(context.Background())
|
||||
if err != nil {
|
||||
return fmt.Errorf("About call failed: %w", err)
|
||||
return fmt.Errorf("about call failed: %w", err)
|
||||
}
|
||||
if u == nil {
|
||||
return errors.New("nil usage returned")
|
||||
|
||||
+1
-1
@@ -489,7 +489,7 @@ func resolveExitCode(err error) {
|
||||
os.Exit(exitcode.TransferExceeded)
|
||||
case fserrors.ShouldRetry(err):
|
||||
os.Exit(exitcode.RetryError)
|
||||
case fserrors.IsNoRetryError(err):
|
||||
case fserrors.IsNoRetryError(err), fserrors.IsNoLowLevelRetryError(err):
|
||||
os.Exit(exitcode.NoRetryError)
|
||||
case fserrors.IsFatalError(err):
|
||||
os.Exit(exitcode.FatalError)
|
||||
|
||||
+3
-4
@@ -1,7 +1,6 @@
|
||||
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
|
||||
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
|
||||
// +build cmount
|
||||
// +build cgo
|
||||
// +build linux darwin freebsd windows
|
||||
// +build linux,cgo darwin,cgo freebsd,cgo windows
|
||||
|
||||
package cmount
|
||||
|
||||
@@ -13,12 +12,12 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/billziss-gh/cgofuse/fuse"
|
||||
"github.com/rclone/rclone/cmd/mountlib"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/log"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/winfsp/cgofuse/fuse"
|
||||
)
|
||||
|
||||
const fhUnset = ^uint64(0)
|
||||
|
||||
+3
-4
@@ -2,10 +2,9 @@
|
||||
//
|
||||
// This uses the cgo based cgofuse library
|
||||
|
||||
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
|
||||
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
|
||||
// +build cmount
|
||||
// +build cgo
|
||||
// +build linux darwin freebsd windows
|
||||
// +build linux,cgo darwin,cgo freebsd,cgo windows
|
||||
|
||||
package cmount
|
||||
|
||||
@@ -18,12 +17,12 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/billziss-gh/cgofuse/fuse"
|
||||
"github.com/rclone/rclone/cmd/mountlib"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/buildinfo"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/winfsp/cgofuse/fuse"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
//go:build cmount && cgo && (linux || darwin || freebsd || windows) && (!race || !windows)
|
||||
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows) && (!race || !windows)
|
||||
// +build cmount
|
||||
// +build cgo
|
||||
// +build linux darwin freebsd windows
|
||||
// +build linux,cgo darwin,cgo freebsd,cgo windows
|
||||
// +build !race !windows
|
||||
|
||||
// FIXME this doesn't work with the race detector under Windows either
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
// Build for cmount for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
//go:build (!linux && !darwin && !freebsd && !windows) || !brew || !cgo || !cmount
|
||||
// +build !linux,!darwin,!freebsd,!windows !brew !cgo !cmount
|
||||
//go:build !((linux && cgo && cmount) || (darwin && cgo && cmount) || (freebsd && cgo && cmount) || (windows && cmount))
|
||||
// +build !linux !cgo !cmount
|
||||
// +build !darwin !cgo !cmount
|
||||
// +build !freebsd !cgo !cmount
|
||||
// +build !windows !cmount
|
||||
|
||||
package cmount
|
||||
@@ -1,5 +1,5 @@
|
||||
//go:build cmount && cgo && windows
|
||||
// +build cmount,cgo,windows
|
||||
//go:build cmount && windows
|
||||
// +build cmount,windows
|
||||
|
||||
package cmount
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ rclone.org website.`,
|
||||
var description = map[string]string{}
|
||||
var addDescription func(root *cobra.Command)
|
||||
addDescription = func(root *cobra.Command) {
|
||||
name := strings.Replace(root.CommandPath(), " ", "_", -1) + ".md"
|
||||
name := strings.ReplaceAll(root.CommandPath(), " ", "_") + ".md"
|
||||
description[name] = root.Short
|
||||
for _, c := range root.Commands() {
|
||||
addDescription(c)
|
||||
@@ -93,11 +93,11 @@ rclone.org website.`,
|
||||
base := strings.TrimSuffix(name, path.Ext(name))
|
||||
data := frontmatter{
|
||||
Date: now,
|
||||
Title: strings.Replace(base, "_", " ", -1),
|
||||
Title: strings.ReplaceAll(base, "_", " "),
|
||||
Description: description[name],
|
||||
Slug: base,
|
||||
URL: "/commands/" + strings.ToLower(base) + "/",
|
||||
Source: strings.Replace(strings.Replace(base, "rclone", "cmd", -1), "_", "/", -1) + "/",
|
||||
Source: strings.ReplaceAll(strings.ReplaceAll(base, "rclone", "cmd"), "_", "/") + "/",
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err := frontmatterTemplate.Execute(&buf, data)
|
||||
|
||||
@@ -65,10 +65,10 @@ at all, then 1 PiB is set as both the total and the free size.
|
||||
To run rclone @ on Windows, you will need to
|
||||
download and install [WinFsp](http://www.secfs.net/winfsp/).
|
||||
|
||||
[WinFsp](https://github.com/billziss-gh/winfsp) is an open-source
|
||||
[WinFsp](https://github.com/winfsp/winfsp) is an open-source
|
||||
Windows File System Proxy which makes it easy to write user space file
|
||||
systems for Windows. It provides a FUSE emulation layer which rclone
|
||||
uses combination with [cgofuse](https://github.com/billziss-gh/cgofuse).
|
||||
uses combination with [cgofuse](https://github.com/winfsp/cgofuse).
|
||||
Both of these packages are by Bill Zissimopoulos who was very helpful
|
||||
during the implementation of rclone @ for Windows.
|
||||
|
||||
@@ -218,7 +218,7 @@ from Microsoft's Sysinternals suite, which has option |-s| to start
|
||||
processes as the SYSTEM account. Another alternative is to run the mount
|
||||
command from a Windows Scheduled Task, or a Windows Service, configured
|
||||
to run as the SYSTEM account. A third alternative is to use the
|
||||
[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)).
|
||||
[WinFsp.Launcher infrastructure](https://github.com/winfsp/winfsp/wiki/WinFsp-Service-Architecture)).
|
||||
Note that when running rclone as another user, it will not use
|
||||
the configuration file from your profile unless you tell it to
|
||||
with the [|--config|](https://rclone.org/docs/#config-config-file) option.
|
||||
|
||||
+13
-8
@@ -78,6 +78,17 @@ type MountPoint struct {
|
||||
ErrChan <-chan error
|
||||
}
|
||||
|
||||
// NewMountPoint makes a new mounting structure
|
||||
func NewMountPoint(mount MountFn, mountPoint string, f fs.Fs, mountOpt *Options, vfsOpt *vfscommon.Options) *MountPoint {
|
||||
return &MountPoint{
|
||||
MountFn: mount,
|
||||
MountPoint: mountPoint,
|
||||
Fs: f,
|
||||
MountOpt: *mountOpt,
|
||||
VFSOpt: *vfsOpt,
|
||||
}
|
||||
}
|
||||
|
||||
// Global constants
|
||||
const (
|
||||
MaxLeafSize = 1024 // don't pass file names longer than this
|
||||
@@ -167,14 +178,7 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm
|
||||
defer cmd.StartStats()()
|
||||
}
|
||||
|
||||
mnt := &MountPoint{
|
||||
MountFn: mount,
|
||||
MountPoint: args[1],
|
||||
Fs: cmd.NewFsDir(args),
|
||||
MountOpt: Opt,
|
||||
VFSOpt: vfsflags.Opt,
|
||||
}
|
||||
|
||||
mnt := NewMountPoint(mount, args[1], cmd.NewFsDir(args), &Opt, &vfsflags.Opt)
|
||||
daemon, err := mnt.Mount()
|
||||
|
||||
// Wait for foreground mount, if any...
|
||||
@@ -253,6 +257,7 @@ func (m *MountPoint) Mount() (daemon *os.Process, err error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to mount FUSE fs: %w", err)
|
||||
}
|
||||
m.MountedOn = time.Now()
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
|
||||
+3
-12
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
"github.com/rclone/rclone/vfs"
|
||||
"github.com/rclone/rclone/vfs/vfsflags"
|
||||
)
|
||||
|
||||
@@ -117,23 +116,15 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
VFS := vfs.New(fdst, &vfsOpt)
|
||||
_, unmountFn, err := mountFn(VFS, mountPoint, &mountOpt)
|
||||
mnt := NewMountPoint(mountFn, mountPoint, fdst, &mountOpt, &vfsOpt)
|
||||
_, err = mnt.Mount()
|
||||
if err != nil {
|
||||
log.Printf("mount FAILED: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add mount to list if mount point was successfully created
|
||||
liveMounts[mountPoint] = &MountPoint{
|
||||
MountPoint: mountPoint,
|
||||
MountedOn: time.Now(),
|
||||
MountFn: mountFn,
|
||||
UnmountFn: unmountFn,
|
||||
MountOpt: mountOpt,
|
||||
VFSOpt: vfsOpt,
|
||||
Fs: fdst,
|
||||
}
|
||||
liveMounts[mountPoint] = mnt
|
||||
|
||||
fs.Debugf(nil, "Mount for %s created at %s using %s", fdst.String(), mountPoint, mountType)
|
||||
return nil, nil
|
||||
|
||||
+56
-39
@@ -42,15 +42,31 @@ builds an in memory representation. rclone ncdu can be used during
|
||||
this scanning phase and you will see it building up the directory
|
||||
structure as it goes along.
|
||||
|
||||
Here are the keys - press '?' to toggle the help on and off
|
||||
You can interact with the user interface using key presses,
|
||||
press '?' to toggle the help on and off. The supported keys are:
|
||||
|
||||
` + strings.Join(helpText()[1:], "\n ") + `
|
||||
|
||||
Listed files/directories may be prefixed by a one-character flag,
|
||||
some of them combined with a description in brackes at end of line.
|
||||
These flags have the following meaning:
|
||||
|
||||
e means this is an empty directory, i.e. contains no files (but
|
||||
may contain empty subdirectories)
|
||||
~ means this is a directory where some of the files (possibly in
|
||||
subdirectories) have unknown size, and therefore the directory
|
||||
size may be underestimated (and average size inaccurate, as it
|
||||
is average of the files with known sizes).
|
||||
. means an error occurred while reading a subdirectory, and
|
||||
therefore the directory size may be underestimated (and average
|
||||
size inaccurate)
|
||||
! means an error occurred while reading this directory
|
||||
|
||||
This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
|
||||
rclone remotes. It is missing lots of features at the moment
|
||||
but is useful as it stands.
|
||||
|
||||
Note that it might take some time to delete big files/folders. The
|
||||
Note that it might take some time to delete big files/directories. The
|
||||
UI won't respond in the meantime since the deletion is done synchronously.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
@@ -283,9 +299,9 @@ func (u *UI) biggestEntry() (biggest int64) {
|
||||
return
|
||||
}
|
||||
for i := range u.entries {
|
||||
size, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if size > biggest {
|
||||
biggest = size
|
||||
attrs, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if attrs.Size > biggest {
|
||||
biggest = attrs.Size
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -297,8 +313,8 @@ func (u *UI) hasEmptyDir() bool {
|
||||
return false
|
||||
}
|
||||
for i := range u.entries {
|
||||
_, count, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if isDir && count == 0 {
|
||||
attrs, _ := u.d.AttrI(u.sortPerm[i])
|
||||
if attrs.IsDir && attrs.Count == 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -343,9 +359,9 @@ func (u *UI) Draw() error {
|
||||
if y >= h-1 {
|
||||
break
|
||||
}
|
||||
size, count, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
|
||||
attrs, err := u.d.AttrI(u.sortPerm[n])
|
||||
fg := termbox.ColorWhite
|
||||
if entriesHaveErrors {
|
||||
if attrs.EntriesHaveErrors {
|
||||
fg = termbox.ColorYellow
|
||||
}
|
||||
if err != nil {
|
||||
@@ -356,15 +372,19 @@ func (u *UI) Draw() error {
|
||||
fg, bg = bg, fg
|
||||
}
|
||||
mark := ' '
|
||||
if isDir {
|
||||
if attrs.IsDir {
|
||||
mark = '/'
|
||||
}
|
||||
fileFlag := ' '
|
||||
message := ""
|
||||
if !readable {
|
||||
if !attrs.Readable {
|
||||
message = " [not read yet]"
|
||||
}
|
||||
if entriesHaveErrors {
|
||||
if attrs.CountUnknownSize > 0 {
|
||||
message = fmt.Sprintf(" [%d of %d files have unknown size, size may be underestimated]", attrs.CountUnknownSize, attrs.Count)
|
||||
fileFlag = '~'
|
||||
}
|
||||
if attrs.EntriesHaveErrors {
|
||||
message = " [some subdirectories could not be read, size may be underestimated]"
|
||||
fileFlag = '.'
|
||||
}
|
||||
@@ -374,32 +394,29 @@ func (u *UI) Draw() error {
|
||||
}
|
||||
extras := ""
|
||||
if u.showCounts {
|
||||
ss := operations.CountStringField(count, u.humanReadable, 9) + " "
|
||||
if count > 0 {
|
||||
ss := operations.CountStringField(attrs.Count, u.humanReadable, 9) + " "
|
||||
if attrs.Count > 0 {
|
||||
extras += ss
|
||||
} else {
|
||||
extras += strings.Repeat(" ", len(ss))
|
||||
}
|
||||
}
|
||||
var averageSize float64
|
||||
if count > 0 {
|
||||
averageSize = float64(size) / float64(count)
|
||||
}
|
||||
if u.showDirAverageSize {
|
||||
ss := operations.SizeStringField(int64(averageSize), u.humanReadable, 9) + " "
|
||||
if averageSize > 0 {
|
||||
avg := attrs.AverageSize()
|
||||
ss := operations.SizeStringField(int64(avg), u.humanReadable, 9) + " "
|
||||
if avg > 0 {
|
||||
extras += ss
|
||||
} else {
|
||||
extras += strings.Repeat(" ", len(ss))
|
||||
}
|
||||
}
|
||||
if showEmptyDir {
|
||||
if isDir && count == 0 && fileFlag == ' ' {
|
||||
if attrs.IsDir && attrs.Count == 0 && fileFlag == ' ' {
|
||||
fileFlag = 'e'
|
||||
}
|
||||
}
|
||||
if u.showGraph {
|
||||
bars := (size + perBar/2 - 1) / perBar
|
||||
bars := (attrs.Size + perBar/2 - 1) / perBar
|
||||
// clip if necessary - only happens during startup
|
||||
if bars > 10 {
|
||||
bars = 10
|
||||
@@ -408,7 +425,7 @@ func (u *UI) Draw() error {
|
||||
}
|
||||
extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
|
||||
}
|
||||
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
|
||||
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(attrs.Size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
|
||||
y++
|
||||
}
|
||||
}
|
||||
@@ -559,14 +576,14 @@ type ncduSort struct {
|
||||
// Less is part of sort.Interface.
|
||||
func (ds *ncduSort) Less(i, j int) bool {
|
||||
var iAvgSize, jAvgSize float64
|
||||
isize, icount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
|
||||
jsize, jcount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
|
||||
iattrs, _ := ds.d.AttrI(ds.sortPerm[i])
|
||||
jattrs, _ := ds.d.AttrI(ds.sortPerm[j])
|
||||
iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
|
||||
if icount > 0 {
|
||||
iAvgSize = float64(isize / icount)
|
||||
if iattrs.Count > 0 {
|
||||
iAvgSize = iattrs.AverageSize()
|
||||
}
|
||||
if jcount > 0 {
|
||||
jAvgSize = float64(jsize / jcount)
|
||||
if jattrs.Count > 0 {
|
||||
jAvgSize = jattrs.AverageSize()
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -575,33 +592,33 @@ func (ds *ncduSort) Less(i, j int) bool {
|
||||
case ds.u.sortByName > 0:
|
||||
break
|
||||
case ds.u.sortBySize < 0:
|
||||
if isize != jsize {
|
||||
return isize < jsize
|
||||
if iattrs.Size != jattrs.Size {
|
||||
return iattrs.Size < jattrs.Size
|
||||
}
|
||||
case ds.u.sortBySize > 0:
|
||||
if isize != jsize {
|
||||
return isize > jsize
|
||||
if iattrs.Size != jattrs.Size {
|
||||
return iattrs.Size > jattrs.Size
|
||||
}
|
||||
case ds.u.sortByCount < 0:
|
||||
if icount != jcount {
|
||||
return icount < jcount
|
||||
if iattrs.Count != jattrs.Count {
|
||||
return iattrs.Count < jattrs.Count
|
||||
}
|
||||
case ds.u.sortByCount > 0:
|
||||
if icount != jcount {
|
||||
return icount > jcount
|
||||
if iattrs.Count != jattrs.Count {
|
||||
return iattrs.Count > jattrs.Count
|
||||
}
|
||||
case ds.u.sortByAverageSize < 0:
|
||||
if iAvgSize != jAvgSize {
|
||||
return iAvgSize < jAvgSize
|
||||
}
|
||||
// if avgSize is equal, sort by size
|
||||
return isize < jsize
|
||||
return iattrs.Size < jattrs.Size
|
||||
case ds.u.sortByAverageSize > 0:
|
||||
if iAvgSize != jAvgSize {
|
||||
return iAvgSize > jAvgSize
|
||||
}
|
||||
// if avgSize is equal, sort by size
|
||||
return isize > jsize
|
||||
return iattrs.Size > jattrs.Size
|
||||
}
|
||||
// if everything equal, sort by name
|
||||
return iname < jname
|
||||
|
||||
+51
-8
@@ -16,14 +16,42 @@ type Dir struct {
|
||||
parent *Dir
|
||||
path string
|
||||
mu sync.Mutex
|
||||
count int64
|
||||
size int64
|
||||
count int64
|
||||
countUnknownSize int64
|
||||
entries fs.DirEntries
|
||||
dirs map[string]*Dir
|
||||
readError error
|
||||
entriesHaveErrors bool
|
||||
}
|
||||
|
||||
// Attrs contains accumulated properties for a directory entry
|
||||
//
|
||||
// Files with unknown size are counted separately but also included
|
||||
// in the total count. They are not included in the size, i.e. treated
|
||||
// as empty files, which means the size may be underestimated.
|
||||
type Attrs struct {
|
||||
Size int64
|
||||
Count int64
|
||||
CountUnknownSize int64
|
||||
IsDir bool
|
||||
Readable bool
|
||||
EntriesHaveErrors bool
|
||||
}
|
||||
|
||||
// AverageSize calculates average size of files in directory
|
||||
//
|
||||
// If there are files with unknown size, this returns the average over
|
||||
// files with known sizes, which means it may be under- or
|
||||
// overestimated.
|
||||
func (a *Attrs) AverageSize() float64 {
|
||||
countKnownSize := a.Count - a.CountUnknownSize
|
||||
if countKnownSize > 0 {
|
||||
return float64(a.Size) / float64(countKnownSize)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Parent returns the directory above this one
|
||||
func (d *Dir) Parent() *Dir {
|
||||
// no locking needed since these are write once in newDir()
|
||||
@@ -49,7 +77,13 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
|
||||
for _, entry := range entries {
|
||||
if o, ok := entry.(fs.Object); ok {
|
||||
d.count++
|
||||
d.size += o.Size()
|
||||
size := o.Size()
|
||||
if size < 0 {
|
||||
// Some backends may return -1 because size of object is not known
|
||||
d.countUnknownSize++
|
||||
} else {
|
||||
d.size += size
|
||||
}
|
||||
}
|
||||
}
|
||||
// Set my directory entry in parent
|
||||
@@ -62,8 +96,9 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
|
||||
// Accumulate counts in parents
|
||||
for ; parent != nil; parent = parent.parent {
|
||||
parent.mu.Lock()
|
||||
parent.count += d.count
|
||||
parent.size += d.size
|
||||
parent.count += d.count
|
||||
parent.countUnknownSize += d.countUnknownSize
|
||||
if d.readError != nil {
|
||||
parent.entriesHaveErrors = true
|
||||
}
|
||||
@@ -91,17 +126,24 @@ func (d *Dir) Remove(i int) {
|
||||
// Call with d.mu held
|
||||
func (d *Dir) remove(i int) {
|
||||
size := d.entries[i].Size()
|
||||
countUnknownSize := int64(0)
|
||||
if size < 0 {
|
||||
size = 0
|
||||
countUnknownSize = 1
|
||||
}
|
||||
count := int64(1)
|
||||
|
||||
subDir, ok := d.getDir(i)
|
||||
if ok {
|
||||
size = subDir.size
|
||||
count = subDir.count
|
||||
countUnknownSize = subDir.countUnknownSize
|
||||
delete(d.dirs, path.Base(subDir.path))
|
||||
}
|
||||
|
||||
d.size -= size
|
||||
d.count -= count
|
||||
d.countUnknownSize -= countUnknownSize
|
||||
d.entries = append(d.entries[:i], d.entries[i+1:]...)
|
||||
|
||||
dir := d
|
||||
@@ -111,6 +153,7 @@ func (d *Dir) remove(i int) {
|
||||
parent.dirs[path.Base(dir.path)] = dir
|
||||
parent.size -= size
|
||||
parent.count -= count
|
||||
parent.countUnknownSize -= countUnknownSize
|
||||
dir = parent
|
||||
parent.mu.Unlock()
|
||||
}
|
||||
@@ -151,19 +194,19 @@ func (d *Dir) Attr() (size int64, count int64) {
|
||||
}
|
||||
|
||||
// AttrI returns the size, count and flags for the i-th directory entry
|
||||
func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
|
||||
func (d *Dir) AttrI(i int) (attrs Attrs, err error) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
subDir, isDir := d.getDir(i)
|
||||
|
||||
if !isDir {
|
||||
return d.entries[i].Size(), 0, false, true, d.entriesHaveErrors, d.readError
|
||||
return Attrs{d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors}, d.readError
|
||||
}
|
||||
if subDir == nil {
|
||||
return 0, 0, true, false, false, nil
|
||||
return Attrs{0, 0, 0, true, false, false}, nil
|
||||
}
|
||||
size, count = subDir.Attr()
|
||||
return size, count, true, true, subDir.entriesHaveErrors, subDir.readError
|
||||
size, count := subDir.Attr()
|
||||
return Attrs{size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors}, subDir.readError
|
||||
}
|
||||
|
||||
// Scan the Fs passed in, returning a root directory channel and an
|
||||
|
||||
+1
-1
@@ -290,7 +290,7 @@ func list(ctx context.Context) error {
|
||||
if !ok {
|
||||
return errors.New("bad JSON")
|
||||
}
|
||||
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.Replace(info["Path"].(string), "/", "-", -1))
|
||||
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.ReplaceAll(info["Path"].(string), "/", "-"))
|
||||
fmt.Printf("%s\n\n", info["Help"])
|
||||
if authRequired := info["AuthRequired"]; authRequired != nil {
|
||||
if authRequired.(bool) {
|
||||
|
||||
@@ -274,7 +274,6 @@ func (vol *Volume) mount(id string) error {
|
||||
if _, err := vol.mnt.Mount(); err != nil {
|
||||
return err
|
||||
}
|
||||
vol.mnt.MountedOn = time.Now()
|
||||
vol.mountReqs[id] = nil
|
||||
vol.drv.monChan <- false // ask monitor to refresh channels
|
||||
return nil
|
||||
|
||||
@@ -43,7 +43,7 @@ var shellUnEscapeRegex = regexp.MustCompile(`\\(.)`)
|
||||
|
||||
// Unescape a string that was escaped by rclone
|
||||
func shellUnEscape(str string) string {
|
||||
str = strings.Replace(str, "'\n'", "\n", -1)
|
||||
str = strings.ReplaceAll(str, "'\n'", "\n")
|
||||
str = shellUnEscapeRegex.ReplaceAllString(str, `$1`)
|
||||
return str
|
||||
}
|
||||
|
||||
+29
-4
@@ -24,26 +24,51 @@ func init() {
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "size remote:path",
|
||||
Short: `Prints the total size and number of objects in remote:path.`,
|
||||
Long: `
|
||||
Counts objects in the path and calculates the total size. Prints the
|
||||
result to standard output.
|
||||
|
||||
By default the output is in human-readable format, but shows values in
|
||||
both human-readable format as well as the raw numbers (global option
|
||||
` + "`--human-readable`" + ` is not considered). Use option ` + "`--json`" + `
|
||||
to format output as JSON instead.
|
||||
|
||||
Recurses by default, use ` + "`--max-depth 1`" + ` to stop the
|
||||
recursion.
|
||||
|
||||
Some backends do not always provide file sizes, see for example
|
||||
[Google Photos](/googlephotos/#size) and
|
||||
[Google Drive](/drive/#limitations-of-google-docs).
|
||||
Rclone will then show a notice in the log indicating how many such
|
||||
files were encountered, and count them in as empty files in the output
|
||||
of the size command.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
fsrc := cmd.NewFsSrc(args)
|
||||
cmd.Run(false, false, command, func() error {
|
||||
var err error
|
||||
var results struct {
|
||||
Count int64 `json:"count"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Count int64 `json:"count"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Sizeless int64 `json:"sizeless"`
|
||||
}
|
||||
|
||||
results.Count, results.Bytes, err = operations.Count(context.Background(), fsrc)
|
||||
results.Count, results.Bytes, results.Sizeless, err = operations.Count(context.Background(), fsrc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if results.Sizeless > 0 {
|
||||
fs.Logf(fsrc, "Size may be underestimated due to %d objects with unknown size", results.Sizeless)
|
||||
}
|
||||
if jsonOutput {
|
||||
return json.NewEncoder(os.Stdout).Encode(results)
|
||||
}
|
||||
fmt.Printf("Total objects: %s (%d)\n", fs.CountSuffix(results.Count), results.Count)
|
||||
fmt.Printf("Total size: %s (%d Byte)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
|
||||
if results.Sizeless > 0 {
|
||||
fmt.Printf("Total objects with unknown size: %s (%d)\n", fs.CountSuffix(results.Sizeless), results.Sizeless)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
},
|
||||
|
||||
+158
-22
@@ -5,6 +5,7 @@ package makefiles
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -16,7 +17,9 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/lib/file"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -29,37 +32,51 @@ var (
|
||||
minFileNameLength = 4
|
||||
maxFileNameLength = 12
|
||||
seed = int64(1)
|
||||
zero = false
|
||||
sparse = false
|
||||
ascii = false
|
||||
pattern = false
|
||||
chargen = false
|
||||
|
||||
// Globals
|
||||
randSource *rand.Rand
|
||||
source io.Reader
|
||||
directoriesToCreate int
|
||||
totalDirectories int
|
||||
fileNames = map[string]struct{}{} // keep a note of which file name we've used already
|
||||
)
|
||||
|
||||
func init() {
|
||||
test.Command.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.IntVarP(cmdFlags, &numberOfFiles, "files", "", numberOfFiles, "Number of files to create")
|
||||
flags.IntVarP(cmdFlags, &averageFilesPerDirectory, "files-per-directory", "", averageFilesPerDirectory, "Average number of files per directory")
|
||||
flags.IntVarP(cmdFlags, &maxDepth, "max-depth", "", maxDepth, "Maximum depth of directory hierarchy")
|
||||
flags.FVarP(cmdFlags, &minFileSize, "min-file-size", "", "Minimum size of file to create")
|
||||
flags.FVarP(cmdFlags, &maxFileSize, "max-file-size", "", "Maximum size of files to create")
|
||||
flags.IntVarP(cmdFlags, &minFileNameLength, "min-name-length", "", minFileNameLength, "Minimum size of file names")
|
||||
flags.IntVarP(cmdFlags, &maxFileNameLength, "max-name-length", "", maxFileNameLength, "Maximum size of file names")
|
||||
flags.Int64VarP(cmdFlags, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)")
|
||||
test.Command.AddCommand(makefilesCmd)
|
||||
makefilesFlags := makefilesCmd.Flags()
|
||||
flags.IntVarP(makefilesFlags, &numberOfFiles, "files", "", numberOfFiles, "Number of files to create")
|
||||
flags.IntVarP(makefilesFlags, &averageFilesPerDirectory, "files-per-directory", "", averageFilesPerDirectory, "Average number of files per directory")
|
||||
flags.IntVarP(makefilesFlags, &maxDepth, "max-depth", "", maxDepth, "Maximum depth of directory hierarchy")
|
||||
flags.FVarP(makefilesFlags, &minFileSize, "min-file-size", "", "Minimum size of file to create")
|
||||
flags.FVarP(makefilesFlags, &maxFileSize, "max-file-size", "", "Maximum size of files to create")
|
||||
flags.IntVarP(makefilesFlags, &minFileNameLength, "min-name-length", "", minFileNameLength, "Minimum size of file names")
|
||||
flags.IntVarP(makefilesFlags, &maxFileNameLength, "max-name-length", "", maxFileNameLength, "Maximum size of file names")
|
||||
|
||||
test.Command.AddCommand(makefileCmd)
|
||||
makefileFlags := makefileCmd.Flags()
|
||||
|
||||
// Common flags to makefiles and makefile
|
||||
for _, f := range []*pflag.FlagSet{makefilesFlags, makefileFlags} {
|
||||
flags.Int64VarP(f, &seed, "seed", "", seed, "Seed for the random number generator (0 for random)")
|
||||
flags.BoolVarP(f, &zero, "zero", "", zero, "Fill files with ASCII 0x00")
|
||||
flags.BoolVarP(f, &sparse, "sparse", "", sparse, "Make the files sparse (appear to be filled with ASCII 0x00)")
|
||||
flags.BoolVarP(f, &ascii, "ascii", "", ascii, "Fill files with random ASCII printable bytes only")
|
||||
flags.BoolVarP(f, &pattern, "pattern", "", pattern, "Fill files with a periodic pattern")
|
||||
flags.BoolVarP(f, &chargen, "chargen", "", chargen, "Fill files with a ASCII chargen pattern")
|
||||
}
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
var makefilesCmd = &cobra.Command{
|
||||
Use: "makefiles <dir>",
|
||||
Short: `Make a random file hierarchy in a directory`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
if seed == 0 {
|
||||
seed = time.Now().UnixNano()
|
||||
fs.Logf(nil, "Using random seed = %d", seed)
|
||||
}
|
||||
randSource = rand.New(rand.NewSource(seed))
|
||||
commonInit()
|
||||
outputDirectory := args[0]
|
||||
directoriesToCreate = numberOfFiles / averageFilesPerDirectory
|
||||
averageSize := (minFileSize + maxFileSize) / 2
|
||||
@@ -73,13 +90,130 @@ var commandDefinition = &cobra.Command{
|
||||
totalBytes := int64(0)
|
||||
for i := 0; i < numberOfFiles; i++ {
|
||||
dir := dirs[randSource.Intn(len(dirs))]
|
||||
totalBytes += writeFile(dir, fileName())
|
||||
size := int64(minFileSize)
|
||||
if maxFileSize > minFileSize {
|
||||
size += randSource.Int63n(int64(maxFileSize - minFileSize))
|
||||
}
|
||||
writeFile(dir, fileName(), size)
|
||||
totalBytes += size
|
||||
}
|
||||
dt := time.Since(start)
|
||||
fs.Logf(nil, "Written %viB in %v at %viB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
|
||||
fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
|
||||
},
|
||||
}
|
||||
|
||||
var makefileCmd = &cobra.Command{
|
||||
Use: "makefile <size> [<file>]+ [flags]",
|
||||
Short: `Make files with random contents of the size given`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1e6, command, args)
|
||||
commonInit()
|
||||
var size fs.SizeSuffix
|
||||
err := size.Set(args[0])
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to parse size %q: %v", args[0], err)
|
||||
}
|
||||
start := time.Now()
|
||||
fs.Logf(nil, "Creating %d files of size %v.", len(args[1:]), size)
|
||||
totalBytes := int64(0)
|
||||
for _, filePath := range args[1:] {
|
||||
dir := filepath.Dir(filePath)
|
||||
name := filepath.Base(filePath)
|
||||
writeFile(dir, name, int64(size))
|
||||
totalBytes += int64(size)
|
||||
}
|
||||
dt := time.Since(start)
|
||||
fs.Logf(nil, "Written %vB in %v at %vB/s.", fs.SizeSuffix(totalBytes), dt.Round(time.Millisecond), fs.SizeSuffix((totalBytes*int64(time.Second))/int64(dt)))
|
||||
},
|
||||
}
|
||||
|
||||
func bool2int(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// common initialisation for makefiles and makefile
|
||||
func commonInit() {
|
||||
if seed == 0 {
|
||||
seed = time.Now().UnixNano()
|
||||
fs.Logf(nil, "Using random seed = %d", seed)
|
||||
}
|
||||
randSource = rand.New(rand.NewSource(seed))
|
||||
if bool2int(zero)+bool2int(sparse)+bool2int(ascii)+bool2int(pattern)+bool2int(chargen) > 1 {
|
||||
log.Fatal("Can only supply one of --zero, --sparse, --ascii, --pattern or --chargen")
|
||||
}
|
||||
switch {
|
||||
case zero, sparse:
|
||||
source = zeroReader{}
|
||||
case ascii:
|
||||
source = asciiReader{}
|
||||
case pattern:
|
||||
source = readers.NewPatternReader(math.MaxInt64)
|
||||
case chargen:
|
||||
source = &chargenReader{}
|
||||
default:
|
||||
source = randSource
|
||||
}
|
||||
if minFileSize > maxFileSize {
|
||||
maxFileSize = minFileSize
|
||||
}
|
||||
}
|
||||
|
||||
type zeroReader struct{}
|
||||
|
||||
// Read a chunk of zeroes
|
||||
func (zeroReader) Read(p []byte) (n int, err error) {
|
||||
for i := range p {
|
||||
p[i] = 0
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
type asciiReader struct{}
|
||||
|
||||
// Read a chunk of printable ASCII characters
|
||||
func (asciiReader) Read(p []byte) (n int, err error) {
|
||||
n, err = randSource.Read(p)
|
||||
for i := range p[:n] {
|
||||
p[i] = (p[i] % (0x7F - 0x20)) + 0x20
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
type chargenReader struct {
|
||||
start byte // offset from startChar to start line with
|
||||
written byte // chars in line so far
|
||||
}
|
||||
|
||||
// Read a chunk of printable ASCII characters in chargen format
|
||||
func (r *chargenReader) Read(p []byte) (n int, err error) {
|
||||
const (
|
||||
startChar = 0x20 // ' '
|
||||
endChar = 0x7E // '~' inclusive
|
||||
charsPerLine = 72
|
||||
)
|
||||
for i := range p {
|
||||
if r.written >= charsPerLine {
|
||||
r.start++
|
||||
if r.start > endChar-startChar {
|
||||
r.start = 0
|
||||
}
|
||||
p[i] = '\n'
|
||||
r.written = 0
|
||||
} else {
|
||||
c := r.start + r.written + startChar
|
||||
if c > endChar {
|
||||
c -= endChar - startChar + 1
|
||||
}
|
||||
p[i] = c
|
||||
r.written++
|
||||
}
|
||||
}
|
||||
return len(p), err
|
||||
}
|
||||
|
||||
// fileName creates a unique random file or directory name
|
||||
func fileName() (name string) {
|
||||
for {
|
||||
@@ -134,7 +268,7 @@ func (d *dir) list(path string, output []string) []string {
|
||||
}
|
||||
|
||||
// writeFile writes a random file at dir/name
|
||||
func writeFile(dir, name string) int64 {
|
||||
func writeFile(dir, name string, size int64) {
|
||||
err := file.MkdirAll(dir, 0777)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to make directory %q: %v", dir, err)
|
||||
@@ -144,8 +278,11 @@ func writeFile(dir, name string) int64 {
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to open file %q: %v", path, err)
|
||||
}
|
||||
size := randSource.Int63n(int64(maxFileSize-minFileSize)) + int64(minFileSize)
|
||||
_, err = io.CopyN(fd, randSource, size)
|
||||
if sparse {
|
||||
err = fd.Truncate(size)
|
||||
} else {
|
||||
_, err = io.CopyN(fd, source, size)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to write %v bytes to file %q: %v", size, path, err)
|
||||
}
|
||||
@@ -154,5 +291,4 @@ func writeFile(dir, name string) int64 {
|
||||
log.Fatalf("Failed to close file %q: %v", path, err)
|
||||
}
|
||||
fs.Infof(path, "Written file size %v", fs.SizeSuffix(size))
|
||||
return size
|
||||
}
|
||||
@@ -25,7 +25,7 @@ var commandDefinition = &cobra.Command{
|
||||
cmd.Run(false, false, command, func() error {
|
||||
ctx := context.Background()
|
||||
ci := fs.GetConfig(context.Background())
|
||||
objects, _, err := operations.Count(ctx, fsrc)
|
||||
objects, _, _, err := operations.Count(ctx, fsrc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -112,8 +112,11 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="Backblaze B2" home="https://www.backblaze.com/b2/cloud-storage.html" config="/b2/" >}}
|
||||
{{< provider name="Box" home="https://www.box.com/" config="/box/" >}}
|
||||
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
|
||||
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
|
||||
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
|
||||
{{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
|
||||
{{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/sftp/#c14" >}}
|
||||
{{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}}
|
||||
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
|
||||
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
|
||||
{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
|
||||
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
|
||||
@@ -126,6 +129,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}}
|
||||
{{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
|
||||
{{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}}
|
||||
{{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}
|
||||
{{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
|
||||
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
|
||||
{{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
|
||||
@@ -166,7 +170,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}}
|
||||
{{< /provider_list >}}
|
||||
|
||||
Links
|
||||
## Links
|
||||
|
||||
* {{< icon "fa fa-home" >}} [Home page](https://rclone.org/)
|
||||
* {{< icon "fab fa-github" >}} [GitHub project page for source and bug tracker](https://github.com/rclone/rclone)
|
||||
|
||||
@@ -334,6 +334,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -575,3 +575,33 @@ put them back in again.` >}}
|
||||
* Vincent Murphy <vdm@vdm.ie>
|
||||
* ctrl-q <34975747+ctrl-q@users.noreply.github.com>
|
||||
* Nil Alexandrov <nalexand@akamai.com>
|
||||
* GuoXingbin <101376330+guoxingbin@users.noreply.github.com>
|
||||
* Berkan Teber <berkan@berkanteber.com>
|
||||
* Tobias Klauser <tklauser@distanz.ch>
|
||||
* KARBOWSKI Piotr <piotr.karbowski@gmail.com>
|
||||
* GH <geeklihui@foxmail.com>
|
||||
* rafma0 <int.main@gmail.com>
|
||||
* Adrien Rey-Jarthon <jobs@adrienjarthon.com>
|
||||
* Nick Gooding <73336146+nickgooding@users.noreply.github.com>
|
||||
* Leroy van Logchem <lr.vanlogchem@gmail.com>
|
||||
* Zsolt Ero <zsolt.ero@gmail.com>
|
||||
* Lesmiscore <nao20010128@gmail.com>
|
||||
* ehsantdy <ehsan.tadayon@arvancloud.com>
|
||||
* SwazRGB <65694696+swazrgb@users.noreply.github.com>
|
||||
* Mateusz Puczyński <mati6095@gmail.com>
|
||||
* Michael C Tiernan - MIT-Research Computing Project <mtiernan@mit.edu>
|
||||
* Kaspian <34658474+KaspianDev@users.noreply.github.com>
|
||||
* Werner <EvilOlaf@users.noreply.github.com>
|
||||
* Hugal31 <hugo.laloge@gmail.com>
|
||||
* Christian Galo <36752715+cgalo5758@users.noreply.github.com>
|
||||
* Erik van Velzen <erik@evanv.nl>
|
||||
* Derek Battams <derek@battams.ca>
|
||||
* SimonLiu <simonliu009@users.noreply.github.com>
|
||||
* Hugo Laloge <hla@lescompanions.com>
|
||||
* Mr-Kanister <68117355+Mr-Kanister@users.noreply.github.com>
|
||||
* Rob Pickerill <r.pickerill@gmail.com>
|
||||
* Andrey <to.merge@gmail.com>
|
||||
* Eric Wolf <19wolf@gmail.com>
|
||||
* Nick <nick.naumann@mailbox.tu-dresden.de>
|
||||
* Jason Zheng <jszheng17@gmail.com>
|
||||
* Matthew Vernon <mvernon@wikimedia.org>
|
||||
@@ -522,8 +522,7 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
## Azure Storage Emulator Support
|
||||
|
||||
|
||||
+6
-2
@@ -173,6 +173,11 @@ the file instead of hiding it.
|
||||
Old versions of files, where available, are visible using the
|
||||
`--b2-versions` flag.
|
||||
|
||||
It is also possible to view a bucket as it was at a certain point in time,
|
||||
using the `--b2-version-at` flag. This will show the file versions as they
|
||||
were at that time, showing files that have been deleted afterwards, and
|
||||
hiding files that were created since.
|
||||
|
||||
If you wish to remove all the old versions then you can use the
|
||||
`rclone cleanup remote:bucket` command which will delete all the old
|
||||
versions of files, leaving the current ones intact. You can also
|
||||
@@ -561,7 +566,6 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ Optional Flags:
|
||||
|
||||
Arbitrary rclone flags may be specified on the
|
||||
[bisync command line](/commands/rclone_bisync/), for example
|
||||
`rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
|
||||
`rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
|
||||
Note that interactions of various rclone flags with bisync process flow
|
||||
has not been fully tested yet.
|
||||
|
||||
@@ -363,6 +363,7 @@ Bisync is considered _BETA_ and has been tested with the following backends:
|
||||
- OneDrive
|
||||
- S3
|
||||
- SFTP
|
||||
- Yandex Disk
|
||||
|
||||
It has not been fully tested with other services yet.
|
||||
If it works, or sorta works, please let us know and we'll update the list.
|
||||
@@ -700,7 +701,7 @@ consider using the flag
|
||||
|
||||
Google docs exist as virtual files on Google Drive and cannot be transferred
|
||||
to other filesystems natively. While it is possible to export a Google doc to
|
||||
a normal file (with `.xlsx` extension, for example), it's not possible
|
||||
a normal file (with `.xlsx` extension, for example), it is not possible
|
||||
to import a normal file back into a Google document.
|
||||
|
||||
Bisync's handling of Google Doc files is to flag them in the run log output
|
||||
|
||||
+1
-2
@@ -466,6 +466,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -5,6 +5,48 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.58.1 - 2022-04-29
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.58.1)
|
||||
|
||||
* Bug Fixes
|
||||
* build: Update github.com/billziss-gh to github.com/winfsp (Nick Craig-Wood)
|
||||
* filter: Fix timezone of `--min-age`/`-max-age` from UTC to local as documented (Nick Craig-Wood)
|
||||
* rc/js: Correct RC method names (Sơn Trần-Nguyễn)
|
||||
* docs
|
||||
* Fix some links to command pages (albertony)
|
||||
* Add `--multi-thread-streams` note to `--transfers`. (Zsolt Ero)
|
||||
* Mount
|
||||
* Fix `--devname` and fusermount: unknown option 'fsname' when mounting via rc (Nick Craig-Wood)
|
||||
* VFS
|
||||
* Remove wording which suggests VFS is only for mounting (Nick Craig-Wood)
|
||||
* Dropbox
|
||||
* Fix retries of multipart uploads with incorrect_offset error (Nick Craig-Wood)
|
||||
* Google Cloud Storage
|
||||
* Use the s3 pacer to speed up transactions (Nick Craig-Wood)
|
||||
* pacer: Default the Google pacer to a burst of 100 to fix gcs pacing (Nick Craig-Wood)
|
||||
* Jottacloud
|
||||
* Fix scope in token request (albertony)
|
||||
* Netstorage
|
||||
* Fix unescaped HTML in documentation (Nick Craig-Wood)
|
||||
* Make levels of headings consistent (Nick Craig-Wood)
|
||||
* Add support contacts to netstorage doc (Nil Alexandrov)
|
||||
* Onedrive
|
||||
* Note that sharepoint also changes web files (.html, .aspx) (GH)
|
||||
* Putio
|
||||
* Handle rate limit errors (Berkan Teber)
|
||||
* Fix multithread download and other ranged requests (rafma0)
|
||||
* S3
|
||||
* Add ChinaMobile EOS to provider list (GuoXingbin)
|
||||
* Sync providers in config description with providers (Nick Craig-Wood)
|
||||
* SFTP
|
||||
* Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr)
|
||||
* Note that Scaleway C14 is deprecating SFTP in favor of S3 (Adrien Rey-Jarthon)
|
||||
* Storj
|
||||
* Fix bucket creation on Move (Nick Craig-Wood)
|
||||
* WebDAV
|
||||
* Don't override Referer if user sets it (Nick Craig-Wood)
|
||||
|
||||
## v1.58.0 - 2022-03-18
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.57.0...v1.58.0)
|
||||
|
||||
+24
-7
@@ -50,6 +50,7 @@ See the following for detailed instructions for
|
||||
* [HDFS](/hdfs/)
|
||||
* [HTTP](/http/)
|
||||
* [Hubic](/hubic/)
|
||||
* [Internet Archive](/internetarchive/)
|
||||
* [Jottacloud](/jottacloud/)
|
||||
* [Koofr](/koofr/)
|
||||
* [Mail.ru Cloud](/mailru/)
|
||||
@@ -787,7 +788,9 @@ See `--compare-dest` and `--backup-dir`.
|
||||
|
||||
### --dedupe-mode MODE ###
|
||||
|
||||
Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean.
|
||||
Mode to run dedupe command in. One of `interactive`, `skip`, `first`,
|
||||
`newest`, `oldest`, `rename`. The default is `interactive`.
|
||||
See the dedupe command for more information as to what these options mean.
|
||||
|
||||
### --disable FEATURE,FEATURE,... ###
|
||||
|
||||
@@ -939,22 +942,22 @@ unit prefix appended to the value (e.g. `9.762Ki`), while in more textual output
|
||||
the full unit is shown (e.g. `9.762 KiB`). For counts the SI standard notation is
|
||||
used, e.g. prefix `k` for kilo. Used with file counts, `1k` means 1000 files.
|
||||
|
||||
The various [list](commands/rclone_ls/) commands output raw numbers by default.
|
||||
The various [list](/commands/rclone_ls/) commands output raw numbers by default.
|
||||
Option `--human-readable` will make them output values in human-readable format
|
||||
instead (with the short unit prefix).
|
||||
|
||||
The [about](commands/rclone_about/) command outputs human-readable by default,
|
||||
The [about](/commands/rclone_about/) command outputs human-readable by default,
|
||||
with a command-specific option `--full` to output the raw numbers instead.
|
||||
|
||||
Command [size](commands/rclone_size/) outputs both human-readable and raw numbers
|
||||
Command [size](/commands/rclone_size/) outputs both human-readable and raw numbers
|
||||
in the same output.
|
||||
|
||||
The [tree](commands/rclone_tree/) command also considers `--human-readable`, but
|
||||
The [tree](/commands/rclone_tree/) command also considers `--human-readable`, but
|
||||
it will not use the exact same notation as the other commands: It rounds to one
|
||||
decimal, and uses single letter suffix, e.g. `K` instead of `Ki`. The reason for
|
||||
this is that it relies on an external library.
|
||||
|
||||
The interactive command [ncdu](commands/rclone_ncdu/) shows human-readable by
|
||||
The interactive command [ncdu](/commands/rclone_ncdu/) shows human-readable by
|
||||
default, and responds to key `u` for toggling human-readable format.
|
||||
|
||||
### --ignore-case-sync ###
|
||||
@@ -1085,7 +1088,12 @@ have a signal to rotate logs.
|
||||
|
||||
### --log-format LIST ###
|
||||
|
||||
Comma separated list of log format options. Accepted options are `date`, `time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other keywords will be silently ignored. `pid` will tag log messages with process identifier which useful with `rclone mount --daemon`. Other accepted options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants). The default log format is "`date`,`time`".
|
||||
Comma separated list of log format options. Accepted options are `date`,
|
||||
`time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other
|
||||
keywords will be silently ignored. `pid` will tag log messages with process
|
||||
identifier which useful with `rclone mount --daemon`. Other accepted
|
||||
options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants).
|
||||
The default log format is "`date`,`time`".
|
||||
|
||||
### --log-level LEVEL ###
|
||||
|
||||
@@ -1793,6 +1801,8 @@ of timeouts or bigger if you have lots of bandwidth and a fast remote.
|
||||
|
||||
The default is to run 4 file transfers in parallel.
|
||||
|
||||
Look at --multi-thread-streams if you would like to control single file transfers.
|
||||
|
||||
### -u, --update ###
|
||||
|
||||
This forces rclone to skip any files which exist on the destination
|
||||
@@ -1864,6 +1874,9 @@ With `-vv` rclone will become very verbose telling you about every
|
||||
file it considers and transfers. Please send bug reports with a log
|
||||
with this setting.
|
||||
|
||||
When setting verbosity as an environment variable, use
|
||||
`RCLONE_VERBOSE=1` or `RCLONE_VERBOSE=2` for `-v` and `-vv` respectively.
|
||||
|
||||
### -V, --version ###
|
||||
|
||||
Prints the version number
|
||||
@@ -2229,6 +2242,10 @@ override the environment variable setting.
|
||||
Or to always use the trash in drive `--drive-use-trash`, set
|
||||
`RCLONE_DRIVE_USE_TRASH=true`.
|
||||
|
||||
Verbosity is slightly different, the environment variable
|
||||
equivalent of `--verbose` or `-v` is `RCLONE_VERBOSE=1`,
|
||||
or for `-vv`, `RCLONE_VERBOSE=2`.
|
||||
|
||||
The same parser is used for the options and the environment variables
|
||||
so they take exactly the same form.
|
||||
|
||||
|
||||
@@ -1375,8 +1375,11 @@ and upload the files if you prefer.
|
||||
|
||||
### Limitations of Google Docs
|
||||
|
||||
Google docs will appear as size -1 in `rclone ls` and as size 0 in
|
||||
anything which uses the VFS layer, e.g. `rclone mount`, `rclone serve`.
|
||||
Google docs will appear as size -1 in `rclone ls`, `rclone ncdu` etc,
|
||||
and as size 0 in anything which uses the VFS layer, e.g. `rclone mount`
|
||||
and `rclone serve`. When calculating directory totals, e.g. in
|
||||
`rclone size` and `rclone ncdu`, they will be counted in as empty
|
||||
files.
|
||||
|
||||
This is because rclone can't find out the size of the Google docs
|
||||
without downloading them.
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ the node running rclone would need to have lots of bandwidth.
|
||||
|
||||
The syncs would be incremental (on a file by file basis).
|
||||
|
||||
Eg
|
||||
e.g.
|
||||
|
||||
rclone sync -i drive:Folder s3:bucket
|
||||
|
||||
|
||||
@@ -192,6 +192,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -131,7 +131,11 @@ The regular expressions used are as defined in the [Go regular
|
||||
expression reference](https://golang.org/pkg/regexp/syntax/). Regular
|
||||
expressions should be enclosed in `{{` `}}`. They will match only the
|
||||
last path segment if the glob doesn't start with `/` or the whole path
|
||||
name if it does.
|
||||
name if it does. Note that rclone does not attempt to parse the
|
||||
supplied regular expression, meaning that using any regular expression
|
||||
filter will prevent rclone from using [directory filter rules](#directory_filter),
|
||||
as it will instead check every path against
|
||||
the supplied regular expression(s).
|
||||
|
||||
Here is how the `{{regexp}}` is transformed into an full regular
|
||||
expression to match the entire path:
|
||||
@@ -247,7 +251,7 @@ currently a means provided to pass regular expression filter options into
|
||||
rclone directly though character class filter rules contain character
|
||||
classes. [Go regular expression reference](https://golang.org/pkg/regexp/syntax/)
|
||||
|
||||
### How filter rules are applied to directories
|
||||
### How filter rules are applied to directories {#directory_filter}
|
||||
|
||||
Rclone commands are applied to path/file names not
|
||||
directories. The entire contents of a directory can be matched
|
||||
@@ -263,6 +267,10 @@ recurse into subdirectories. This potentially optimises access to a remote
|
||||
by avoiding listing unnecessary directories. Whether optimisation is
|
||||
desirable depends on the specific filter rules and source remote content.
|
||||
|
||||
If any [regular expression filters](#regexp) are in use, then no
|
||||
directory recursion optimisation is possible, as rclone must check
|
||||
every path against the supplied regular expression(s).
|
||||
|
||||
Directory recursion optimisation occurs if either:
|
||||
|
||||
* A source remote does not support the rclone `ListR` primitive. local,
|
||||
|
||||
@@ -337,6 +337,7 @@ and may be set in the config file.
|
||||
--ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited
|
||||
--ftp-disable-epsv Disable using EPSV even if server advertises support
|
||||
--ftp-disable-mlsd Disable using MLSD even if server advertises support
|
||||
--ftp-disable-utf8 Disable using UTF-8 even if server advertises support
|
||||
--ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
|
||||
--ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
|
||||
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
|
||||
|
||||
+12
-2
@@ -267,6 +267,17 @@ Properties:
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --ftp-disable-utf8
|
||||
|
||||
Disable using UTF-8 even if server advertises support.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: disable_utf8
|
||||
- Env Var: RCLONE_FTP_DISABLE_UTF8
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --ftp-writing-mdtm
|
||||
|
||||
Use MDTM to set modification time (VsFtpd quirk)
|
||||
@@ -395,8 +406,7 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
The implementation of : `--dump headers`,
|
||||
`--dump bodies`, `--dump auth` for debugging isn't the same as
|
||||
|
||||
@@ -609,6 +609,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -220,6 +220,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
+92
-38
@@ -3,11 +3,11 @@ title: "Install"
|
||||
description: "Rclone Installation"
|
||||
---
|
||||
|
||||
# Install #
|
||||
# Install
|
||||
|
||||
Rclone is a Go program and comes as a single binary file.
|
||||
|
||||
## Quickstart ##
|
||||
## Quickstart
|
||||
|
||||
* [Download](/downloads/) the relevant binary.
|
||||
* Extract the `rclone` executable, `rclone.exe` on Windows, from the archive.
|
||||
@@ -22,20 +22,20 @@ run `rclone -h`.
|
||||
Already installed rclone can be easily updated to the latest version
|
||||
using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
|
||||
|
||||
## Script installation ##
|
||||
## Script installation
|
||||
|
||||
To install rclone on Linux/macOS/BSD systems, run:
|
||||
|
||||
curl https://rclone.org/install.sh | sudo bash
|
||||
sudo -v ; curl https://rclone.org/install.sh | sudo bash
|
||||
|
||||
For beta installation, run:
|
||||
|
||||
curl https://rclone.org/install.sh | sudo bash -s beta
|
||||
sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
|
||||
|
||||
Note that this script checks the version of rclone installed first and
|
||||
won't re-download if not needed.
|
||||
|
||||
## Linux installation from precompiled binary ##
|
||||
## Linux installation from precompiled binary
|
||||
|
||||
Fetch and unpack
|
||||
|
||||
@@ -59,7 +59,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
|
||||
|
||||
rclone config
|
||||
|
||||
## macOS installation with brew ##
|
||||
## macOS installation with brew
|
||||
|
||||
brew install rclone
|
||||
|
||||
@@ -68,7 +68,7 @@ NOTE: This version of rclone will not support `mount` any more (see
|
||||
on macOS, either install a precompiled binary or enable the relevant option
|
||||
when [installing from source](#install-from-source).
|
||||
|
||||
## macOS installation from precompiled binary, using curl ##
|
||||
## macOS installation from precompiled binary, using curl
|
||||
|
||||
To avoid problems with macOS gatekeeper enforcing the binary to be signed and
|
||||
notarized it is enough to download with `curl`.
|
||||
@@ -96,20 +96,20 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
|
||||
|
||||
rclone config
|
||||
|
||||
## macOS installation from precompiled binary, using a web browser ##
|
||||
## macOS installation from precompiled binary, using a web browser
|
||||
|
||||
When downloading a binary with a web browser, the browser will set the macOS
|
||||
gatekeeper quarantine attribute. Starting from Catalina, when attempting to run
|
||||
`rclone`, a pop-up will appear saying:
|
||||
|
||||
“rclone” cannot be opened because the developer cannot be verified.
|
||||
"rclone" cannot be opened because the developer cannot be verified.
|
||||
macOS cannot verify that this app is free from malware.
|
||||
|
||||
The simplest fix is to run
|
||||
|
||||
xattr -d com.apple.quarantine rclone
|
||||
|
||||
## Install with docker ##
|
||||
## Install with docker
|
||||
|
||||
The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
|
||||
These images are autobuilt by docker hub from the rclone source based
|
||||
@@ -188,39 +188,93 @@ ls ~/data/mount
|
||||
kill %1
|
||||
```
|
||||
|
||||
## Install from source ##
|
||||
## Install from source
|
||||
|
||||
Make sure you have at least [Go](https://golang.org/) go1.15
|
||||
installed. [Download go](https://golang.org/dl/) if necessary. The
|
||||
latest release is recommended. Then
|
||||
Make sure you have git and [Go](https://golang.org/) installed.
|
||||
Go version 1.16 or newer is required, latest release is recommended.
|
||||
You can get it from your package manager, or download it from
|
||||
[golang.org/dl](https://golang.org/dl/). Then you can run the following:
|
||||
|
||||
```sh
|
||||
```
|
||||
git clone https://github.com/rclone/rclone.git
|
||||
cd rclone
|
||||
go build
|
||||
# If on macOS and mount is wanted, instead run: make GOTAGS=cmount
|
||||
./rclone version
|
||||
```
|
||||
|
||||
This will leave you a checked out version of rclone you can modify and
|
||||
send pull requests with. If you use `make` instead of `go build` then
|
||||
the rclone build will have the correct version information in it.
|
||||
This will check out the rclone source in subfolder rclone, which you can later
|
||||
modify and send pull requests with. Then it will build the rclone executable
|
||||
in the same folder. As an initial check you can now run `./rclone version`
|
||||
(`.\rclone version` on Windows).
|
||||
|
||||
You can also build the latest stable rclone with:
|
||||
Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/)
|
||||
command will not be available unless you specify additional build tag `cmount`.
|
||||
|
||||
go get github.com/rclone/rclone
|
||||
```
|
||||
go build -tags cmount
|
||||
```
|
||||
|
||||
or the latest version (equivalent to the beta) with
|
||||
This assumes you have a GCC compatible C compiler (GCC or Clang) in your PATH,
|
||||
as it uses [cgo](https://pkg.go.dev/cmd/cgo). But on Windows, the
|
||||
[cgofuse](https://github.com/winfsp/cgofuse) library that the cmount
|
||||
implementation is based on, also supports building
|
||||
[without cgo](https://github.com/golang/go/wiki/WindowsDLLs), i.e. by setting
|
||||
environment variable CGO_ENABLED to value 0 (static linking). This is how the
|
||||
official Windows release of rclone is being built, starting with version 1.59.
|
||||
It is still possible to build with cgo on Windows as well, by using the MinGW
|
||||
port of GCC, e.g. by installing it in a [MSYS2](https://www.msys2.org)
|
||||
distribution (make sure you install it in the classic mingw64 subsystem, the
|
||||
ucrt64 version is not compatible).
|
||||
|
||||
go get github.com/rclone/rclone@master
|
||||
Additionally, on Windows, you must install the third party utility
|
||||
[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected.
|
||||
If building with cgo, you must also set environment variable CPATH pointing to
|
||||
the fuse include directory within the WinFsp installation
|
||||
(normally `C:\Program Files (x86)\WinFsp\inc\fuse`).
|
||||
|
||||
These will build the binary in `$(go env GOPATH)/bin`
|
||||
(`~/go/bin/rclone` by default) after downloading the source to the go
|
||||
module cache. Note - do **not** use the `-u` flag here. This causes go
|
||||
to try to update the dependencies that rclone uses and sometimes these
|
||||
don't work with the current version of rclone.
|
||||
You may also add arguments `-ldflags -s` (with or without `-tags cmount`),
|
||||
to omit symbol table and debug information, making the executable file smaller,
|
||||
and `-trimpath` to remove references to local file system paths. This is how
|
||||
the official rclone releases are built.
|
||||
|
||||
## Installation with Ansible ##
|
||||
```
|
||||
go build -trimpath -ldflags -s -tags cmount
|
||||
```
|
||||
|
||||
Instead of executing the `go build` command directly, you can run it via the
|
||||
Makefile, which also sets version information and copies the resulting rclone
|
||||
executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
|
||||
corresponds to `~/go/bin/rclone` by default).
|
||||
|
||||
```
|
||||
make
|
||||
```
|
||||
|
||||
To include mount command on macOS and Windows with Makefile build:
|
||||
|
||||
```
|
||||
make GOTAGS=cmount
|
||||
```
|
||||
|
||||
As an alternative you can download the source, build and install rclone in one
|
||||
operation, as a regular Go package. The source will be stored it in the Go
|
||||
module cache, and the resulting executable will be in your GOPATH bin folder
|
||||
(`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).
|
||||
|
||||
With Go version 1.17 or newer:
|
||||
|
||||
```
|
||||
go install github.com/rclone/rclone@latest
|
||||
```
|
||||
|
||||
With Go versions older than 1.17 (do **not** use the `-u` flag, it causes Go to
|
||||
try to update the dependencies that rclone uses and sometimes these don't work
|
||||
with the current version):
|
||||
|
||||
```
|
||||
go get github.com/rclone/rclone
|
||||
```
|
||||
|
||||
## Installation with Ansible
|
||||
|
||||
This can be done with [Stefan Weichinger's ansible
|
||||
role](https://github.com/stefangweichinger/ansible-rclone).
|
||||
@@ -236,7 +290,7 @@ Instructions
|
||||
- rclone
|
||||
```
|
||||
|
||||
## Portable installation ##
|
||||
## Portable installation
|
||||
|
||||
As mentioned [above](https://rclone.org/install/#quickstart), rclone is single
|
||||
executable (`rclone`, or `rclone.exe` on Windows) that you can download as a
|
||||
@@ -314,7 +368,7 @@ the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
|
||||
utility from Microsoft's Sysinternals suite, which takes option `-s` to
|
||||
execute commands as the `SYSTEM` user.
|
||||
|
||||
#### Start from Startup folder ###
|
||||
#### Start from Startup folder
|
||||
|
||||
To quickly execute an rclone command you can simply create a standard
|
||||
Windows Explorer shortcut for the complete rclone command you want to run. If you
|
||||
@@ -329,7 +383,7 @@ functionality to set it to run as different user, or to set conditions or
|
||||
actions on certain events. Setting up a scheduled task as described below
|
||||
will often give you better results.
|
||||
|
||||
#### Start from Task Scheduler ###
|
||||
#### Start from Task Scheduler
|
||||
|
||||
Task Scheduler is an administrative tool built into Windows, and it can be used to
|
||||
configure rclone to be started automatically in a highly configurable way, e.g.
|
||||
@@ -339,14 +393,14 @@ be available to all users it can run as the `SYSTEM` user.
|
||||
For technical information, see
|
||||
https://docs.microsoft.com/windows/win32/taskschd/task-scheduler-start-page.
|
||||
|
||||
#### Run as service ###
|
||||
#### Run as service
|
||||
|
||||
For running rclone at system startup, you can create a Windows service that executes
|
||||
your rclone command, as an alternative to scheduled task configured to run at startup.
|
||||
|
||||
##### Mount command built-in service integration ####
|
||||
##### Mount command built-in service integration
|
||||
|
||||
For mount commands, Rclone has a built-in Windows service integration via the third-party
|
||||
For mount commands, rclone has a built-in Windows service integration via the third-party
|
||||
WinFsp library it uses. Registering as a regular Windows service easy, as you just have to
|
||||
execute the built-in PowerShell command `New-Service` (requires administrative privileges).
|
||||
|
||||
@@ -366,7 +420,7 @@ Windows standard methods for managing network drives. This is currently not
|
||||
officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later
|
||||
it should be possible through path rewriting as described [here](https://github.com/rclone/rclone/issues/3340).
|
||||
|
||||
##### Third-party service integration #####
|
||||
##### Third-party service integration
|
||||
|
||||
To Windows service running any rclone command, the excellent third-party utility
|
||||
[NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used.
|
||||
|
||||
+10
-6
@@ -12,7 +12,7 @@ set -e
|
||||
#when adding a tool to the list make sure to also add its corresponding command further in the script
|
||||
unzip_tools_list=('unzip' '7z' 'busybox')
|
||||
|
||||
usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
|
||||
usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
|
||||
|
||||
#check for beta flag
|
||||
if [ -n "$1" ] && [ "$1" != "beta" ]; then
|
||||
@@ -81,6 +81,8 @@ case $OS in
|
||||
;;
|
||||
Darwin)
|
||||
OS='osx'
|
||||
binTgtDir=/usr/local/bin
|
||||
man1TgtDir=/usr/local/share/man/man1
|
||||
;;
|
||||
SunOS)
|
||||
OS='solaris'
|
||||
@@ -171,12 +173,14 @@ case "$OS" in
|
||||
;;
|
||||
'osx')
|
||||
#binary
|
||||
mkdir -p /usr/local/bin
|
||||
cp rclone /usr/local/bin/rclone.new
|
||||
mv /usr/local/bin/rclone.new /usr/local/bin/rclone
|
||||
mkdir -m 0555 -p ${binTgtDir}
|
||||
cp rclone ${binTgtDir}/rclone.new
|
||||
mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone
|
||||
chmod a=x ${binTgtDir}/rclone
|
||||
#manual
|
||||
mkdir -p /usr/local/share/man/man1
|
||||
cp rclone.1 /usr/local/share/man/man1/
|
||||
mkdir -m 0555 -p ${man1TgtDir}
|
||||
cp rclone.1 ${man1TgtDir}
|
||||
chmod a=r ${man1TgtDir}/rclone.1
|
||||
;;
|
||||
*)
|
||||
echo 'OS not supported'
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
---
|
||||
title: "Internet Archive"
|
||||
description: "Rclone docs for Internet Archive"
|
||||
---
|
||||
|
||||
# {{< icon "fa fa-archive" >}} Internet Archive
|
||||
|
||||
The Internet Archive backend utilizes Items on [archive.org](https://archive.org/)
|
||||
|
||||
Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.html) for the API this backend uses.
|
||||
|
||||
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
|
||||
command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
|
||||
|
||||
Once you have made a remote (see the provider specific section above)
|
||||
you can use it like this:
|
||||
|
||||
Unlike S3, listing up all items uploaded by you isn't supported.
|
||||
|
||||
Make a new item
|
||||
|
||||
rclone mkdir remote:item
|
||||
|
||||
List the contents of a item
|
||||
|
||||
rclone ls remote:item
|
||||
|
||||
Sync `/home/local/directory` to the remote item, deleting any excess
|
||||
files in the item.
|
||||
|
||||
rclone sync -i /home/local/directory remote:item
|
||||
|
||||
## Notes
|
||||
Because of Internet Archive's architecture, it enqueues write operations (and extra post-processings) in a per-item queue. You can check item's queue at https://catalogd.archive.org/history/item-name-here . Because of that, all uploads/deletes will not show up immediately and takes some time to be available.
|
||||
The per-item queue is enqueued to an another queue, Item Deriver Queue. [You can check the status of Item Deriver Queue here.](https://catalogd.archive.org/catalog.php?whereami=1) This queue has a limit, and it may block you from uploading, or even deleting. You should avoid uploading a lot of small files for better behavior.
|
||||
|
||||
You can optionally wait for the server's processing to finish, by setting non-zero value to `wait_archive` key.
|
||||
By making it wait, rclone can do normal file comparison.
|
||||
Make sure to set a large enough value (e.g. `30m0s` for smaller files) as it can take a long time depending on server's queue.
|
||||
|
||||
## Configuration
|
||||
|
||||
Here is an example of making an internetarchive configuration.
|
||||
Most applies to the other providers as well, any differences are described [below](#providers).
|
||||
|
||||
First run
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> remote
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
XX / InternetArchive Items
|
||||
\ (internetarchive)
|
||||
Storage> internetarchive
|
||||
Option access_key_id.
|
||||
IAS3 Access Key.
|
||||
Leave blank for anonymous access.
|
||||
You can find one here: https://archive.org/account/s3.php
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> XXXX
|
||||
Option secret_access_key.
|
||||
IAS3 Secret Key (password).
|
||||
Leave blank for anonymous access.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> XXXX
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> y
|
||||
Option endpoint.
|
||||
IAS3 Endpoint.
|
||||
Leave blank for default value.
|
||||
Enter a string value. Press Enter for the default (https://s3.us.archive.org).
|
||||
endpoint>
|
||||
Option front_endpoint.
|
||||
Host of InternetArchive Frontend.
|
||||
Leave blank for default value.
|
||||
Enter a string value. Press Enter for the default (https://archive.org).
|
||||
front_endpoint>
|
||||
Option disable_checksum.
|
||||
Don't store MD5 checksum with object metadata.
|
||||
Normally rclone will calculate the MD5 checksum of the input before
|
||||
uploading it so it can ask the server to check the object against checksum.
|
||||
This is great for data integrity checking but can cause long delays for
|
||||
large files to start uploading.
|
||||
Enter a boolean value (true or false). Press Enter for the default (true).
|
||||
disable_checksum> true
|
||||
Option encoding.
|
||||
The encoding for the backend.
|
||||
See the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot).
|
||||
encoding>
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
--------------------
|
||||
[remote]
|
||||
type = internetarchive
|
||||
access_key_id = XXXX
|
||||
secret_access_key = XXXX
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/internetarchive/internetarchive.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the standard options specific to internetarchive (Internet Archive).
|
||||
|
||||
#### --internetarchive-access-key-id
|
||||
|
||||
IAS3 Access Key.
|
||||
|
||||
Leave blank for anonymous access.
|
||||
You can find one here: https://archive.org/account/s3.php
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: access_key_id
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --internetarchive-secret-access-key
|
||||
|
||||
IAS3 Secret Key (password).
|
||||
|
||||
Leave blank for anonymous access.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: secret_access_key
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
### Advanced options
|
||||
|
||||
Here are the advanced options specific to internetarchive (Internet Archive).
|
||||
|
||||
#### --internetarchive-endpoint
|
||||
|
||||
IAS3 Endpoint.
|
||||
|
||||
Leave blank for default value.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT
|
||||
- Type: string
|
||||
- Default: "https://s3.us.archive.org"
|
||||
|
||||
#### --internetarchive-front-endpoint
|
||||
|
||||
Host of InternetArchive Frontend.
|
||||
|
||||
Leave blank for default value.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: front_endpoint
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT
|
||||
- Type: string
|
||||
- Default: "https://archive.org"
|
||||
|
||||
#### --internetarchive-disable-checksum
|
||||
|
||||
Don't ask the server to test against MD5 checksum calculated by rclone.
|
||||
Normally rclone will calculate the MD5 checksum of the input before
|
||||
uploading it so it can ask the server to check the object against checksum.
|
||||
This is great for data integrity checking but can cause long delays for
|
||||
large files to start uploading.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: disable_checksum
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM
|
||||
- Type: bool
|
||||
- Default: true
|
||||
|
||||
#### --internetarchive-wait-archive
|
||||
|
||||
Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish.
|
||||
Only enable if you need to be guaranteed to be reflected after write operations.
|
||||
0 to disable waiting. No errors to be thrown in case of timeout.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: wait_archive
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE
|
||||
- Type: Duration
|
||||
- Default: 0s
|
||||
|
||||
#### --internetarchive-encoding
|
||||
|
||||
The encoding for the backend.
|
||||
|
||||
See the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: encoding
|
||||
- Env Var: RCLONE_INTERNETARCHIVE_ENCODING
|
||||
- Type: MultiEncoder
|
||||
- Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
+22
-12
@@ -3,8 +3,7 @@ title: "Akamai Netstorage"
|
||||
description: "Rclone docs for Akamai NetStorage"
|
||||
---
|
||||
|
||||
{{< icon "fas fa-database" >}} Akamai NetStorage
|
||||
-------------------------------------------------
|
||||
# {{< icon "fas fa-database" >}} Akamai NetStorage
|
||||
|
||||
Paths are specified as `remote:`
|
||||
You may put subdirectories in too, e.g. `remote:/path/to/dir`.
|
||||
@@ -19,6 +18,8 @@ See all buckets
|
||||
rclone lsd remote:
|
||||
The initial setup for Netstorage involves getting an account and secret. Use `rclone config` to walk you through the setup process.
|
||||
|
||||
## Configuration
|
||||
|
||||
Here's an example of how to make a remote called `ns1`.
|
||||
|
||||
1. To begin the interactive configuration process, enter this command:
|
||||
@@ -110,28 +111,31 @@ y/e/d> y
|
||||
|
||||
This remote is called `ns1` and can now be used.
|
||||
|
||||
### Example operations
|
||||
## Example operations
|
||||
|
||||
Get started with rclone and NetStorage with these examples. For additional rclone commands, visit https://rclone.org/commands/.
|
||||
|
||||
##### See contents of a directory in your project
|
||||
### See contents of a directory in your project
|
||||
|
||||
rclone lsd ns1:/974012/testing/
|
||||
|
||||
##### Sync the contents local with remote
|
||||
### Sync the contents local with remote
|
||||
|
||||
rclone sync . ns1:/974012/testing/
|
||||
|
||||
##### Upload local content to remote
|
||||
### Upload local content to remote
|
||||
rclone copy notes.txt ns1:/974012/testing/
|
||||
|
||||
##### Delete content on remote
|
||||
### Delete content on remote
|
||||
rclone delete ns1:/974012/testing/notes.txt
|
||||
|
||||
##### Move or copy content between CP codes.
|
||||
### Move or copy content between CP codes.
|
||||
|
||||
Your credentials must have access to two CP codes on the same remote. You can't perform operations between different remotes.
|
||||
|
||||
rclone move ns1:/974012/testing/notes.txt ns1:/974450/testing2/
|
||||
|
||||
## Features
|
||||
|
||||
### Symlink Support
|
||||
|
||||
@@ -152,7 +156,7 @@ With NetStorage, directories can exist in one of two forms:
|
||||
|
||||
Rclone will intercept all file uploads and mkdir commands for the NetStorage remote and will explicitly issue the mkdir command for each directory in the uploading path. This will help with the interoperability with the other Akamai services such as SFTP and the Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly.
|
||||
|
||||
### ListR Feature
|
||||
### `--fast-list` / ListR support
|
||||
|
||||
NetStorage remote supports the ListR feature by using the "list" NetStorage API action to return a lexicographical list of all objects within the specified CP code, recursing into subdirectories as they're encountered.
|
||||
|
||||
@@ -164,7 +168,7 @@ There are pros and cons of using the ListR method, refer to [rclone documentatio
|
||||
|
||||
**Note**: There is a known limitation that "lsf -R" will display number of files in the directory and directory size as -1 when ListR method is used. The workaround is to pass "--disable listR" flag if these numbers are important in the output.
|
||||
|
||||
### Purge Feature
|
||||
### Purge
|
||||
|
||||
NetStorage remote supports the purge feature by using the "quick-delete" NetStorage API action. The quick-delete action is disabled by default for security reasons and can be enabled for the account through the Akamai portal. Rclone will first try to use quick-delete action for the purge command and if this functionality is disabled then will fall back to a standard delete method.
|
||||
|
||||
@@ -179,7 +183,7 @@ Here are the standard options specific to netstorage (Akamai NetStorage).
|
||||
|
||||
Domain+path of NetStorage host to connect to.
|
||||
|
||||
Format should be <domain>/<internal folders>
|
||||
Format should be `<domain>/<internal folders>`
|
||||
|
||||
Properties:
|
||||
|
||||
@@ -271,6 +275,12 @@ You can create a symbolic link in ObjectStore with the symlink action.
|
||||
The desired path location (including applicable sub-directories) ending in
|
||||
the object that will be the target of the symlink (for example, /links/mylink).
|
||||
Include the file extension for the object, if applicable.
|
||||
rclone backend symlink <src> <path>
|
||||
`rclone backend symlink <src> <path>`
|
||||
|
||||
## Support
|
||||
|
||||
If you have any questions or issues, please contact [Akamai Technical Support
|
||||
via Control Center or by
|
||||
phone](https://control.akamai.com/apps/support-ui/#/contact-support).
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
@@ -629,7 +629,7 @@ are converted you will no longer need the ignore options above.
|
||||
It is a [known](https://github.com/OneDrive/onedrive-api-docs/issues/1068) issue
|
||||
that Sharepoint (not OneDrive or OneDrive for Business) may return "item not
|
||||
found" errors when users try to replace or delete uploaded files; this seems to
|
||||
mainly affect Office files (.docx, .xlsx, etc.). As a workaround, you may use
|
||||
mainly affect Office files (.docx, .xlsx, etc.) and web files (.html, .aspx, etc.). As a workaround, you may use
|
||||
the `--backup-dir <BACKUP_DIR>` command line argument so rclone moves the
|
||||
files to be replaced/deleted into a given backup directory (instead of directly
|
||||
replacing/deleting them). For example, to instruct rclone to move the files into
|
||||
|
||||
@@ -177,7 +177,6 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
|
||||
+32
-29
@@ -32,6 +32,7 @@ Here is an overview of the major features of each cloud storage system.
|
||||
| HDFS | - | Yes | No | No | - |
|
||||
| HTTP | - | No | No | No | R |
|
||||
| Hubic | MD5 | Yes | No | No | R/W |
|
||||
| Internet Archive | MD5, SHA1, CRC32 | Yes | No | No | - |
|
||||
| Jottacloud | MD5 | Yes | Yes | No | R |
|
||||
| Koofr | MD5 | No | Yes | No | - |
|
||||
| Mail.ru Cloud | Mailru ⁶ | Yes | Yes | No | - |
|
||||
@@ -301,35 +302,36 @@ list of all possible values by passing an invalid value to this
|
||||
flag, e.g. `--local-encoding "help"`. The command `rclone help flags encoding`
|
||||
will show you the defaults for the backends.
|
||||
|
||||
| Encoding | Characters |
|
||||
| --------- | ---------- |
|
||||
| Asterisk | `*` |
|
||||
| BackQuote | `` ` `` |
|
||||
| BackSlash | `\` |
|
||||
| Colon | `:` |
|
||||
| CrLf | CR 0x0D, LF 0x0A |
|
||||
| Ctl | All control characters 0x00-0x1F |
|
||||
| Del | DEL 0x7F |
|
||||
| Dollar | `$` |
|
||||
| Dot | `.` or `..` as entire string |
|
||||
| DoubleQuote | `"` |
|
||||
| Hash | `#` |
|
||||
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) |
|
||||
| LeftCrLfHtVt | CR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a string |
|
||||
| LeftPeriod | `.` on the left of a string |
|
||||
| LeftSpace | SPACE on the left of a string |
|
||||
| LeftTilde | `~` on the left of a string |
|
||||
| LtGt | `<`, `>` |
|
||||
| None | No characters are encoded |
|
||||
| Percent | `%` |
|
||||
| Pipe | \| |
|
||||
| Question | `?` |
|
||||
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string |
|
||||
| RightPeriod | `.` on the right of a string |
|
||||
| RightSpace | SPACE on the right of a string |
|
||||
| SingleQuote | `'` |
|
||||
| Slash | `/` |
|
||||
| SquareBracket | `[`, `]` |
|
||||
| Encoding | Characters | Encoded as |
|
||||
| --------- | ---------- | ---------- |
|
||||
| Asterisk | `*` | `*` |
|
||||
| BackQuote | `` ` `` | ``` |
|
||||
| BackSlash | `\` | `\` |
|
||||
| Colon | `:` | `:` |
|
||||
| CrLf | CR 0x0D, LF 0x0A | `␍`, `␊` |
|
||||
| Ctl | All control characters 0x00-0x1F | `␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟` |
|
||||
| Del | DEL 0x7F | `␡` |
|
||||
| Dollar | `$` | `$` |
|
||||
| Dot | `.` or `..` as entire string | `.`, `..` |
|
||||
| DoubleQuote | `"` | `"` |
|
||||
| Hash | `#` | `#` |
|
||||
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | `�` |
|
||||
| LeftCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string | `␍`, `␊`, `␉`, `␋` |
|
||||
| LeftPeriod | `.` on the left of a string | `.` |
|
||||
| LeftSpace | SPACE on the left of a string | `␠` |
|
||||
| LeftTilde | `~` on the left of a string | `~` |
|
||||
| LtGt | `<`, `>` | `<`, `>` |
|
||||
| None | No characters are encoded | |
|
||||
| Percent | `%` | `%` |
|
||||
| Pipe | \| | `|` |
|
||||
| Question | `?` | `?` |
|
||||
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string | `␍`, `␊`, `␉`, `␋` |
|
||||
| RightPeriod | `.` on the right of a string | `.` |
|
||||
| RightSpace | SPACE on the right of a string | `␠` |
|
||||
| Semicolon | `;` | `;` |
|
||||
| SingleQuote | `'` | `'` |
|
||||
| Slash | `/` | `/` |
|
||||
| SquareBracket | `[`, `]` | `[`, `]` |
|
||||
|
||||
##### Encoding example: FTP
|
||||
|
||||
@@ -426,6 +428,7 @@ upon backend-specific capabilities.
|
||||
| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes |
|
||||
| HTTP | No | No | No | No | No | No | No | No | No | Yes |
|
||||
| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No |
|
||||
| Internet Archive | No | Yes | No | No | Yes | Yes | No | Yes | Yes | No |
|
||||
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
|
||||
| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| Mega | Yes | No | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
|
||||
@@ -127,3 +127,12 @@ Properties:
|
||||
- Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
|
||||
## Limitations
|
||||
|
||||
put.io has rate limiting. When you hit a limit, rclone automatically
|
||||
retries after waiting the amount of time requested by the server.
|
||||
|
||||
If you want to avoid ever hitting these limits, you may use the
|
||||
`--tpslimit` flag with a low number. Note that the imposed limits
|
||||
may be different for different operations, and may change over time.
|
||||
@@ -318,6 +318,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -150,6 +150,16 @@ use these methods. The alternative is to use `--rc-user` and
|
||||
|
||||
Default Off.
|
||||
|
||||
### --rc-baseurl
|
||||
|
||||
Prefix for URLs.
|
||||
|
||||
Default is root
|
||||
|
||||
### --rc-template
|
||||
|
||||
User-specified template.
|
||||
|
||||
## Accessing the remote control via the rclone rc command {#api-rc}
|
||||
|
||||
Rclone itself implements the remote control protocol in its `rclone
|
||||
@@ -1285,6 +1295,7 @@ Returns:
|
||||
|
||||
- count - number of files
|
||||
- bytes - number of bytes in those files
|
||||
- sizeless - number of files with unknown size, included in count but not accounted for in bytes
|
||||
|
||||
See the [size command](/commands/rclone_size/) command for more information on the above.
|
||||
|
||||
|
||||
@@ -92,3 +92,24 @@ Configuration file is stored at:
|
||||
Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and
|
||||
place it in the correct place (use `rclone config file` on the remote
|
||||
box to find out where).
|
||||
|
||||
## Configuring using SSH Tunnel ##
|
||||
|
||||
Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 53682 to local machine by using the following command:
|
||||
```
|
||||
ssh -L localhost:53682:localhost:53682 username@remote_server
|
||||
```
|
||||
Then on the headless box run `rclone` config and answer `Y` to the `Use
|
||||
auto config?` question.
|
||||
|
||||
```
|
||||
...
|
||||
Remote config
|
||||
Use auto config?
|
||||
* Say Y if not sure
|
||||
* Say N if you are working on a remote or headless machine
|
||||
y) Yes (default)
|
||||
n) No
|
||||
y/n> y
|
||||
```
|
||||
Then copy and paste the auth url `http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx` to the browser on your local machine, complete the auth and it is done.
|
||||
+626
-27
@@ -11,6 +11,9 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#configuration" start="true" >}}
|
||||
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
|
||||
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
|
||||
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
|
||||
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
|
||||
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
|
||||
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
|
||||
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
|
||||
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
|
||||
@@ -69,7 +72,7 @@ name> remote
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -566,7 +569,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
|
||||
#### --s3-provider
|
||||
|
||||
@@ -585,6 +588,10 @@ Properties:
|
||||
- Alibaba Cloud Object Storage System (OSS) formerly Aliyun
|
||||
- "Ceph"
|
||||
- Ceph Object Storage
|
||||
- "ChinaMobile"
|
||||
- China Mobile Ecloud Elastic Object Storage (EOS)
|
||||
- "ArvanCloud"
|
||||
- Arvan Cloud Object Storage (AOS)
|
||||
- "DigitalOcean"
|
||||
- Digital Ocean Spaces
|
||||
- "Dreamhost"
|
||||
@@ -826,7 +833,7 @@ Properties:
|
||||
|
||||
- Config: region
|
||||
- Env Var: RCLONE_S3_REGION
|
||||
- Provider: !AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS
|
||||
- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -851,6 +858,96 @@ Properties:
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --s3-endpoint
|
||||
|
||||
Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_S3_ENDPOINT
|
||||
- Provider: ChinaMobile
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
- "eos-wuxi-1.cmecloud.cn"
|
||||
- The default endpoint - a good choice if you are unsure.
|
||||
- East China (Suzhou)
|
||||
- "eos-jinan-1.cmecloud.cn"
|
||||
- East China (Jinan)
|
||||
- "eos-ningbo-1.cmecloud.cn"
|
||||
- East China (Hangzhou)
|
||||
- "eos-shanghai-1.cmecloud.cn"
|
||||
- East China (Shanghai-1)
|
||||
- "eos-zhengzhou-1.cmecloud.cn"
|
||||
- Central China (Zhengzhou)
|
||||
- "eos-hunan-1.cmecloud.cn"
|
||||
- Central China (Changsha-1)
|
||||
- "eos-zhuzhou-1.cmecloud.cn"
|
||||
- Central China (Changsha-2)
|
||||
- "eos-guangzhou-1.cmecloud.cn"
|
||||
- South China (Guangzhou-2)
|
||||
- "eos-dongguan-1.cmecloud.cn"
|
||||
- South China (Guangzhou-3)
|
||||
- "eos-beijing-1.cmecloud.cn"
|
||||
- North China (Beijing-1)
|
||||
- "eos-beijing-2.cmecloud.cn"
|
||||
- North China (Beijing-2)
|
||||
- "eos-beijing-4.cmecloud.cn"
|
||||
- North China (Beijing-3)
|
||||
- "eos-huhehaote-1.cmecloud.cn"
|
||||
- North China (Huhehaote)
|
||||
- "eos-chengdu-1.cmecloud.cn"
|
||||
- Southwest China (Chengdu)
|
||||
- "eos-chongqing-1.cmecloud.cn"
|
||||
- Southwest China (Chongqing)
|
||||
- "eos-guiyang-1.cmecloud.cn"
|
||||
- Southwest China (Guiyang)
|
||||
- "eos-xian-1.cmecloud.cn"
|
||||
- Nouthwest China (Xian)
|
||||
- "eos-yunnan.cmecloud.cn"
|
||||
- Yunnan China (Kunming)
|
||||
- "eos-yunnan-2.cmecloud.cn"
|
||||
- Yunnan China (Kunming-2)
|
||||
- "eos-tianjin-1.cmecloud.cn"
|
||||
- Tianjin China (Tianjin)
|
||||
- "eos-jilin-1.cmecloud.cn"
|
||||
- Jilin China (Changchun)
|
||||
- "eos-hubei-1.cmecloud.cn"
|
||||
- Hubei China (Xiangyan)
|
||||
- "eos-jiangxi-1.cmecloud.cn"
|
||||
- Jiangxi China (Nanchang)
|
||||
- "eos-gansu-1.cmecloud.cn"
|
||||
- Gansu China (Lanzhou)
|
||||
- "eos-shanxi-1.cmecloud.cn"
|
||||
- Shanxi China (Taiyuan)
|
||||
- eos-liaoning-1.cmecloud.cn"
|
||||
- Liaoning China (Shenyang)
|
||||
- "eos-hebei-1.cmecloud.cn"
|
||||
- Hebei China (Shijiazhuang)
|
||||
- "eos-fujian-1.cmecloud.cn"
|
||||
- Fujian China (Xiamen)
|
||||
- "eos-guangxi-1.cmecloud.cn"
|
||||
- Guangxi China (Nanning)
|
||||
- "eos-anhui-1.cmecloud.cn"
|
||||
- Anhui China (Huainan)
|
||||
|
||||
#### --s3-endpoint
|
||||
|
||||
Endpoint for Arvan Cloud Object Storage (AOS) API.
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_S3_ENDPOINT
|
||||
- Provider: ArvanCloud
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
- "s3.ir-thr-at1.arvanstorage.com"
|
||||
- The default endpoint - a good choice if you are unsure.
|
||||
- Tehran Iran (Asiatech)
|
||||
- "s3.ir-tbz-sh1.arvanstorage.com"
|
||||
- Tabriz Iran (Shahriar)
|
||||
|
||||
|
||||
|
||||
#### --s3-endpoint
|
||||
|
||||
Endpoint for IBM COS S3 API.
|
||||
@@ -1220,7 +1317,7 @@ Properties:
|
||||
|
||||
- Config: endpoint
|
||||
- Env Var: RCLONE_S3_ENDPOINT
|
||||
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp
|
||||
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1250,6 +1347,8 @@ Properties:
|
||||
- Wasabi AP Northeast 1 (Tokyo) endpoint
|
||||
- "s3.ap-northeast-2.wasabisys.com"
|
||||
- Wasabi AP Northeast 2 (Osaka) endpoint
|
||||
- "s3.ir-thr-at1.arvanstorage.com"
|
||||
- ArvanCloud Tehran Iran (Asiatech) endpoint
|
||||
|
||||
#### --s3-location-constraint
|
||||
|
||||
@@ -1457,7 +1556,7 @@ Properties:
|
||||
|
||||
- Config: location_constraint
|
||||
- Env Var: RCLONE_S3_LOCATION_CONSTRAINT
|
||||
- Provider: !AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS
|
||||
- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
@@ -1529,7 +1628,7 @@ Properties:
|
||||
|
||||
- Config: server_side_encryption
|
||||
- Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION
|
||||
- Provider: AWS,Ceph,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1611,6 +1710,38 @@ Properties:
|
||||
|
||||
#### --s3-storage-class
|
||||
|
||||
The storage class to use when storing new objects in ChinaMobile.
|
||||
|
||||
- Config: storage_class
|
||||
- Env Var: RCLONE_S3_STORAGE_CLASS
|
||||
- Provider: ChinaMobile
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
- ""
|
||||
- Default
|
||||
- "STANDARD"
|
||||
- Standard storage class
|
||||
- "GLACIER"
|
||||
- Archive storage mode
|
||||
- "STANDARD_IA"
|
||||
- Infrequent access storage mode
|
||||
|
||||
#### --s3-storage-class
|
||||
|
||||
The storage class to use when storing new objects in ArvanCloud.
|
||||
|
||||
- Config: storage_class
|
||||
- Env Var: RCLONE_S3_STORAGE_CLASS
|
||||
- Provider: ArvanCloud
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
- "STANDARD"
|
||||
- Standard storage class
|
||||
|
||||
#### --s3-storage-class
|
||||
|
||||
The storage class to use when storing new objects in Tencent COS.
|
||||
|
||||
Properties:
|
||||
@@ -1653,7 +1784,7 @@ Properties:
|
||||
|
||||
### Advanced options
|
||||
|
||||
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
|
||||
|
||||
#### --s3-bucket-acl
|
||||
|
||||
@@ -1705,7 +1836,7 @@ Properties:
|
||||
|
||||
- Config: sse_customer_algorithm
|
||||
- Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM
|
||||
- Provider: AWS,Ceph,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1722,7 +1853,7 @@ Properties:
|
||||
|
||||
- Config: sse_customer_key
|
||||
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY
|
||||
- Provider: AWS,Ceph,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -1740,7 +1871,7 @@ Properties:
|
||||
|
||||
- Config: sse_customer_key_md5
|
||||
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5
|
||||
- Provider: AWS,Ceph,Minio
|
||||
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
|
||||
- Type: string
|
||||
- Required: false
|
||||
- Examples:
|
||||
@@ -2315,7 +2446,7 @@ Options:
|
||||
|
||||
{{< rem autogenerated options stop >}}
|
||||
|
||||
### Anonymous access to public buckets ###
|
||||
### Anonymous access to public buckets
|
||||
|
||||
If you want to use rclone to access a public bucket, configure with a
|
||||
blank `access_key_id` and `secret_access_key`. Your config should end
|
||||
@@ -2349,12 +2480,19 @@ You will be able to list and copy data but not upload it.
|
||||
This is the provider used as main example and described in the [configuration](#configuration) section above.
|
||||
|
||||
### AWS Snowball Edge
|
||||
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring
|
||||
bulk data back to AWS. Its main software interface is S3 object storage.
|
||||
|
||||
To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'
|
||||
be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as
|
||||
the snowball device does not support query parameter based authentication.
|
||||
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware
|
||||
appliance used for transferring bulk data back to AWS. Its main
|
||||
software interface is S3 object storage.
|
||||
|
||||
To use rclone with AWS Snowball Edge devices, configure as standard
|
||||
for an 'S3 Compatible Service'.
|
||||
|
||||
If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise
|
||||
you will run into authentication header issues as the snowball device
|
||||
does not support query parameter based authentication.
|
||||
|
||||
With rclone v1.59 or later setting `upload_cutoff` should not be necessary.
|
||||
|
||||
eg.
|
||||
```
|
||||
@@ -2393,10 +2531,11 @@ server_side_encryption =
|
||||
storage_class =
|
||||
```
|
||||
|
||||
If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you
|
||||
may need to supply the parameter `--s3-upload-cutoff 0` or put this in
|
||||
the config file as `upload_cutoff 0` to work around a bug which causes
|
||||
uploading of small files to fail.
|
||||
If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a
|
||||
version of rclone before v1.59 then you may need to supply the
|
||||
parameter `--s3-upload-cutoff 0` or put this in the config file as
|
||||
`upload_cutoff 0` to work around a bug which causes uploading of small
|
||||
files to fail.
|
||||
|
||||
Note also that Ceph sometimes puts `/` in the passwords it gives
|
||||
users. If you read the secret access key using the command line tools
|
||||
@@ -2423,6 +2562,106 @@ removed).
|
||||
Because this is a json dump, it is encoding the `/` as `\/`, so if you
|
||||
use the secret key as `xxxxxx/xxxx` it will work fine.
|
||||
|
||||
### Cloudflare R2 {#cloudflare-r2}
|
||||
|
||||
[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage
|
||||
allows developers to store large amounts of unstructured data without
|
||||
the costly egress bandwidth fees associated with typical cloud storage
|
||||
services.
|
||||
|
||||
Here is an example of making a Cloudflare R2 configuration. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
Note that all buckets are private, and all are stored in the same
|
||||
"auto" region. It is necessary to use Cloudflare workers to share the
|
||||
content of a bucket publicly.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> r2
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
...
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
|
||||
\ (s3)
|
||||
...
|
||||
Storage> s3
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
...
|
||||
XX / Cloudflare R2 Storage
|
||||
\ (Cloudflare)
|
||||
...
|
||||
provider> Cloudflare
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth> 1
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> ACCESS_KEY
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> SECRET_ACCESS_KEY
|
||||
Option region.
|
||||
Region to connect to.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
|
||||
\ (auto)
|
||||
region> 1
|
||||
Option endpoint.
|
||||
Endpoint for S3 API.
|
||||
Required when using an S3 clone.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This will leave your config looking something like:
|
||||
|
||||
```
|
||||
[r2]
|
||||
type = s3
|
||||
provider = Cloudflare
|
||||
access_key_id = ACCESS_KEY
|
||||
secret_access_key = SECRET_ACCESS_KEY
|
||||
region = auto
|
||||
endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
|
||||
acl = private
|
||||
```
|
||||
|
||||
Now run `rclone lsf r2:` to see your buckets and `rclone lsf
|
||||
r2:bucket` to look within a bucket.
|
||||
|
||||
### Dreamhost
|
||||
|
||||
Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is
|
||||
@@ -2521,7 +2760,7 @@ Choose a number from below, or type in your own value
|
||||
\ "alias"
|
||||
2 / Amazon Drive
|
||||
\ "amazon cloud drive"
|
||||
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS)
|
||||
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS)
|
||||
\ "s3"
|
||||
4 / Backblaze B2
|
||||
\ "b2"
|
||||
@@ -2778,6 +3017,9 @@ server_side_encryption =
|
||||
storage_class =
|
||||
```
|
||||
|
||||
[C14 Cold Storage](https://www.online.net/en/storage/c14-cold-storage) is the low-cost S3 Glacier alternative from Scaleway and it works the same way as on S3 by accepting the "GLACIER" `storage_class`.
|
||||
So you can configure your remote with the `storage_class = GLACIER` option to upload directly to C14. Don't forget that in this state you can't read files back after, you will need to restore them to "STANDARD" storage_class first before being able to read them (see "restore" section above)
|
||||
|
||||
### Seagate Lyve Cloud {#lyve}
|
||||
|
||||
[Seagate Lyve Cloud](https://www.seagate.com/gb/en/services/cloud/storage/) is an S3
|
||||
@@ -2803,7 +3045,7 @@ Choose `s3` backend
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -2990,7 +3232,7 @@ name> wasabi
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon S3 (also Dreamhost, Ceph, Minio)
|
||||
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -3104,7 +3346,7 @@ Type of storage to configure.
|
||||
Enter a string value. Press Enter for the default ("").
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -3194,6 +3436,364 @@ d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
### China Mobile Ecloud Elastic Object Storage (EOS) {#china-mobile-ecloud-eos}
|
||||
|
||||
Here is an example of making an [China Mobile Ecloud Elastic Object Storage (EOS)](https:///ecloud.10086.cn/home/product-introduction/eos/)
|
||||
configuration. First run:
|
||||
|
||||
rclone config
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
name> ChinaMobile
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
...
|
||||
5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
|
||||
\ (s3)
|
||||
...
|
||||
Storage> s3
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
...
|
||||
4 / China Mobile Ecloud Elastic Object Storage (EOS)
|
||||
\ (ChinaMobile)
|
||||
...
|
||||
provider> ChinaMobile
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth>
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> accesskeyid
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> secretaccesskey
|
||||
Option endpoint.
|
||||
Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
/ The default endpoint - a good choice if you are unsure.
|
||||
1 | East China (Suzhou)
|
||||
\ (eos-wuxi-1.cmecloud.cn)
|
||||
2 / East China (Jinan)
|
||||
\ (eos-jinan-1.cmecloud.cn)
|
||||
3 / East China (Hangzhou)
|
||||
\ (eos-ningbo-1.cmecloud.cn)
|
||||
4 / East China (Shanghai-1)
|
||||
\ (eos-shanghai-1.cmecloud.cn)
|
||||
5 / Central China (Zhengzhou)
|
||||
\ (eos-zhengzhou-1.cmecloud.cn)
|
||||
6 / Central China (Changsha-1)
|
||||
\ (eos-hunan-1.cmecloud.cn)
|
||||
7 / Central China (Changsha-2)
|
||||
\ (eos-zhuzhou-1.cmecloud.cn)
|
||||
8 / South China (Guangzhou-2)
|
||||
\ (eos-guangzhou-1.cmecloud.cn)
|
||||
9 / South China (Guangzhou-3)
|
||||
\ (eos-dongguan-1.cmecloud.cn)
|
||||
10 / North China (Beijing-1)
|
||||
\ (eos-beijing-1.cmecloud.cn)
|
||||
11 / North China (Beijing-2)
|
||||
\ (eos-beijing-2.cmecloud.cn)
|
||||
12 / North China (Beijing-3)
|
||||
\ (eos-beijing-4.cmecloud.cn)
|
||||
13 / North China (Huhehaote)
|
||||
\ (eos-huhehaote-1.cmecloud.cn)
|
||||
14 / Southwest China (Chengdu)
|
||||
\ (eos-chengdu-1.cmecloud.cn)
|
||||
15 / Southwest China (Chongqing)
|
||||
\ (eos-chongqing-1.cmecloud.cn)
|
||||
16 / Southwest China (Guiyang)
|
||||
\ (eos-guiyang-1.cmecloud.cn)
|
||||
17 / Nouthwest China (Xian)
|
||||
\ (eos-xian-1.cmecloud.cn)
|
||||
18 / Yunnan China (Kunming)
|
||||
\ (eos-yunnan.cmecloud.cn)
|
||||
19 / Yunnan China (Kunming-2)
|
||||
\ (eos-yunnan-2.cmecloud.cn)
|
||||
20 / Tianjin China (Tianjin)
|
||||
\ (eos-tianjin-1.cmecloud.cn)
|
||||
21 / Jilin China (Changchun)
|
||||
\ (eos-jilin-1.cmecloud.cn)
|
||||
22 / Hubei China (Xiangyan)
|
||||
\ (eos-hubei-1.cmecloud.cn)
|
||||
23 / Jiangxi China (Nanchang)
|
||||
\ (eos-jiangxi-1.cmecloud.cn)
|
||||
24 / Gansu China (Lanzhou)
|
||||
\ (eos-gansu-1.cmecloud.cn)
|
||||
25 / Shanxi China (Taiyuan)
|
||||
\ (eos-shanxi-1.cmecloud.cn)
|
||||
26 / Liaoning China (Shenyang)
|
||||
\ (eos-liaoning-1.cmecloud.cn)
|
||||
27 / Hebei China (Shijiazhuang)
|
||||
\ (eos-hebei-1.cmecloud.cn)
|
||||
28 / Fujian China (Xiamen)
|
||||
\ (eos-fujian-1.cmecloud.cn)
|
||||
29 / Guangxi China (Nanning)
|
||||
\ (eos-guangxi-1.cmecloud.cn)
|
||||
30 / Anhui China (Huainan)
|
||||
\ (eos-anhui-1.cmecloud.cn)
|
||||
endpoint> 1
|
||||
Option location_constraint.
|
||||
Location constraint - must match endpoint.
|
||||
Used when creating buckets only.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / East China (Suzhou)
|
||||
\ (wuxi1)
|
||||
2 / East China (Jinan)
|
||||
\ (jinan1)
|
||||
3 / East China (Hangzhou)
|
||||
\ (ningbo1)
|
||||
4 / East China (Shanghai-1)
|
||||
\ (shanghai1)
|
||||
5 / Central China (Zhengzhou)
|
||||
\ (zhengzhou1)
|
||||
6 / Central China (Changsha-1)
|
||||
\ (hunan1)
|
||||
7 / Central China (Changsha-2)
|
||||
\ (zhuzhou1)
|
||||
8 / South China (Guangzhou-2)
|
||||
\ (guangzhou1)
|
||||
9 / South China (Guangzhou-3)
|
||||
\ (dongguan1)
|
||||
10 / North China (Beijing-1)
|
||||
\ (beijing1)
|
||||
11 / North China (Beijing-2)
|
||||
\ (beijing2)
|
||||
12 / North China (Beijing-3)
|
||||
\ (beijing4)
|
||||
13 / North China (Huhehaote)
|
||||
\ (huhehaote1)
|
||||
14 / Southwest China (Chengdu)
|
||||
\ (chengdu1)
|
||||
15 / Southwest China (Chongqing)
|
||||
\ (chongqing1)
|
||||
16 / Southwest China (Guiyang)
|
||||
\ (guiyang1)
|
||||
17 / Nouthwest China (Xian)
|
||||
\ (xian1)
|
||||
18 / Yunnan China (Kunming)
|
||||
\ (yunnan)
|
||||
19 / Yunnan China (Kunming-2)
|
||||
\ (yunnan2)
|
||||
20 / Tianjin China (Tianjin)
|
||||
\ (tianjin1)
|
||||
21 / Jilin China (Changchun)
|
||||
\ (jilin1)
|
||||
22 / Hubei China (Xiangyan)
|
||||
\ (hubei1)
|
||||
23 / Jiangxi China (Nanchang)
|
||||
\ (jiangxi1)
|
||||
24 / Gansu China (Lanzhou)
|
||||
\ (gansu1)
|
||||
25 / Shanxi China (Taiyuan)
|
||||
\ (shanxi1)
|
||||
26 / Liaoning China (Shenyang)
|
||||
\ (liaoning1)
|
||||
27 / Hebei China (Shijiazhuang)
|
||||
\ (hebei1)
|
||||
28 / Fujian China (Xiamen)
|
||||
\ (fujian1)
|
||||
29 / Guangxi China (Nanning)
|
||||
\ (guangxi1)
|
||||
30 / Anhui China (Huainan)
|
||||
\ (anhui1)
|
||||
location_constraint> 1
|
||||
Option acl.
|
||||
Canned ACL used when creating buckets and storing or copying objects.
|
||||
This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
|
||||
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
Note that this ACL is applied when server-side copying objects as S3
|
||||
doesn't copy the ACL from the source but rather writes a fresh one.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
/ Owner gets FULL_CONTROL.
|
||||
1 | No one else has access rights (default).
|
||||
\ (private)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
2 | The AllUsers group gets READ access.
|
||||
\ (public-read)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
3 | The AllUsers group gets READ and WRITE access.
|
||||
| Granting this on a bucket is generally not recommended.
|
||||
\ (public-read-write)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
4 | The AuthenticatedUsers group gets READ access.
|
||||
\ (authenticated-read)
|
||||
/ Object owner gets FULL_CONTROL.
|
||||
acl> private
|
||||
Option server_side_encryption.
|
||||
The server-side encryption algorithm used when storing this object in S3.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / None
|
||||
\ ()
|
||||
2 / AES256
|
||||
\ (AES256)
|
||||
server_side_encryption>
|
||||
Option storage_class.
|
||||
The storage class to use when storing new objects in ChinaMobile.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / Default
|
||||
\ ()
|
||||
2 / Standard storage class
|
||||
\ (STANDARD)
|
||||
3 / Archive storage mode
|
||||
\ (GLACIER)
|
||||
4 / Infrequent access storage mode
|
||||
\ (STANDARD_IA)
|
||||
storage_class>
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
--------------------
|
||||
[ChinaMobile]
|
||||
type = s3
|
||||
provider = ChinaMobile
|
||||
access_key_id = accesskeyid
|
||||
secret_access_key = secretaccesskey
|
||||
endpoint = eos-wuxi-1.cmecloud.cn
|
||||
location_constraint = wuxi1
|
||||
acl = private
|
||||
--------------------
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
### ArvanCloud
|
||||
|
||||
[ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage.
|
||||
It gives you access to backup and archived files and allows sharing.
|
||||
Files like profile image in the app, images sent by users or scanned documents can be stored securely and easily in our Object Storage service.
|
||||
|
||||
ArvanCloud provides an S3 interface which can be configured for use with
|
||||
rclone like this.
|
||||
|
||||
```
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
n/s> n
|
||||
name> ArvanCloud
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value
|
||||
[snip]
|
||||
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Enter AWS credentials in the next step
|
||||
\ "false"
|
||||
2 / Get AWS credentials from the environment (env vars or IAM)
|
||||
\ "true"
|
||||
env_auth> 1
|
||||
AWS Access Key ID - leave blank for anonymous access or runtime credentials.
|
||||
access_key_id> YOURACCESSKEY
|
||||
AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
|
||||
secret_access_key> YOURSECRETACCESSKEY
|
||||
Region to connect to.
|
||||
Choose a number from below, or type in your own value
|
||||
/ The default endpoint - a good choice if you are unsure.
|
||||
1 | US Region, Northern Virginia, or Pacific Northwest.
|
||||
| Leave location constraint empty.
|
||||
\ "us-east-1"
|
||||
[snip]
|
||||
region>
|
||||
Endpoint for S3 API.
|
||||
Leave blank if using ArvanCloud to use the default endpoint for the region.
|
||||
Specify if using an S3 clone such as Ceph.
|
||||
endpoint> s3.arvanstorage.com
|
||||
Location constraint - must be set to match the Region. Used when creating buckets only.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Empty for Iran-Tehran Region.
|
||||
\ ""
|
||||
[snip]
|
||||
location_constraint>
|
||||
Canned ACL used when creating buckets and/or storing objects in S3.
|
||||
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Owner gets FULL_CONTROL. No one else has access rights (default).
|
||||
\ "private"
|
||||
[snip]
|
||||
acl>
|
||||
The server-side encryption algorithm used when storing this object in S3.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / None
|
||||
\ ""
|
||||
2 / AES256
|
||||
\ "AES256"
|
||||
server_side_encryption>
|
||||
The storage class to use when storing objects in S3.
|
||||
Choose a number from below, or type in your own value
|
||||
1 / Default
|
||||
\ ""
|
||||
2 / Standard storage class
|
||||
\ "STANDARD"
|
||||
storage_class>
|
||||
Remote config
|
||||
--------------------
|
||||
[ArvanCloud]
|
||||
env_auth = false
|
||||
access_key_id = YOURACCESSKEY
|
||||
secret_access_key = YOURSECRETACCESSKEY
|
||||
region = ir-thr-at1
|
||||
endpoint = s3.arvanstorage.com
|
||||
location_constraint =
|
||||
acl =
|
||||
server_side_encryption =
|
||||
storage_class =
|
||||
--------------------
|
||||
y) Yes this is OK
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This will leave the config file looking like this.
|
||||
|
||||
```
|
||||
[ArvanCloud]
|
||||
type = s3
|
||||
provider = ArvanCloud
|
||||
env_auth = false
|
||||
access_key_id = YOURACCESSKEY
|
||||
secret_access_key = YOURSECRETACCESSKEY
|
||||
region =
|
||||
endpoint = s3.arvanstorage.com
|
||||
location_constraint =
|
||||
acl =
|
||||
server_side_encryption =
|
||||
storage_class =
|
||||
```
|
||||
|
||||
### Tencent COS {#tencent-cos}
|
||||
|
||||
[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.
|
||||
@@ -3227,7 +3827,7 @@ Choose a number from below, or type in your own value
|
||||
\ "alias"
|
||||
3 / Amazon Drive
|
||||
\ "amazon cloud drive"
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
|
||||
\ "s3"
|
||||
[snip]
|
||||
Storage> s3
|
||||
@@ -3440,5 +4040,4 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
@@ -11,7 +11,6 @@ Protocol](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol).
|
||||
The SFTP backend can be used with a number of different providers:
|
||||
|
||||
{{< provider_list >}}
|
||||
{{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/sftp/#c14">}}
|
||||
{{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net">}}
|
||||
{{< /provider_list >}}
|
||||
|
||||
@@ -676,12 +675,6 @@ with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`
|
||||
Note that `--timeout` and `--contimeout` are both supported.
|
||||
|
||||
|
||||
## C14 {#c14}
|
||||
|
||||
C14 is supported through the SFTP backend.
|
||||
|
||||
See [C14's documentation](https://www.online.net/en/storage/c14-cold-storage)
|
||||
|
||||
## rsync.net {#rsync-net}
|
||||
|
||||
rsync.net is supported through the SFTP backend.
|
||||
|
||||
@@ -251,6 +251,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -415,8 +415,7 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
## Known issues
|
||||
|
||||
|
||||
@@ -280,6 +280,5 @@ this capability cannot determine free space for an rclone mount or
|
||||
use policy `mfs` (most free space) as a member of an rclone union
|
||||
remote.
|
||||
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
|
||||
See [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
|
||||
|
||||
@@ -73,6 +73,7 @@
|
||||
<a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe"></i> HDFS (Hadoop Distributed Filesystem)</a>
|
||||
<a class="dropdown-item" href="/http/"><i class="fa fa-globe"></i> HTTP</a>
|
||||
<a class="dropdown-item" href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a>
|
||||
<a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive"></i> Internet Archive</a>
|
||||
<a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud"></i> Jottacloud</a>
|
||||
<a class="dropdown-item" href="/koofr/"><i class="fa fa-suitcase"></i> Koofr</a>
|
||||
<a class="dropdown-item" href="/mailru/"><i class="fa fa-at"></i> Mail.ru Cloud</a>
|
||||
|
||||
@@ -1 +1 @@
|
||||
v1.58.0
|
||||
v1.59.0
|
||||
Loaded 100 of 154 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user