Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4010380ea8 | ||
|
|
c138e52a57 | ||
|
|
e22ce597ad | ||
|
|
79bd9e7913 | ||
|
|
32f9393ac8 | ||
|
|
f97c876eb1 | ||
|
|
9b43836e19 | ||
|
|
ff817e8764 | ||
|
|
3c63dec849 | ||
|
|
33876c5806 | ||
|
|
fa3b444341 | ||
|
|
e5fc424955 | ||
|
|
06badeffa3 | ||
|
|
eb71d1be18 | ||
|
|
7506a3c84c | ||
|
|
831abd3406 | ||
|
|
9c08cd80c7 | ||
|
|
948db193a2 | ||
|
|
72bc3f5079 | ||
|
|
bf8a428fbd | ||
|
|
05cc6f829b | ||
|
|
af73833773 | ||
|
|
3167a63780 | ||
|
|
1d9795daa6 | ||
|
|
03ea89adf0 |
No files matched your search
@@ -23,9 +23,6 @@ jobs:
|
||||
build:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 60
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -106,6 +103,7 @@ jobs:
|
||||
check-latest: true
|
||||
|
||||
- name: Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
|
||||
echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
|
||||
@@ -114,6 +112,7 @@ jobs:
|
||||
if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
|
||||
|
||||
- name: Install Libraries on Linux
|
||||
shell: bash
|
||||
run: |
|
||||
sudo modprobe fuse
|
||||
sudo chmod 666 /dev/fuse
|
||||
@@ -123,6 +122,7 @@ jobs:
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
|
||||
- name: Install Libraries on macOS
|
||||
shell: bash
|
||||
run: |
|
||||
# https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
|
||||
# https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
|
||||
@@ -151,6 +151,7 @@ jobs:
|
||||
if: matrix.os == 'windows-latest'
|
||||
|
||||
- name: Print Go version and environment
|
||||
shell: bash
|
||||
run: |
|
||||
printf "Using go at: $(which go)\n"
|
||||
printf "Go version: $(go version)\n"
|
||||
@@ -162,24 +163,29 @@ jobs:
|
||||
env
|
||||
|
||||
- name: Build rclone
|
||||
shell: bash
|
||||
run: |
|
||||
make
|
||||
|
||||
- name: Rclone version
|
||||
shell: bash
|
||||
run: |
|
||||
rclone version
|
||||
|
||||
- name: Run tests
|
||||
shell: bash
|
||||
run: |
|
||||
make quicktest
|
||||
if: matrix.quicktest
|
||||
|
||||
- name: Race test
|
||||
shell: bash
|
||||
run: |
|
||||
make racequicktest
|
||||
if: matrix.racequicktest
|
||||
|
||||
- name: Run librclone tests
|
||||
shell: bash
|
||||
run: |
|
||||
make -C librclone/ctest test
|
||||
make -C librclone/ctest clean
|
||||
@@ -187,12 +193,14 @@ jobs:
|
||||
if: matrix.librclonetest
|
||||
|
||||
- name: Compile all architectures test
|
||||
shell: bash
|
||||
run: |
|
||||
make
|
||||
make compile_all
|
||||
if: matrix.compile_all
|
||||
|
||||
- name: Deploy built binaries
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
|
||||
make ci_beta
|
||||
@@ -211,6 +219,7 @@ jobs:
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
shell: bash
|
||||
run: |
|
||||
echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
@@ -305,6 +314,7 @@ jobs:
|
||||
go-version: '>=1.24.0-rc.1'
|
||||
|
||||
- name: Set global environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "VERSION=$(make version)" >> $GITHUB_ENV
|
||||
|
||||
@@ -323,6 +333,7 @@ jobs:
|
||||
run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
|
||||
|
||||
- name: arm-v7a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
@@ -336,6 +347,7 @@ jobs:
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a .
|
||||
|
||||
- name: arm64-v8a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
@@ -348,6 +360,7 @@ jobs:
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a .
|
||||
|
||||
- name: x86 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
@@ -360,6 +373,7 @@ jobs:
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 .
|
||||
|
||||
- name: x64 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
|
||||
Generated
+149
-82
@@ -81,7 +81,7 @@
|
||||
<header id="title-block-header">
|
||||
<h1 class="title">rclone(1) User Manual</h1>
|
||||
<p class="author">Nick Craig-Wood</p>
|
||||
<p class="date">Jun 17, 2025</p>
|
||||
<p class="date">Jun 27, 2025</p>
|
||||
</header>
|
||||
<h1 id="name">NAME</h1>
|
||||
<p>rclone - manage files on cloud storage</p>
|
||||
@@ -426,6 +426,7 @@ choco install rclone</code></pre>
|
||||
<p><a href="https://repology.org/project/rclone/versions"><img src="https://repology.org/badge/vertical-allrepos/rclone.svg?columns=3" alt="Packaging status" /></a></p>
|
||||
<h2 id="docker">Docker installation</h2>
|
||||
<p>The rclone developers maintain a <a href="https://hub.docker.com/r/rclone/rclone">docker image for rclone</a>.</p>
|
||||
<p><strong>Note:</strong> We also now offer a paid version of rclone with enterprise-grade security and zero CVEs through our partner <a href="https://securebuild.com/blog/introducing-securebuild">SecureBuild</a>. If you are interested, check out their website and the <a href="https://securebuild.com/images/rclone">Rclone SecureBuild Image</a>.</p>
|
||||
<p>These images are built as part of the release process based on a minimal Alpine Linux.</p>
|
||||
<p>The <code>:latest</code> tag will always point to the latest stable release. You can use the <code>:beta</code> tag to get the latest build from master. You can also use version tags, e.g. <code>:1.49.1</code>, <code>:1.49</code> or <code>:1</code>.</p>
|
||||
<pre><code>$ docker pull rclone/rclone:latest
|
||||
@@ -2682,15 +2683,15 @@ X-User-Defined </code></pre>
|
||||
<pre><code>rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,command=echo"
|
||||
// Output: stories/The Quick Brown Fox!.txt</code></pre>
|
||||
<pre><code>rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20250617</code></pre>
|
||||
// Output: stories/The Quick Brown Fox!-20250618</code></pre>
|
||||
<pre><code>rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-17 0551PM</code></pre>
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-18 0148PM</code></pre>
|
||||
<pre><code>rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,regex=[\\.\\w]/ab"
|
||||
// Output: ababababababab/ababab ababababab ababababab ababab!abababab</code></pre>
|
||||
<p>Multiple transformations can be used in sequence, applied in the order they are specified on the command line.</p>
|
||||
<p>The <code>--name-transform</code> flag is also available in <code>sync</code>, <code>copy</code>, and <code>move</code>.</p>
|
||||
<h1 id="files-vs-directories">Files vs Directories</h1>
|
||||
<p>By default <code>--name-transform</code> will only apply to file names. The means only the leaf file name will be transformed. However some of the transforms would be better applied to the whole path or just directories. To choose which which part of the file path is affected some tags can be added to the <code>--name-transform</code></p>
|
||||
<p>By default <code>--name-transform</code> will only apply to file names. The means only the leaf file name will be transformed. However some of the transforms would be better applied to the whole path or just directories. To choose which which part of the file path is affected some tags can be added to the <code>--name-transform</code>.</p>
|
||||
<table>
|
||||
<colgroup>
|
||||
<col style="width: 50%" />
|
||||
@@ -2718,7 +2719,7 @@ X-User-Defined </code></pre>
|
||||
</tbody>
|
||||
</table>
|
||||
<p>This is used by adding the tag into the transform name like this: <code>--name-transform file,prefix=ABC</code> or <code>--name-transform dir,prefix=DEF</code>.</p>
|
||||
<p>For some conversions using all is more likely to be useful, for example <code>--name-transform all,nfc</code></p>
|
||||
<p>For some conversions using all is more likely to be useful, for example <code>--name-transform all,nfc</code>.</p>
|
||||
<p>Note that <code>--name-transform</code> may not add path separators <code>/</code> to the name. This will cause an error.</p>
|
||||
<h1 id="ordering-and-conflicts">Ordering and Conflicts</h1>
|
||||
<ul>
|
||||
@@ -2739,16 +2740,7 @@ X-User-Defined </code></pre>
|
||||
</ul>
|
||||
<h1 id="race-conditions-and-non-deterministic-behavior">Race Conditions and Non-Deterministic Behavior</h1>
|
||||
<p>Some transformations, such as <code>replace=old:new</code>, may introduce conflicts where multiple source files map to the same destination name. This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these. * If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic. * Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results.</p>
|
||||
<ul>
|
||||
<li>To minimize risks, users should:
|
||||
<ul>
|
||||
<li>Carefully review transformations that may introduce conflicts.</li>
|
||||
<li>Use <code>--dry-run</code> to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).</li>
|
||||
<li>Avoid transformations that cause multiple distinct source files to map to the same destination name.</li>
|
||||
<li>Consider disabling concurrency with <code>--transfers=1</code> if necessary.</li>
|
||||
<li>Certain transformations (e.g. <code>prefix</code>) will have a multiplying effect every time they are used. Avoid these when using <code>bisync</code>.</li>
|
||||
</ul></li>
|
||||
</ul>
|
||||
<p>To minimize risks, users should: * Carefully review transformations that may introduce conflicts. * Use <code>--dry-run</code> to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations). * Avoid transformations that cause multiple distinct source files to map to the same destination name. * Consider disabling concurrency with <code>--transfers=1</code> if necessary. * Certain transformations (e.g. <code>prefix</code>) will have a multiplying effect every time they are used. Avoid these when using <code>bisync</code>.</p>
|
||||
<pre><code>rclone convmv dest:path --name-transform XXX [flags]</code></pre>
|
||||
<h2 id="options-48">Options</h2>
|
||||
<pre><code> --create-empty-src-dirs Create empty source dirs on destination after move
|
||||
@@ -13239,7 +13231,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--tpslimit float Limit HTTP transactions per second to this
|
||||
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
|
||||
--use-cookies Enable session cookiejar
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.0")</code></pre>
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.2")</code></pre>
|
||||
<h2 id="performance">Performance</h2>
|
||||
<p>Flags helpful for increasing performance.</p>
|
||||
<pre><code> --buffer-size SizeSuffix In memory buffer size when reading files for each --transfer (default 16Mi)
|
||||
@@ -13661,6 +13653,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
|
||||
--ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-http-proxy string URL for HTTP CONNECT proxy
|
||||
--ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s)
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-no-check-upload Don't check the upload is OK
|
||||
@@ -21368,6 +21361,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="box-client-credentials">--box-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -22622,6 +22616,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="sharefile-client-credentials">--sharefile-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -23395,7 +23390,7 @@ upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"</c
|
||||
<p>See the <a href="https://rclone.org/docs/#metadata">metadata</a> docs for more info.</p>
|
||||
<h1 id="doi">DOI</h1>
|
||||
<p>The DOI remote is a read only remote for reading files from digital object identifiers (DOI).</p>
|
||||
<p>Currently, the DOI backend supports supports DOIs hosted with: - <a href="https://inveniosoftware.org/products/rdm/">InvenioRDM</a> - <a href="https://zenodo.org">Zenodo</a> - <a href="https://data.caltech.edu">CaltechDATA</a> - <a href="https://inveniosoftware.org/showcase/">Other InvenioRDM repositories</a> - <a href="https://dataverse.org">Dataverse</a> - <a href="https://dataverse.harvard.edu">Harvard Dataverse</a> - <a href="https://dataverse.org/installations">Other Dataverse repositories</a></p>
|
||||
<p>Currently, the DOI backend supports DOIs hosted with: - <a href="https://inveniosoftware.org/products/rdm/">InvenioRDM</a> - <a href="https://zenodo.org">Zenodo</a> - <a href="https://data.caltech.edu">CaltechDATA</a> - <a href="https://inveniosoftware.org/showcase/">Other InvenioRDM repositories</a> - <a href="https://dataverse.org">Dataverse</a> - <a href="https://dataverse.harvard.edu">Harvard Dataverse</a> - <a href="https://dataverse.org/installations">Other Dataverse repositories</a></p>
|
||||
<p>Paths are specified as <code>remote:path</code></p>
|
||||
<p>Paths may be as deep as required, e.g. <code>remote:directory/subdirectory</code>.</p>
|
||||
<h2 id="configuration-12">Configuration</h2>
|
||||
@@ -23756,6 +23751,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="dropbox-client-credentials">--dropbox-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -24724,6 +24720,16 @@ rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=IXs2wc8OJOz7SYLBk47
|
||||
<li>Type: string</li>
|
||||
<li>Required: false</li>
|
||||
</ul>
|
||||
<h4 id="ftp-http-proxy">--ftp-http-proxy</h4>
|
||||
<p>URL for HTTP CONNECT proxy</p>
|
||||
<p>Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: http_proxy</li>
|
||||
<li>Env Var: RCLONE_FTP_HTTP_PROXY</li>
|
||||
<li>Type: string</li>
|
||||
<li>Required: false</li>
|
||||
</ul>
|
||||
<h4 id="ftp-no-check-upload">--ftp-no-check-upload</h4>
|
||||
<p>Don't check the upload is OK</p>
|
||||
<p>Normally rclone will try to check the upload exists after it has uploaded a file to make sure the size and modification time are as expected.</p>
|
||||
@@ -25651,6 +25657,7 @@ ya29.c.c0ASRK0GbAFEewXD [truncated]</code></pre>
|
||||
<h4 id="gcs-client-credentials">--gcs-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -26354,6 +26361,7 @@ trashed=false and 'c' in parents</code></pre>
|
||||
<h4 id="drive-client-credentials">--drive-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -27429,6 +27437,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="gphotos-client-credentials">--gphotos-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -27596,6 +27605,13 @@ y/e/d> y</code></pre>
|
||||
<p>Rclone cannot delete files anywhere except under <code>album</code>.</p>
|
||||
<h3 id="deleting-albums">Deleting albums</h3>
|
||||
<p>The Google Photos API does not support deleting albums - see <a href="https://issuetracker.google.com/issues/135714733">bug #135714733</a>.</p>
|
||||
<h2 id="making-your-own-client_id-1">Making your own client_id</h2>
|
||||
<p>When you use rclone with Google photos in its default configuration you are using rclone's client_id. This is shared between all the rclone users. There is a global rate limit on the number of queries per second that each client_id can do set by Google.</p>
|
||||
<p>If there is a problem with this client_id (eg quota too low or the client_id stops working) then you can make your own.</p>
|
||||
<p>Please follow the steps in <a href="https://rclone.org/drive/#making-your-own-client-id">the google drive docs</a>. You will need these scopes instead of the drive ones detailed:</p>
|
||||
<pre><code>https://www.googleapis.com/auth/photoslibrary.appendonly
|
||||
https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata
|
||||
https://www.googleapis.com/auth/photoslibrary.edit.appcreateddata</code></pre>
|
||||
<h1 id="hasher">Hasher</h1>
|
||||
<p>Hasher is a special overlay backend to create remotes which handle checksums for other remotes. It's main functions include: - Emulate hash types unimplemented by backends - Cache checksums to help with slow hashing of large local or (S)FTP files - Warm up checksum cache from external SUM files</p>
|
||||
<h2 id="getting-started-1">Getting started</h2>
|
||||
@@ -28151,6 +28167,7 @@ rclone lsd remote:/users/test/path</code></pre>
|
||||
<h4 id="hidrive-client-credentials">--hidrive-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -29403,6 +29420,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="jottacloud-client-credentials">--jottacloud-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -30162,6 +30180,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="mailru-client-credentials">--mailru-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -32064,7 +32083,10 @@ y/e/d> y</code></pre>
|
||||
<h4 id="creating-client-id-for-onedrive-personal">Creating Client ID for OneDrive Personal</h4>
|
||||
<p>To create your own Client ID, please follow these steps:</p>
|
||||
<ol type="1">
|
||||
<li>Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click <code>New registration</code>.</li>
|
||||
<li>Open https://portal.azure.com/?quickstart=true#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/Overview and then under the <code>Add</code> menu click <code>App registration</code>.
|
||||
<ul>
|
||||
<li>If you have not created an Azure account, you will be prompted to. This is free, but you need to provide a phone number, address, and credit card for identity verification.</li>
|
||||
</ul></li>
|
||||
<li>Enter a name for your app, choose account type <code>Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)</code>, select <code>Web</code> in <code>Redirect URI</code>, then type (do not copy and paste) <code>http://localhost:53682/</code> and click Register. Copy and keep the <code>Application (client) ID</code> under the app name for later use.</li>
|
||||
<li>Under <code>manage</code> select <code>Certificates & secrets</code>, click <code>New client secret</code>. Enter a description (can be anything) and set <code>Expires</code> to 24 months. Copy and keep that secret <em>Value</em> for later use (you <em>won't</em> be able to see this value afterwards).</li>
|
||||
<li>Under <code>manage</code> select <code>API permissions</code>, click <code>Add a permission</code> and select <code>Microsoft Graph</code> then select <code>delegated permissions</code>.</li>
|
||||
@@ -32302,6 +32324,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="onedrive-client-credentials">--onedrive-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -32635,75 +32658,75 @@ rclone rc vfs/refresh recursive=true</code></pre>
|
||||
<p>Permissions are also supported, if <code>--onedrive-metadata-permissions</code> is set. The accepted values for <code>--onedrive-metadata-permissions</code> are "<code>read</code>", "<code>write</code>", "<code>read,write</code>", and "<code>off</code>" (the default). "<code>write</code>" supports adding new permissions, updating the "role" of existing permissions, and removing permissions. Updating and removing require the Permission ID to be known, so it is recommended to use "<code>read,write</code>" instead of "<code>write</code>" if you wish to update/remove permissions.</p>
|
||||
<p>Permissions are read/written in JSON format using the same schema as the <a href="https://learn.microsoft.com/en-us/onedrive/developer/rest-api/resources/permission?view=odsp-graph-online">OneDrive API</a>, which differs slightly between OneDrive Personal and Business.</p>
|
||||
<p>Example for OneDrive Personal:</p>
|
||||
<div class="sourceCode" id="cb1405"><pre class="sourceCode json"><code class="sourceCode json"><span id="cb1405-1"><a href="#cb1405-1" aria-hidden="true"></a><span class="ot">[</span></span>
|
||||
<span id="cb1405-2"><a href="#cb1405-2" aria-hidden="true"></a> <span class="fu">{</span></span>
|
||||
<span id="cb1405-3"><a href="#cb1405-3" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"1234567890ABC!123"</span><span class="fu">,</span></span>
|
||||
<span id="cb1405-4"><a href="#cb1405-4" aria-hidden="true"></a> <span class="dt">"grantedTo"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1405-5"><a href="#cb1405-5" aria-hidden="true"></a> <span class="dt">"user"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1405-6"><a href="#cb1405-6" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"ryan@contoso.com"</span></span>
|
||||
<span id="cb1405-7"><a href="#cb1405-7" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1405-8"><a href="#cb1405-8" aria-hidden="true"></a> <span class="dt">"application"</span><span class="fu">:</span> <span class="fu">{},</span></span>
|
||||
<span id="cb1405-9"><a href="#cb1405-9" aria-hidden="true"></a> <span class="dt">"device"</span><span class="fu">:</span> <span class="fu">{}</span></span>
|
||||
<span id="cb1405-10"><a href="#cb1405-10" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1405-11"><a href="#cb1405-11" aria-hidden="true"></a> <span class="dt">"invitation"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1405-12"><a href="#cb1405-12" aria-hidden="true"></a> <span class="dt">"email"</span><span class="fu">:</span> <span class="st">"ryan@contoso.com"</span></span>
|
||||
<span id="cb1405-13"><a href="#cb1405-13" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1405-14"><a href="#cb1405-14" aria-hidden="true"></a> <span class="dt">"link"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1405-15"><a href="#cb1405-15" aria-hidden="true"></a> <span class="dt">"webUrl"</span><span class="fu">:</span> <span class="st">"https://1drv.ms/t/s!1234567890ABC"</span></span>
|
||||
<span id="cb1405-16"><a href="#cb1405-16" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1405-17"><a href="#cb1405-17" aria-hidden="true"></a> <span class="dt">"roles"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1405-18"><a href="#cb1405-18" aria-hidden="true"></a> <span class="st">"read"</span></span>
|
||||
<span id="cb1405-19"><a href="#cb1405-19" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1405-20"><a href="#cb1405-20" aria-hidden="true"></a> <span class="dt">"shareId"</span><span class="fu">:</span> <span class="st">"s!1234567890ABC"</span></span>
|
||||
<span id="cb1405-21"><a href="#cb1405-21" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1405-22"><a href="#cb1405-22" aria-hidden="true"></a><span class="ot">]</span></span></code></pre></div>
|
||||
<p>Example for OneDrive Business:</p>
|
||||
<div class="sourceCode" id="cb1406"><pre class="sourceCode json"><code class="sourceCode json"><span id="cb1406-1"><a href="#cb1406-1" aria-hidden="true"></a><span class="ot">[</span></span>
|
||||
<span id="cb1406-2"><a href="#cb1406-2" aria-hidden="true"></a> <span class="fu">{</span></span>
|
||||
<span id="cb1406-3"><a href="#cb1406-3" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"48d31887-5fad-4d73-a9f5-3c356e68a038"</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-4"><a href="#cb1406-4" aria-hidden="true"></a> <span class="dt">"grantedToIdentities"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1406-5"><a href="#cb1406-5" aria-hidden="true"></a> <span class="fu">{</span></span>
|
||||
<span id="cb1406-6"><a href="#cb1406-6" aria-hidden="true"></a> <span class="dt">"user"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-7"><a href="#cb1406-7" aria-hidden="true"></a> <span class="dt">"displayName"</span><span class="fu">:</span> <span class="st">"ryan@contoso.com"</span></span>
|
||||
<span id="cb1406-8"><a href="#cb1406-8" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-9"><a href="#cb1406-9" aria-hidden="true"></a> <span class="dt">"application"</span><span class="fu">:</span> <span class="fu">{},</span></span>
|
||||
<span id="cb1406-10"><a href="#cb1406-10" aria-hidden="true"></a> <span class="dt">"device"</span><span class="fu">:</span> <span class="fu">{}</span></span>
|
||||
<span id="cb1406-11"><a href="#cb1406-11" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1406-12"><a href="#cb1406-12" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-13"><a href="#cb1406-13" aria-hidden="true"></a> <span class="dt">"link"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-14"><a href="#cb1406-14" aria-hidden="true"></a> <span class="dt">"type"</span><span class="fu">:</span> <span class="st">"view"</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-15"><a href="#cb1406-15" aria-hidden="true"></a> <span class="dt">"scope"</span><span class="fu">:</span> <span class="st">"users"</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-16"><a href="#cb1406-16" aria-hidden="true"></a> <span class="dt">"webUrl"</span><span class="fu">:</span> <span class="st">"https://contoso.sharepoint.com/:w:/t/design/a577ghg9hgh737613bmbjf839026561fmzhsr85ng9f3hjck2t5s"</span></span>
|
||||
<span id="cb1406-17"><a href="#cb1406-17" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-18"><a href="#cb1406-18" aria-hidden="true"></a> <span class="dt">"roles"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1406-19"><a href="#cb1406-19" aria-hidden="true"></a> <span class="st">"read"</span></span>
|
||||
<span id="cb1406-20"><a href="#cb1406-20" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-21"><a href="#cb1406-21" aria-hidden="true"></a> <span class="dt">"shareId"</span><span class="fu">:</span> <span class="st">"u!LKj1lkdlals90j1nlkascl"</span></span>
|
||||
<span id="cb1406-22"><a href="#cb1406-22" aria-hidden="true"></a> <span class="fu">}</span><span class="ot">,</span></span>
|
||||
<span id="cb1406-23"><a href="#cb1406-23" aria-hidden="true"></a> <span class="fu">{</span></span>
|
||||
<span id="cb1406-24"><a href="#cb1406-24" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"5D33DD65C6932946"</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-25"><a href="#cb1406-25" aria-hidden="true"></a> <span class="dt">"grantedTo"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-26"><a href="#cb1406-26" aria-hidden="true"></a> <span class="dt">"user"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-27"><a href="#cb1406-27" aria-hidden="true"></a> <span class="dt">"displayName"</span><span class="fu">:</span> <span class="st">"John Doe"</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-28"><a href="#cb1406-28" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"efee1b77-fb3b-4f65-99d6-274c11914d12"</span></span>
|
||||
<span id="cb1406-29"><a href="#cb1406-29" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-30"><a href="#cb1406-30" aria-hidden="true"></a> <span class="dt">"application"</span><span class="fu">:</span> <span class="fu">{},</span></span>
|
||||
<span id="cb1406-31"><a href="#cb1406-31" aria-hidden="true"></a> <span class="dt">"device"</span><span class="fu">:</span> <span class="fu">{}</span></span>
|
||||
<span id="cb1406-32"><a href="#cb1406-32" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-33"><a href="#cb1406-33" aria-hidden="true"></a> <span class="dt">"roles"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1406-34"><a href="#cb1406-34" aria-hidden="true"></a> <span class="st">"owner"</span></span>
|
||||
<span id="cb1406-35"><a href="#cb1406-35" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-36"><a href="#cb1406-36" aria-hidden="true"></a> <span class="dt">"shareId"</span><span class="fu">:</span> <span class="st">"FWxc1lasfdbEAGM5fI7B67aB5ZMPDMmQ11U"</span></span>
|
||||
<span id="cb1406-37"><a href="#cb1406-37" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1406-38"><a href="#cb1406-38" aria-hidden="true"></a><span class="ot">]</span></span></code></pre></div>
|
||||
<span id="cb1406-3"><a href="#cb1406-3" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"1234567890ABC!123"</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-4"><a href="#cb1406-4" aria-hidden="true"></a> <span class="dt">"grantedTo"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-5"><a href="#cb1406-5" aria-hidden="true"></a> <span class="dt">"user"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-6"><a href="#cb1406-6" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"ryan@contoso.com"</span></span>
|
||||
<span id="cb1406-7"><a href="#cb1406-7" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-8"><a href="#cb1406-8" aria-hidden="true"></a> <span class="dt">"application"</span><span class="fu">:</span> <span class="fu">{},</span></span>
|
||||
<span id="cb1406-9"><a href="#cb1406-9" aria-hidden="true"></a> <span class="dt">"device"</span><span class="fu">:</span> <span class="fu">{}</span></span>
|
||||
<span id="cb1406-10"><a href="#cb1406-10" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-11"><a href="#cb1406-11" aria-hidden="true"></a> <span class="dt">"invitation"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-12"><a href="#cb1406-12" aria-hidden="true"></a> <span class="dt">"email"</span><span class="fu">:</span> <span class="st">"ryan@contoso.com"</span></span>
|
||||
<span id="cb1406-13"><a href="#cb1406-13" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-14"><a href="#cb1406-14" aria-hidden="true"></a> <span class="dt">"link"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1406-15"><a href="#cb1406-15" aria-hidden="true"></a> <span class="dt">"webUrl"</span><span class="fu">:</span> <span class="st">"https://1drv.ms/t/s!1234567890ABC"</span></span>
|
||||
<span id="cb1406-16"><a href="#cb1406-16" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1406-17"><a href="#cb1406-17" aria-hidden="true"></a> <span class="dt">"roles"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1406-18"><a href="#cb1406-18" aria-hidden="true"></a> <span class="st">"read"</span></span>
|
||||
<span id="cb1406-19"><a href="#cb1406-19" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1406-20"><a href="#cb1406-20" aria-hidden="true"></a> <span class="dt">"shareId"</span><span class="fu">:</span> <span class="st">"s!1234567890ABC"</span></span>
|
||||
<span id="cb1406-21"><a href="#cb1406-21" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1406-22"><a href="#cb1406-22" aria-hidden="true"></a><span class="ot">]</span></span></code></pre></div>
|
||||
<p>Example for OneDrive Business:</p>
|
||||
<div class="sourceCode" id="cb1407"><pre class="sourceCode json"><code class="sourceCode json"><span id="cb1407-1"><a href="#cb1407-1" aria-hidden="true"></a><span class="ot">[</span></span>
|
||||
<span id="cb1407-2"><a href="#cb1407-2" aria-hidden="true"></a> <span class="fu">{</span></span>
|
||||
<span id="cb1407-3"><a href="#cb1407-3" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"48d31887-5fad-4d73-a9f5-3c356e68a038"</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-4"><a href="#cb1407-4" aria-hidden="true"></a> <span class="dt">"grantedToIdentities"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1407-5"><a href="#cb1407-5" aria-hidden="true"></a> <span class="fu">{</span></span>
|
||||
<span id="cb1407-6"><a href="#cb1407-6" aria-hidden="true"></a> <span class="dt">"user"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1407-7"><a href="#cb1407-7" aria-hidden="true"></a> <span class="dt">"displayName"</span><span class="fu">:</span> <span class="st">"ryan@contoso.com"</span></span>
|
||||
<span id="cb1407-8"><a href="#cb1407-8" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1407-9"><a href="#cb1407-9" aria-hidden="true"></a> <span class="dt">"application"</span><span class="fu">:</span> <span class="fu">{},</span></span>
|
||||
<span id="cb1407-10"><a href="#cb1407-10" aria-hidden="true"></a> <span class="dt">"device"</span><span class="fu">:</span> <span class="fu">{}</span></span>
|
||||
<span id="cb1407-11"><a href="#cb1407-11" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1407-12"><a href="#cb1407-12" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-13"><a href="#cb1407-13" aria-hidden="true"></a> <span class="dt">"link"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1407-14"><a href="#cb1407-14" aria-hidden="true"></a> <span class="dt">"type"</span><span class="fu">:</span> <span class="st">"view"</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-15"><a href="#cb1407-15" aria-hidden="true"></a> <span class="dt">"scope"</span><span class="fu">:</span> <span class="st">"users"</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-16"><a href="#cb1407-16" aria-hidden="true"></a> <span class="dt">"webUrl"</span><span class="fu">:</span> <span class="st">"https://contoso.sharepoint.com/:w:/t/design/a577ghg9hgh737613bmbjf839026561fmzhsr85ng9f3hjck2t5s"</span></span>
|
||||
<span id="cb1407-17"><a href="#cb1407-17" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1407-18"><a href="#cb1407-18" aria-hidden="true"></a> <span class="dt">"roles"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1407-19"><a href="#cb1407-19" aria-hidden="true"></a> <span class="st">"read"</span></span>
|
||||
<span id="cb1407-20"><a href="#cb1407-20" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-21"><a href="#cb1407-21" aria-hidden="true"></a> <span class="dt">"shareId"</span><span class="fu">:</span> <span class="st">"u!LKj1lkdlals90j1nlkascl"</span></span>
|
||||
<span id="cb1407-22"><a href="#cb1407-22" aria-hidden="true"></a> <span class="fu">}</span><span class="ot">,</span></span>
|
||||
<span id="cb1407-23"><a href="#cb1407-23" aria-hidden="true"></a> <span class="fu">{</span></span>
|
||||
<span id="cb1407-24"><a href="#cb1407-24" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"5D33DD65C6932946"</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-25"><a href="#cb1407-25" aria-hidden="true"></a> <span class="dt">"grantedTo"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1407-26"><a href="#cb1407-26" aria-hidden="true"></a> <span class="dt">"user"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1407-27"><a href="#cb1407-27" aria-hidden="true"></a> <span class="dt">"displayName"</span><span class="fu">:</span> <span class="st">"John Doe"</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-28"><a href="#cb1407-28" aria-hidden="true"></a> <span class="dt">"id"</span><span class="fu">:</span> <span class="st">"efee1b77-fb3b-4f65-99d6-274c11914d12"</span></span>
|
||||
<span id="cb1407-29"><a href="#cb1407-29" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1407-30"><a href="#cb1407-30" aria-hidden="true"></a> <span class="dt">"application"</span><span class="fu">:</span> <span class="fu">{},</span></span>
|
||||
<span id="cb1407-31"><a href="#cb1407-31" aria-hidden="true"></a> <span class="dt">"device"</span><span class="fu">:</span> <span class="fu">{}</span></span>
|
||||
<span id="cb1407-32"><a href="#cb1407-32" aria-hidden="true"></a> <span class="fu">},</span></span>
|
||||
<span id="cb1407-33"><a href="#cb1407-33" aria-hidden="true"></a> <span class="dt">"roles"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||
<span id="cb1407-34"><a href="#cb1407-34" aria-hidden="true"></a> <span class="st">"owner"</span></span>
|
||||
<span id="cb1407-35"><a href="#cb1407-35" aria-hidden="true"></a> <span class="ot">]</span><span class="fu">,</span></span>
|
||||
<span id="cb1407-36"><a href="#cb1407-36" aria-hidden="true"></a> <span class="dt">"shareId"</span><span class="fu">:</span> <span class="st">"FWxc1lasfdbEAGM5fI7B67aB5ZMPDMmQ11U"</span></span>
|
||||
<span id="cb1407-37"><a href="#cb1407-37" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1407-38"><a href="#cb1407-38" aria-hidden="true"></a><span class="ot">]</span></span></code></pre></div>
|
||||
<p>To write permissions, pass in a "permissions" metadata key using this same format. The <a href="https://rclone.org/docs/#metadata-mapper"><code>--metadata-mapper</code></a> tool can be very helpful for this.</p>
|
||||
<p>When adding permissions, an email address can be provided in the <code>User.ID</code> or <code>DisplayName</code> properties of <code>grantedTo</code> or <code>grantedToIdentities</code>. Alternatively, an ObjectID can be provided in <code>User.ID</code>. At least one valid recipient must be provided in order to add a permission for a user. Creating a Public Link is also supported, if <code>Link.Scope</code> is set to <code>"anonymous"</code>.</p>
|
||||
<p>Example request to add a "read" permission with <code>--metadata-mapper</code>:</p>
|
||||
<div class="sourceCode" id="cb1407"><pre class="sourceCode json"><code class="sourceCode json"><span id="cb1407-1"><a href="#cb1407-1" aria-hidden="true"></a><span class="fu">{</span></span>
|
||||
<span id="cb1407-2"><a href="#cb1407-2" aria-hidden="true"></a> <span class="dt">"Metadata"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1407-3"><a href="#cb1407-3" aria-hidden="true"></a> <span class="dt">"permissions"</span><span class="fu">:</span> <span class="st">"[{</span><span class="ch">\"</span><span class="st">grantedToIdentities</span><span class="ch">\"</span><span class="st">:[{</span><span class="ch">\"</span><span class="st">user</span><span class="ch">\"</span><span class="st">:{</span><span class="ch">\"</span><span class="st">id</span><span class="ch">\"</span><span class="st">:</span><span class="ch">\"</span><span class="st">ryan@contoso.com</span><span class="ch">\"</span><span class="st">}}],</span><span class="ch">\"</span><span class="st">roles</span><span class="ch">\"</span><span class="st">:[</span><span class="ch">\"</span><span class="st">read</span><span class="ch">\"</span><span class="st">]}]"</span></span>
|
||||
<span id="cb1407-4"><a href="#cb1407-4" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1407-5"><a href="#cb1407-5" aria-hidden="true"></a><span class="fu">}</span></span></code></pre></div>
|
||||
<div class="sourceCode" id="cb1408"><pre class="sourceCode json"><code class="sourceCode json"><span id="cb1408-1"><a href="#cb1408-1" aria-hidden="true"></a><span class="fu">{</span></span>
|
||||
<span id="cb1408-2"><a href="#cb1408-2" aria-hidden="true"></a> <span class="dt">"Metadata"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||
<span id="cb1408-3"><a href="#cb1408-3" aria-hidden="true"></a> <span class="dt">"permissions"</span><span class="fu">:</span> <span class="st">"[{</span><span class="ch">\"</span><span class="st">grantedToIdentities</span><span class="ch">\"</span><span class="st">:[{</span><span class="ch">\"</span><span class="st">user</span><span class="ch">\"</span><span class="st">:{</span><span class="ch">\"</span><span class="st">id</span><span class="ch">\"</span><span class="st">:</span><span class="ch">\"</span><span class="st">ryan@contoso.com</span><span class="ch">\"</span><span class="st">}}],</span><span class="ch">\"</span><span class="st">roles</span><span class="ch">\"</span><span class="st">:[</span><span class="ch">\"</span><span class="st">read</span><span class="ch">\"</span><span class="st">]}]"</span></span>
|
||||
<span id="cb1408-4"><a href="#cb1408-4" aria-hidden="true"></a> <span class="fu">}</span></span>
|
||||
<span id="cb1408-5"><a href="#cb1408-5" aria-hidden="true"></a><span class="fu">}</span></span></code></pre></div>
|
||||
<p>Note that adding a permission can fail if a conflicting permission already exists for the file/folder.</p>
|
||||
<p>To update an existing permission, include both the Permission ID and the new <code>roles</code> to be assigned. <code>roles</code> is the only property that can be changed.</p>
|
||||
<p>To remove permissions, pass in a blob containing only the permissions you wish to keep (which can be empty, to remove all.) Note that the <code>owner</code> role will be ignored, as it cannot be removed.</p>
|
||||
@@ -35077,6 +35100,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="pcloud-client-credentials">--pcloud-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -35686,6 +35710,7 @@ y/e/d> </code></pre>
|
||||
<h4 id="premiumizeme-client-credentials">--premiumizeme-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -36077,6 +36102,7 @@ e/n/d/r/c/s/q> q</code></pre>
|
||||
<h4 id="putio-client-credentials">--putio-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -39099,6 +39125,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="yandex-client-credentials">--yandex-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -39322,6 +39349,7 @@ y/e/d> </code></pre>
|
||||
<h4 id="zoho-client-credentials">--zoho-client-credentials</h4>
|
||||
<p>Use client credentials OAuth flow.</p>
|
||||
<p>This will use the OAUTH2 client Credentials Flow as described in RFC 6749.</p>
|
||||
<p>Note that this option is NOT supported by all backends.</p>
|
||||
<p>Properties:</p>
|
||||
<ul>
|
||||
<li>Config: client_credentials</li>
|
||||
@@ -40062,6 +40090,45 @@ $ tree /tmp/c
|
||||
<li>"error": return an error based on option value</li>
|
||||
</ul>
|
||||
<h1 id="changelog-1">Changelog</h1>
|
||||
<h2 id="v1.70.2---2025-06-27">v1.70.2 - 2025-06-27</h2>
|
||||
<p><a href="https://github.com/rclone/rclone/compare/v1.70.1...v1.70.2">See commits</a></p>
|
||||
<ul>
|
||||
<li>Bug Fixes
|
||||
<ul>
|
||||
<li>convmv: Make --dry-run logs less noisy (nielash)</li>
|
||||
<li>sync: Avoid copying dir metadata to itself (nielash)</li>
|
||||
<li>build: Bump github.com/go-chi/chi/v5 from 5.2.1 to 5.2.2 to fix GHSA-vrw8-fxc6-2r93 (dependabot[bot])</li>
|
||||
<li>convmv: Fix moving to unicode-equivalent name (nielash)</li>
|
||||
<li>log: Fix deadlock when using systemd logging (Nick Craig-Wood)</li>
|
||||
<li>pacer: Fix nil pointer deref in RetryError (Nick Craig-Wood)</li>
|
||||
<li>doc fixes (Ali Zein Yousuf, Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Local
|
||||
<ul>
|
||||
<li>Fix --skip-links on Windows when skipping Junction points (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Combine
|
||||
<ul>
|
||||
<li>Fix directory not found errors with ListP interface (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Mega
|
||||
<ul>
|
||||
<li>Fix tls handshake failure (necaran)</li>
|
||||
</ul></li>
|
||||
<li>Pikpak
|
||||
<ul>
|
||||
<li>Fix uploads fail with "aws-chunked encoding is not supported" error (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
</ul>
|
||||
<h2 id="v1.70.1---2025-06-19">v1.70.1 - 2025-06-19</h2>
|
||||
<p><a href="https://github.com/rclone/rclone/compare/v1.70.0...v1.70.1">See commits</a></p>
|
||||
<ul>
|
||||
<li>Bug Fixes
|
||||
<ul>
|
||||
<li>convmv: Fix spurious "error running command echo" on Windows (Nick Craig-Wood)</li>
|
||||
<li>doc fixes (albertony, Ed Craig-Wood, jinjingroad)</li>
|
||||
</ul></li>
|
||||
</ul>
|
||||
<h2 id="v1.70.0---2025-06-17">v1.70.0 - 2025-06-17</h2>
|
||||
<p><a href="https://github.com/rclone/rclone/compare/v1.69.0...v1.70.0">See commits</a></p>
|
||||
<ul>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
% rclone(1) User Manual
|
||||
% Nick Craig-Wood
|
||||
% Jun 17, 2025
|
||||
% Jun 27, 2025
|
||||
|
||||
# NAME
|
||||
|
||||
@@ -512,6 +512,12 @@ package is here.
|
||||
|
||||
The rclone developers maintain a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
|
||||
|
||||
**Note:** We also now offer a paid version of rclone with
|
||||
enterprise-grade security and zero CVEs through our partner
|
||||
[SecureBuild](https://securebuild.com/blog/introducing-securebuild).
|
||||
If you are interested, check out their website and the [Rclone
|
||||
SecureBuild Image](https://securebuild.com/images/rclone).
|
||||
|
||||
These images are built as part of the release process based on a
|
||||
minimal Alpine Linux.
|
||||
|
||||
@@ -4452,12 +4458,12 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,command=e
|
||||
|
||||
```
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20250617
|
||||
// Output: stories/The Quick Brown Fox!-20250618
|
||||
```
|
||||
|
||||
```
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-17 0551PM
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-18 0148PM
|
||||
```
|
||||
|
||||
```
|
||||
@@ -4465,17 +4471,15 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,regex=[\\
|
||||
// Output: ababababababab/ababab ababababab ababababab ababab!abababab
|
||||
```
|
||||
|
||||
|
||||
|
||||
Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
|
||||
|
||||
The `--name-transform` flag is also available in `sync`, `copy`, and `move`.
|
||||
|
||||
# Files vs Directories ##
|
||||
# Files vs Directories
|
||||
|
||||
By default `--name-transform` will only apply to file names. The means only the leaf file name will be transformed.
|
||||
However some of the transforms would be better applied to the whole path or just directories.
|
||||
To choose which which part of the file path is affected some tags can be added to the `--name-transform`
|
||||
To choose which which part of the file path is affected some tags can be added to the `--name-transform`.
|
||||
|
||||
| Tag | Effect |
|
||||
|------|------|
|
||||
@@ -4485,11 +4489,11 @@ To choose which which part of the file path is affected some tags can be added t
|
||||
|
||||
This is used by adding the tag into the transform name like this: `--name-transform file,prefix=ABC` or `--name-transform dir,prefix=DEF`.
|
||||
|
||||
For some conversions using all is more likely to be useful, for example `--name-transform all,nfc`
|
||||
For some conversions using all is more likely to be useful, for example `--name-transform all,nfc`.
|
||||
|
||||
Note that `--name-transform` may not add path separators `/` to the name. This will cause an error.
|
||||
|
||||
# Ordering and Conflicts ##
|
||||
# Ordering and Conflicts
|
||||
|
||||
* Transformations will be applied in the order specified by the user.
|
||||
* If the `file` tag is in use (the default) then only the leaf name of files will be transformed.
|
||||
@@ -4504,19 +4508,19 @@ user, allowing for intentional use cases (e.g., trimming one prefix before addin
|
||||
* Users should be aware that certain combinations may lead to unexpected results and should verify
|
||||
transformations using `--dry-run` before execution.
|
||||
|
||||
# Race Conditions and Non-Deterministic Behavior ##
|
||||
# Race Conditions and Non-Deterministic Behavior
|
||||
|
||||
Some transformations, such as `replace=old:new`, may introduce conflicts where multiple source files map to the same destination name.
|
||||
This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these.
|
||||
* If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic.
|
||||
* Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results.
|
||||
|
||||
* To minimize risks, users should:
|
||||
* Carefully review transformations that may introduce conflicts.
|
||||
* Use `--dry-run` to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
|
||||
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
|
||||
* Consider disabling concurrency with `--transfers=1` if necessary.
|
||||
* Certain transformations (e.g. `prefix`) will have a multiplying effect every time they are used. Avoid these when using `bisync`.
|
||||
To minimize risks, users should:
|
||||
* Carefully review transformations that may introduce conflicts.
|
||||
* Use `--dry-run` to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
|
||||
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
|
||||
* Consider disabling concurrency with `--transfers=1` if necessary.
|
||||
* Certain transformations (e.g. `prefix`) will have a multiplying effect every time they are used. Avoid these when using `bisync`.
|
||||
|
||||
|
||||
|
||||
@@ -22400,7 +22404,7 @@ Flags for general networking and HTTP stuff.
|
||||
--tpslimit float Limit HTTP transactions per second to this
|
||||
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
|
||||
--use-cookies Enable session cookiejar
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.2")
|
||||
```
|
||||
|
||||
|
||||
@@ -22882,6 +22886,7 @@ Backend-only flags (these can be set in the config file also).
|
||||
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
|
||||
--ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-http-proxy string URL for HTTP CONNECT proxy
|
||||
--ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s)
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-no-check-upload Don't check the upload is OK
|
||||
@@ -33549,6 +33554,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -35368,6 +35375,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -36631,7 +36640,7 @@ See the [metadata](https://rclone.org/docs/#metadata) docs for more info.
|
||||
|
||||
The DOI remote is a read only remote for reading files from digital object identifiers (DOI).
|
||||
|
||||
Currently, the DOI backend supports supports DOIs hosted with:
|
||||
Currently, the DOI backend supports DOIs hosted with:
|
||||
- [InvenioRDM](https://inveniosoftware.org/products/rdm/)
|
||||
- [Zenodo](https://zenodo.org)
|
||||
- [CaltechDATA](https://data.caltech.edu)
|
||||
@@ -37110,6 +37119,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -38548,6 +38559,20 @@ Properties:
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --ftp-http-proxy
|
||||
|
||||
URL for HTTP CONNECT proxy
|
||||
|
||||
Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: http_proxy
|
||||
- Env Var: RCLONE_FTP_HTTP_PROXY
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --ftp-no-check-upload
|
||||
|
||||
Don't check the upload is OK
|
||||
@@ -39591,6 +39616,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -40404,6 +40431,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -41939,6 +41968,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -42246,6 +42277,25 @@ Rclone cannot delete files anywhere except under `album`.
|
||||
|
||||
The Google Photos API does not support deleting albums - see [bug #135714733](https://issuetracker.google.com/issues/135714733).
|
||||
|
||||
## Making your own client_id
|
||||
|
||||
When you use rclone with Google photos in its default configuration you
|
||||
are using rclone's client_id. This is shared between all the rclone
|
||||
users. There is a global rate limit on the number of queries per
|
||||
second that each client_id can do set by Google.
|
||||
|
||||
If there is a problem with this client_id (eg quota too low or the
|
||||
client_id stops working) then you can make your own.
|
||||
|
||||
Please follow the steps in [the google drive docs](https://rclone.org/drive/#making-your-own-client-id).
|
||||
You will need these scopes instead of the drive ones detailed:
|
||||
|
||||
```
|
||||
https://www.googleapis.com/auth/photoslibrary.appendonly
|
||||
https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata
|
||||
https://www.googleapis.com/auth/photoslibrary.edit.appcreateddata
|
||||
```
|
||||
|
||||
# Hasher
|
||||
|
||||
Hasher is a special overlay backend to create remotes which handle
|
||||
@@ -43128,6 +43178,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -44697,6 +44749,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -45573,6 +45627,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -48306,7 +48362,8 @@ For example, you might see throttling.
|
||||
|
||||
To create your own Client ID, please follow these steps:
|
||||
|
||||
1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`.
|
||||
1. Open https://portal.azure.com/?quickstart=true#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/Overview and then under the `Add` menu click `App registration`.
|
||||
* If you have not created an Azure account, you will be prompted to. This is free, but you need to provide a phone number, address, and credit card for identity verification.
|
||||
2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI`, then type (do not copy and paste) `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use.
|
||||
3. Under `manage` select `Certificates & secrets`, click `New client secret`. Enter a description (can be anything) and set `Expires` to 24 months. Copy and keep that secret _Value_ for later use (you _won't_ be able to see this value afterwards).
|
||||
4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`.
|
||||
@@ -48559,6 +48616,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -52187,6 +52246,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -52988,6 +53049,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -53584,6 +53647,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -58004,6 +58069,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -58304,6 +58371,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -59071,6 +59140,35 @@ Options:
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.70.2 - 2025-06-27
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.1...v1.70.2)
|
||||
|
||||
* Bug Fixes
|
||||
* convmv: Make --dry-run logs less noisy (nielash)
|
||||
* sync: Avoid copying dir metadata to itself (nielash)
|
||||
* build: Bump github.com/go-chi/chi/v5 from 5.2.1 to 5.2.2 to fix GHSA-vrw8-fxc6-2r93 (dependabot[bot])
|
||||
* convmv: Fix moving to unicode-equivalent name (nielash)
|
||||
* log: Fix deadlock when using systemd logging (Nick Craig-Wood)
|
||||
* pacer: Fix nil pointer deref in RetryError (Nick Craig-Wood)
|
||||
* doc fixes (Ali Zein Yousuf, Nick Craig-Wood)
|
||||
* Local
|
||||
* Fix --skip-links on Windows when skipping Junction points (Nick Craig-Wood)
|
||||
* Combine
|
||||
* Fix directory not found errors with ListP interface (Nick Craig-Wood)
|
||||
* Mega
|
||||
* Fix tls handshake failure (necaran)
|
||||
* Pikpak
|
||||
* Fix uploads fail with "aws-chunked encoding is not supported" error (Nick Craig-Wood)
|
||||
|
||||
## v1.70.1 - 2025-06-19
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.0...v1.70.1)
|
||||
|
||||
* Bug Fixes
|
||||
* convmv: Fix spurious "error running command echo" on Windows (Nick Craig-Wood)
|
||||
* doc fixes (albertony, Ed Craig-Wood, jinjingroad)
|
||||
|
||||
## v1.70.0 - 2025-06-17
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.70.0)
|
||||
|
||||
Generated
+122
-21
@@ -1,6 +1,6 @@
|
||||
rclone(1) User Manual
|
||||
Nick Craig-Wood
|
||||
Jun 17, 2025
|
||||
Jun 27, 2025
|
||||
|
||||
NAME
|
||||
|
||||
@@ -491,6 +491,10 @@ Docker installation
|
||||
|
||||
The rclone developers maintain a docker image for rclone.
|
||||
|
||||
Note: We also now offer a paid version of rclone with enterprise-grade
|
||||
security and zero CVEs through our partner SecureBuild. If you are
|
||||
interested, check out their website and the Rclone SecureBuild Image.
|
||||
|
||||
These images are built as part of the release process based on a minimal
|
||||
Alpine Linux.
|
||||
|
||||
@@ -4145,10 +4149,10 @@ Examples:
|
||||
// Output: stories/The Quick Brown Fox!.txt
|
||||
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20250617
|
||||
// Output: stories/The Quick Brown Fox!-20250618
|
||||
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-17 0551PM
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-18 0148PM
|
||||
|
||||
rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,regex=[\\.\\w]/ab"
|
||||
// Output: ababababababab/ababab ababababab ababababab ababab!abababab
|
||||
@@ -4164,7 +4168,7 @@ By default --name-transform will only apply to file names. The means
|
||||
only the leaf file name will be transformed. However some of the
|
||||
transforms would be better applied to the whole path or just
|
||||
directories. To choose which which part of the file path is affected
|
||||
some tags can be added to the --name-transform
|
||||
some tags can be added to the --name-transform.
|
||||
|
||||
-----------------------------------------------------------------------
|
||||
Tag Effect
|
||||
@@ -4184,7 +4188,7 @@ This is used by adding the tag into the transform name like this:
|
||||
--name-transform file,prefix=ABC or --name-transform dir,prefix=DEF.
|
||||
|
||||
For some conversions using all is more likely to be useful, for example
|
||||
--name-transform all,nfc
|
||||
--name-transform all,nfc.
|
||||
|
||||
Note that --name-transform may not add path separators / to the name.
|
||||
This will cause an error.
|
||||
@@ -4223,16 +4227,14 @@ be non-deterministic. * Running rclone check after a sync using such
|
||||
transformations may erroneously report missing or differing files due to
|
||||
overwritten results.
|
||||
|
||||
- To minimize risks, users should:
|
||||
- Carefully review transformations that may introduce conflicts.
|
||||
- Use --dry-run to inspect changes before executing a sync (but
|
||||
keep in mind that it won't show the effect of non-deterministic
|
||||
transformations).
|
||||
- Avoid transformations that cause multiple distinct source files
|
||||
to map to the same destination name.
|
||||
- Consider disabling concurrency with --transfers=1 if necessary.
|
||||
- Certain transformations (e.g. prefix) will have a multiplying
|
||||
effect every time they are used. Avoid these when using bisync.
|
||||
To minimize risks, users should: * Carefully review transformations that
|
||||
may introduce conflicts. * Use --dry-run to inspect changes before
|
||||
executing a sync (but keep in mind that it won't show the effect of
|
||||
non-deterministic transformations). * Avoid transformations that cause
|
||||
multiple distinct source files to map to the same destination name. *
|
||||
Consider disabling concurrency with --transfers=1 if necessary. *
|
||||
Certain transformations (e.g. prefix) will have a multiplying effect
|
||||
every time they are used. Avoid these when using bisync.
|
||||
|
||||
rclone convmv dest:path --name-transform XXX [flags]
|
||||
|
||||
@@ -21961,7 +21963,7 @@ Flags for general networking and HTTP stuff.
|
||||
--tpslimit float Limit HTTP transactions per second to this
|
||||
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
|
||||
--use-cookies Enable session cookiejar
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.2")
|
||||
|
||||
Performance
|
||||
|
||||
@@ -22413,6 +22415,7 @@ Backend-only flags (these can be set in the config file also).
|
||||
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
|
||||
--ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-http-proxy string URL for HTTP CONNECT proxy
|
||||
--ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s)
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-no-check-upload Don't check the upload is OK
|
||||
@@ -32909,6 +32912,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -34747,6 +34752,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -35978,9 +35985,9 @@ DOI
|
||||
The DOI remote is a read only remote for reading files from digital
|
||||
object identifiers (DOI).
|
||||
|
||||
Currently, the DOI backend supports supports DOIs hosted with: -
|
||||
InvenioRDM - Zenodo - CaltechDATA - Other InvenioRDM repositories -
|
||||
Dataverse - Harvard Dataverse - Other Dataverse repositories
|
||||
Currently, the DOI backend supports DOIs hosted with: - InvenioRDM -
|
||||
Zenodo - CaltechDATA - Other InvenioRDM repositories - Dataverse -
|
||||
Harvard Dataverse - Other Dataverse repositories
|
||||
|
||||
Paths are specified as remote:path
|
||||
|
||||
@@ -36438,6 +36445,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -37853,6 +37862,20 @@ Properties:
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
--ftp-http-proxy
|
||||
|
||||
URL for HTTP CONNECT proxy
|
||||
|
||||
Set this to a URL for an HTTP proxy which supports the HTTP CONNECT
|
||||
verb.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: http_proxy
|
||||
- Env Var: RCLONE_FTP_HTTP_PROXY
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
--ftp-no-check-upload
|
||||
|
||||
Don't check the upload is OK
|
||||
@@ -38891,6 +38914,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -39745,6 +39770,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -41311,6 +41338,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -41617,6 +41646,23 @@ Deleting albums
|
||||
The Google Photos API does not support deleting albums - see bug
|
||||
#135714733.
|
||||
|
||||
Making your own client_id
|
||||
|
||||
When you use rclone with Google photos in its default configuration you
|
||||
are using rclone's client_id. This is shared between all the rclone
|
||||
users. There is a global rate limit on the number of queries per second
|
||||
that each client_id can do set by Google.
|
||||
|
||||
If there is a problem with this client_id (eg quota too low or the
|
||||
client_id stops working) then you can make your own.
|
||||
|
||||
Please follow the steps in the google drive docs. You will need these
|
||||
scopes instead of the drive ones detailed:
|
||||
|
||||
https://www.googleapis.com/auth/photoslibrary.appendonly
|
||||
https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata
|
||||
https://www.googleapis.com/auth/photoslibrary.edit.appcreateddata
|
||||
|
||||
Hasher
|
||||
|
||||
Hasher is a special overlay backend to create remotes which handle
|
||||
@@ -42487,6 +42533,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -44163,6 +44211,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -45064,6 +45114,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -47830,8 +47882,11 @@ Creating Client ID for OneDrive Personal
|
||||
To create your own Client ID, please follow these steps:
|
||||
|
||||
1. Open
|
||||
https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade
|
||||
and then click New registration.
|
||||
https://portal.azure.com/?quickstart=true#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/Overview
|
||||
and then under the Add menu click App registration.
|
||||
- If you have not created an Azure account, you will be prompted
|
||||
to. This is free, but you need to provide a phone number,
|
||||
address, and credit card for identity verification.
|
||||
2. Enter a name for your app, choose account type
|
||||
Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox),
|
||||
select Web in Redirect URI, then type (do not copy and paste)
|
||||
@@ -48114,6 +48169,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -51840,6 +51897,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -52644,6 +52703,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -53228,6 +53289,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -57679,6 +57742,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -57974,6 +58039,8 @@ Use client credentials OAuth flow.
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC
|
||||
6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
@@ -58734,6 +58801,40 @@ Options:
|
||||
|
||||
Changelog
|
||||
|
||||
v1.70.2 - 2025-06-27
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- convmv: Make --dry-run logs less noisy (nielash)
|
||||
- sync: Avoid copying dir metadata to itself (nielash)
|
||||
- build: Bump github.com/go-chi/chi/v5 from 5.2.1 to 5.2.2 to fix
|
||||
GHSA-vrw8-fxc6-2r93 (dependabot[bot])
|
||||
- convmv: Fix moving to unicode-equivalent name (nielash)
|
||||
- log: Fix deadlock when using systemd logging (Nick Craig-Wood)
|
||||
- pacer: Fix nil pointer deref in RetryError (Nick Craig-Wood)
|
||||
- doc fixes (Ali Zein Yousuf, Nick Craig-Wood)
|
||||
- Local
|
||||
- Fix --skip-links on Windows when skipping Junction points (Nick
|
||||
Craig-Wood)
|
||||
- Combine
|
||||
- Fix directory not found errors with ListP interface (Nick
|
||||
Craig-Wood)
|
||||
- Mega
|
||||
- Fix tls handshake failure (necaran)
|
||||
- Pikpak
|
||||
- Fix uploads fail with "aws-chunked encoding is not supported"
|
||||
error (Nick Craig-Wood)
|
||||
|
||||
v1.70.1 - 2025-06-19
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- convmv: Fix spurious "error running command echo" on Windows
|
||||
(Nick Craig-Wood)
|
||||
- doc fixes (albertony, Ed Craig-Wood, jinjingroad)
|
||||
|
||||
v1.70.0 - 2025-06-17
|
||||
|
||||
See commits
|
||||
|
||||
@@ -39,7 +39,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
* Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
|
||||
* Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
|
||||
* FileLu [:page_facing_up:](https://rclone.org/filelu/)
|
||||
* Files.com [:page_facing_up:](https://rclone.org/filescom/)
|
||||
* FlashBlade [:page_facing_up:](https://rclone.org/s3/#pure-storage-flashblade)
|
||||
* FTP [:page_facing_up:](https://rclone.org/ftp/)
|
||||
@@ -112,7 +111,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
* Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
|
||||
* Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
|
||||
* Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
|
||||
* The local filesystem [:page_facing_up:](https://rclone.org/local/)
|
||||
|
||||
Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
|
||||
|
||||
+15
-104
@@ -72,7 +72,6 @@ const (
|
||||
emulatorAccount = "devstoreaccount1"
|
||||
emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
||||
emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
|
||||
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -560,11 +559,6 @@ type Fs struct {
|
||||
pacer *fs.Pacer // To pace and retry the API calls
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
publicAccess container.PublicAccessType // Container Public Access Level
|
||||
|
||||
// user delegation cache
|
||||
userDelegationMu sync.Mutex
|
||||
userDelegation *service.UserDelegationCredential
|
||||
userDelegationExpiry time.Time
|
||||
}
|
||||
|
||||
// Object describes an azure object
|
||||
@@ -990,38 +984,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
f.cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
case opt.UseAZ:
|
||||
var options = azidentity.AzureCLICredentialOptions{}
|
||||
f.cred, err = azidentity.NewAzureCLICredential(&options)
|
||||
@@ -1726,38 +1688,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
return f.deleteContainer(ctx, container)
|
||||
}
|
||||
|
||||
// Get a user delegation which is valid for at least sasCopyValidity
|
||||
//
|
||||
// This value is cached in f
|
||||
func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCredential, error) {
|
||||
f.userDelegationMu.Lock()
|
||||
defer f.userDelegationMu.Unlock()
|
||||
|
||||
if f.userDelegation != nil && time.Until(f.userDelegationExpiry) > sasCopyValidity {
|
||||
return f.userDelegation, nil
|
||||
}
|
||||
|
||||
// Validity window
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(2 * sasCopyValidity)
|
||||
startStr := start.Format(time.RFC3339)
|
||||
expiryStr := expiry.Format(time.RFC3339)
|
||||
|
||||
// Acquire user delegation key from the service client
|
||||
info := service.KeyInfo{
|
||||
Start: &startStr,
|
||||
Expiry: &expiryStr,
|
||||
}
|
||||
userDelegationKey, err := f.svc.GetUserDelegationCredential(ctx, info, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get user delegation key: %w", err)
|
||||
}
|
||||
|
||||
f.userDelegation = userDelegationKey
|
||||
f.userDelegationExpiry = expiry
|
||||
return f.userDelegation, nil
|
||||
}
|
||||
|
||||
// getAuth gets auth to copy o.
|
||||
//
|
||||
// tokenOK is used to signal that token based auth (Microsoft Entra
|
||||
@@ -1769,7 +1699,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
|
||||
// URL (not a SAS) and token will be empty.
|
||||
//
|
||||
// If tokenOK is true it may also return a token for the auth.
|
||||
func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err error) {
|
||||
func (o *Object) getAuth(ctx context.Context, tokenOK bool, noAuth bool) (srcURL string, token *string, err error) {
|
||||
f := o.fs
|
||||
srcBlobSVC := o.getBlobSVC()
|
||||
srcURL = srcBlobSVC.URL()
|
||||
@@ -1778,47 +1708,29 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
|
||||
case noAuth:
|
||||
// If same storage account then no auth needed
|
||||
case f.cred != nil:
|
||||
// Generate a User Delegation SAS URL using Azure AD credentials
|
||||
userDelegationKey, err := f.getUserDelegation(ctx)
|
||||
if !tokenOK {
|
||||
return srcURL, token, errors.New("not supported: Microsoft Entra ID")
|
||||
}
|
||||
options := policy.TokenRequestOptions{}
|
||||
accessToken, err := f.cred.GetToken(ctx, options)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("sas creation: %w", err)
|
||||
return srcURL, token, fmt.Errorf("failed to create access token: %w", err)
|
||||
}
|
||||
|
||||
// Build the SAS values
|
||||
perms := sas.BlobPermissions{Read: true}
|
||||
container, containerPath := o.split()
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(sasCopyValidity)
|
||||
vals := sas.BlobSignatureValues{
|
||||
StartTime: start,
|
||||
ExpiryTime: expiry,
|
||||
Permissions: perms.String(),
|
||||
ContainerName: container,
|
||||
BlobName: containerPath,
|
||||
}
|
||||
|
||||
// Sign with the delegation key
|
||||
queryParameters, err := vals.SignWithUserDelegation(userDelegationKey)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("signing SAS with user delegation failed: %w", err)
|
||||
}
|
||||
|
||||
// Append the SAS to the URL
|
||||
srcURL = srcBlobSVC.URL() + "?" + queryParameters.Encode()
|
||||
token = &accessToken.Token
|
||||
case f.sharedKeyCred != nil:
|
||||
// Generate a short lived SAS URL if using shared key credentials
|
||||
expiry := time.Now().Add(sasCopyValidity)
|
||||
expiry := time.Now().Add(time.Hour)
|
||||
sasOptions := blob.GetSASURLOptions{}
|
||||
srcURL, err = srcBlobSVC.GetSASURL(sas.BlobPermissions{Read: true}, expiry, &sasOptions)
|
||||
if err != nil {
|
||||
return srcURL, fmt.Errorf("failed to create SAS URL: %w", err)
|
||||
return srcURL, token, fmt.Errorf("failed to create SAS URL: %w", err)
|
||||
}
|
||||
case f.anonymous || f.opt.SASURL != "":
|
||||
// If using a SASURL or anonymous, no need for any extra auth
|
||||
default:
|
||||
return srcURL, errors.New("unknown authentication type")
|
||||
return srcURL, token, errors.New("unknown authentication type")
|
||||
}
|
||||
return srcURL, nil
|
||||
return srcURL, token, nil
|
||||
}
|
||||
|
||||
// Do multipart parallel copy.
|
||||
@@ -1839,7 +1751,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
|
||||
o.fs = f
|
||||
o.remote = remote
|
||||
|
||||
srcURL, err := src.getAuth(ctx, false)
|
||||
srcURL, token, err := src.getAuth(ctx, true, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("multipart copy: %w", err)
|
||||
}
|
||||
@@ -1883,8 +1795,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
|
||||
Count: partSize,
|
||||
},
|
||||
// Specifies the authorization scheme and signature for the copy source.
|
||||
// We use SAS URLs as this doesn't seem to work always
|
||||
// CopySourceAuthorization: token,
|
||||
CopySourceAuthorization: token,
|
||||
// CPKInfo *blob.CPKInfo
|
||||
// CPKScopeInfo *blob.CPKScopeInfo
|
||||
}
|
||||
@@ -1954,7 +1865,7 @@ func (f *Fs) copySinglepart(ctx context.Context, remote, dstContainer, dstPath s
|
||||
dstBlobSVC := f.getBlobSVC(dstContainer, dstPath)
|
||||
|
||||
// Get the source auth - none needed for same storage account
|
||||
srcURL, err := src.getAuth(ctx, f == src.fs)
|
||||
srcURL, _, err := src.getAuth(ctx, false, f == src.fs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("single part copy: source auth: %w", err)
|
||||
}
|
||||
|
||||
@@ -569,38 +569,6 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, errors.New("no authentication method configured")
|
||||
}
|
||||
|
||||
@@ -1673,21 +1673,6 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) {
|
||||
return o.getMetaDataListing(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// If using versionAt we need to list the find the correct version.
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
info, err := o.getMetaDataListing(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.Action == "hide" {
|
||||
// Rerturn object not found error if the current version is deleted.
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
_, info, err = o.getOrHead(ctx, "HEAD", nil)
|
||||
return info, err
|
||||
}
|
||||
|
||||
@@ -446,14 +446,14 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
|
||||
t.Run("List", func(t *testing.T) {
|
||||
fstest.CheckListing(t, f, test.want)
|
||||
})
|
||||
|
||||
t.Run("NewObject", func(t *testing.T) {
|
||||
gotObj, gotErr := f.NewObject(ctx, fileName)
|
||||
assert.Equal(t, test.wantErr, gotErr)
|
||||
if gotErr == nil {
|
||||
assert.Equal(t, test.wantSize, gotObj.Size())
|
||||
}
|
||||
})
|
||||
// b2 NewObject doesn't work with VersionAt
|
||||
//t.Run("NewObject", func(t *testing.T) {
|
||||
// gotObj, gotErr := f.NewObject(ctx, fileName)
|
||||
// assert.Equal(t, test.wantErr, gotErr)
|
||||
// if gotErr == nil {
|
||||
// assert.Equal(t, test.wantSize, gotObj.Size())
|
||||
// }
|
||||
//})
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -52,7 +52,7 @@ func (f *Fs) testUploadTimeout(t *testing.T) {
|
||||
ci.Timeout = saveTimeout
|
||||
}()
|
||||
ci.LowLevelRetries = 1
|
||||
ci.Timeout = fs.Duration(idleTimeout)
|
||||
ci.Timeout = idleTimeout
|
||||
|
||||
upload := func(concurrency int, shutTimeout time.Duration) (obj fs.Object, err error) {
|
||||
fixFs := deriveFs(ctx, t, f, settings{
|
||||
|
||||
@@ -421,9 +421,6 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
if src.Size() == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
return uploadFile(ctx, f, in, src.Remote(), options...)
|
||||
}
|
||||
|
||||
@@ -662,9 +659,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
if src.Size() == 0 {
|
||||
return fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
srcRemote := o.Remote()
|
||||
|
||||
@@ -676,7 +670,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
var resp *client.UploadResult
|
||||
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
var res *http.Response
|
||||
res, resp, err = o.fs.ik.Upload(ctx, in, client.UploadParam{
|
||||
FileName: fileName,
|
||||
@@ -731,7 +725,7 @@ func uploadFile(ctx context.Context, f *Fs, in io.Reader, srcRemote string, opti
|
||||
UseUniqueFileName := new(bool)
|
||||
*UseUniqueFileName = false
|
||||
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
var res *http.Response
|
||||
var err error
|
||||
res, _, err = f.ik.Upload(ctx, in, client.UploadParam{
|
||||
@@ -800,10 +794,35 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side move operations.
|
||||
//
|
||||
// This is stored with the remote path given.
|
||||
//
|
||||
// It returns the destination Object and a possible error.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantMove
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
file, err := srcObj.Open(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return uploadFile(ctx, f, file, remote)
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied.
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Purger = &Fs{}
|
||||
_ fs.PublicLinker = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
_ fs.Copier = &Fs{}
|
||||
)
|
||||
@@ -461,7 +461,7 @@ func translateErrorsDir(err error) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// translateErrorsObject translates Koofr errors to rclone errors (for an object operation)
|
||||
// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
|
||||
func translateErrorsObject(err error) error {
|
||||
switch err := err.(type) {
|
||||
case httpclient.InvalidStatusError:
|
||||
|
||||
@@ -617,36 +617,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
case 1:
|
||||
// upload file using link from first step
|
||||
var res *http.Response
|
||||
var location string
|
||||
|
||||
// Check to see if we are being redirected
|
||||
opts := &rest.Opts{
|
||||
Method: "HEAD",
|
||||
RootURL: getFirstStepResult.Data.SignURL,
|
||||
Options: options,
|
||||
NoRedirect: true,
|
||||
}
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
res, err = o.fs.srv.Call(ctx, opts)
|
||||
return o.fs.shouldRetry(ctx, res, err)
|
||||
})
|
||||
if res != nil {
|
||||
location = res.Header.Get("Location")
|
||||
if location != "" {
|
||||
// set the URL to the new Location
|
||||
opts.RootURL = location
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("head upload URL: %w", err)
|
||||
}
|
||||
|
||||
file := io.MultiReader(bytes.NewReader(first10mBytes), in)
|
||||
|
||||
opts.Method = "PUT"
|
||||
opts.Body = file
|
||||
opts.ContentLength = &size
|
||||
opts := &rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: getFirstStepResult.Data.SignURL,
|
||||
Options: options,
|
||||
Body: file,
|
||||
ContentLength: &size,
|
||||
}
|
||||
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
res, err = o.fs.srv.Call(ctx, opts)
|
||||
|
||||
@@ -305,12 +305,6 @@ only useful for reading.
|
||||
Help: "The last status change time.",
|
||||
}},
|
||||
},
|
||||
{
|
||||
Name: "hashes",
|
||||
Help: `Comma separated list of supported checksum types.`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
},
|
||||
{
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -337,7 +331,6 @@ type Options struct {
|
||||
NoSparse bool `config:"no_sparse"`
|
||||
NoSetModTime bool `config:"no_set_modtime"`
|
||||
TimeType timeType `config:"time_type"`
|
||||
Hashes fs.CommaSepList `config:"hashes"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
NoClone bool `config:"no_clone"`
|
||||
}
|
||||
@@ -1028,19 +1021,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
if len(f.opt.Hashes) > 0 {
|
||||
// Return only configured hashes.
|
||||
// Note: Could have used hash.SupportOnly to limit supported hashes for all hash related features.
|
||||
var supported hash.Set
|
||||
for _, hashName := range f.opt.Hashes {
|
||||
var ht hash.Type
|
||||
if err := ht.Set(hashName); err != nil {
|
||||
fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
|
||||
}
|
||||
supported.Add(ht)
|
||||
}
|
||||
return supported
|
||||
}
|
||||
return hash.Supported()
|
||||
}
|
||||
|
||||
|
||||
@@ -634,7 +634,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
|
||||
return
|
||||
}
|
||||
|
||||
// itemToDirEntry converts API item to rclone directory entry
|
||||
// itemToEntry converts API item to rclone directory entry
|
||||
// The dirSize return value is:
|
||||
//
|
||||
// <0 - for a file or in case of error
|
||||
|
||||
@@ -155,7 +155,6 @@ func (f *Fs) getFile(ctx context.Context, ID string) (info *api.File, err error)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
|
||||
if err == nil && !info.Links.ApplicationOctetStream.Valid() {
|
||||
time.Sleep(5 * time.Second)
|
||||
return true, errors.New("no link")
|
||||
}
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
|
||||
@@ -1,333 +0,0 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/rclone/rclone/backend/pikpak/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
bufferSize = 1024 * 1024 // default size of the pages used in the reader
|
||||
bufferCacheSize = 64 // max number of buffers to keep in cache
|
||||
bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long
|
||||
)
|
||||
|
||||
// bufferPool is a global pool of buffers
|
||||
var (
|
||||
bufferPool *pool.Pool
|
||||
bufferPoolOnce sync.Once
|
||||
)
|
||||
|
||||
// get a buffer pool
|
||||
func getPool() *pool.Pool {
|
||||
bufferPoolOnce.Do(func() {
|
||||
ci := fs.GetConfig(context.Background())
|
||||
// Initialise the buffer pool when used
|
||||
bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)
|
||||
})
|
||||
return bufferPool
|
||||
}
|
||||
|
||||
// NewRW gets a pool.RW using the multipart pool
|
||||
func NewRW() *pool.RW {
|
||||
return pool.NewRW(getPool())
|
||||
}
|
||||
|
||||
// Upload does a multipart upload in parallel
|
||||
func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
|
||||
// make concurrency machinery
|
||||
tokens := pacer.NewTokenDispenser(w.con)
|
||||
|
||||
uploadCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
defer atexit.OnError(&err, func() {
|
||||
cancel()
|
||||
fs.Debugf(w.o, "multipart upload: Cancelling...")
|
||||
errCancel := w.Abort(ctx)
|
||||
if errCancel != nil {
|
||||
fs.Debugf(w.o, "multipart upload: failed to cancel: %v", errCancel)
|
||||
}
|
||||
})()
|
||||
|
||||
var (
|
||||
g, gCtx = errgroup.WithContext(uploadCtx)
|
||||
finished = false
|
||||
off int64
|
||||
size = w.size
|
||||
chunkSize = w.chunkSize
|
||||
)
|
||||
|
||||
// Do the accounting manually
|
||||
in, acc := accounting.UnWrapAccounting(w.in)
|
||||
|
||||
for partNum := int64(0); !finished; partNum++ {
|
||||
// Get a block of memory from the pool and token which limits concurrency.
|
||||
tokens.Get()
|
||||
rw := NewRW()
|
||||
if acc != nil {
|
||||
rw.SetAccounting(acc.AccountRead)
|
||||
}
|
||||
|
||||
free := func() {
|
||||
// return the memory and token
|
||||
_ = rw.Close() // Can't return an error
|
||||
tokens.Put()
|
||||
}
|
||||
|
||||
// Fail fast, in case an errgroup managed function returns an error
|
||||
// gCtx is cancelled. There is no point in uploading all the other parts.
|
||||
if gCtx.Err() != nil {
|
||||
free()
|
||||
break
|
||||
}
|
||||
|
||||
// Read the chunk
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, in, chunkSize)
|
||||
if err == io.EOF {
|
||||
if n == 0 && partNum != 0 { // end if no data and if not first chunk
|
||||
free()
|
||||
break
|
||||
}
|
||||
finished = true
|
||||
} else if err != nil {
|
||||
free()
|
||||
return fmt.Errorf("multipart upload: failed to read source: %w", err)
|
||||
}
|
||||
|
||||
partNum := partNum
|
||||
partOff := off
|
||||
off += n
|
||||
g.Go(func() (err error) {
|
||||
defer free()
|
||||
fs.Debugf(w.o, "multipart upload: starting chunk %d size %v offset %v/%v", partNum, fs.SizeSuffix(n), fs.SizeSuffix(partOff), fs.SizeSuffix(size))
|
||||
_, err = w.WriteChunk(gCtx, int32(partNum), rw)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
err = g.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = w.Close(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multipart upload: failed to finalise: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// state of ChunkWriter
|
||||
type pikpakChunkWriter struct {
|
||||
chunkSize int64
|
||||
size int64
|
||||
con int
|
||||
f *Fs
|
||||
o *Object
|
||||
in io.Reader
|
||||
mu sync.Mutex
|
||||
completedParts []types.CompletedPart
|
||||
client *s3.Client
|
||||
mOut *s3.CreateMultipartUploadOutput
|
||||
}
|
||||
|
||||
func (f *Fs) newChunkWriter(ctx context.Context, remote string, size int64, p *api.ResumableParams, in io.Reader, options ...fs.OpenOption) (w *pikpakChunkWriter, err error) {
|
||||
// Temporary Object under construction
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
|
||||
// calculate size of parts
|
||||
chunkSize := f.opt.ChunkSize
|
||||
|
||||
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
|
||||
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
|
||||
// 48 GiB which seems like a not too unreasonable limit.
|
||||
if size == -1 {
|
||||
warnStreamUpload.Do(func() {
|
||||
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
|
||||
f.opt.ChunkSize, fs.SizeSuffix(int64(chunkSize)*int64(maxUploadParts)))
|
||||
})
|
||||
} else {
|
||||
chunkSize = chunksize.Calculator(o, size, maxUploadParts, chunkSize)
|
||||
}
|
||||
|
||||
client, err := f.newS3Client(ctx, p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create upload client: %w", err)
|
||||
}
|
||||
w = &pikpakChunkWriter{
|
||||
chunkSize: int64(chunkSize),
|
||||
size: size,
|
||||
con: max(1, f.opt.UploadConcurrency),
|
||||
f: f,
|
||||
o: o,
|
||||
in: in,
|
||||
completedParts: make([]types.CompletedPart, 0),
|
||||
client: client,
|
||||
}
|
||||
|
||||
req := &s3.CreateMultipartUploadInput{
|
||||
Bucket: &p.Bucket,
|
||||
Key: &p.Key,
|
||||
}
|
||||
// Apply upload options
|
||||
for _, option := range options {
|
||||
key, value := option.Header()
|
||||
lowerKey := strings.ToLower(key)
|
||||
switch lowerKey {
|
||||
case "":
|
||||
// ignore
|
||||
case "cache-control":
|
||||
req.CacheControl = aws.String(value)
|
||||
case "content-disposition":
|
||||
req.ContentDisposition = aws.String(value)
|
||||
case "content-encoding":
|
||||
req.ContentEncoding = aws.String(value)
|
||||
case "content-type":
|
||||
req.ContentType = aws.String(value)
|
||||
}
|
||||
}
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
w.mOut, err = w.client.CreateMultipartUpload(ctx, req)
|
||||
return w.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create multipart upload failed: %w", err)
|
||||
}
|
||||
fs.Debugf(w.o, "multipart upload: %q initiated", *w.mOut.UploadId)
|
||||
return
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (w *pikpakChunkWriter) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if fserrors.ShouldRetry(err) {
|
||||
return true, err
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// add a part number and etag to the completed parts
|
||||
func (w *pikpakChunkWriter) addCompletedPart(part types.CompletedPart) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.completedParts = append(w.completedParts, part)
|
||||
}
|
||||
|
||||
// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
|
||||
func (w *pikpakChunkWriter) WriteChunk(ctx context.Context, chunkNumber int32, reader io.ReadSeeker) (currentChunkSize int64, err error) {
|
||||
if chunkNumber < 0 {
|
||||
err := fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
|
||||
return -1, err
|
||||
}
|
||||
|
||||
partNumber := chunkNumber + 1
|
||||
var res *s3.UploadPartOutput
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
// Discover the size by seeking to the end
|
||||
currentChunkSize, err = reader.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// rewind the reader on retry and after reading md5
|
||||
_, err := reader.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
res, err = w.client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: w.mOut.Bucket,
|
||||
Key: w.mOut.Key,
|
||||
UploadId: w.mOut.UploadId,
|
||||
PartNumber: &partNumber,
|
||||
Body: reader,
|
||||
})
|
||||
if err != nil {
|
||||
if chunkNumber <= 8 {
|
||||
return w.shouldRetry(ctx, err)
|
||||
}
|
||||
// retry all chunks once have done the first few
|
||||
return true, err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return -1, fmt.Errorf("failed to upload chunk %d with %v bytes: %w", partNumber, currentChunkSize, err)
|
||||
}
|
||||
|
||||
w.addCompletedPart(types.CompletedPart{
|
||||
PartNumber: &partNumber,
|
||||
ETag: res.ETag,
|
||||
})
|
||||
|
||||
fs.Debugf(w.o, "multipart upload: wrote chunk %d with %v bytes", partNumber, currentChunkSize)
|
||||
return currentChunkSize, err
|
||||
}
|
||||
|
||||
// Abort the multipart upload
|
||||
func (w *pikpakChunkWriter) Abort(ctx context.Context) (err error) {
|
||||
// Abort the upload session
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
_, err = w.client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: w.mOut.Bucket,
|
||||
Key: w.mOut.Key,
|
||||
UploadId: w.mOut.UploadId,
|
||||
})
|
||||
return w.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to abort multipart upload %q: %w", *w.mOut.UploadId, err)
|
||||
}
|
||||
fs.Debugf(w.o, "multipart upload: %q aborted", *w.mOut.UploadId)
|
||||
return
|
||||
}
|
||||
|
||||
// Close and finalise the multipart upload
|
||||
func (w *pikpakChunkWriter) Close(ctx context.Context) (err error) {
|
||||
// sort the completed parts by part number
|
||||
sort.Slice(w.completedParts, func(i, j int) bool {
|
||||
return *w.completedParts[i].PartNumber < *w.completedParts[j].PartNumber
|
||||
})
|
||||
// Finalise the upload session
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
_, err = w.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: w.mOut.Bucket,
|
||||
Key: w.mOut.Key,
|
||||
UploadId: w.mOut.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: w.completedParts,
|
||||
},
|
||||
})
|
||||
return w.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to complete multipart upload: %w", err)
|
||||
}
|
||||
fs.Debugf(w.o, "multipart upload: %q finished", *w.mOut.UploadId)
|
||||
return
|
||||
}
|
||||
+31
-127
@@ -41,10 +41,12 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/rclone/rclone/backend/pikpak/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
@@ -64,22 +66,17 @@ import (
|
||||
|
||||
// Constants
|
||||
const (
|
||||
clientID = "YUMx5nI8ZU8Ap8pm"
|
||||
clientVersion = "2.0.0"
|
||||
packageName = "mypikpak.com"
|
||||
defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
taskWaitTime = 500 * time.Millisecond
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
rootURL = "https://api-drive.mypikpak.com"
|
||||
|
||||
maxUploadParts = 10000 // Part number must be an integer between 1 and 10000, inclusive.
|
||||
defaultChunkSize = fs.SizeSuffix(1024 * 1024 * 5) // Part size should be in [100KB, 5GB]
|
||||
minChunkSize = 100 * fs.Kibi
|
||||
maxChunkSize = 5 * fs.Gibi
|
||||
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
|
||||
maxUploadCutoff = 5 * fs.Gibi // maximum allowed size for singlepart uploads
|
||||
clientID = "YUMx5nI8ZU8Ap8pm"
|
||||
clientVersion = "2.0.0"
|
||||
packageName = "mypikpak.com"
|
||||
defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
taskWaitTime = 500 * time.Millisecond
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
rootURL = "https://api-drive.mypikpak.com"
|
||||
minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize)
|
||||
defaultUploadConcurrency = manager.DefaultUploadConcurrency
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -226,14 +223,6 @@ Fill in for rclone to use a non root folder as its starting point.
|
||||
Help: "Files bigger than this will be cached on disk to calculate hash if required.",
|
||||
Default: fs.SizeSuffix(10 * 1024 * 1024),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: `Cutoff for switching to chunked upload.
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Chunk size for multipart uploads.
|
||||
@@ -252,7 +241,7 @@ large file of known size to stay below the 10,000 chunks limit.
|
||||
|
||||
Increasing the chunk size decreases the accuracy of the progress
|
||||
statistics displayed with "-P" flag.`,
|
||||
Default: defaultChunkSize,
|
||||
Default: minChunkSize,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
@@ -268,7 +257,7 @@ in memory.
|
||||
If you are uploading small numbers of large files over high-speed links
|
||||
and these uploads do not fully utilize your bandwidth, then increasing
|
||||
this may help to speed up the transfers.`,
|
||||
Default: 4,
|
||||
Default: defaultUploadConcurrency,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
@@ -305,7 +294,6 @@ type Options struct {
|
||||
NoMediaLink bool `config:"no_media_link"`
|
||||
HashMemoryThreshold fs.SizeSuffix `config:"hash_memory_limit"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
@@ -479,11 +467,6 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
|
||||
// when a zero-byte file was uploaded with an invalid captcha token
|
||||
f.rst.captcha.Invalidate()
|
||||
return true, err
|
||||
} else if strings.Contains(apiErr.Reason, "idx.shub.mypikpak.com") && apiErr.Code == 500 {
|
||||
// internal server error: Post "http://idx.shub.mypikpak.com": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
|
||||
// This typically happens when trying to retrieve a gcid for which no record exists.
|
||||
// No retry is needed in this case.
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -541,39 +524,6 @@ func (f *Fs) newClientWithPacer(ctx context.Context) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
if cs < minChunkSize {
|
||||
return fmt.Errorf("%s is less than %s", cs, minChunkSize)
|
||||
}
|
||||
if cs > maxChunkSize {
|
||||
return fmt.Errorf("%s is greater than %s", cs, maxChunkSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
||||
err = checkUploadChunkSize(cs)
|
||||
if err == nil {
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func checkUploadCutoff(cs fs.SizeSuffix) error {
|
||||
if cs > maxUploadCutoff {
|
||||
return fmt.Errorf("%s is greater than %s", cs, maxUploadCutoff)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
||||
err = checkUploadCutoff(cs)
|
||||
if err == nil {
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// newFs partially constructs Fs from the path
|
||||
//
|
||||
// It constructs a valid Fs but doesn't attempt to figure out whether
|
||||
@@ -581,17 +531,11 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
||||
func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
if err := configstruct.Set(m, opt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = checkUploadChunkSize(opt.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pikpak: chunk size: %w", err)
|
||||
}
|
||||
err = checkUploadCutoff(opt.UploadCutoff)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pikpak: upload cutoff: %w", err)
|
||||
if opt.ChunkSize < minChunkSize {
|
||||
return nil, fmt.Errorf("chunk size must be at least %s", minChunkSize)
|
||||
}
|
||||
|
||||
root := parsePath(path)
|
||||
@@ -1316,7 +1260,9 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fs) newS3Client(ctx context.Context, p *api.ResumableParams) (s3Client *s3.Client, err error) {
|
||||
func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable) (err error) {
|
||||
p := resumable.Params
|
||||
|
||||
// Create a credentials provider
|
||||
creds := credentials.NewStaticCredentialsProvider(p.AccessKeyID, p.AccessKeySecret, p.SecurityToken)
|
||||
|
||||
@@ -1326,64 +1272,22 @@ func (f *Fs) newS3Client(ctx context.Context, p *api.ResumableParams) (s3Client
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ci := fs.GetConfig(ctx)
|
||||
cfg.RetryMaxAttempts = ci.LowLevelRetries
|
||||
cfg.HTTPClient = getClient(ctx, &f.opt)
|
||||
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String("https://mypikpak.com/")
|
||||
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
|
||||
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
|
||||
})
|
||||
return client, nil
|
||||
}
|
||||
partSize := chunksize.Calculator(name, size, int(manager.MaxUploadParts), f.opt.ChunkSize)
|
||||
|
||||
func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable, options ...fs.OpenOption) (err error) {
|
||||
p := resumable.Params
|
||||
|
||||
if size < 0 || size >= int64(f.opt.UploadCutoff) {
|
||||
mu, err := f.newChunkWriter(ctx, name, size, p, in, options...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multipart upload failed to initialise: %w", err)
|
||||
}
|
||||
return mu.Upload(ctx)
|
||||
}
|
||||
|
||||
// upload singlepart
|
||||
client, err := f.newS3Client(ctx, p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create upload client: %w", err)
|
||||
}
|
||||
req := &s3.PutObjectInput{
|
||||
// Create an uploader with custom options
|
||||
uploader := manager.NewUploader(client, func(u *manager.Uploader) {
|
||||
u.PartSize = int64(partSize)
|
||||
u.Concurrency = f.opt.UploadConcurrency
|
||||
})
|
||||
// Perform an upload
|
||||
_, err = uploader.Upload(ctx, &s3.PutObjectInput{
|
||||
Bucket: &p.Bucket,
|
||||
Key: &p.Key,
|
||||
Body: io.NopCloser(in),
|
||||
}
|
||||
// Apply upload options
|
||||
for _, option := range options {
|
||||
key, value := option.Header()
|
||||
lowerKey := strings.ToLower(key)
|
||||
switch lowerKey {
|
||||
case "":
|
||||
// ignore
|
||||
case "cache-control":
|
||||
req.CacheControl = aws.String(value)
|
||||
case "content-disposition":
|
||||
req.ContentDisposition = aws.String(value)
|
||||
case "content-encoding":
|
||||
req.ContentEncoding = aws.String(value)
|
||||
case "content-type":
|
||||
req.ContentType = aws.String(value)
|
||||
}
|
||||
}
|
||||
var s3opts = []func(*s3.Options){}
|
||||
// Can't retry single part uploads as only have an io.Reader
|
||||
s3opts = append(s3opts, func(o *s3.Options) {
|
||||
o.RetryMaxAttempts = 1
|
||||
})
|
||||
err = f.pacer.CallNoRetry(func() (bool, error) {
|
||||
_, err = client.PutObject(ctx, req, s3opts...)
|
||||
return f.shouldRetry(ctx, nil, err)
|
||||
Body: in,
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -1441,7 +1345,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, gcid string,
|
||||
if uploadType == api.UploadTypeForm && new.Form != nil {
|
||||
err = f.uploadByForm(ctx, in, req.Name, size, new.Form, options...)
|
||||
} else if uploadType == api.UploadTypeResumable && new.Resumable != nil {
|
||||
err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable, options...)
|
||||
err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable)
|
||||
} else {
|
||||
err = fmt.Errorf("no method available for uploading: %+v", new)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Test PikPak filesystem interface
|
||||
package pikpak
|
||||
package pikpak_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/backend/pikpak"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
@@ -12,23 +12,6 @@ import (
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestPikPak:",
|
||||
NilObject: (*Object)(nil),
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: minChunkSize,
|
||||
MaxChunkSize: maxChunkSize,
|
||||
},
|
||||
NilObject: (*pikpak.Object)(nil),
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
return f.setUploadChunkSize(cs)
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
return f.setUploadCutoff(cs)
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
+3
-27
@@ -185,9 +185,6 @@ var providerOption = fs.Option{
|
||||
}, {
|
||||
Value: "Qiniu",
|
||||
Help: "Qiniu Object Storage (Kodo)",
|
||||
}, {
|
||||
Value: "Zata",
|
||||
Help: "Zata (S3 compatible Gateway)",
|
||||
}, {
|
||||
Value: "Other",
|
||||
Help: "Any other S3 compatible provider",
|
||||
@@ -493,14 +490,6 @@ func init() {
|
||||
Value: "ap-northeast-1",
|
||||
Help: "Northeast Asia Region 1.\nNeeds location constraint ap-northeast-1.",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region where you can connect with.\n",
|
||||
Provider: "Zata",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "us-east-1",
|
||||
Help: "Indore, Madhya Pradesh, India",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region where your bucket will be created and your data stored.\n",
|
||||
@@ -587,7 +576,7 @@ func init() {
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
|
||||
Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega,Zata",
|
||||
Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
|
||||
@@ -1391,14 +1380,6 @@ func init() {
|
||||
Value: "s3-ap-northeast-1.qiniucs.com",
|
||||
Help: "Northeast Asia Endpoint 1",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for Zata Object Storage.",
|
||||
Provider: "Zata",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "idr01.zata.ai",
|
||||
Help: "South Asia Endpoint",
|
||||
}},
|
||||
}, {
|
||||
// Selectel endpoints: https://docs.selectel.ru/en/cloud/object-storage/manage/domains/#s3-api-domains
|
||||
Name: "endpoint",
|
||||
@@ -1411,7 +1392,7 @@ func init() {
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
|
||||
Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox,Zata",
|
||||
Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "objects-us-east-1.dream.io",
|
||||
Help: "Dream Objects endpoint",
|
||||
@@ -2635,7 +2616,7 @@ The parameter should be a date, "2006-01-02", datetime "2006-01-02
|
||||
Note that when using this no file write operations are permitted,
|
||||
so you can't upload files or delete them.
|
||||
|
||||
See [the time option docs](/docs/#time-options) for valid formats.
|
||||
See [the time option docs](/docs/#time-option) for valid formats.
|
||||
`,
|
||||
Default: fs.Time{},
|
||||
Advanced: true,
|
||||
@@ -3650,11 +3631,6 @@ func setQuirks(opt *Options) {
|
||||
urlEncodeListings = false
|
||||
virtualHostStyle = false
|
||||
useAlreadyExists = false // untested
|
||||
case "Zata":
|
||||
useMultipartEtag = false
|
||||
mightGzip = false
|
||||
useUnsignedPayload = false
|
||||
useAlreadyExists = false
|
||||
case "Exaba":
|
||||
virtualHostStyle = false
|
||||
case "GCS":
|
||||
|
||||
@@ -111,8 +111,7 @@ func init() {
|
||||
encoder.EncodeSlash |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeDoubleQuote |
|
||||
encoder.EncodeInvalidUtf8 |
|
||||
encoder.EncodeDot),
|
||||
encoder.EncodeInvalidUtf8),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
+63
-214
@@ -222,45 +222,15 @@ E.g. the second example above should be rewritten as:
|
||||
Help: "Windows Command Prompt",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "hashes",
|
||||
Help: `Comma separated list of supported checksum types.`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "md5sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read MD5 hashes.\n\nLeave blank for autodetect.",
|
||||
Help: "The command used to read md5 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "sha1sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read SHA-1 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "crc32sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read CRC-32 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "sha256sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read SHA-256 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "blake3sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read BLAKE3 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "xxh3sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read XXH3 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "xxh128sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read XXH128 hashes.\n\nLeave blank for autodetect.",
|
||||
Help: "The command used to read sha1 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_links",
|
||||
@@ -565,14 +535,8 @@ type Options struct {
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
ShellType string `config:"shell_type"`
|
||||
Hashes fs.CommaSepList `config:"hashes"`
|
||||
Md5sumCommand string `config:"md5sum_command"`
|
||||
Sha1sumCommand string `config:"sha1sum_command"`
|
||||
Crc32sumCommand string `config:"crc32sum_command"`
|
||||
Sha256sumCommand string `config:"sha256sum_command"`
|
||||
Blake3sumCommand string `config:"blake3sum_command"`
|
||||
Xxh3sumCommand string `config:"xxh3sum_command"`
|
||||
Xxh128sumCommand string `config:"xxh128sum_command"`
|
||||
SkipLinks bool `config:"skip_links"`
|
||||
Subsystem string `config:"subsystem"`
|
||||
ServerCommand string `config:"server_command"`
|
||||
@@ -621,18 +585,13 @@ type Fs struct {
|
||||
|
||||
// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
size int64 // size of the object
|
||||
modTime uint32 // modification time of the object as unix time
|
||||
mode os.FileMode // mode bits from the file
|
||||
md5sum *string // Cached MD5 checksum
|
||||
sha1sum *string // Cached SHA-1 checksum
|
||||
crc32sum *string // Cached CRC-32 checksum
|
||||
sha256sum *string // Cached SHA-256 checksum
|
||||
blake3sum *string // Cached BLAKE3 checksum
|
||||
xxh3sum *string // Cached XXH3 checksum
|
||||
xxh128sum *string // Cached XXH128 checksum
|
||||
fs *Fs
|
||||
remote string
|
||||
size int64 // size of the object
|
||||
modTime uint32 // modification time of the object as unix time
|
||||
mode os.FileMode // mode bits from the file
|
||||
md5sum *string // Cached MD5 checksum
|
||||
sha1sum *string // Cached SHA1 checksum
|
||||
}
|
||||
|
||||
// conn encapsulates an ssh client and corresponding sftp client
|
||||
@@ -932,7 +891,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
User: opt.User,
|
||||
Auth: []ssh.AuthMethod{},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: time.Duration(f.ci.ConnectTimeout),
|
||||
Timeout: f.ci.ConnectTimeout,
|
||||
ClientVersion: "SSH-2.0-" + f.ci.UserAgent,
|
||||
}
|
||||
|
||||
@@ -1684,112 +1643,14 @@ func (f *Fs) Hashes() hash.Set {
|
||||
return *f.cachedHashes
|
||||
}
|
||||
|
||||
hashTypesSupported := hash.NewHashSet()
|
||||
f.cachedHashes = &hashTypesSupported
|
||||
hashSet := hash.NewHashSet()
|
||||
f.cachedHashes = &hashSet
|
||||
|
||||
if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported {
|
||||
return hashTypesSupported
|
||||
}
|
||||
|
||||
hashTypes := hash.NewHashSet()
|
||||
if len(f.opt.Hashes) > 0 {
|
||||
for _, hashName := range f.opt.Hashes {
|
||||
var hashType hash.Type
|
||||
if err := hashType.Set(hashName); err != nil {
|
||||
fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
|
||||
}
|
||||
hashTypes.Add(hashType)
|
||||
}
|
||||
} else {
|
||||
hashTypes.Add(hash.MD5, hash.SHA1)
|
||||
}
|
||||
|
||||
hashCommands := map[hash.Type]struct {
|
||||
option *string
|
||||
emptyHash string
|
||||
hashCommands []struct{ hashFile, hashEmpty string }
|
||||
}{
|
||||
hash.MD5: {
|
||||
&f.opt.Md5sumCommand,
|
||||
"d41d8cd98f00b204e9800998ecf8427e",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"md5sum", "md5sum"},
|
||||
{"md5 -r", "md5 -r"},
|
||||
{"rclone md5sum", "rclone md5sum"},
|
||||
},
|
||||
},
|
||||
hash.SHA1: {
|
||||
&f.opt.Sha1sumCommand,
|
||||
"da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"sha1sum", "sha1sum"},
|
||||
{"sha1 -r", "sha1 -r"},
|
||||
{"rclone sha1sum", "rclone sha1sum"},
|
||||
},
|
||||
},
|
||||
hash.CRC32: {
|
||||
&f.opt.Sha1sumCommand,
|
||||
"00000000",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"crc32", "crc32"},
|
||||
{"rclone hashsum crc32", "rclone hashsum crc32"},
|
||||
},
|
||||
},
|
||||
hash.SHA256: {
|
||||
&f.opt.Sha256sumCommand,
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"sha256sum", "sha1sum"},
|
||||
{"sha256 -r", "sha1 -r"},
|
||||
{"rclone hashsum sha256", "rclone hashsum sha256"},
|
||||
},
|
||||
},
|
||||
hash.BLAKE3: {
|
||||
&f.opt.Blake3sumCommand,
|
||||
"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"b3sum", "b3sum"},
|
||||
{"rclone hashsum blake3", "rclone hashsum blake3"},
|
||||
},
|
||||
},
|
||||
hash.XXH3: {
|
||||
&f.opt.Xxh3sumCommand,
|
||||
"2d06800538d394c2",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
// The xxhsum tool uses a non-standard prefix "XXH3_" preceding the hash output for the 64-bit variant
|
||||
// of XXH3, to avoid confusion with the older 64-bit algorithm XXH64. This was introduced in version
|
||||
// 0.8.3 released Dec 30, 2024. Older versions only supported the alternative BSD style output format,
|
||||
// otherwise optional with argument --tag. We are currently not expecting these output formats and can
|
||||
// therefore not use the "xxhsum -H3" command or its xxh3sum alias directly.
|
||||
//{"xxh3sum", "xxh3sum"},
|
||||
//{"xxhsum -H3", "xxhsum -H3"},
|
||||
{"rclone hashsum xxh3", "rclone hashsum xxh3"},
|
||||
},
|
||||
},
|
||||
hash.XXH128: {
|
||||
&f.opt.Xxh128sumCommand,
|
||||
"99aa06d3014798d86001c324468d497f",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"xxh128sum", "xxh128sum"},
|
||||
{"xxhsum -H2", "xxhsum -H2"},
|
||||
{"rclone hashsum xxh128", "rclone hashsum xxh128"},
|
||||
},
|
||||
},
|
||||
}
|
||||
if f.shellType == "powershell" {
|
||||
for _, hashType := range []hash.Type{hash.MD5, hash.SHA1, hash.SHA256} {
|
||||
if entry, ok := hashCommands[hashType]; ok {
|
||||
entry.hashCommands = append(hashCommands[hashType].hashCommands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
fmt.Sprintf("&{param($Path);Get-FileHash -Algorithm %v -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}", hashType),
|
||||
fmt.Sprintf("Get-FileHash -Algorithm %v -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", hashType),
|
||||
})
|
||||
hashCommands[hashType] = entry
|
||||
}
|
||||
}
|
||||
return hashSet
|
||||
}
|
||||
|
||||
// look for a hash command which works
|
||||
checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool {
|
||||
if *hashCommand == hashCommandNotSupported {
|
||||
return false
|
||||
@@ -1818,25 +1679,55 @@ func (f *Fs) Hashes() hash.Set {
|
||||
}
|
||||
|
||||
changed := false
|
||||
for _, hashType := range hashTypes.Array() {
|
||||
if entry, ok := hashCommands[hashType]; ok {
|
||||
if works := checkHash(hashType, entry.hashCommands, entry.emptyHash, entry.option, &changed); works {
|
||||
hashTypesSupported.Add(hashType)
|
||||
}
|
||||
}
|
||||
md5Commands := []struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
{"md5sum", "md5sum"},
|
||||
{"md5 -r", "md5 -r"},
|
||||
{"rclone md5sum", "rclone md5sum"},
|
||||
}
|
||||
sha1Commands := []struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
{"sha1sum", "sha1sum"},
|
||||
{"sha1 -r", "sha1 -r"},
|
||||
{"rclone sha1sum", "rclone sha1sum"},
|
||||
}
|
||||
if f.shellType == "powershell" {
|
||||
md5Commands = append(md5Commands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
"&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
|
||||
"Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
|
||||
})
|
||||
|
||||
sha1Commands = append(sha1Commands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
"&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
|
||||
"Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
|
||||
})
|
||||
}
|
||||
|
||||
md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
|
||||
sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
|
||||
|
||||
if changed {
|
||||
// Save permanently in config to avoid the extra work next time
|
||||
for _, hashType := range hashTypes.Array() {
|
||||
if entry, ok := hashCommands[hashType]; ok {
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set %vsum_command to override)", hashType, *entry.option, hashType)
|
||||
f.m.Set(fmt.Sprintf("%vsum_command", hashType), *entry.option)
|
||||
}
|
||||
}
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand)
|
||||
f.m.Set("md5sum_command", f.opt.Md5sumCommand)
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand)
|
||||
f.m.Set("sha1sum_command", f.opt.Sha1sumCommand)
|
||||
}
|
||||
|
||||
return hashTypesSupported
|
||||
if sha1Works {
|
||||
hashSet.Add(hash.SHA1)
|
||||
}
|
||||
if md5Works {
|
||||
hashSet.Add(hash.MD5)
|
||||
}
|
||||
|
||||
return hashSet
|
||||
}
|
||||
|
||||
// About gets usage stats
|
||||
@@ -1971,43 +1862,17 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
_ = o.fs.Hashes()
|
||||
|
||||
var hashCmd string
|
||||
switch r {
|
||||
case hash.MD5:
|
||||
if r == hash.MD5 {
|
||||
if o.md5sum != nil {
|
||||
return *o.md5sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Md5sumCommand
|
||||
case hash.SHA1:
|
||||
} else if r == hash.SHA1 {
|
||||
if o.sha1sum != nil {
|
||||
return *o.sha1sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Sha1sumCommand
|
||||
case hash.CRC32:
|
||||
if o.crc32sum != nil {
|
||||
return *o.crc32sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Crc32sumCommand
|
||||
case hash.SHA256:
|
||||
if o.sha256sum != nil {
|
||||
return *o.sha256sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Sha256sumCommand
|
||||
case hash.BLAKE3:
|
||||
if o.blake3sum != nil {
|
||||
return *o.blake3sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Blake3sumCommand
|
||||
case hash.XXH3:
|
||||
if o.xxh3sum != nil {
|
||||
return *o.xxh3sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Xxh3sumCommand
|
||||
case hash.XXH128:
|
||||
if o.xxh128sum != nil {
|
||||
return *o.xxh128sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Xxh128sumCommand
|
||||
default:
|
||||
} else {
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
if hashCmd == "" || hashCmd == hashCommandNotSupported {
|
||||
@@ -2024,21 +1889,10 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
}
|
||||
hashString := parseHash(outBytes)
|
||||
fs.Debugf(o, "Parsed hash: %s", hashString)
|
||||
switch r {
|
||||
case hash.MD5:
|
||||
if r == hash.MD5 {
|
||||
o.md5sum = &hashString
|
||||
case hash.SHA1:
|
||||
} else if r == hash.SHA1 {
|
||||
o.sha1sum = &hashString
|
||||
case hash.CRC32:
|
||||
o.crc32sum = &hashString
|
||||
case hash.SHA256:
|
||||
o.sha256sum = &hashString
|
||||
case hash.BLAKE3:
|
||||
o.blake3sum = &hashString
|
||||
case hash.XXH3:
|
||||
o.xxh3sum = &hashString
|
||||
case hash.XXH128:
|
||||
o.xxh128sum = &hashString
|
||||
}
|
||||
return hashString, nil
|
||||
}
|
||||
@@ -2103,7 +1957,7 @@ func (f *Fs) remoteShellPath(remote string) string {
|
||||
}
|
||||
|
||||
// Converts a byte array from the SSH session returned by
|
||||
// an invocation of hash command to a hash string
|
||||
// an invocation of md5sum/sha1sum to a hash string
|
||||
// as expected by the rest of this application
|
||||
func parseHash(bytes []byte) string {
|
||||
// For strings with backslash *sum writes a leading \
|
||||
@@ -2332,11 +2186,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Clear the hash cache since we are about to update the object
|
||||
o.md5sum = nil
|
||||
o.sha1sum = nil
|
||||
o.crc32sum = nil
|
||||
o.sha256sum = nil
|
||||
o.blake3sum = nil
|
||||
o.xxh3sum = nil
|
||||
o.xxh128sum = nil
|
||||
c, err := o.fs.getSftpConnection(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Update: %w", err)
|
||||
|
||||
@@ -38,7 +38,7 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
|
||||
|
||||
d := &smb2.Dialer{}
|
||||
if f.opt.UseKerberos {
|
||||
cl, err := createKerberosClient(f.opt.KerberosCCache)
|
||||
cl, err := getKerberosClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+40
-56
@@ -14,81 +14,65 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
kerberosClient sync.Map // map[string]*client.Client
|
||||
kerberosErr sync.Map // map[string]error
|
||||
kerberosClient *client.Client
|
||||
kerberosErr error
|
||||
kerberosOnce sync.Once
|
||||
)
|
||||
|
||||
func resolveCcachePath(ccachePath string) (string, error) {
|
||||
if ccachePath == "" {
|
||||
ccachePath = os.Getenv("KRB5CCNAME")
|
||||
// getKerberosClient returns a Kerberos client that can be used to authenticate.
|
||||
func getKerberosClient() (*client.Client, error) {
|
||||
if kerberosClient == nil || kerberosErr == nil {
|
||||
kerberosOnce.Do(func() {
|
||||
kerberosClient, kerberosErr = createKerberosClient()
|
||||
})
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(ccachePath, ":"):
|
||||
parts := strings.SplitN(ccachePath, ":", 2)
|
||||
prefix, path := parts[0], parts[1]
|
||||
switch prefix {
|
||||
case "FILE":
|
||||
return path, nil
|
||||
case "DIR":
|
||||
primary, err := os.ReadFile(filepath.Join(path, "primary"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(path, strings.TrimSpace(string(primary))), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
|
||||
}
|
||||
case ccachePath == "":
|
||||
u, err := user.Current()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "/tmp/krb5cc_" + u.Uid, nil
|
||||
default:
|
||||
return ccachePath, nil
|
||||
}
|
||||
return kerberosClient, kerberosErr
|
||||
}
|
||||
|
||||
func loadKerberosConfig() (*config.Config, error) {
|
||||
// createKerberosClient creates a new Kerberos client.
|
||||
func createKerberosClient() (*client.Client, error) {
|
||||
cfgPath := os.Getenv("KRB5_CONFIG")
|
||||
if cfgPath == "" {
|
||||
cfgPath = "/etc/krb5.conf"
|
||||
}
|
||||
return config.Load(cfgPath)
|
||||
}
|
||||
|
||||
// createKerberosClient creates a new Kerberos client.
|
||||
func createKerberosClient(ccachePath string) (*client.Client, error) {
|
||||
ccachePath, err := resolveCcachePath(ccachePath)
|
||||
cfg, err := config.Load(cfgPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check if we already have a client or an error for this ccache path
|
||||
if errVal, ok := kerberosErr.Load(ccachePath); ok {
|
||||
return nil, errVal.(error)
|
||||
}
|
||||
if clientVal, ok := kerberosClient.Load(ccachePath); ok {
|
||||
return clientVal.(*client.Client), nil
|
||||
// Determine the ccache location from the environment, falling back to the
|
||||
// default location.
|
||||
ccachePath := os.Getenv("KRB5CCNAME")
|
||||
switch {
|
||||
case strings.Contains(ccachePath, ":"):
|
||||
parts := strings.SplitN(ccachePath, ":", 2)
|
||||
switch parts[0] {
|
||||
case "FILE":
|
||||
ccachePath = parts[1]
|
||||
case "DIR":
|
||||
primary, err := os.ReadFile(filepath.Join(parts[1], "primary"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ccachePath = filepath.Join(parts[1], strings.TrimSpace(string(primary)))
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
|
||||
}
|
||||
case ccachePath == "":
|
||||
u, err := user.Current()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ccachePath = "/tmp/krb5cc_" + u.Uid
|
||||
}
|
||||
|
||||
// create a new client if not found in the map
|
||||
cfg, err := loadKerberosConfig()
|
||||
if err != nil {
|
||||
kerberosErr.Store(ccachePath, err)
|
||||
return nil, err
|
||||
}
|
||||
ccache, err := credentials.LoadCCache(ccachePath)
|
||||
if err != nil {
|
||||
kerberosErr.Store(ccachePath, err)
|
||||
return nil, err
|
||||
}
|
||||
cl, err := client.NewFromCCache(ccache, cfg)
|
||||
if err != nil {
|
||||
kerberosErr.Store(ccachePath, err)
|
||||
return nil, err
|
||||
}
|
||||
kerberosClient.Store(ccachePath, cl)
|
||||
return cl, nil
|
||||
|
||||
return client.NewFromCCache(ccache, cfg)
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package smb
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestResolveCcachePath(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Setup: files for FILE and DIR modes
|
||||
fileCcache := filepath.Join(tmpDir, "file_ccache")
|
||||
err := os.WriteFile(fileCcache, []byte{}, 0600)
|
||||
assert.NoError(t, err)
|
||||
|
||||
dirCcache := filepath.Join(tmpDir, "dir_ccache")
|
||||
err = os.Mkdir(dirCcache, 0755)
|
||||
assert.NoError(t, err)
|
||||
err = os.WriteFile(filepath.Join(dirCcache, "primary"), []byte("ticket"), 0600)
|
||||
assert.NoError(t, err)
|
||||
dirCcacheTicket := filepath.Join(dirCcache, "ticket")
|
||||
err = os.WriteFile(dirCcacheTicket, []byte{}, 0600)
|
||||
assert.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ccachePath string
|
||||
envKRB5CCNAME string
|
||||
expected string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "FILE: prefix from env",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "FILE:" + fileCcache,
|
||||
expected: fileCcache,
|
||||
},
|
||||
{
|
||||
name: "DIR: prefix from env",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "DIR:" + dirCcache,
|
||||
expected: dirCcacheTicket,
|
||||
},
|
||||
{
|
||||
name: "Unsupported prefix",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "MEMORY:/bad/path",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "Direct file path (no prefix)",
|
||||
ccachePath: "/tmp/myccache",
|
||||
expected: "/tmp/myccache",
|
||||
},
|
||||
{
|
||||
name: "Default to /tmp/krb5cc_<uid>",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "",
|
||||
expected: "/tmp/krb5cc_",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Setenv("KRB5CCNAME", tt.envKRB5CCNAME)
|
||||
result, err := resolveCcachePath(tt.ccachePath)
|
||||
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -107,20 +107,6 @@ Set to 0 to keep connections indefinitely.
|
||||
Help: "Whether the server is configured to be case-insensitive.\n\nAlways true on Windows shares.",
|
||||
Default: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "kerberos_ccache",
|
||||
Help: `Path to the Kerberos credential cache (krb5cc).
|
||||
|
||||
Overrides the default KRB5CCNAME environment variable and allows this
|
||||
instance of the SMB backend to use a different Kerberos cache file.
|
||||
This is useful when mounting multiple SMB with different credentials
|
||||
or running in multi-user environments.
|
||||
|
||||
Supported formats:
|
||||
- FILE:/path/to/ccache – Use the specified file.
|
||||
- DIR:/path/to/ccachedir – Use the primary file inside the specified directory.
|
||||
- /path/to/ccache – Interpreted as a file path.`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -151,7 +137,6 @@ type Options struct {
|
||||
Domain string `config:"domain"`
|
||||
SPN string `config:"spn"`
|
||||
UseKerberos bool `config:"use_kerberos"`
|
||||
KerberosCCache string `config:"kerberos_ccache"`
|
||||
HideSpecial bool `config:"hide_special_share"`
|
||||
CaseInsensitive bool `config:"case_insensitive"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/smb"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
@@ -19,9 +18,6 @@ func TestIntegration(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIntegration2(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("skipping as -remote is set")
|
||||
}
|
||||
krb5Dir := t.TempDir()
|
||||
t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
|
||||
t.Setenv("KRB5CCNAME", filepath.Join(krb5Dir, "ccache"))
|
||||
@@ -30,24 +26,3 @@ func TestIntegration2(t *testing.T) {
|
||||
NilObject: (*smb.Object)(nil),
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegration3(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("skipping as -remote is set")
|
||||
}
|
||||
|
||||
krb5Dir := t.TempDir()
|
||||
t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
|
||||
ccache := filepath.Join(krb5Dir, "ccache")
|
||||
t.Setenv("RCLONE_TEST_CUSTOM_CCACHE_LOCATION", ccache)
|
||||
|
||||
name := "TestSMBKerberosCcache"
|
||||
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":rclone",
|
||||
NilObject: (*smb.Object)(nil),
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "kerberos_ccache", Value: ccache},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -491,8 +491,8 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
|
||||
ApplicationCredentialName: opt.ApplicationCredentialName,
|
||||
ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
|
||||
EndpointType: swift.EndpointType(opt.EndpointType),
|
||||
ConnectTimeout: time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transport
|
||||
Timeout: time.Duration(10 * ci.Timeout), // Use the timeouts in the transport
|
||||
ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
|
||||
Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
|
||||
Transport: fshttp.NewTransport(ctx),
|
||||
FetchUntilEmptyPage: opt.FetchUntilEmptyPage,
|
||||
PartialPageFetchThreshold: opt.PartialPageFetchThreshold,
|
||||
|
||||
@@ -1550,7 +1550,7 @@ func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string
|
||||
extraHeaders := map[string]string{}
|
||||
if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
|
||||
if o.fs.useOCMtime {
|
||||
extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
|
||||
extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", o.modTime.Unix())
|
||||
}
|
||||
// Set one upload checksum
|
||||
// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
|
||||
|
||||
+1
-2
@@ -12,5 +12,4 @@
|
||||
<seb•ɑƬ•chezwam•ɖɵʈ•org>
|
||||
<allllaboutyou@gmail.com>
|
||||
<psycho@feltzv.fr>
|
||||
<afw5059@gmail.com>
|
||||
<piyushgarg80>
|
||||
<afw5059@gmail.com>
|
||||
@@ -278,7 +278,7 @@ func testBisync(t *testing.T, path1, path2 string) {
|
||||
}
|
||||
bisync.Colors = true
|
||||
time.Local = bisync.TZ
|
||||
ci.FsCacheExpireDuration = fs.Duration(5 * time.Hour)
|
||||
ci.FsCacheExpireDuration = 5 * time.Hour
|
||||
|
||||
baseDir, err := os.Getwd()
|
||||
require.NoError(t, err, "get current directory")
|
||||
|
||||
+2
-2
@@ -55,7 +55,7 @@ type Options struct {
|
||||
Compare CompareOpt
|
||||
CompareFlag string
|
||||
DebugName string
|
||||
MaxLock fs.Duration
|
||||
MaxLock time.Duration
|
||||
ConflictResolve Prefer
|
||||
ConflictLoser ConflictLoserAction
|
||||
ConflictSuffixFlag string
|
||||
@@ -146,7 +146,7 @@ func init() {
|
||||
flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.Compare.DownloadHash, "download-hash", "", Opt.Compare.DownloadHash, "Compute hash by downloading when otherwise unavailable. (warning: may be slow and use lots of data!)", "")
|
||||
flags.FVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
|
||||
flags.DurationVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", Opt.MaxLock, "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
|
||||
flags.FVarP(cmdFlags, &Opt.ConflictResolve, "conflict-resolve", "", "Automatically resolve conflicts by preferring the version that is: "+ConflictResolveList+" (default: none)", "")
|
||||
flags.FVarP(cmdFlags, &Opt.ConflictLoser, "conflict-loser", "", "Action to take on the loser of a sync conflict (when there is a winner) or on both files (when there is no winner): "+ConflictLoserList+" (default: num)", "")
|
||||
flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/terminal"
|
||||
)
|
||||
|
||||
const basicallyforever = fs.Duration(200 * 365 * 24 * time.Hour)
|
||||
const basicallyforever = 200 * 365 * 24 * time.Hour
|
||||
|
||||
var stopRenewal func()
|
||||
|
||||
@@ -66,9 +66,9 @@ func (b *bisyncRun) removeLockFile() {
|
||||
}
|
||||
|
||||
func (b *bisyncRun) setLockFileExpiration() {
|
||||
if b.opt.MaxLock > 0 && b.opt.MaxLock < fs.Duration(2*time.Minute) {
|
||||
if b.opt.MaxLock > 0 && b.opt.MaxLock < 2*time.Minute {
|
||||
fs.Logf(nil, Color(terminal.YellowFg, "--max-lock cannot be shorter than 2 minutes (unless 0.) Changing --max-lock from %v to %v"), b.opt.MaxLock, 2*time.Minute)
|
||||
b.opt.MaxLock = fs.Duration(2 * time.Minute)
|
||||
b.opt.MaxLock = 2 * time.Minute
|
||||
} else if b.opt.MaxLock <= 0 {
|
||||
b.opt.MaxLock = basicallyforever
|
||||
}
|
||||
@@ -80,7 +80,7 @@ func (b *bisyncRun) renewLockFile() {
|
||||
data.Session = b.basePath
|
||||
data.PID = strconv.Itoa(os.Getpid())
|
||||
data.TimeRenewed = time.Now()
|
||||
data.TimeExpires = time.Now().Add(time.Duration(b.opt.MaxLock))
|
||||
data.TimeExpires = time.Now().Add(b.opt.MaxLock)
|
||||
|
||||
// save data file
|
||||
df, err := os.Create(b.lockFile)
|
||||
@@ -131,7 +131,7 @@ func (b *bisyncRun) startLockRenewal() func() {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ticker := time.NewTicker(time.Duration(b.opt.MaxLock) - time.Minute)
|
||||
ticker := time.NewTicker(b.opt.MaxLock - time.Minute)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
|
||||
@@ -138,7 +138,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
|
||||
if b.SyncCI != nil {
|
||||
fs.Infoc(nil, Color(terminal.YellowFg, "Telling Sync to wrap up early."))
|
||||
b.SyncCI.MaxTransfer = 1
|
||||
b.SyncCI.MaxDuration = fs.Duration(1 * time.Second)
|
||||
b.SyncCI.MaxDuration = 1 * time.Second
|
||||
b.SyncCI.CutoffMode = fs.CutoffModeSoft
|
||||
gracePeriod := 30 * time.Second // TODO: flag to customize this?
|
||||
if !waitFor("Canceling Sync if not done in", gracePeriod, func() bool { return b.CleanupCompleted }) {
|
||||
|
||||
+2
-2
@@ -376,8 +376,8 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
|
||||
return files.Save(queueFile)
|
||||
}
|
||||
|
||||
func naptime(totalWait fs.Duration) {
|
||||
expireTime := time.Now().Add(time.Duration(totalWait))
|
||||
func naptime(totalWait time.Duration) {
|
||||
expireTime := time.Now().Add(totalWait)
|
||||
fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST"))
|
||||
for i := 0; time.Until(expireTime) > 0; i++ {
|
||||
if i > 0 && i%10 == 0 {
|
||||
|
||||
+1
-1
@@ -73,7 +73,7 @@ you what happened to it. These are reminiscent of diff files.
|
||||
- |* path| means path was present in source and destination but different.
|
||||
- |! path| means there was an error reading or hashing the source or dest.
|
||||
|
||||
The default number of parallel checks is 8. See the [--checkers](/docs/#checkers-int)
|
||||
The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
|
||||
option for more information.
|
||||
`, "|", "`")
|
||||
|
||||
|
||||
+1
-1
@@ -288,7 +288,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
|
||||
accounting.GlobalStats().ResetErrors()
|
||||
}
|
||||
if ci.RetriesInterval > 0 {
|
||||
time.Sleep(time.Duration(ci.RetriesInterval))
|
||||
time.Sleep(ci.RetriesInterval)
|
||||
}
|
||||
}
|
||||
stopStats()
|
||||
|
||||
+4
-20
@@ -8,23 +8,18 @@ import (
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
createEmptySrcDirs = false
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -95,30 +90,19 @@ for more info.
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
|
||||
|
||||
**Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
|
||||
|
||||
`, "|", "`") + operationsflags.Help(),
|
||||
`, "|", "`"),
|
||||
Annotations: map[string]string{
|
||||
"groups": "Copy,Filter,Listing,Important",
|
||||
},
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(ctx, fdst, fsrc, createEmptySrcDirs)
|
||||
return sync.CopyDir(context.Background(), fdst, fsrc, createEmptySrcDirs)
|
||||
}
|
||||
return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)
|
||||
return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
+3
-24
@@ -6,21 +6,12 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -55,8 +46,7 @@ the destination.
|
||||
*If you are looking to copy just a byte range of a file, please see 'rclone cat --offset X --count Y'*
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
|
||||
` + operationsflags.Help(),
|
||||
`,
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.35",
|
||||
"groups": "Copy,Filter,Listing,Important",
|
||||
@@ -65,21 +55,10 @@ the destination.
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(ctx, fdst, fsrc, false)
|
||||
return sync.CopyDir(context.Background(), fdst, fsrc, false)
|
||||
}
|
||||
return operations.CopyFile(ctx, fdst, fsrc, dstFileName, srcFileName)
|
||||
return operations.CopyFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
@@ -240,7 +240,7 @@ to run as the SYSTEM account. A third alternative is to use the
|
||||
Read more in the [install documentation](https://rclone.org/install/).
|
||||
Note that when running rclone as another user, it will not use
|
||||
the configuration file from your profile unless you tell it to
|
||||
with the [`--config`](https://rclone.org/docs/#config-string) option.
|
||||
with the [`--config`](https://rclone.org/docs/#config-config-file) option.
|
||||
Note also that it is now the SYSTEM account that will have the owner
|
||||
permissions, and other accounts will have permissions according to the
|
||||
group or others scopes. As mentioned above, these will then not get the
|
||||
|
||||
+3
-20
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -17,8 +16,6 @@ import (
|
||||
var (
|
||||
deleteEmptySrcDirs = false
|
||||
createEmptySrcDirs = false
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -26,8 +23,6 @@ func init() {
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "")
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -71,8 +66,7 @@ for more info.
|
||||
|--dry-run| or the |--interactive|/|-i| flag.
|
||||
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
|
||||
|
||||
`, "|", "`") + operationsflags.Help(),
|
||||
`, "|", "`"),
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.19",
|
||||
"groups": "Filter,Listing,Important,Copy",
|
||||
@@ -81,21 +75,10 @@ for more info.
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
|
||||
return sync.MoveDir(context.Background(), fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
|
||||
}
|
||||
return operations.MoveFile(ctx, fdst, fsrc, srcFileName, srcFileName)
|
||||
return operations.MoveFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
+3
-24
@@ -6,21 +6,12 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -56,8 +47,7 @@ successful transfer.
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
|
||||
|
||||
` + operationsflags.Help(),
|
||||
`,
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.35",
|
||||
"groups": "Filter,Listing,Important,Copy",
|
||||
@@ -67,21 +57,10 @@ successful transfer.
|
||||
fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
|
||||
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(ctx, fdst, fsrc, false, false)
|
||||
return sync.MoveDir(context.Background(), fdst, fsrc, false, false)
|
||||
}
|
||||
return operations.MoveFile(ctx, fdst, fsrc, dstFileName, srcFileName)
|
||||
return operations.MoveFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
+50
-101
@@ -95,44 +95,61 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
|
||||
if err != nil {
|
||||
return fmt.Errorf("send output failed: %w", err)
|
||||
}
|
||||
case "md5sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.MD5, args)
|
||||
case "sha1sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.SHA1, args)
|
||||
case "crc32":
|
||||
return c.handleHashsumCommand(ctx, out, hash.CRC32, args)
|
||||
case "sha256sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.SHA256, args)
|
||||
case "b3sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.BLAKE3, args)
|
||||
case "xxh128sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
|
||||
case "xxhsum":
|
||||
argv := strings.SplitN(args, " ", 2)
|
||||
if len(argv) == 0 || argv[0] != "-H2" {
|
||||
return fmt.Errorf("%q not implemented", command)
|
||||
case "md5sum", "sha1sum":
|
||||
ht := hash.MD5
|
||||
if binary == "sha1sum" {
|
||||
ht = hash.SHA1
|
||||
}
|
||||
if len(argv) > 1 {
|
||||
args = argv[1]
|
||||
} else {
|
||||
args = ""
|
||||
if !c.vfs.Fs().Hashes().Contains(ht) {
|
||||
return fmt.Errorf("%v hash not supported", ht)
|
||||
}
|
||||
return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
|
||||
case "rclone":
|
||||
argv := strings.SplitN(args, " ", 3)
|
||||
if len(argv) > 1 && argv[0] == "hashsum" {
|
||||
var ht hash.Type
|
||||
if err := ht.Set(argv[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(argv) > 2 {
|
||||
args = argv[2]
|
||||
var hashSum string
|
||||
if args == "" {
|
||||
// empty hash for no input
|
||||
if ht == hash.MD5 {
|
||||
hashSum = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
} else {
|
||||
args = ""
|
||||
hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
}
|
||||
args = "-"
|
||||
} else {
|
||||
node, err := c.vfs.Stat(args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed finding file %q: %w", args, err)
|
||||
}
|
||||
if node.IsDir() {
|
||||
return errors.New("can't hash directory")
|
||||
}
|
||||
o, ok := node.DirEntry().(fs.ObjectInfo)
|
||||
if !ok {
|
||||
fs.Debugf(args, "File uploading - reading hash from VFS cache")
|
||||
in, err := node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs open failed: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = in.Close()
|
||||
}()
|
||||
h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
|
||||
}
|
||||
_, err = io.Copy(h, in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs copy failed: %w", err)
|
||||
}
|
||||
hashSum = h.Sums()[ht]
|
||||
} else {
|
||||
hashSum, err = o.Hash(ctx, ht)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed: %w", err)
|
||||
}
|
||||
}
|
||||
return c.handleHashsumCommand(ctx, out, ht, args)
|
||||
}
|
||||
return fmt.Errorf("%q not implemented", command)
|
||||
_, err = fmt.Fprintf(out, "%s %s\n", hashSum, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("send output failed: %w", err)
|
||||
}
|
||||
case "echo":
|
||||
// Special cases for legacy rclone command detection.
|
||||
// Before rclone v1.49.0 the sftp backend used "echo 'abc' | md5sum" when
|
||||
@@ -172,74 +189,6 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleHashsumCommand is a helper to execCommand for common functionality of hashsum related commands
|
||||
func (c *conn) handleHashsumCommand(ctx context.Context, out io.Writer, ht hash.Type, args string) (err error) {
|
||||
if !c.vfs.Fs().Hashes().Contains(ht) {
|
||||
return fmt.Errorf("%v hash not supported", ht)
|
||||
}
|
||||
var hashSum string
|
||||
if args == "" {
|
||||
// empty hash for no input
|
||||
switch ht {
|
||||
case hash.MD5:
|
||||
hashSum = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
case hash.SHA1:
|
||||
hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
case hash.CRC32:
|
||||
hashSum = "00000000"
|
||||
case hash.SHA256:
|
||||
hashSum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
case hash.BLAKE3:
|
||||
hashSum = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"
|
||||
case hash.XXH3:
|
||||
hashSum = "2d06800538d394c2"
|
||||
case hash.XXH128:
|
||||
hashSum = "99aa06d3014798d86001c324468d497f"
|
||||
default:
|
||||
return fmt.Errorf("%v hash not implemented", ht)
|
||||
}
|
||||
args = "-"
|
||||
} else {
|
||||
node, err := c.vfs.Stat(args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed finding file %q: %w", args, err)
|
||||
}
|
||||
if node.IsDir() {
|
||||
return errors.New("can't hash directory")
|
||||
}
|
||||
o, ok := node.DirEntry().(fs.ObjectInfo)
|
||||
if !ok {
|
||||
fs.Debugf(args, "File uploading - reading hash from VFS cache")
|
||||
in, err := node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs open failed: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = in.Close()
|
||||
}()
|
||||
h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
|
||||
}
|
||||
_, err = io.Copy(h, in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs copy failed: %w", err)
|
||||
}
|
||||
hashSum = h.Sums()[ht]
|
||||
} else {
|
||||
hashSum, err = o.Hash(ctx, ht)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "%s %s\n", hashSum, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("send output failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// handle a new incoming channel request
|
||||
func (c *conn) handleChannel(newChannel ssh.NewChannel) {
|
||||
fs.Debugf(c.what, "Incoming channel: %s\n", newChannel.ChannelType())
|
||||
|
||||
+160
-14
@@ -3,9 +3,13 @@ package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
mutex "sync" // renamed as "sync" already in use
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
@@ -15,7 +19,7 @@ import (
|
||||
|
||||
var (
|
||||
createEmptySrcDirs = false
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
opt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
@@ -23,15 +27,119 @@ func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)
|
||||
// TODO: add same flags to move and copy
|
||||
}
|
||||
|
||||
var lock mutex.Mutex
|
||||
|
||||
func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, srcOk := src.(fs.Object)
|
||||
_, dstOk := dst.(fs.Object)
|
||||
var filename string
|
||||
if !srcOk && !dstOk {
|
||||
return
|
||||
} else if srcOk && !dstOk {
|
||||
filename = src.String()
|
||||
} else {
|
||||
filename = dst.String()
|
||||
}
|
||||
|
||||
if sigil.Writer(opt) != nil {
|
||||
operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)
|
||||
}
|
||||
if opt.Combined != nil {
|
||||
operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
|
||||
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
|
||||
}
|
||||
if opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
}
|
||||
}
|
||||
|
||||
// GetSyncLoggerOpt gets the options corresponding to the logger flags
|
||||
func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
opt.LoggerFn = syncLoggerFn
|
||||
if opt.TimeFormat == "max" {
|
||||
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
|
||||
}
|
||||
opt.SetListFormat(ctx, command.Flags())
|
||||
opt.NewListJSON(ctx, fdst, "")
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
|
||||
close := func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return opt, close, nil
|
||||
}
|
||||
|
||||
func anyNotBlank(s ...string) bool {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "sync source:path dest:path",
|
||||
Short: `Make source and dest identical, modifying destination only.`,
|
||||
// Warning! "|" will be replaced by backticks below
|
||||
Long: strings.ReplaceAll(`Sync the source to the destination, changing the destination
|
||||
Long: `Sync the source to the destination, changing the destination
|
||||
only. Doesn't transfer files that are identical on source and
|
||||
destination, testing by size and modification time or MD5SUM.
|
||||
Destination is updated to match source, including deleting files
|
||||
@@ -40,7 +148,7 @@ want to delete files from destination, use the
|
||||
[copy](/commands/rclone_copy/) command instead.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
|--dry-run| or the |--interactive|/|i| flag.
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
|
||||
rclone sync --interactive SOURCE remote:DESTINATION
|
||||
|
||||
@@ -63,18 +171,55 @@ destination that is inside the source directory.
|
||||
|
||||
Rclone will sync the modification times of files and directories if
|
||||
the backend supports it. If metadata syncing is required then use the
|
||||
|--metadata| flag.
|
||||
` + "`--metadata`" + ` flag.
|
||||
|
||||
Note that the modification time and metadata for the root directory
|
||||
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
|
||||
for more info.
|
||||
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
|
||||
**Note**: Use the |rclone dedupe| command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
|
||||
|
||||
`, "|", "`") + operationsflags.Help(),
|
||||
## Logger Flags
|
||||
|
||||
The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` +
|
||||
"`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or
|
||||
stdout if it is ` + "`-`" + `) supplied. What they write is described in the
|
||||
help below. For example ` + "`--differ`" + ` will write all paths which are
|
||||
present on both the source and destination but different.
|
||||
|
||||
The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- ` + "`= path`" + ` means path was found in source and destination and was identical
|
||||
- ` + "`- path`" + ` means path was missing on the source, so only in the destination
|
||||
- ` + "`+ path`" + ` means path was missing on the destination, so only in the source
|
||||
- ` + "`* path`" + ` means path was present in source and destination but different.
|
||||
- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.
|
||||
|
||||
The ` + "`--dest-after`" + ` flag writes a list file using the same format flags
|
||||
as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the sync.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `
|
||||
- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during the sync, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
`,
|
||||
Annotations: map[string]string{
|
||||
"groups": "Sync,Copy,Filter,Listing,Important",
|
||||
},
|
||||
@@ -83,14 +228,15 @@ See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,
|
||||
loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {
|
||||
ctx = operations.WithSyncLogger(ctx, opt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
|
||||
@@ -14,13 +14,13 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
pollInterval = fs.Duration(10 * time.Second)
|
||||
pollInterval = 10 * time.Second
|
||||
)
|
||||
|
||||
func init() {
|
||||
test.Command.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.FVarP(cmdFlags, &pollInterval, "poll-interval", "", "Time to wait between polling for changes", "")
|
||||
flags.DurationVarP(cmdFlags, &pollInterval, "poll-interval", "", pollInterval, "Time to wait between polling for changes", "")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -39,7 +39,7 @@ var commandDefinition = &cobra.Command{
|
||||
if do := features.ChangeNotify; do != nil {
|
||||
pollChan := make(chan time.Duration)
|
||||
do(ctx, changeNotify, pollChan)
|
||||
pollChan <- time.Duration(pollInterval)
|
||||
pollChan <- pollInterval
|
||||
fs.Logf(nil, "Waiting for changes, polling every %v", pollInterval)
|
||||
} else {
|
||||
return errors.New("poll-interval is not supported by this remote")
|
||||
|
||||
@@ -40,7 +40,7 @@ var (
|
||||
checkStreaming bool
|
||||
checkBase32768 bool
|
||||
all bool
|
||||
uploadWait fs.Duration
|
||||
uploadWait time.Duration
|
||||
positionLeftRe = regexp.MustCompile(`(?s)^(.*)-position-left-([[:xdigit:]]+)$`)
|
||||
positionMiddleRe = regexp.MustCompile(`(?s)^position-middle-([[:xdigit:]]+)-(.*)-$`)
|
||||
positionRightRe = regexp.MustCompile(`(?s)^position-right-([[:xdigit:]]+)-(.*)$`)
|
||||
@@ -52,7 +52,7 @@ func init() {
|
||||
flags.StringVarP(cmdFlags, &writeJSON, "write-json", "", "", "Write results to file", "")
|
||||
flags.BoolVarP(cmdFlags, &checkNormalization, "check-normalization", "", false, "Check UTF-8 Normalization", "")
|
||||
flags.BoolVarP(cmdFlags, &checkControl, "check-control", "", false, "Check control characters", "")
|
||||
flags.FVarP(cmdFlags, &uploadWait, "upload-wait", "", "Wait after writing a file", "")
|
||||
flags.DurationVarP(cmdFlags, &uploadWait, "upload-wait", "", 0, "Wait after writing a file", "")
|
||||
flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", false, "Check max filename length", "")
|
||||
flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", false, "Check uploads with indeterminate file size", "")
|
||||
flags.BoolVarP(cmdFlags, &checkBase32768, "check-base32768", "", false, "Check can store all possible base32768 characters", "")
|
||||
@@ -204,7 +204,7 @@ func (r *results) writeFile(path string) (fs.Object, error) {
|
||||
src := object.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
|
||||
obj, err := r.f.Put(r.ctx, bytes.NewBufferString(contents), src)
|
||||
if uploadWait > 0 {
|
||||
time.Sleep(time.Duration(uploadWait))
|
||||
time.Sleep(uploadWait)
|
||||
}
|
||||
return obj, err
|
||||
}
|
||||
|
||||
@@ -123,8 +123,8 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
|
||||
{{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
|
||||
{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
|
||||
{{< provider name="FileLu Cloud Storage" home="https://filelu.com/" config="/filelu/" >}}
|
||||
{{< provider name="Files.com" home="https://www.files.com/" config="/filescom/" >}}
|
||||
{{< provider name="FileLu Cloud Storage" home="https://filelu.com/" config="/filelu/" >}}
|
||||
{{< provider name="FlashBlade" home="https://www.purestorage.com/products/unstructured-data-storage.html" config="/s3/#pure-storage-flashblade" >}}
|
||||
{{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
|
||||
{{< provider name="Gofile" home="https://gofile.io/" config="/gofile/" >}}
|
||||
@@ -197,7 +197,6 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
|
||||
{{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}
|
||||
{{< provider name="Zoho WorkDrive" home="https://www.zoho.com/workdrive/" config="/zoho/" >}}
|
||||
{{< provider name="Zata" home="https://zata.ai/" config="/s3/#Zata" end="true" >}}
|
||||
{{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}}
|
||||
{{< /provider_list >}}
|
||||
|
||||
|
||||
+1
-10
@@ -788,6 +788,7 @@ put them back in again.` >}}
|
||||
* Adithya Kumar <akumar42@protonmail.com>
|
||||
* Tayo-pasedaRJ <138471223+Tayo-pasedaRJ@users.noreply.github.com>
|
||||
* Peter Kreuser <logo@kreuser.name>
|
||||
* Piyush <piyushgarg80>
|
||||
* fotile96 <fotile96@users.noreply.github.com>
|
||||
* Luc Ritchie <luc.ritchie@gmail.com>
|
||||
* cynful <cynful@users.noreply.github.com>
|
||||
@@ -981,13 +982,3 @@ put them back in again.` >}}
|
||||
* Flora Thiebaut <johann.thiebaut@gmail.com>
|
||||
* kingston125 <support@filelu.com>
|
||||
* Ser-Bul <30335009+Ser-Bul@users.noreply.github.com>
|
||||
* jinjingroad <jinjingroad@sina.com>
|
||||
* necaran <55765083+necaran@users.noreply.github.com>
|
||||
* Marvin Rösch <marvinroesch99@gmail.com>
|
||||
* curlwget <curlwget@icloud.com>
|
||||
* $@M@RTH_ <165886349+Samarth1015@users.noreply.github.com>
|
||||
* Ali Zein Yousuf <azy5030@gmail.com>
|
||||
* Davide Bizzarri <davide.bizzarri@willhaben.at>
|
||||
* Ross Smith II <ross@smithii.com>
|
||||
* Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com>
|
||||
* Sudipto Baral <sudiptobaral.me@gmail.com>
|
||||
@@ -297,16 +297,6 @@ be explicitly specified using exactly one of the `msi_object_id`,
|
||||
If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
|
||||
set, this is is equivalent to using `env_auth`.
|
||||
|
||||
#### Fedrated Identity Credentials
|
||||
|
||||
If these variables are set, rclone will authenticate with fedrated identity.
|
||||
|
||||
- `tenant_id`: tenant_id to authenticate in storage
|
||||
- `client_id`: client ID of the application the user will authenticate to storage
|
||||
- `msi_client_id`: managed identity client ID of the application the user will authenticate to
|
||||
|
||||
By default "api://AzureADTokenExchange" is used as scope for token retrieval over MSI. This token is then exchanged for actual storage token using 'tenant_id' and 'client_id'.
|
||||
|
||||
#### Azure CLI tool `az` {#use_az}
|
||||
|
||||
Set to use the [Azure CLI tool `az`](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
|
||||
@@ -295,16 +295,6 @@ be explicitly specified using exactly one of the `msi_object_id`,
|
||||
|
||||
If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
|
||||
set, this is is equivalent to using `env_auth`.
|
||||
|
||||
#### Fedrated Identity Credentials
|
||||
|
||||
If these variables are set, rclone will authenticate with fedrated identity.
|
||||
|
||||
- `tenant_id`: tenant_id to authenticate in storage
|
||||
- `client_id`: client ID of the application the user will authenticate to storage
|
||||
- `msi_client_id`: managed identity client ID of the application the user will authenticate to
|
||||
|
||||
By default "api://AzureADTokenExchange" is used as scope for token retrieval over MSI. This token is then exchanged for actual storage token using 'tenant_id' and 'client_id'.
|
||||
|
||||
#### Azure CLI tool `az` {#use_az}
|
||||
Set to use the [Azure CLI tool `az`](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
|
||||
@@ -635,7 +635,7 @@ directory separators.) To address this particular issue, an additional
|
||||
`2006-01-02 0304PM`.
|
||||
|
||||
Note that `--conflict-suffix` is entirely separate from rclone's main
|
||||
[`--sufix`](/docs/#suffix-string) flag. This is intentional, as users may wish
|
||||
[`--sufix`](/docs/#suffix-suffix) flag. This is intentional, as users may wish
|
||||
to use both flags simultaneously, if also using
|
||||
[`--backup-dir`](#backup-dir1-and-backup-dir2).
|
||||
|
||||
@@ -810,7 +810,7 @@ without requiring the user to get involved and run a `--resync`. (See also:
|
||||
|
||||
### --backup-dir1 and --backup-dir2
|
||||
|
||||
As of `v1.66`, [`--backup-dir`](/docs/#backup-dir-string) is supported in bisync.
|
||||
As of `v1.66`, [`--backup-dir`](/docs/#backup-dir-dir) is supported in bisync.
|
||||
Because `--backup-dir` must be a non-overlapping path on the same remote,
|
||||
Bisync has introduced new `--backup-dir1` and `--backup-dir2` flags to support
|
||||
separate backup-dirs for `Path1` and `Path2` (bisyncing between different
|
||||
@@ -841,7 +841,7 @@ In the event of a [rename due to a sync conflict](#conflict-loser), the
|
||||
rename is not considered a delete, unless a previous conflict with the same
|
||||
name already exists and would get overwritten.
|
||||
|
||||
See also: [`--suffix`](/docs/#suffix-string),
|
||||
See also: [`--suffix`](/docs/#suffix-suffix),
|
||||
[`--suffix-keep-extension`](/docs/#suffix-keep-extension)
|
||||
|
||||
## Operation
|
||||
@@ -1831,7 +1831,7 @@ about _Unison_ and synchronization in general.
|
||||
* Final listings are now generated from sync results, to avoid needing to re-list
|
||||
* Bisync is now much more resilient to changes that happen during a bisync run, and far less prone to critical errors / undetected changes
|
||||
* Bisync is now capable of rolling a file listing back in cases of uncertainty, essentially marking the file as needing to be rechecked next time.
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color) (`AUTO`|`NEVER`|`ALWAYS`)
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`)
|
||||
* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`,
|
||||
for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors).
|
||||
* Fixed handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)
|
||||
|
||||
@@ -390,6 +390,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -5,25 +5,6 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.70.3 - 2025-07-09
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.2...v1.70.3)
|
||||
|
||||
* Bug Fixes
|
||||
* check: Fix difference report (was reporting error counts) (albertony)
|
||||
* march: Fix deadlock when using `--no-traverse` (Nick Craig-Wood)
|
||||
* doc fixes (albertony, Nick Craig-Wood)
|
||||
* Azure Blob
|
||||
* Fix server side copy error "requires exactly one scope" (Nick Craig-Wood)
|
||||
* B2
|
||||
* Fix finding objects when using `--b2-version-at` (Davide Bizzarri)
|
||||
* Linkbox
|
||||
* Fix upload error "user upload file not exist" (Nick Craig-Wood)
|
||||
* Pikpak
|
||||
* Improve error handling for missing links and unrecoverable 500s (wiserain)
|
||||
* WebDAV
|
||||
* Fix setting modtime to that of local object instead of remote (WeidiDeng)
|
||||
|
||||
## v1.70.2 - 2025-06-27
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.1...v1.70.2)
|
||||
@@ -68,7 +49,7 @@ description: "Rclone Changelog"
|
||||
* New Features
|
||||
* Add [`--max-connections`](/docs/#max-connections-n) to control maximum backend concurrency (Nick Craig-Wood)
|
||||
* Add [`--max-buffer-memory`](/docs/#max-buffer-memory) to limit total buffer memory usage (Nick Craig-Wood)
|
||||
* Add transform library and [`--name-transform`](/docs/#name-transform-stringarray) flag (nielash)
|
||||
* Add transform library and [`--name-transform`](/docs/#name-transform-command-xxxx) flag (nielash)
|
||||
* sync: Implement [`--list-cutoff`](/docs/#list-cutoff) to allow on disk sorting for reduced memory use (Nick Craig-Wood)
|
||||
* accounting: Add listed stat for number of directory entries listed (Nick Craig-Wood)
|
||||
* backend: Skip hash calculation when the hashType is None (Oleksiy Stashok)
|
||||
@@ -777,7 +758,7 @@ description: "Rclone Changelog"
|
||||
* Final listings are now generated from sync results, to avoid needing to re-list (nielash)
|
||||
* Bisync is now much more resilient to changes that happen during a bisync run, and far less prone to critical errors / undetected changes (nielash)
|
||||
* Bisync is now capable of rolling a file listing back in cases of uncertainty, essentially marking the file as needing to be rechecked next time. (nielash)
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-autoneveralways) (`AUTO`|`NEVER`|`ALWAYS`) (nielash)
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`) (nielash)
|
||||
* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`, for performance improvements and less risk of error. (nielash)
|
||||
* `--resync` is now much more efficient (especially for users of `--create-empty-src-dirs`) (nielash)
|
||||
* Google Docs (and other files of unknown size) are now supported (with the same options as in `sync`) (nielash)
|
||||
@@ -789,7 +770,7 @@ instead of of `--size-only`, when `check` is not available.
|
||||
* A new `--max-lock` setting allows lock files to automatically renew and expire, for better automatic recovery when a run is interrupted. (nielash)
|
||||
* Bisync now supports auto-resolving sync conflicts and customizing rename behavior with new [`--conflict-resolve`](#conflict-resolve), [`--conflict-loser`](#conflict-loser), and [`--conflict-suffix`](#conflict-suffix) flags. (nielash)
|
||||
* A new [`--resync-mode`](#resync-mode) flag allows more control over which version of a file gets kept during a `--resync`. (nielash)
|
||||
* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-duration) (when [`--resilient`](#resilient) is set.) (nielash)
|
||||
* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time) (when [`--resilient`](#resilient) is set.) (nielash)
|
||||
* Clarify file operation directions in dry-run logs (Kyle Reynolds)
|
||||
* Local
|
||||
* Fix cleanRootPath on Windows after go1.21.4 stdlib update (nielash)
|
||||
@@ -3040,7 +3021,7 @@ instead of of `--size-only`, when `check` is not available.
|
||||
* New Features
|
||||
* The [VFS layer](/commands/rclone_mount/#vfs-virtual-file-system) was heavily reworked for this release - see below for more details
|
||||
* Interactive mode [-i/--interactive](/docs/#interactive) for destructive operations (fishbullet)
|
||||
* Add [--bwlimit-file](/docs/#bwlimit-file-bwtimetable) flag to limit speeds of individual file transfers (Nick Craig-Wood)
|
||||
* Add [--bwlimit-file](/docs/#bwlimit-file-bandwidth-spec) flag to limit speeds of individual file transfers (Nick Craig-Wood)
|
||||
* Transfers are sorted by start time in the stats and progress output (Max Sum)
|
||||
* Make sure backends expand `~` and environment vars in file names they use (Nick Craig-Wood)
|
||||
* Add [--refresh-times](/docs/#refresh-times) flag to set modtimes on hashless backends (Nick Craig-Wood)
|
||||
|
||||
@@ -339,6 +339,7 @@ rclone [flags]
|
||||
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
|
||||
--ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-http-proxy string URL for HTTP CONNECT proxy
|
||||
--ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s)
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-no-check-upload Don't check the upload is OK
|
||||
@@ -997,7 +998,7 @@ rclone [flags]
|
||||
--use-json-log Use json log format
|
||||
--use-mmap Use mmap allocator (see docs)
|
||||
--use-server-modtime Use server modified time instead of object metadata
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.2")
|
||||
-v, --verbose count Print lots more stuff (repeat for more)
|
||||
-V, --version Print the version number
|
||||
--webdav-auth-redirect Preserve authentication on redirect
|
||||
|
||||
@@ -221,12 +221,12 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,command=e
|
||||
|
||||
```
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
|
||||
// Output: stories/The Quick Brown Fox!-20250617
|
||||
// Output: stories/The Quick Brown Fox!-20250618
|
||||
```
|
||||
|
||||
```
|
||||
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-17 0551PM
|
||||
// Output: stories/The Quick Brown Fox!-2025-06-18 0148PM
|
||||
```
|
||||
|
||||
```
|
||||
@@ -234,17 +234,15 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,regex=[\\
|
||||
// Output: ababababababab/ababab ababababab ababababab ababab!abababab
|
||||
```
|
||||
|
||||
|
||||
|
||||
Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
|
||||
|
||||
The `--name-transform` flag is also available in `sync`, `copy`, and `move`.
|
||||
|
||||
# Files vs Directories ##
|
||||
# Files vs Directories
|
||||
|
||||
By default `--name-transform` will only apply to file names. The means only the leaf file name will be transformed.
|
||||
However some of the transforms would be better applied to the whole path or just directories.
|
||||
To choose which which part of the file path is affected some tags can be added to the `--name-transform`
|
||||
To choose which which part of the file path is affected some tags can be added to the `--name-transform`.
|
||||
|
||||
| Tag | Effect |
|
||||
|------|------|
|
||||
@@ -254,11 +252,11 @@ To choose which which part of the file path is affected some tags can be added t
|
||||
|
||||
This is used by adding the tag into the transform name like this: `--name-transform file,prefix=ABC` or `--name-transform dir,prefix=DEF`.
|
||||
|
||||
For some conversions using all is more likely to be useful, for example `--name-transform all,nfc`
|
||||
For some conversions using all is more likely to be useful, for example `--name-transform all,nfc`.
|
||||
|
||||
Note that `--name-transform` may not add path separators `/` to the name. This will cause an error.
|
||||
|
||||
# Ordering and Conflicts ##
|
||||
# Ordering and Conflicts
|
||||
|
||||
* Transformations will be applied in the order specified by the user.
|
||||
* If the `file` tag is in use (the default) then only the leaf name of files will be transformed.
|
||||
@@ -273,19 +271,19 @@ user, allowing for intentional use cases (e.g., trimming one prefix before addin
|
||||
* Users should be aware that certain combinations may lead to unexpected results and should verify
|
||||
transformations using `--dry-run` before execution.
|
||||
|
||||
# Race Conditions and Non-Deterministic Behavior ##
|
||||
# Race Conditions and Non-Deterministic Behavior
|
||||
|
||||
Some transformations, such as `replace=old:new`, may introduce conflicts where multiple source files map to the same destination name.
|
||||
This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these.
|
||||
* If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic.
|
||||
* Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results.
|
||||
|
||||
* To minimize risks, users should:
|
||||
* Carefully review transformations that may introduce conflicts.
|
||||
* Use `--dry-run` to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
|
||||
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
|
||||
* Consider disabling concurrency with `--transfers=1` if necessary.
|
||||
* Certain transformations (e.g. `prefix`) will have a multiplying effect every time they are used. Avoid these when using `bisync`.
|
||||
To minimize risks, users should:
|
||||
* Carefully review transformations that may introduce conflicts.
|
||||
* Use `--dry-run` to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
|
||||
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
|
||||
* Consider disabling concurrency with `--transfers=1` if necessary.
|
||||
* Certain transformations (e.g. `prefix`) will have a multiplying effect every time they are used. Avoid these when using `bisync`.
|
||||
|
||||
|
||||
|
||||
|
||||
+497
-521
File diff suppressed because it is too large.
Load diff
@@ -699,6 +699,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -305,6 +305,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
+1
-1
@@ -233,7 +233,7 @@ Restart-Service hns
|
||||
It is likely you have more than 10,000 files that need to be
|
||||
synced. By default, rclone only gets 10,000 files ahead in a sync so as
|
||||
not to use up too much memory. You can change this default with the
|
||||
[--max-backlog](/docs/#max-backlog-int) flag.
|
||||
[--max-backlog](/docs/#max-backlog-n) flag.
|
||||
|
||||
### Rclone is using too much memory or appears to have a memory leak
|
||||
|
||||
|
||||
@@ -99,21 +99,6 @@ excess files in the directory.
|
||||
|
||||
rclone sync --interactive /home/local/directory remote:dir
|
||||
|
||||
### Hashes
|
||||
|
||||
In December 2024 `files.com` started [supporting more checksums](https://www.files.com/blog/2024/11/01/new-modern-checksum-options-now-available-with-opt).
|
||||
|
||||
However if you want hashes you **must** enable them in your site by
|
||||
visiting the "File Integrity" section in "Data Governance" to
|
||||
configure your settings and select the checksum algorithms that meet
|
||||
your requirements.
|
||||
|
||||
Rclone currently only support CRC32 and MD5 for use in syncing but
|
||||
selecting more checksums will not affect rclone's operations.
|
||||
|
||||
For use with rclone, selecting at least MD5 is recommended so rclone
|
||||
can do an end to end integrity check.
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/filescom/filescom.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
|
||||
@@ -683,7 +683,7 @@ Default units are `KiB` but abbreviations `B`, `K`, `M`, `G`, `T` or `P` are val
|
||||
E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50 KiB
|
||||
size or larger.
|
||||
|
||||
See [the size option docs](/docs/#size-options) for more info.
|
||||
See [the size option docs](/docs/#size-option) for more info.
|
||||
|
||||
### `--max-size` - Don't transfer any file larger than this
|
||||
|
||||
@@ -693,7 +693,7 @@ Default units are `KiB` but abbreviations `B`, `K`, `M`, `G`, `T` or `P` are val
|
||||
E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1 GiB
|
||||
size or smaller.
|
||||
|
||||
See [the size option docs](/docs/#size-options) for more info.
|
||||
See [the size option docs](/docs/#size-option) for more info.
|
||||
|
||||
### `--max-age` - Don't transfer any file older than this
|
||||
|
||||
@@ -704,7 +704,7 @@ Controls the maximum age of files within the scope of an rclone command.
|
||||
E.g. `rclone ls remote: --max-age 2d` lists files on `remote:` of 2 days
|
||||
old or less.
|
||||
|
||||
See [the time option docs](/docs/#time-options) for valid formats.
|
||||
See [the time option docs](/docs/#time-option) for valid formats.
|
||||
|
||||
### `--min-age` - Don't transfer any file younger than this
|
||||
|
||||
@@ -716,7 +716,7 @@ Controls the minimum age of files within the scope of an rclone command.
|
||||
E.g. `rclone ls remote: --min-age 2d` lists files on `remote:` of 2 days
|
||||
old or more.
|
||||
|
||||
See [the time option docs](/docs/#time-options) for valid formats.
|
||||
See [the time option docs](/docs/#time-option) for valid formats.
|
||||
|
||||
### `--hash-filter` - Deterministically select a subset of files {#hash-filter}
|
||||
|
||||
|
||||
@@ -119,7 +119,7 @@ Flags for general networking and HTTP stuff.
|
||||
--tpslimit float Limit HTTP transactions per second to this
|
||||
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
|
||||
--use-cookies Enable session cookiejar
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.0")
|
||||
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.2")
|
||||
```
|
||||
|
||||
|
||||
@@ -601,6 +601,7 @@ Backend-only flags (these can be set in the config file also).
|
||||
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
|
||||
--ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-http-proxy string URL for HTTP CONNECT proxy
|
||||
--ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s)
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-no-check-upload Don't check the upload is OK
|
||||
|
||||
@@ -433,6 +433,20 @@ Properties:
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --ftp-http-proxy
|
||||
|
||||
URL for HTTP CONNECT proxy
|
||||
|
||||
Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
|
||||
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: http_proxy
|
||||
- Env Var: RCLONE_FTP_HTTP_PROXY
|
||||
- Type: string
|
||||
- Required: false
|
||||
|
||||
#### --ftp-no-check-upload
|
||||
|
||||
Don't check the upload is OK
|
||||
|
||||
@@ -679,6 +679,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -324,6 +324,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -288,6 +288,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -491,9 +491,9 @@ the locations that rclone will use.
|
||||
|
||||
To override them set the corresponding options (as command-line arguments, or as
|
||||
[environment variables](https://rclone.org/docs/#environment-variables)):
|
||||
- [--config](https://rclone.org/docs/#config-string)
|
||||
- [--cache-dir](https://rclone.org/docs/#cache-dir-string)
|
||||
- [--temp-dir](https://rclone.org/docs/#temp-dir-string)
|
||||
- [--config](https://rclone.org/docs/#config-config-file)
|
||||
- [--cache-dir](https://rclone.org/docs/#cache-dir-dir)
|
||||
- [--temp-dir](https://rclone.org/docs/#temp-dir-dir)
|
||||
|
||||
## Autostart
|
||||
|
||||
@@ -546,7 +546,7 @@ NOTE: Remember that when rclone runs as the `SYSTEM` user, the user profile
|
||||
that it sees will not be yours. This means that if you normally run rclone with
|
||||
configuration file in the default location, to be able to use the same configuration
|
||||
when running as the system user you must explicitly tell rclone where to find
|
||||
it with the [`--config`](https://rclone.org/docs/#config-string) option,
|
||||
it with the [`--config`](https://rclone.org/docs/#config-config-file) option,
|
||||
or else it will look in the system users profile path (`C:\Windows\System32\config\systemprofile`).
|
||||
To test your command manually from a Command Prompt, you can run it with
|
||||
the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
|
||||
|
||||
@@ -257,7 +257,7 @@ flag.
|
||||
Note that Jottacloud requires the MD5 hash before upload so if the
|
||||
source does not have an MD5 checksum then the file will be cached
|
||||
temporarily on disk (in location given by
|
||||
[--temp-dir](/docs/#temp-dir-string)) before it is uploaded.
|
||||
[--temp-dir](/docs/#temp-dir-dir)) before it is uploaded.
|
||||
Small files will be cached in memory - see the
|
||||
[--jottacloud-md5-memory-limit](#jottacloud-md5-memory-limit) flag.
|
||||
When uploading from local disk the source checksum is always available,
|
||||
@@ -383,6 +383,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -300,6 +300,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -386,6 +386,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -377,8 +377,6 @@ will show you the defaults for the backends.
|
||||
| Exclamation | `!` | `!` |
|
||||
| Hash | `#` | `#` |
|
||||
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | `�` |
|
||||
| ForceNFC | All invalid NFC characters | Their valid NFC equivalents |
|
||||
| ForceNFD | All invalid NFD characters | Their valid NFD equivalents |
|
||||
| LeftCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string | `␍`, `␊`, `␉`, `␋` |
|
||||
| LeftPeriod | `.` on the left of a string | `.` |
|
||||
| LeftSpace | SPACE on the left of a string | `␠` |
|
||||
@@ -518,7 +516,7 @@ upon backend-specific capabilities.
|
||||
| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes |
|
||||
| HTTP | No | No | No | No | No | No | No | No | No | No | Yes |
|
||||
| iCloud Drive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| ImageKit | Yes | No | Yes | No | No | No | No | No | No | No | Yes |
|
||||
| ImageKit | Yes | Yes | Yes | No | No | No | No | No | No | No | Yes |
|
||||
| Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No |
|
||||
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
|
||||
|
||||
@@ -230,6 +230,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -195,6 +195,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -192,6 +192,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
+4
-4
@@ -546,8 +546,8 @@ And this is equivalent to `/tmp/dir`
|
||||
|
||||
```
|
||||
{
|
||||
"type": "local",
|
||||
"_root": "/tmp/dir"
|
||||
type = "local",
|
||||
_root = "/tmp/dir"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -2359,8 +2359,8 @@ e.g.
|
||||
"fs": "/tmp",
|
||||
"remote": 3
|
||||
},
|
||||
"status": 400,
|
||||
"path": "operations/rmdir"
|
||||
"status": 400
|
||||
"path": "operations/rmdir",
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -43,7 +43,6 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="Synology C2 Object Storage" home="https://c2.synology.com/en-global/object-storage/overview" config="/s3/#synology-c2" >}}
|
||||
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
|
||||
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}}
|
||||
{{< provider name="Zata" home="https://zata.ai/" config="/s3/#Zata" end="true" >}}
|
||||
{{< /provider_list >}}
|
||||
|
||||
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
|
||||
@@ -6041,155 +6040,3 @@ d) Delete this remote
|
||||
|
||||
y/e/d> y
|
||||
```
|
||||
### Zata Object Storage {#Zata}
|
||||
|
||||
[Zata Object Storage](https://zata.ai/) provides a secure, S3-compatible cloud storage solution designed for scalability and performance, ideal for a variety of data storage needs.
|
||||
|
||||
First run:
|
||||
|
||||
```
|
||||
rclone config
|
||||
```
|
||||
|
||||
```
|
||||
This will guide you through an interactive setup process:
|
||||
|
||||
e) Edit existing remote
|
||||
n) New remote
|
||||
d) Delete remote
|
||||
r) Rename remote
|
||||
c) Copy remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
e/n/d/r/c/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> my zata storage
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
|
||||
Storage> s3
|
||||
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
|
||||
XX / Zata (S3 compatible Gateway)
|
||||
\ (Zata)
|
||||
|
||||
provider> Zata
|
||||
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth>
|
||||
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> "your key"
|
||||
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> "your secret key"
|
||||
|
||||
Option region.
|
||||
Region where you can connect with.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / Indore, Madhya Pradesh, India
|
||||
\ (us-east-1)
|
||||
region> 1
|
||||
|
||||
Option endpoint.
|
||||
Endpoint for Zata Object Storage.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / South Asia Endpoint
|
||||
\ (idr01.zata.ai)
|
||||
endpoint> 1
|
||||
|
||||
Option location_constraint.
|
||||
Location constraint - must be set to match the Region.
|
||||
Leave blank if not sure. Used when creating buckets only.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
location_constraint>
|
||||
|
||||
Option acl.
|
||||
Canned ACL used when creating buckets and storing or copying objects.
|
||||
This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
|
||||
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
Note that this ACL is applied when server-side copying objects as S3
|
||||
doesn't copy the ACL from the source but rather writes a fresh one.
|
||||
If the acl is an empty string then no X-Amz-Acl: header is added and
|
||||
the default (private) will be used.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
/ Owner gets FULL_CONTROL.
|
||||
1 | No one else has access rights (default).
|
||||
\ (private)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
2 | The AllUsers group gets READ access.
|
||||
\ (public-read)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
3 | The AllUsers group gets READ and WRITE access.
|
||||
| Granting this on a bucket is generally not recommended.
|
||||
\ (public-read-write)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
4 | The AuthenticatedUsers group gets READ access.
|
||||
\ (authenticated-read)
|
||||
/ Object owner gets FULL_CONTROL.
|
||||
5 | Bucket owner gets READ access.
|
||||
| If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
\ (bucket-owner-read)
|
||||
/ Both the object owner and the bucket owner get FULL_CONTROL over the object.
|
||||
6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
\ (bucket-owner-full-control)
|
||||
acl>
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n>
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: Zata
|
||||
- access_key_id: xxx
|
||||
- secret_access_key: xxx
|
||||
- region: us-east-1
|
||||
- endpoint: idr01.zata.ai
|
||||
Keep this "my zata storage" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d>
|
||||
|
||||
```
|
||||
This will leave the config file looking like this.
|
||||
```
|
||||
[my zata storage]
|
||||
type = s3
|
||||
provider = Zata
|
||||
access_key_id = xxx
|
||||
secret_access_key = xxx
|
||||
region = us-east-1
|
||||
endpoint = idr01.zata.ai
|
||||
|
||||
```
|
||||
+20
-24
@@ -318,30 +318,29 @@ is able to use checksumming if the same login has shell access,
|
||||
and can execute remote commands. If there is a command that can
|
||||
calculate compatible checksums on the remote system, Rclone can
|
||||
then be configured to execute this whenever a checksum is needed,
|
||||
and read back the results. By default MD5 and SHA-1 are considered,
|
||||
but also CRC32, SHA-256, BLAKE3, XXH3 and XXH128 are supported,
|
||||
option `hashes` can be set to specify which to consider.
|
||||
and read back the results. Currently MD5 and SHA-1 are supported.
|
||||
|
||||
Normally this requires an external utility being available on
|
||||
the server. E.g. for MD5 checksums, by default rclone will try commands
|
||||
`md5sum`, `md5` and `rclone md5sum`, and the first one found
|
||||
usable will be picked. These utilities normally need to be in the
|
||||
remote's PATH to be found.
|
||||
the server. By default rclone will try commands `md5sum`, `md5`
|
||||
and `rclone md5sum` for MD5 checksums, and the first one found usable
|
||||
will be picked. Same with `sha1sum`, `sha1` and `rclone sha1sum`
|
||||
commands for SHA-1 checksums. These utilities normally need to
|
||||
be in the remote's PATH to be found.
|
||||
|
||||
In some cases the shell itself is capable of calculating checksums.
|
||||
PowerShell is an example of such a shell. If rclone detects that the
|
||||
remote shell is PowerShell, which means it most probably is a
|
||||
Windows OpenSSH server, rclone will use a predefined script block
|
||||
to produce the checksums for MD5, SHA-1 and SHA-256 when no external
|
||||
checksum commands are found (see [shell access](#shell-access)). This
|
||||
assumes PowerShell version 4.0 or newer.
|
||||
to produce the checksums when no external checksum commands are found
|
||||
(see [shell access](#shell-access)). This assumes PowerShell version
|
||||
4.0 or newer.
|
||||
|
||||
The options `md5sum_command`, `sha1_command`, etc. can be used to customize
|
||||
the commands to be executed for calculation of checksums. You can for
|
||||
example set a specific path to where the md5sum executable are located,
|
||||
or specify some other tool that print checksums in compatible format.
|
||||
The value can include command-line arguments, or even shell script blocks
|
||||
as with PowerShell. Rclone has subcommands [hashsum](/commands/rclone_hashsum/),
|
||||
The options `md5sum_command` and `sha1_command` can be used to customize
|
||||
the command to be executed for calculation of checksums. You can for
|
||||
example set a specific path to where md5sum and sha1sum executables
|
||||
are located, or use them to specify some other tools that print checksums
|
||||
in compatible format. The value can include command-line arguments,
|
||||
or even shell script blocks as with PowerShell. Rclone has subcommands
|
||||
[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/)
|
||||
that use compatible format, which means if you have an rclone executable
|
||||
on the server it can be used. As mentioned above, they will be automatically
|
||||
@@ -357,14 +356,11 @@ configuration, so next time it will use the same. Value `none`
|
||||
will be set if none of the default commands could be used for a specific
|
||||
algorithm, and this algorithm will not be supported by the remote.
|
||||
|
||||
Disabling the checksumming completely may be required if you are connecting to
|
||||
SFTP servers which are not under your control, and to which the execution of
|
||||
remote shell commands is prohibited. Set the configuration option `disable_hashcheck`
|
||||
to `true` to disable checksumming entirely (you get the same effect by setting
|
||||
option `hashes` to `none` or options `md5sum_command`, `sha1_command` etc.
|
||||
to `none`). Set option `shell_type` to `none` to not only disable checksumming,
|
||||
but also disable all other functionality that are based on remote shell command
|
||||
execution.
|
||||
Disabling the checksumming may be required if you are connecting to SFTP servers
|
||||
which are not under your control, and to which the execution of remote shell
|
||||
commands is prohibited. Set the configuration option `disable_hashcheck`
|
||||
to `true` to disable checksumming entirely, or set `shell_type` to `none`
|
||||
to disable all functionality based on remote shell command execution.
|
||||
|
||||
### Modification times and hashes
|
||||
|
||||
|
||||
@@ -252,6 +252,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -57,7 +57,7 @@ off donation.
|
||||
Thank you very much to our sponsors:
|
||||
|
||||
{{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}}
|
||||
{{< sponsor src="/img/logos/filescom-enterprise-grade-workflows.png" width="300" height="200" title="Start Your Free Trial Today" link="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone">}}
|
||||
{{< sponsor src="/img/logos/filescom-enterprise-grade-workflows.png" width="300" height="200" title="Start Your Free Trial Today" link="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=banner">}}
|
||||
{{< sponsor src="/img/logos/sia.svg" width="200" height="200" title="Visit our sponsor sia" link="https://sia.tech">}}
|
||||
{{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}}
|
||||
{{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
|
||||
|
||||
@@ -192,6 +192,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -230,6 +230,8 @@ Use client credentials OAuth flow.
|
||||
|
||||
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
|
||||
|
||||
Note that this option is NOT supported by all backends.
|
||||
|
||||
Properties:
|
||||
|
||||
- Config: client_credentials
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
Gold Sponsor
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<a href="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone" target="_blank" rel="noopener" title="Start Your Free Trial Today"><img style="max-width: 100%; height: auto;" src="/img/logos/filescom-enterprise-grade-workflows.png"></a><br />
|
||||
<a href="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=banner" target="_blank" rel="noopener" title="Start Your Free Trial Today"><img style="max-width: 100%; height: auto;" src="/img/logos/filescom-enterprise-grade-workflows.png"></a><br />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -66,8 +66,8 @@
|
||||
<a class="dropdown-item" href="/koofr/#digi-storage"><i class="fa fa-cloud fa-fw"></i> Digi Storage</a>
|
||||
<a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a>
|
||||
<a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a>
|
||||
<a class="dropdown-item" href="/filelu/"><i class="fa fa-folder"></i> FileLu Cloud Storage</a>
|
||||
<a class="dropdown-item" href="/filescom/"><i class="fa fa-brands fa-files-pinwheel fa-fw"></i> Files.com</a>
|
||||
<a class="dropdown-item" href="/filelu/"><i class="fa fa-brands fa-files-pinwheel fa-fw"></i> Files.com</a>
|
||||
<a class="dropdown-item" href="/filescom/"><i class="fa fa-cloud fa-fw"></i> FileLu Cloud Storage</a>
|
||||
<a class="dropdown-item" href="/ftp/"><i class="fa fa-file fa-fw"></i> FTP</a>
|
||||
<a class="dropdown-item" href="/gofile/"><i class="fa fa-folder fa-fw"></i> Gofile</a>
|
||||
<a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google fa-fw"></i> Google Cloud Storage</a>
|
||||
|
||||
@@ -1 +1 @@
|
||||
v1.71.0
|
||||
v1.70.2
|
||||
@@ -304,7 +304,7 @@ func (acc *Account) ServerSideTransferEnd(n int64) {
|
||||
acc.stats.Bytes(n)
|
||||
}
|
||||
|
||||
// serverSideEnd accounts for non specific server-side data
|
||||
// serverSideEnd accounts for non specific server side data
|
||||
func (acc *Account) serverSideEnd(n int64) {
|
||||
// Account for bytes unless we are checking
|
||||
if !acc.checking {
|
||||
@@ -312,13 +312,13 @@ func (acc *Account) serverSideEnd(n int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// ServerSideCopyEnd accounts for a read of n bytes in a server-side copy
|
||||
// ServerSideCopyEnd accounts for a read of n bytes in a sever side copy
|
||||
func (acc *Account) ServerSideCopyEnd(n int64) {
|
||||
acc.stats.AddServerSideCopy(n)
|
||||
acc.serverSideEnd(n)
|
||||
}
|
||||
|
||||
// ServerSideMoveEnd accounts for a read of n bytes in a server-side move
|
||||
// ServerSideMoveEnd accounts for a read of n bytes in a sever side move
|
||||
func (acc *Account) ServerSideMoveEnd(n int64) {
|
||||
acc.stats.AddServerSideMove(n)
|
||||
acc.serverSideEnd(n)
|
||||
|
||||
Vendored
+2
-3
@@ -6,7 +6,6 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
@@ -26,8 +25,8 @@ func createOnFirstUse() {
|
||||
once.Do(func() {
|
||||
ci := fs.GetConfig(context.Background())
|
||||
c = cache.New()
|
||||
c.SetExpireDuration(time.Duration(ci.FsCacheExpireDuration))
|
||||
c.SetExpireInterval(time.Duration(ci.FsCacheExpireInterval))
|
||||
c.SetExpireDuration(ci.FsCacheExpireDuration)
|
||||
c.SetExpireInterval(ci.FsCacheExpireInterval)
|
||||
c.SetFinalizer(func(value any) {
|
||||
if s, ok := value.(fs.Shutdowner); ok {
|
||||
_ = fs.CountError(context.Background(), s.Shutdown(context.Background()))
|
||||
|
||||
+10
-10
@@ -570,12 +570,12 @@ type ConfigInfo struct {
|
||||
IgnoreTimes bool `config:"ignore_times"`
|
||||
IgnoreExisting bool `config:"ignore_existing"`
|
||||
IgnoreErrors bool `config:"ignore_errors"`
|
||||
ModifyWindow Duration `config:"modify_window"`
|
||||
ModifyWindow time.Duration `config:"modify_window"`
|
||||
Checkers int `config:"checkers"`
|
||||
Transfers int `config:"transfers"`
|
||||
ConnectTimeout Duration `config:"contimeout"` // Connect timeout
|
||||
Timeout Duration `config:"timeout"` // Data channel timeout
|
||||
ExpectContinueTimeout Duration `config:"expect_continue_timeout"`
|
||||
ConnectTimeout time.Duration `config:"contimeout"` // Connect timeout
|
||||
Timeout time.Duration `config:"timeout"` // Data channel timeout
|
||||
ExpectContinueTimeout time.Duration `config:"expect_continue_timeout"`
|
||||
Dump DumpFlags `config:"dump"`
|
||||
InsecureSkipVerify bool `config:"no_check_certificate"` // Skip server certificate verification
|
||||
DeleteMode DeleteMode `config:"delete_mode"`
|
||||
@@ -584,7 +584,7 @@ type ConfigInfo struct {
|
||||
TrackRenames bool `config:"track_renames"` // Track file renames.
|
||||
TrackRenamesStrategy string `config:"track_renames_strategy"` // Comma separated list of strategies used to track renames
|
||||
Retries int `config:"retries"` // High-level retries
|
||||
RetriesInterval Duration `config:"retries_sleep"`
|
||||
RetriesInterval time.Duration `config:"retries_sleep"`
|
||||
LowLevelRetries int `config:"low_level_retries"`
|
||||
UpdateOlder bool `config:"update"` // Skip files that are newer on the destination
|
||||
NoGzip bool `config:"no_gzip_encoding"` // Disable compression
|
||||
@@ -623,7 +623,7 @@ type ConfigInfo struct {
|
||||
PasswordCommand SpaceSepList `config:"password_command"`
|
||||
UseServerModTime bool `config:"use_server_modtime"`
|
||||
MaxTransfer SizeSuffix `config:"max_transfer"`
|
||||
MaxDuration Duration `config:"max_duration"`
|
||||
MaxDuration time.Duration `config:"max_duration"`
|
||||
CutoffMode CutoffMode `config:"cutoff_mode"`
|
||||
MaxBacklog int `config:"max_backlog"`
|
||||
MaxStatsGroups int `config:"max_stats_groups"`
|
||||
@@ -652,11 +652,11 @@ type ConfigInfo struct {
|
||||
RefreshTimes bool `config:"refresh_times"`
|
||||
NoConsole bool `config:"no_console"`
|
||||
TrafficClass uint8 `config:"traffic_class"`
|
||||
FsCacheExpireDuration Duration `config:"fs_cache_expire_duration"`
|
||||
FsCacheExpireInterval Duration `config:"fs_cache_expire_interval"`
|
||||
FsCacheExpireDuration time.Duration `config:"fs_cache_expire_duration"`
|
||||
FsCacheExpireInterval time.Duration `config:"fs_cache_expire_interval"`
|
||||
DisableHTTP2 bool `config:"disable_http2"`
|
||||
HumanReadable bool `config:"human_readable"`
|
||||
KvLockTime Duration `config:"kv_lock_time"` // maximum time to keep key-value database locked by process
|
||||
KvLockTime time.Duration `config:"kv_lock_time"` // maximum time to keep key-value database locked by process
|
||||
DisableHTTPKeepAlives bool `config:"disable_http_keep_alives"`
|
||||
Metadata bool `config:"metadata"`
|
||||
ServerSideAcrossConfigs bool `config:"server_side_across_configs"`
|
||||
@@ -766,7 +766,7 @@ func InitialLogLevel() LogLevel {
|
||||
// TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise
|
||||
func (ci *ConfigInfo) TimeoutOrInfinite() time.Duration {
|
||||
if ci.Timeout > 0 {
|
||||
return time.Duration(ci.Timeout)
|
||||
return ci.Timeout
|
||||
}
|
||||
return ModTimeNotSupported
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ func FileExists(ctx context.Context, fs Fs, remote string) (bool, error) {
|
||||
// GetModifyWindow calculates the maximum modify window between the given Fses
|
||||
// and the Config.ModifyWindow parameter.
|
||||
func GetModifyWindow(ctx context.Context, fss ...Info) time.Duration {
|
||||
window := time.Duration(GetConfig(ctx).ModifyWindow)
|
||||
window := GetConfig(ctx).ModifyWindow
|
||||
for _, f := range fss {
|
||||
if f != nil {
|
||||
precision := f.Precision()
|
||||
|
||||
+2
-2
@@ -27,10 +27,10 @@ func NewDialer(ctx context.Context) *Dialer {
|
||||
ci := fs.GetConfig(ctx)
|
||||
dialer := &Dialer{
|
||||
Dialer: net.Dialer{
|
||||
Timeout: time.Duration(ci.ConnectTimeout),
|
||||
Timeout: ci.ConnectTimeout,
|
||||
KeepAlive: 30 * time.Second,
|
||||
},
|
||||
timeout: time.Duration(ci.Timeout),
|
||||
timeout: ci.Timeout,
|
||||
tclass: int(ci.TrafficClass),
|
||||
}
|
||||
if ci.BindAddr != nil {
|
||||
|
||||
+3
-3
@@ -58,8 +58,8 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
|
||||
t.Proxy = http.ProxyFromEnvironment
|
||||
t.MaxIdleConnsPerHost = 2 * (ci.Checkers + ci.Transfers + 1)
|
||||
t.MaxIdleConns = 2 * t.MaxIdleConnsPerHost
|
||||
t.TLSHandshakeTimeout = time.Duration(ci.ConnectTimeout)
|
||||
t.ResponseHeaderTimeout = time.Duration(ci.Timeout)
|
||||
t.TLSHandshakeTimeout = ci.ConnectTimeout
|
||||
t.ResponseHeaderTimeout = ci.Timeout
|
||||
t.DisableKeepAlives = ci.DisableHTTPKeepAlives
|
||||
|
||||
// TLS Config
|
||||
@@ -109,7 +109,7 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
|
||||
return NewDialer(ctx).DialContext(reqCtx, network, addr)
|
||||
}
|
||||
t.IdleConnTimeout = 60 * time.Second
|
||||
t.ExpectContinueTimeout = time.Duration(ci.ExpectContinueTimeout)
|
||||
t.ExpectContinueTimeout = ci.ExpectContinueTimeout
|
||||
|
||||
if ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
|
||||
fs.Debugf(nil, "You have specified to dump information. Please be noted that the "+
|
||||
|
||||
@@ -16,8 +16,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/jzelinskie/whirlpool"
|
||||
"github.com/zeebo/blake3"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
// Type indicates a standard hashing algorithm
|
||||
@@ -93,27 +91,8 @@ var (
|
||||
|
||||
// SHA512 indicates SHA-512 support
|
||||
SHA512 Type
|
||||
|
||||
// BLAKE3 indicates BLAKE3 support
|
||||
BLAKE3 Type
|
||||
|
||||
// XXH3 indicates XXH3 support, also known as XXH3-64, a variant of xxHash
|
||||
XXH3 Type
|
||||
|
||||
// XXH128 indicates XXH128 support, also known as XXH3-128, a variant of xxHash
|
||||
XXH128 Type
|
||||
)
|
||||
|
||||
type xxh128Hasher struct {
|
||||
xxh3.Hasher
|
||||
}
|
||||
|
||||
// Sum overrides xxh3.Sum to return value based on Sum128 instead of the default Sum64.
|
||||
func (h *xxh128Hasher) Sum(b []byte) []byte {
|
||||
buf := h.Sum128().Bytes()
|
||||
return buf[:]
|
||||
}
|
||||
|
||||
func init() {
|
||||
MD5 = RegisterHash("md5", "MD5", 32, md5.New)
|
||||
SHA1 = RegisterHash("sha1", "SHA-1", 40, sha1.New)
|
||||
@@ -121,9 +100,6 @@ func init() {
|
||||
CRC32 = RegisterHash("crc32", "CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() })
|
||||
SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New)
|
||||
SHA512 = RegisterHash("sha512", "SHA-512", 128, sha512.New)
|
||||
BLAKE3 = RegisterHash("blake3", "BLAKE3", 64, func() hash.Hash { return blake3.New() })
|
||||
XXH3 = RegisterHash("xxh3", "XXH3", 16, func() hash.Hash { return xxh3.New() })
|
||||
XXH128 = RegisterHash("xxh128", "XXH128", 32, func() hash.Hash { return &xxh128Hasher{} })
|
||||
}
|
||||
|
||||
// Supported returns a set of all the supported hashes by
|
||||
|
||||
@@ -78,9 +78,6 @@ var hashTestSet = []hashTest{
|
||||
hash.CRC32: "a6041d7e",
|
||||
hash.SHA256: "c839e57675862af5c21bd0a15413c3ec579e0d5522dab600bc6c3489b05b8f54",
|
||||
hash.SHA512: "008e7e9b5d94d37bf5e07c955890f730f137a41b8b0db16cb535a9b4cb5632c2bccff31685ec470130fe10e2258a0ab50ab587472258f3132ccf7d7d59fb91db",
|
||||
hash.BLAKE3: "0a7276a407a3be1b4d31488318ee05a335aad5a3b82c4420e592a8178c9e86bb",
|
||||
hash.XXH3: "4b83b0c51c543525",
|
||||
hash.XXH128: "438de241a57d684214f67657f7aad93b",
|
||||
},
|
||||
},
|
||||
// Empty data set
|
||||
@@ -93,9 +90,6 @@ var hashTestSet = []hashTest{
|
||||
hash.CRC32: "00000000",
|
||||
hash.SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
hash.SHA512: "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
|
||||
hash.BLAKE3: "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
|
||||
hash.XXH3: "2d06800538d394c2",
|
||||
hash.XXH128: "99aa06d3014798d86001c324468d497f",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
+29
-56
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
@@ -290,7 +291,6 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
|
||||
srcPrev, dstPrev fs.DirEntry
|
||||
srcPrevName, dstPrevName string
|
||||
src, dst fs.DirEntry
|
||||
srcHasMore, dstHasMore = true, true
|
||||
srcName, dstName string
|
||||
)
|
||||
srcDone := func() {
|
||||
@@ -311,14 +311,14 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
|
||||
}
|
||||
// Reload src and dst if needed - we set them to nil if used
|
||||
if src == nil {
|
||||
src, srcHasMore = <-srcChan
|
||||
src = <-srcChan
|
||||
srcName = m.srcKey(src)
|
||||
}
|
||||
if dst == nil {
|
||||
dst, dstHasMore = <-dstChan
|
||||
dst = <-dstChan
|
||||
dstName = m.dstKey(dst)
|
||||
}
|
||||
if !srcHasMore && !dstHasMore {
|
||||
if src == nil && dst == nil {
|
||||
break
|
||||
}
|
||||
if src != nil && srcPrev != nil {
|
||||
@@ -419,65 +419,38 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
|
||||
// If NoTraverse is set, then try to find a matching object
|
||||
// for each item in the srcList to head dst object
|
||||
if m.NoTraverse && !m.NoCheckDest {
|
||||
startedDst = true
|
||||
workers := ci.Checkers
|
||||
originalSrcChan := srcChan
|
||||
srcChan = make(chan fs.DirEntry, 100)
|
||||
|
||||
type matchTask struct {
|
||||
src fs.DirEntry // src object to find in destination
|
||||
dstMatch chan<- fs.DirEntry // channel to receive matching dst object or nil
|
||||
}
|
||||
matchTasks := make(chan matchTask, workers)
|
||||
dstMatches := make(chan (<-chan fs.DirEntry), workers)
|
||||
|
||||
// Create the tasks from the originalSrcChan. These are put into matchTasks for
|
||||
// processing and dstMatches so they can be retrieved in order.
|
||||
go func() {
|
||||
for src := range originalSrcChan {
|
||||
srcChan <- src
|
||||
dstMatch := make(chan fs.DirEntry, 1)
|
||||
matchTasks <- matchTask{
|
||||
src: src,
|
||||
dstMatch: dstMatch,
|
||||
}
|
||||
dstMatches <- dstMatch
|
||||
}
|
||||
close(matchTasks)
|
||||
}()
|
||||
|
||||
// Get the tasks from the queue and find a matching object.
|
||||
var workerWg sync.WaitGroup
|
||||
for range workers {
|
||||
workerWg.Add(1)
|
||||
go func() {
|
||||
defer workerWg.Done()
|
||||
for t := range matchTasks {
|
||||
leaf := path.Base(t.src.Remote())
|
||||
dst, err := m.Fdst.NewObject(m.Ctx, path.Join(job.dstRemote, leaf))
|
||||
if err != nil {
|
||||
dst = nil
|
||||
}
|
||||
t.dstMatch <- dst
|
||||
}
|
||||
}()
|
||||
ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.dstKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Close dstResults when all the workers have finished
|
||||
go func() {
|
||||
workerWg.Wait()
|
||||
close(dstMatches)
|
||||
}()
|
||||
|
||||
// Read the matches in order and send them to dstChan if found.
|
||||
startedDst = true
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for dstMatch := range dstMatches {
|
||||
dst := <-dstMatch
|
||||
// Note that dst may be nil here
|
||||
// We send these on so we don't deadlock the reader
|
||||
dstChan <- dst
|
||||
defer ls.CleanUp()
|
||||
|
||||
g, gCtx := errgroup.WithContext(m.Ctx)
|
||||
g.SetLimit(ci.Checkers)
|
||||
for src := range originalSrcChan {
|
||||
srcChan <- src
|
||||
if srcObj, ok := src.(fs.Object); ok {
|
||||
g.Go(func() error {
|
||||
leaf := path.Base(srcObj.Remote())
|
||||
dstObj, err := m.Fdst.NewObject(gCtx, path.Join(job.dstRemote, leaf))
|
||||
if err == nil {
|
||||
_ = ls.Add(fs.DirEntries{dstObj}) // ignore errors
|
||||
}
|
||||
return nil // ignore errors
|
||||
})
|
||||
}
|
||||
}
|
||||
dstListErr = g.Wait()
|
||||
sendErr := ls.Send()
|
||||
if dstListErr == nil {
|
||||
dstListErr = sendErr
|
||||
}
|
||||
close(srcChan)
|
||||
close(dstChan)
|
||||
|
||||
@@ -249,7 +249,7 @@ func (c *checkMarch) reportResults(ctx context.Context, err error) error {
|
||||
fs.Logf(c.opt.Fsrc, "%d %s missing", c.srcFilesMissing.Load(), entity)
|
||||
}
|
||||
|
||||
fs.Logf(c.opt.Fdst, "%d differences found", c.differences.Load())
|
||||
fs.Logf(c.opt.Fdst, "%d differences found", accounting.Stats(ctx).GetErrors())
|
||||
if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
|
||||
fs.Logf(c.opt.Fdst, "%d errors while checking", errs)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
mutex "sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
@@ -107,43 +106,6 @@ type LoggerOpt struct {
|
||||
Absolute bool
|
||||
}
|
||||
|
||||
// NewDefaultLoggerFn creates a logger function that writes the sigil and path to configured files that match the sigil
|
||||
func NewDefaultLoggerFn(opt *LoggerOpt) LoggerFn {
|
||||
var lock mutex.Mutex
|
||||
|
||||
return func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, srcOk := src.(fs.Object)
|
||||
_, dstOk := dst.(fs.Object)
|
||||
var filename string
|
||||
if !srcOk && !dstOk {
|
||||
return
|
||||
} else if srcOk && !dstOk {
|
||||
filename = src.String()
|
||||
} else {
|
||||
filename = dst.String()
|
||||
}
|
||||
|
||||
if sigil.Writer(*opt) != nil {
|
||||
SyncFprintf(sigil.Writer(*opt), "%s\n", filename)
|
||||
}
|
||||
if opt.Combined != nil {
|
||||
SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
|
||||
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
|
||||
}
|
||||
if opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger
|
||||
func WithLogger(ctx context.Context, logger LoggerFn) context.Context {
|
||||
return context.WithValue(ctx, loggerKey, logger)
|
||||
|
||||
@@ -3,28 +3,12 @@
|
||||
package operationsflags
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
//go:embed operationsflags.md
|
||||
var help string
|
||||
|
||||
// Help returns the help string cleaned up to simplify appending
|
||||
func Help() string {
|
||||
return strings.TrimSpace(help) + "\n\n"
|
||||
}
|
||||
|
||||
// AddLoggerFlagsOptions contains options for the Logger Flags
|
||||
type AddLoggerFlagsOptions struct {
|
||||
Combined string // a file with file names with leading sigils
|
||||
@@ -36,20 +20,6 @@ type AddLoggerFlagsOptions struct {
|
||||
DestAfter string // files that exist on the destination post-sync
|
||||
}
|
||||
|
||||
// AnySet checks if any of the logger flags have a non-blank value
|
||||
func (o AddLoggerFlagsOptions) AnySet() bool {
|
||||
return anyNotBlank(o.Combined, o.MissingOnSrc, o.MissingOnDst, o.Match, o.Differ, o.ErrFile, o.DestAfter)
|
||||
}
|
||||
|
||||
func anyNotBlank(s ...string) bool {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AddLoggerFlags adds the logger flags to the cmdFlags command
|
||||
func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) {
|
||||
flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync")
|
||||
@@ -73,75 +43,3 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt
|
||||
flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
|
||||
// flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
|
||||
}
|
||||
|
||||
// ConfigureLoggers verifies and sets up writers for log files requested via CLI flags
|
||||
func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
if opt.TimeFormat == "max" {
|
||||
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
|
||||
}
|
||||
opt.SetListFormat(ctx, command.Flags())
|
||||
opt.NewListJSON(ctx, fdst, "")
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := open(flagsOpt.Combined, &opt.Combined); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.Match, &opt.Match); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.Differ, &opt.Differ); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.ErrFile, &opt.Error); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.DestAfter, &opt.DestAfter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
close := func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.NoTraverse && opt.Combined != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
|
||||
}
|
||||
if ci.NoTraverse && opt.MissingOnSrc != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
|
||||
}
|
||||
if ci.NoTraverse && opt.DestAfter != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
|
||||
}
|
||||
|
||||
return close, nil
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
## Logger Flags
|
||||
|
||||
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths,
|
||||
one per line, to the file name (or stdout if it is `-`) supplied. What they write is described
|
||||
in the help below. For example `--differ` will write all paths which are present
|
||||
on both the source and destination but different.
|
||||
|
||||
The `--combined` flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- `= path` means path was found in source and destination and was identical
|
||||
- `- path` means path was missing on the source, so only in the destination
|
||||
- `+ path` means path was missing on the destination, so only in the source
|
||||
- `* path` means path was present in source and destination but different.
|
||||
- `! path` means there was an error reading or hashing the source or dest.
|
||||
|
||||
The `--dest-after` flag writes a list file using the same format flags
|
||||
as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's `--itemize-changes`, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the command is finished.
|
||||
|
||||
When the `--no-traverse` flag is set, all logs involving files that exist only
|
||||
on the destination will be incomplete or completely missing.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- `--max-duration` / `CutoffModeHard`
|
||||
- `--compare-dest` / `--copy-dest`
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use `--retries 1` to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during execution, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
+3
-3
@@ -148,7 +148,7 @@ func (jobs *Jobs) kickExpire() {
|
||||
jobs.mu.Lock()
|
||||
defer jobs.mu.Unlock()
|
||||
if !jobs.expireRunning {
|
||||
time.AfterFunc(time.Duration(jobs.opt.JobExpireInterval), jobs.Expire)
|
||||
time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire)
|
||||
jobs.expireRunning = true
|
||||
}
|
||||
}
|
||||
@@ -160,13 +160,13 @@ func (jobs *Jobs) Expire() {
|
||||
now := time.Now()
|
||||
for ID, job := range jobs.jobs {
|
||||
job.mu.Lock()
|
||||
if job.Finished && now.Sub(job.EndTime) > time.Duration(jobs.opt.JobExpireDuration) {
|
||||
if job.Finished && now.Sub(job.EndTime) > jobs.opt.JobExpireDuration {
|
||||
delete(jobs.jobs, ID)
|
||||
}
|
||||
job.mu.Unlock()
|
||||
}
|
||||
if len(jobs.jobs) != 0 {
|
||||
time.AfterFunc(time.Duration(jobs.opt.JobExpireInterval), jobs.Expire)
|
||||
time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire)
|
||||
jobs.expireRunning = true
|
||||
} else {
|
||||
jobs.expireRunning = false
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestNewJobs(t *testing.T) {
|
||||
func TestJobsKickExpire(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
jobs := newJobs()
|
||||
jobs.opt.JobExpireInterval = fs.Duration(time.Millisecond)
|
||||
jobs.opt.JobExpireInterval = time.Millisecond
|
||||
assert.Equal(t, false, jobs.expireRunning)
|
||||
jobs.kickExpire()
|
||||
jobs.mu.Lock()
|
||||
@@ -41,7 +41,7 @@ func TestJobsExpire(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
wait := make(chan struct{})
|
||||
jobs := newJobs()
|
||||
jobs.opt.JobExpireInterval = fs.Duration(time.Millisecond)
|
||||
jobs.opt.JobExpireInterval = time.Millisecond
|
||||
assert.Equal(t, false, jobs.expireRunning)
|
||||
var gotJobID int64
|
||||
var gotJob *Job
|
||||
@@ -64,7 +64,7 @@ func TestJobsExpire(t *testing.T) {
|
||||
assert.Equal(t, 1, len(jobs.jobs))
|
||||
jobs.mu.Lock()
|
||||
job.mu.Lock()
|
||||
job.EndTime = time.Now().Add(-time.Duration(rc.Opt.JobExpireDuration) - 60*time.Second)
|
||||
job.EndTime = time.Now().Add(-rc.Opt.JobExpireDuration - 60*time.Second)
|
||||
assert.Equal(t, true, jobs.expireRunning)
|
||||
job.mu.Unlock()
|
||||
jobs.mu.Unlock()
|
||||
|
||||
Loaded 100 of 126 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user