Compare commits

...
16 Commits
Author SHA1 Message Date
Nick Craig-Wood 3f0e9f5fca Version v1.71.0 2025-08-22 16:03:16 +01:00
Nick Craig-Wood cfd0d28742 fs: tls: add --client-pass support for encrypted --client-key files
This also widens the supported types

- Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY")
- Unencrypted PKCS#8 ("BEGIN PRIVATE KEY")
- Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY")
- Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected.
2025-08-22 12:19:29 +01:00
Nick Craig-Wood e7a2b322ec ftp: make TLS config default to global TLS config - Fixes #6671
This allows --ca-cert, --client-cert, --no-check-certificate etc to be
used.

This also allows `override.ca_cert = XXX` to be used in the config
file.
2025-08-22 12:19:29 +01:00
Nick Craig-Wood d3a0805a2b fshttp: return *Transport rather than http.RoundTripper from NewTransport
This allows further customization, reading the existing config and is
the Go recommended way "accept interfaces, return structs".
2025-08-22 12:19:29 +01:00
nielash d4edf8ac18 bisync: release from beta
As of v1.71, bisync is officially out of beta.

Some history:

- bisync was born in 2018 as https://github.com/cjnaz/rclonesync-V2
by @cjnaz, written in python.
- In 2021, @ivandeex ported it to go with @cjnaz's support.
https://github.com/rclone/rclone/pull/5164
- It was introduced as an "experimental" feature in v1.58.
https://github.com/rclone/rclone/commit/6210e22ab585eccdf27c12642d342056459e2e6e
- In 2023, bisync needed a new maintainer, and @nielash volunteered.
https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636
- Later in 2023, bisync received a major overhaul and was relabeled "beta"
(from "experimental"). https://github.com/rclone/rclone/pull/7410
- In 2024, integration tests were introduced for bisync (which previously had
only unit tests). https://github.com/rclone/rclone/pull/7693
- As of August 2025, bisync is stable and integration tests are passing on all
of the "flagship" backends.

Development doesn't stop here, of course. But bisync has come a long way since
its "experimental" days, and the "beta" tag is no longer needed.
2025-08-22 12:13:59 +01:00
nielash 87d14b000a bisync: fix markdown formatting issues flagged by linter in docs 2025-08-22 12:13:59 +01:00
nielash 12bded980b bisync: fix --no-slow-hash settings on path2
Before this change, if path2 had slow hashes, and --no-slow-hash or --slow-hash-sync-only
was in use, bisync was erroneously setting path1's hashtype to 'none' instead of
path2's. This change fixes the issue.

See https://forum.rclone.org/t/hashtype-mismatch-with-slow-hash-sync-only-in-onedrive-local-bisync/52138/2?u=nielash
2025-08-22 12:13:59 +01:00
Nick Craig-Wood 6e0e76af9d Add cui to contributors 2025-08-22 12:13:59 +01:00
Nick Craig-Wood 6f9b2f7b9b docs: add code of conduct 2025-08-22 11:42:51 +01:00
cui f61d79396d lib/mmap: convert to using unsafe.Slice to avoid deprecated reflect.SliceHeader 2025-08-22 00:35:50 +01:00
dependabot[bot] 9b22e38450 build: bump golangci/golangci-lint-action from 6 to 8
Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6 to 8.
- [Release notes](https://github.com/golangci/golangci-lint-action/releases)
- [Commits](https://github.com/golangci/golangci-lint-action/compare/v6...v8)

---
updated-dependencies:
- dependency-name: golangci/golangci-lint-action
  dependency-version: '8'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-08-22 00:14:01 +01:00
albertony 9e4fe18830 build: update golangci-lint configuration 2025-08-22 00:14:01 +01:00
albertony ae5cc1ab37 build: ignore revive lint issue var-naming: avoid meaningless package names 2025-08-22 00:14:01 +01:00
albertony d4be38ec02 build: fix lint issue: should omit type error from declaration 2025-08-22 00:14:01 +01:00
albertony 115cff3007 Revert "build: downgrade linter to use go1.24 until it is fixed for go1.25"
This reverts commit 8f84f91666.
2025-08-22 00:14:01 +01:00
albertony 70b862f026 build: migrate golangci-lint configuration to v2 format 2025-08-22 00:14:01 +01:00
61 changed files with 61999 additions and 33780 deletions

No files matched your search

+7 -6
View File
@@ -224,7 +224,7 @@ jobs:
id: setup-go
uses: actions/setup-go@v5
with:
go-version: '1.24'
go-version: '>=1.24.0-rc.1'
check-latest: true
cache: false
@@ -239,13 +239,13 @@ jobs:
restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
- name: Code quality test (Linux)
uses: golangci/golangci-lint-action@v6
uses: golangci/golangci-lint-action@v8
with:
version: latest
skip-cache: true
- name: Code quality test (Windows)
uses: golangci/golangci-lint-action@v6
uses: golangci/golangci-lint-action@v8
env:
GOOS: "windows"
with:
@@ -253,7 +253,7 @@ jobs:
skip-cache: true
- name: Code quality test (macOS)
uses: golangci/golangci-lint-action@v6
uses: golangci/golangci-lint-action@v8
env:
GOOS: "darwin"
with:
@@ -261,7 +261,7 @@ jobs:
skip-cache: true
- name: Code quality test (FreeBSD)
uses: golangci/golangci-lint-action@v6
uses: golangci/golangci-lint-action@v8
env:
GOOS: "freebsd"
with:
@@ -269,7 +269,7 @@ jobs:
skip-cache: true
- name: Code quality test (OpenBSD)
uses: golangci/golangci-lint-action@v6
uses: golangci/golangci-lint-action@v8
env:
GOOS: "openbsd"
with:
@@ -290,6 +290,7 @@ jobs:
MAINTAINERS.md
README.md
RELEASE.md
CODE_OF_CONDUCT.md
docs/content/{authors,bugs,changelog,docs,downloads,faq,filtering,gui,install,licence,overview,privacy}.md
- name: Scan edits of autogenerated files
+129 -127
View File
@@ -1,144 +1,146 @@
# golangci-lint configuration options
version: "2"
linters:
# Configure the linter set. To avoid unexpected results the implicit default
# set is ignored and all the ones to use are explicitly enabled.
default: none
enable:
# Default
- errcheck
- goimports
- revive
- ineffassign
- govet
- unconvert
- ineffassign
- staticcheck
- gosimple
- stylecheck
- unused
- misspell
# Additional
- gocritic
#- prealloc
#- maligned
disable-all: true
- misspell
#- prealloc # TODO
- revive
- unconvert
# Configure checks. Mostly using defaults but with some commented exceptions.
settings:
staticcheck:
# With staticcheck there is only one setting, so to extend the implicit
# default value it must be explicitly included.
checks:
# Default
- all
- -ST1000
- -ST1003
- -ST1016
- -ST1020
- -ST1021
- -ST1022
# Disable quickfix checks
- -QF*
gocritic:
# With gocritic there are different settings, but since enabled-checks
# and disabled-checks cannot both be set, for full customization the
# alternative is to disable all defaults and explicitly enable the ones
# to use.
disable-all: true
enabled-checks:
#- appendAssign # Skip default
- argOrder
- assignOp
- badCall
- badCond
#- captLocal # Skip default
- caseOrder
- codegenComment
#- commentFormatting # Skip default
- defaultCaseOrder
- deprecatedComment
- dupArg
- dupBranchBody
- dupCase
- dupSubExpr
- elseif
#- exitAfterDefer # Skip default
- flagDeref
- flagName
#- ifElseChain # Skip default
- mapKey
- newDeref
- offBy1
- regexpMust
- ruleguard # Enable additional check that are not enabled by default
#- singleCaseSwitch # Skip default
- sloppyLen
- sloppyTypeAssert
- switchTrue
- typeSwitchVar
- underef
- unlambda
- unslice
- valSwap
- wrapperFunc
settings:
ruleguard:
rules: ${base-path}/bin/rules.go
revive:
# With revive there is in reality only one setting, and when at least one
# rule are specified then only these rules will be considered, defaults
# and all others are then implicitly disabled, so must explicitly enable
# all rules to be used.
rules:
- name: blank-imports
disabled: false
- name: context-as-argument
disabled: false
- name: context-keys-type
disabled: false
- name: dot-imports
disabled: false
#- name: empty-block # Skip default
# disabled: true
- name: error-naming
disabled: false
- name: error-return
disabled: false
- name: error-strings
disabled: false
- name: errorf
disabled: false
- name: exported
disabled: false
#- name: increment-decrement # Skip default
# disabled: true
- name: indent-error-flow
disabled: false
- name: package-comments
disabled: false
- name: range
disabled: false
- name: receiver-naming
disabled: false
#- name: redefines-builtin-id # Skip default
# disabled: true
#- name: superfluous-else # Skip default
# disabled: true
- name: time-naming
disabled: false
- name: unexported-return
disabled: false
#- name: unreachable-code # Skip default
# disabled: true
#- name: unused-parameter # Skip default
# disabled: true
- name: var-declaration
disabled: false
- name: var-naming
disabled: false
formatters:
enable:
- goimports
issues:
# Enable some lints excluded by default
exclude-use-default: false
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
max-issues-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
max-same-issues: 0
exclude-rules:
- linters:
- staticcheck
text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
# don't disable the revive messages about comments on exported functions
include:
- EXC0012
- EXC0013
- EXC0014
- EXC0015
run:
# timeout for analysis, e.g. 30s, 5m, default is 1m
# Timeout for total work, e.g. 30s, 5m, 5m30s. Default is 0 (disabled).
timeout: 10m
linters-settings:
revive:
# setting rules seems to disable all the rules, so re-enable them here
rules:
- name: blank-imports
disabled: false
- name: context-as-argument
disabled: false
- name: context-keys-type
disabled: false
- name: dot-imports
disabled: false
- name: empty-block
disabled: true
- name: error-naming
disabled: false
- name: error-return
disabled: false
- name: error-strings
disabled: false
- name: errorf
disabled: false
- name: exported
disabled: false
- name: increment-decrement
disabled: true
- name: indent-error-flow
disabled: false
- name: package-comments
disabled: false
- name: range
disabled: false
- name: receiver-naming
disabled: false
- name: redefines-builtin-id
disabled: true
- name: superfluous-else
disabled: true
- name: time-naming
disabled: false
- name: unexported-return
disabled: false
- name: unreachable-code
disabled: true
- name: unused-parameter
disabled: true
- name: var-declaration
disabled: false
- name: var-naming
disabled: false
stylecheck:
# Only enable the checks performed by the staticcheck stand-alone tool,
# as documented here: https://staticcheck.io/docs/configuration/options/#checks
checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]
gocritic:
# Enable all default checks with some exceptions and some additions (commented).
# Cannot use both enabled-checks and disabled-checks, so must specify all to be used.
disable-all: true
enabled-checks:
#- appendAssign # Enabled by default
- argOrder
- assignOp
- badCall
- badCond
#- captLocal # Enabled by default
- caseOrder
- codegenComment
#- commentFormatting # Enabled by default
- defaultCaseOrder
- deprecatedComment
- dupArg
- dupBranchBody
- dupCase
- dupSubExpr
- elseif
#- exitAfterDefer # Enabled by default
- flagDeref
- flagName
#- ifElseChain # Enabled by default
- mapKey
- newDeref
- offBy1
- regexpMust
- ruleguard # Not enabled by default
#- singleCaseSwitch # Enabled by default
- sloppyLen
- sloppyTypeAssert
- switchTrue
- typeSwitchVar
- underef
- unlambda
- unslice
- valSwap
- wrapperFunc
settings:
ruleguard:
rules: "${configDir}/bin/rules.go"
+80
View File
@@ -0,0 +1,80 @@
# Rclone Code of Conduct
Like the technical community as a whole, the Rclone team and community
is made up of a mixture of professionals and volunteers from all over
the world, working on every aspect of the mission - including
mentorship, teaching, and connecting people.
Diversity is one of our huge strengths, but it can also lead to
communication issues and unhappiness. To that end, we have a few
ground rules that we ask people to adhere to. This code applies
equally to founders, mentors and those seeking help and guidance.
This isn't an exhaustive list of things that you can't do. Rather,
take it in the spirit in which it's intended - a guide to make it
easier to enrich all of us and the technical communities in which we
participate.
This code of conduct applies to all spaces managed by the Rclone
project or Rclone Services Ltd. This includes the issue tracker, the
forum, the GitHub site, the wiki, any other online services or
in-person events. In addition, violations of this code outside these
spaces may affect a person's ability to participate within them.
- **Be friendly and patient.**
- **Be welcoming.** We strive to be a community that welcomes and
supports people of all backgrounds and identities. This includes,
but is not limited to members of any race, ethnicity, culture,
national origin, colour, immigration status, social and economic
class, educational level, sex, sexual orientation, gender identity
and expression, age, size, family status, political belief,
religion, and mental and physical ability.
- **Be considerate.** Your work will be used by other people, and you
in turn will depend on the work of others. Any decision you take
will affect users and colleagues, and you should take those
consequences into account when making decisions. Remember that we're
a world-wide community, so you might not be communicating in someone
else's primary language.
- **Be respectful.** Not all of us will agree all the time, but
disagreement is no excuse for poor behavior and poor manners. We
might all experience some frustration now and then, but we cannot
allow that frustration to turn into a personal attack. It's
important to remember that a community where people feel
uncomfortable or threatened is not a productive one. Members of the
Rclone community should be respectful when dealing with other
members as well as with people outside the Rclone community.
- **Be careful in the words that you choose.** We are a community of
professionals, and we conduct ourselves professionally. Be kind to
others. Do not insult or put down other participants. Harassment and
other exclusionary behavior aren't acceptable. This includes, but is
not limited to:
- Violent threats or language directed against another person.
- Discriminatory jokes and language.
- Posting sexually explicit or violent material.
- Posting (or threatening to post) other people's personally
identifying information ("doxing").
- Personal insults, especially those using racist or sexist terms.
- Unwelcome sexual attention.
- Advocating for, or encouraging, any of the above behavior.
- Repeated harassment of others. In general, if someone asks you to
stop, then stop.
- **When we disagree, try to understand why.** Disagreements, both
social and technical, happen all the time and Rclone is no
exception. It is important that we resolve disagreements and
differing views constructively. Remember that we're different. The
strength of Rclone comes from its varied community, people from a
wide range of backgrounds. Different people have different
perspectives on issues. Being unable to understand why someone holds
a viewpoint doesn't mean that they're wrong. Don't forget that it is
human to err and blaming each other doesn't get us anywhere.
Instead, focus on helping to resolve issues and learning from
mistakes.
If you believe someone is violating the code of conduct, we ask that
you report it by emailing [info@rclone.com](mailto:info@rclone.com).
Original text courtesy of the [Speak Up! project](http://web.archive.org/web/20141109123859/http://speakup.io/coc.html).
## Questions?
If you have questions, please feel free to [contact us](mailto:info@rclone.com).
Generated
+33901 -12521
View File
File diff suppressed because it is too large. Load diff
Generated
+11137 -9288
View File
File diff suppressed because it is too large. Load diff
Generated
+3808 -2127
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -14,7 +14,7 @@ import (
)
// errFileNotFound represent file not found error
var errFileNotFound error = errors.New("file not found")
var errFileNotFound = errors.New("file not found")
// getFileCode retrieves the file code for a given file path
func (f *Fs) getFileCode(ctx context.Context, filePath string) (string, error) {
+10 -3
View File
@@ -283,6 +283,7 @@ type Fs struct {
user string
pass string
dialAddr string
tlsConf *tls.Config // default TLS client config
poolMu sync.Mutex
pool []*ftp.ServerConn
drain *time.Timer // used to drain the pool when we stop using the connections
@@ -408,9 +409,14 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
func (f *Fs) tlsConfig() *tls.Config {
var tlsConfig *tls.Config
if f.opt.TLS || f.opt.ExplicitTLS {
tlsConfig = &tls.Config{
ServerName: f.opt.Host,
InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
if f.tlsConf != nil {
tlsConfig = f.tlsConf.Clone()
} else {
tlsConfig = new(tls.Config)
}
tlsConfig.ServerName = f.opt.Host
if f.opt.SkipVerifyTLSCert {
tlsConfig.InsecureSkipVerify = true
}
if f.opt.TLSCacheSize > 0 {
tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(f.opt.TLSCacheSize)
@@ -671,6 +677,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
dialAddr: dialAddr,
tokens: pacer.NewTokenDispenser(opt.Concurrency),
pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
tlsConf: fshttp.NewTransport(ctx).TLSClientConfig,
}
f.features = (&fs.Features{
CanHaveEmptyDirectories: true,
+1 -1
View File
@@ -6641,7 +6641,7 @@ func (o *Object) uploadMultipart(ctx context.Context, src fs.ObjectInfo, in io.R
return wantETag, gotETag, versionID, ui, err
}
var s3cw *s3ChunkWriter = chunkWriter.(*s3ChunkWriter)
s3cw := chunkWriter.(*s3ChunkWriter)
gotETag = *stringClone(s3cw.eTag)
versionID = stringClone(s3cw.versionID)
+1 -1
View File
@@ -1,7 +1,7 @@
// Package common defines code common to the union and the policies
//
// These need to be defined in a separate package to avoid import loops
package common
package common //nolint:revive // Don't include revive when running golangci-lint because this triggers var-naming: avoid meaningless package names
import "github.com/rclone/rclone/fs"
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env bash
set -euo pipefail
# Create test TLS certificates for use with rclone.
OUT_DIR="${OUT_DIR:-./tls-test}"
CA_SUBJ="${CA_SUBJ:-/C=US/ST=Test/L=Test/O=Test Org/OU=Test Unit/CN=Test Root CA}"
SERVER_CN="${SERVER_CN:-localhost}"
CLIENT_CN="${CLIENT_CN:-Test Client}"
CLIENT_KEY_PASS="${CLIENT_KEY_PASS:-testpassword}"
CA_DAYS=${CA_DAYS:-3650}
SERVER_DAYS=${SERVER_DAYS:-825}
CLIENT_DAYS=${CLIENT_DAYS:-825}
mkdir -p "$OUT_DIR"
cd "$OUT_DIR"
# Create OpenSSL config
# CA extensions
cat > ca_openssl.cnf <<'EOF'
[ ca_ext ]
basicConstraints = critical, CA:true, pathlen:1
keyUsage = critical, keyCertSign, cRLSign
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always,issuer
EOF
# Server extensions (SAN includes localhost + loopback IP)
cat > server_openssl.cnf <<EOF
[ server_ext ]
basicConstraints = critical, CA:false
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
subjectAltName = @alt_names
[ alt_names ]
DNS.1 = ${SERVER_CN}
IP.1 = 127.0.0.1
EOF
# Client extensions (for mTLS client auth)
cat > client_openssl.cnf <<'EOF'
[ client_ext ]
basicConstraints = critical, CA:false
keyUsage = critical, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
EOF
echo "Create CA key, CSR, and self-signed CA cert"
if [ ! -f ca.key.pem ]; then
openssl genrsa -out ca.key.pem 4096
chmod 600 ca.key.pem
fi
openssl req -new -key ca.key.pem -subj "$CA_SUBJ" -out ca.csr.pem
openssl x509 -req -in ca.csr.pem -signkey ca.key.pem \
-sha256 -days "$CA_DAYS" \
-extfile ca_openssl.cnf -extensions ca_ext \
-out ca.cert.pem
echo "Create server key (NO PASSWORD) and cert signed by CA"
openssl genrsa -out server.key.pem 2048
chmod 600 server.key.pem
openssl req -new -key server.key.pem -subj "/CN=${SERVER_CN}" -out server.csr.pem
openssl x509 -req -in server.csr.pem \
-CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \
-out server.cert.pem -days "$SERVER_DAYS" -sha256 \
-extfile server_openssl.cnf -extensions server_ext
echo "Create client key (PASSWORD-PROTECTED), CSR, and cert"
openssl genrsa -aes256 -passout pass:"$CLIENT_KEY_PASS" -out client.key.pem 2048
chmod 600 client.key.pem
openssl req -new -key client.key.pem -passin pass:"$CLIENT_KEY_PASS" \
-subj "/CN=${CLIENT_CN}" -out client.csr.pem
openssl x509 -req -in client.csr.pem \
-CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \
-out client.cert.pem -days "$CLIENT_DAYS" -sha256 \
-extfile client_openssl.cnf -extensions client_ext
echo "Verify chain"
openssl verify -CAfile ca.cert.pem server.cert.pem client.cert.pem
echo "Done"
echo
echo "Summary"
echo "-------"
printf "%-22s %s\n" \
"CA key:" "ca.key.pem" \
"CA cert:" "ca.cert.pem" \
"Server key:" "server.key.pem (no password)" \
"Server CSR:" "server.csr.pem" \
"Server cert:" "server.cert.pem (SAN: ${SERVER_CN}, 127.0.0.1)" \
"Client key:" "client.key.pem (encrypted)" \
"Client CSR:" "client.csr.pem" \
"Client cert:" "client.cert.pem" \
"Client key password:" "$CLIENT_KEY_PASS"
echo
echo "Test rclone server"
echo
echo "rclone serve http -vv --addr :8080 --cert ${OUT_DIR}/server.cert.pem --key ${OUT_DIR}/server.key.pem --client-ca ${OUT_DIR}/ca.cert.pem ."
echo
echo "Test rclone client"
echo
echo "rclone lsf :http: --http-url 'https://localhost:8080' --ca-cert ${OUT_DIR}/ca.cert.pem --client-cert ${OUT_DIR}/client.cert.pem --client-key ${OUT_DIR}/client.key.pem --client-pass \$(rclone obscure $CLIENT_KEY_PASS)"
echo
+1 -3
View File
@@ -141,7 +141,7 @@ func init() {
flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "")
flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "")
flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")
flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "")
flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync.", "")
flags.BoolVarP(cmdFlags, &Opt.Recover, "recover", "", Opt.Recover, "Automatically recover from interruptions without requiring --resync.", "")
flags.StringVarP(cmdFlags, &Opt.CompareFlag, "compare", "", Opt.CompareFlag, "Comma-separated list of bisync-specific compare options ex. 'size,modtime,checksum' (default: 'size,modtime')", "")
flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
@@ -163,7 +163,6 @@ var commandDefinition = &cobra.Command{
Annotations: map[string]string{
"versionIntroduced": "v1.58",
"groups": "Filter,Copy,Important",
"status": "Beta",
},
RunE: func(command *cobra.Command, args []string) error {
// NOTE: avoid putting too much handling here, as it won't apply to the rc.
@@ -191,7 +190,6 @@ var commandDefinition = &cobra.Command{
}
}
fs.Logf(nil, "bisync is IN BETA. Don't use in production!")
cmd.Run(false, true, command, func() error {
err := Bisync(ctx, fs1, fs2, &opt)
if err == ErrBisyncAborted {
+1 -1
View File
@@ -177,7 +177,7 @@ func (b *bisyncRun) setHashType(ci *fs.ConfigInfo) {
}
if (b.opt.Compare.NoSlowHash || b.opt.Compare.SlowHashSyncOnly) && b.fs2.Features().SlowHash {
fs.Infoc(nil, Color(terminal.YellowFg, "Slow hash detected on Path2. Will ignore checksum due to slow-hash settings"))
b.opt.Compare.HashType1 = hash.None
b.opt.Compare.HashType2 = hash.None
} else {
b.opt.Compare.HashType2 = b.fs2.Hashes().GetOne()
if b.opt.Compare.HashType2 != hash.None {
+2 -3
View File
@@ -35,8 +35,7 @@ var rcHelp = makeHelp(`This takes the following parameters
- removeEmptyDirs - remove empty directories at the final cleanup step
- filtersFile - read filtering patterns from a file
- ignoreListingChecksum - Do not use checksums for listings
- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync.
Use at your own risk!
- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync.
- workdir - server directory for history files (default: |~/.cache/rclone/bisync|)
- backupdir1 - --backup-dir for Path1. Must be a non-overlapping path on the same remote.
- backupdir2 - --backup-dir for Path2. Must be a non-overlapping path on the same remote.
@@ -56,7 +55,7 @@ On each successive run it will:
Changes include |New|, |Newer|, |Older|, and |Deleted| files.
- Propagate changes on Path1 to Path2, and vice-versa.
Bisync is **in beta** and is considered an **advanced command**, so use with care.
Bisync is considered an **advanced command**, so use with care.
Make sure you have read and understood the entire [manual](https://rclone.org/bisync)
(especially the [Limitations](https://rclone.org/bisync/#limitations) section) before using,
or data loss can result. Questions can be asked in the [Rclone Forum](https://forum.rclone.org/).
+1
View File
@@ -1003,3 +1003,4 @@ put them back in again.` >}}
- huangnauh <huanglibo2010@gmail.com>
- Lucas Bremgartner <breml@users.noreply.github.com>
- Binbin Qian <qianbinbin@hotmail.com>
- cui <523516579@qq.com>
+365 -245
View File
File diff suppressed because it is too large. Load diff
+2
View File
@@ -390,6 +390,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+105 -6
View File
@@ -6,6 +6,111 @@ description: "Rclone Changelog"
# Changelog
## v1.71.0 - 2025-08-22
[See commits](https://github.com/rclone/rclone/compare/v1.70.0...v1.71.0)
- New S3 providers
- [Exaba](/s3/#exaba) (Nick Craig-Wood)
- We've been working with Exaba and they would love early adopters, self-hosters and testers
- [OVHcloud Object Storage](/s3/#ovhcloud) (Florent Vennetier)
- [Zata](/s3/#Zata) ($@M@RTH_)
- New Features
- Allow [global config to be overriden or set on backend creation](/docs/#globalconfig) (Nick Craig-Wood)
- bisync: Promoted from beta to stable (nielash)
- build
- Update to go1.25 and make go1.24 the minimum required version (Nick Craig-Wood)
- Update all dependencies (Nick Craig-Wood)
- Upgrade golangci-lint to v2 and fix code and config (albertony)
- Add markdown linting to workflow and fix all the docs (albertony)
- Extend check for edits of autogenerated files to all commits in a pull request (albertony)
- Set default shell to bash in build.yml (Ross Smith II)
- config
- Make parsing of `Duration` options consistent (albertony)
- Add error if `RCLONE_CONFIG_PASS` was supplied it but didn't decrypt the config (n4n5)
- copy, copyto: Implement logger flags to store result of sync (Marvin Rösch)
- doc fixes (albertony, Ali Zein Yousuf, Binbin Qian, curlwget, Ed Craig-Wood, jinjingroad, Lucas Bremgartner, n4n5, Nick Craig-Wood)
- hash: Add support for BLAKE3, XXH3, XXH128 (albertony)
- http
- Add `--client-pass` support for encrypted `--client-key` files (Nick Craig-Wood)
- Allow setting of `--http-proxy` from the command line (Nick Craig-Wood)
- lib/mmap: Convert to using unsafe.Slice to avoid deprecated reflect.SliceHeader (cui)
- lib/transform: Avoid empty charmap entry (albertony)
- log: Add log rotation options when using [`--log-file`](/docs/#log-file-string) (Nick Craig-Wood)
- move, moveto: Implement logger flags to store result of sync (Marvin Rösch)
- pool: Unify memory between multipart and asyncreader to use one pool (Nick Craig-Wood)
- purge: Exit with a fatal error if filters are set on `rclone purge` (Nick Craig-Wood)
- rc
- Add `config/unlock` to unlock the config file (n4n5)
- Fix `config/listremotes` to send an empty array instead of nil (n4n5)
- Populate transfer snapshot with "what" value (Timothy Jacobs)
- serve sftp: Add support for more hashes (crc32, sha256, blake3, xxh3, xxh128) (albertony)
- transform: Add `truncate_keep_extension` and `truncate_bytes` (nielash)
- Bug Fixes
- about: Fix potential overflow of about in various backends (Nick Craig-Wood)
- accounting
- Fix incorrect stats with `--transfers=1` (Yu Xin)
- Fix stats (`speed=0` and `eta=nil`) when starting jobs via rc (Nick Craig-Wood)
- bisync
- Fix concurrent runs via rc (nielash)
- Fix `--no-slow-hash` settings on path2 (nielash)
- config: fix problem reading pasted tokens over 4095 bytes (Nick Craig-Wood)
- oauthutil: Fix nil pointer crash when started with expired token (Nick Craig-Wood)
- operations: Fix too many connections open when using `--max-buffer-memory` (Nick Craig-Wood)
- pool: Fix deadlock with `--max-buffer-memory` and multipart transfers (Nick Craig-Wood)
- rc: Fix `operations/check` ignoring `oneWay` parameter (houance)
- Mount
- Fix identification of symlinks in directory listings (huangnauh)
- Local
- Configurable supported hashes (albertony)
- Fix `--copy-links` on Windows when listing Junction points (Anagh Kumar Baranwal)
- Azure Blob
- Add support for client assertion based authentication (Vikas Bhansali)
- Fix deadlock with `--max-connections` with `InvalidBlockOrBlob` errors (Nick Craig-Wood)
- Fix double accounting for multipart uploads (Nick Craig-Wood)
- Azurefiles
- Add support for client assertion based authentication (Vikas Bhansali)
- Fix hash getting erased when modtime is set (nielash)
- Box
- Fix about: cannot unmarshal number 1.0e+18 into Go struct field (Nick Craig-Wood)
- Combine
- Fix directory not found errors with ListP interface (Nick Craig-Wood)
- Filescom
- Document that hashes need to be enabled (Nick Craig-Wood)
- FTP
- Allow insecure TLS ciphers (Anagh Kumar Baranwal)
- Make TLS config default to global TLS config (Nick Craig-Wood)
- This allows `--ca-cert`, `--client-cert` etc to be used.
- This also allows `override.ca_cert = XXX` to be used in the config file.
- Googlephotos
- Added warning for Google Photos compatability-fixes (raider13209)
- Imagekit
- Return correct error when attempting to upload zero length files (Nick Craig-Wood)
- Don't low level retry uploads (Nick Craig-Wood)
- Remove server side Copy method as it was downloading and uploading (Nick Craig-Wood)
- Internetarchive
- Fix server side copy of files with `&` (Nick Craig-Wood)
- Mega
- Fix tls handshake failure (necaran)
- Oracleobjectstorage
- Add read only metadata support (praveen-solanki-oracle)
- Pikpak
- Rewrite upload to bypass AWS S3 manager (wiserain)
- Enhance Move for better handling of error and name collision (wiserain)
- Enhance Copy method to handle name collisions and improve error management (wiserain)
- Add file name integrity check during upload (wiserain)
- S3
- Add `--s3-use-arn-region` flag (Nick Craig-Wood)
- Fix memory leak by cloning strings (liubingrun)
- Fix multipart upload and server side copy when using bucket policy SSE-C (Albin Parou)
- Fix `Content-Type: aws-chunked` causing upload errors with `--metadata` (Nick Craig-Wood)
- SFTP
- Add support for more hashes (crc32, sha256, blake3, xxh3, xxh128) (albertony)
- Smb
- Add `--smb-kerberos-ccache` option to set kerberos ccache per smb backend (Sudipto Baral)
- Refresh Kerberos credentials when ccache file changes (Sudipto Baral)
- Improve multithreaded upload performance using multiple connections (Sudipto Baral)
## v1.70.3 - 2025-07-09
[See commits](https://github.com/rclone/rclone/compare/v1.70.2...v1.70.3)
@@ -39,12 +144,6 @@ description: "Rclone Changelog"
- doc fixes (Ali Zein Yousuf, Nick Craig-Wood)
- Local
- Fix --skip-links on Windows when skipping Junction points (Nick Craig-Wood)
- Combine
- Fix directory not found errors with ListP interface (Nick Craig-Wood)
- Mega
- Fix tls handshake failure (necaran)
- Pikpak
- Fix uploads fail with "aws-chunked encoding is not supported" error (Nick Craig-Wood)
## v1.70.1 - 2025-06-19
+23 -5
View File
@@ -165,6 +165,7 @@ rclone [flags]
--chunker-remote string Remote to chunk/unchunk
--client-cert string Client SSL certificate (PEM) for mutual TLS auth
--client-key string Client SSL private key (PEM) for mutual TLS auth
--client-pass string Password for client SSL private key (PEM) for mutual TLS auth (obscured) (obscured)
--cloudinary-adjust-media-files-extensions Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems (default true)
--cloudinary-api-key string Cloudinary API Key
--cloudinary-api-secret string Cloudinary API Secret
@@ -327,6 +328,7 @@ rclone [flags]
--fix-case Force rename of case insensitive dest to match source
--fs-cache-expire-duration Duration Cache remotes for this long (0 to disable caching) (default 5m0s)
--fs-cache-expire-interval Duration Interval to check for expired remotes (default 1m0s)
--ftp-allow-insecure-tls-ciphers Allow insecure TLS ciphers
--ftp-ask-password Allow asking for FTP password when needed
--ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s)
--ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited
@@ -339,6 +341,7 @@ rclone [flags]
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
--ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
--ftp-host string FTP host to connect to
--ftp-http-proxy string URL for HTTP CONNECT proxy
--ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s)
--ftp-no-check-certificate Do not verify the TLS certificate of the server
--ftp-no-check-upload Don't check the upload is OK
@@ -432,6 +435,7 @@ rclone [flags]
--http-no-escape Do not escape URL metacharacters in path names
--http-no-head Don't use HEAD requests
--http-no-slash Set this if the site doesn't end directories with /
--http-proxy string HTTP proxy URL
--http-url string URL of HTTP host to connect to
--human-readable Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi
--iclouddrive-apple-id string Apple ID
@@ -499,6 +503,7 @@ rclone [flags]
--local-case-sensitive Force the filesystem to report itself as case sensitive
--local-description string Description of the remote
--local-encoding Encoding The encoding for the backend (default Slash,Dot)
--local-hashes CommaSepList Comma separated list of supported checksum types
--local-links Translate symlinks to/from regular files with a '.rclonelink' extension for the local backend
--local-no-check-updated Don't check to see if the files change during upload
--local-no-clone Disable reflink cloning for server-side copies
@@ -510,6 +515,10 @@ rclone [flags]
--local-unicode-normalization Apply unicode NFC normalization to paths and filenames
--local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated)
--log-file string Log everything to this file
--log-file-compress If set, compress rotated log files using gzip
--log-file-max-age Duration Maximum duration to retain old log files (eg "7d") (default 0s)
--log-file-max-backups int Maximum number of old log files to retain
--log-file-max-size SizeSuffix Maximum size of the log file before it's rotated (eg "10M") (default off)
--log-format Bits Comma separated list of log format options (default date,time)
--log-level LogLevel Log level DEBUG|INFO|NOTICE|ERROR (default NOTICE)
--log-systemd Activate systemd integration for the logger
@@ -680,7 +689,8 @@ rclone [flags]
--pikpak-pass string Pikpak password (obscured)
--pikpak-root-folder-id string ID of the root folder
--pikpak-trashed-only Only show files that are in the trash
--pikpak-upload-concurrency int Concurrency for multipart uploads (default 5)
--pikpak-upload-concurrency int Concurrency for multipart uploads (default 4)
--pikpak-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi)
--pikpak-use-trash Send files to the trash instead of deleting permanently (default true)
--pikpak-user string Pikpak username
--pikpak-user-agent string HTTP user agent for pikpak (default "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0")
@@ -820,6 +830,7 @@ rclone [flags]
--s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint
--s3-use-accept-encoding-gzip Accept-Encoding: gzip Whether to send Accept-Encoding: gzip header (default unset)
--s3-use-already-exists Tristate Set if rclone should report BucketAlreadyExists errors on bucket creation (default unset)
--s3-use-arn-region If true, enables arn region support for the service
--s3-use-dual-stack If true use AWS S3 dual-stack endpoint (IPv6 support)
--s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset)
--s3-use-multipart-uploads Tristate Set if rclone should use multipart uploads (default unset)
@@ -833,7 +844,7 @@ rclone [flags]
--seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled)
--seafile-create-library Should rclone create a library if it doesn't exist
--seafile-description string Description of the remote
--seafile-encoding Encoding The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
--seafile-encoding Encoding The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8,Dot)
--seafile-library string Name of the library
--seafile-library-key string Library password (for encrypted libraries only) (obscured)
--seafile-pass string Password (obscured)
@@ -841,15 +852,18 @@ rclone [flags]
--seafile-user string User name (usually email address)
--server-side-across-configs Allow server-side operations (e.g. copy) to work across different configs
--sftp-ask-password Allow asking for SFTP password when needed
--sftp-blake3sum-command string The command used to read BLAKE3 hashes
--sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki)
--sftp-ciphers SpaceSepList Space separated list of ciphers to be used for session encryption, ordered by preference
--sftp-concurrency int The maximum number of outstanding requests for one file (default 64)
--sftp-connections int Maximum number of SFTP simultaneous connections, 0 for unlimited
--sftp-copy-is-hardlink Set to enable server side copies using hardlinks
--sftp-crc32sum-command string The command used to read CRC-32 hashes
--sftp-description string Description of the remote
--sftp-disable-concurrent-reads If set don't use concurrent reads
--sftp-disable-concurrent-writes If set don't use concurrent writes
--sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available
--sftp-hashes CommaSepList Comma separated list of supported checksum types
--sftp-host string SSH host to connect to
--sftp-host-key-algorithms SpaceSepList Space separated list of host key algorithms, ordered by preference
--sftp-http-proxy string URL for HTTP CONNECT proxy
@@ -861,7 +875,7 @@ rclone [flags]
--sftp-key-use-agent When set forces the usage of the ssh-agent
--sftp-known-hosts-file string Optional path to known_hosts file
--sftp-macs SpaceSepList Space separated list of MACs (message authentication code) algorithms, ordered by preference
--sftp-md5sum-command string The command used to read md5 hashes
--sftp-md5sum-command string The command used to read MD5 hashes
--sftp-pass string SSH password, leave blank to use ssh-agent (obscured)
--sftp-path-override string Override path used by SSH shell commands
--sftp-port int SSH port number (default 22)
@@ -870,7 +884,8 @@ rclone [flags]
--sftp-server-command string Specifies the path or command to run a sftp server on the remote host
--sftp-set-env SpaceSepList Environment variables to pass to sftp and commands
--sftp-set-modtime Set the modified time on the remote if set (default true)
--sftp-sha1sum-command string The command used to read sha1 hashes
--sftp-sha1sum-command string The command used to read SHA-1 hashes
--sftp-sha256sum-command string The command used to read SHA-256 hashes
--sftp-shell-type string The type of SSH shell on remote server, if any
--sftp-skip-links Set to skip any symlinks and any other non regular files
--sftp-socks-proxy string Socks 5 proxy host
@@ -879,6 +894,8 @@ rclone [flags]
--sftp-use-fstat If set use fstat instead of stat
--sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods
--sftp-user string SSH username (default "$USER")
--sftp-xxh128sum-command string The command used to read XXH128 hashes
--sftp-xxh3sum-command string The command used to read XXH3 hashes
--sharefile-auth-url string Auth server URL
--sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi)
--sharefile-client-credentials Use client credentials OAuth flow
@@ -905,6 +922,7 @@ rclone [flags]
--smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true)
--smb-host string SMB server hostname to connect to
--smb-idle-timeout Duration Max time before closing idle connections (default 1m0s)
--smb-kerberos-ccache string Path to the Kerberos credential cache (krb5cc)
--smb-pass string SMB password (obscured)
--smb-port int SMB port number (default 445)
--smb-spn string Service principal name
@@ -997,7 +1015,7 @@ rclone [flags]
--use-json-log Use json log format
--use-mmap Use mmap allocator (see docs)
--use-server-modtime Use server modified time instead of object metadata
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.0")
--user-agent string Set the user-agent to a specified string (default "rclone/v1.71.0")
-v, --verbose count Print lots more stuff (repeat for more)
-V, --version Print the version number
--webdav-auth-redirect Preserve authentication on redirect
+2 -3
View File
@@ -1,7 +1,6 @@
---
title: "rclone bisync"
description: "Perform bidirectional synchronization between two paths."
status: Beta
versionIntroduced: v1.58
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/bisync/ and as part of making a release run "make commanddocs"
---
@@ -21,7 +20,7 @@ On each successive run it will:
Changes include `New`, `Newer`, `Older`, and `Deleted` files.
- Propagate changes on Path1 to Path2, and vice-versa.
Bisync is **in beta** and is considered an **advanced command**, so use with care.
Bisync is considered an **advanced command**, so use with care.
Make sure you have read and understood the entire [manual](https://rclone.org/bisync)
(especially the [Limitations](https://rclone.org/bisync/#limitations) section) before using,
or data loss can result. Questions can be asked in the [Rclone Forum](https://forum.rclone.org/).
@@ -56,7 +55,7 @@ rclone bisync remote1:path1 remote2:path2 [flags]
--no-slow-hash Ignore listing checksums only on backends where they are slow
--recover Automatically recover from interruptions without requiring --resync.
--remove-empty-dirs Remove ALL empty directories at the final cleanup step.
--resilient Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!
--resilient Allow future runs to retry after certain less-serious errors, instead of requiring --resync.
-1, --resync Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.
--resync-mode string During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.) (default "none")
--slow-hash-sync-only Ignore slow checksums for listings and deltas, but still consider them during sync calls.
+1 -1
View File
@@ -49,7 +49,7 @@ you what happened to it. These are reminiscent of diff files.
- `* path` means path was present in source and destination but different.
- `! path` means there was an error reading or hashing the source or dest.
The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
The default number of parallel checks is 8. See the [--checkers](/docs/#checkers-int)
option for more information.
+1 -1
View File
@@ -44,7 +44,7 @@ you what happened to it. These are reminiscent of diff files.
- `* path` means path was present in source and destination but different.
- `! path` means there was an error reading or hashing the source or dest.
The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
The default number of parallel checks is 8. See the [--checkers](/docs/#checkers-int)
option for more information.
+13 -15
View File
@@ -221,12 +221,12 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,command=e
```
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{YYYYMMDD}"
// Output: stories/The Quick Brown Fox!-20250617
// Output: stories/The Quick Brown Fox!-20250618
```
```
rclone convmv "stories/The Quick Brown Fox!" --name-transform "date=-{macfriendlytime}"
// Output: stories/The Quick Brown Fox!-2025-06-17 0551PM
// Output: stories/The Quick Brown Fox!-2025-06-18 0148PM
```
```
@@ -234,17 +234,15 @@ rclone convmv "stories/The Quick Brown Fox!.txt" --name-transform "all,regex=[\\
// Output: ababababababab/ababab ababababab ababababab ababab!abababab
```
Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
The `--name-transform` flag is also available in `sync`, `copy`, and `move`.
# Files vs Directories ##
# Files vs Directories
By default `--name-transform` will only apply to file names. The means only the leaf file name will be transformed.
However some of the transforms would be better applied to the whole path or just directories.
To choose which which part of the file path is affected some tags can be added to the `--name-transform`
To choose which which part of the file path is affected some tags can be added to the `--name-transform`.
| Tag | Effect |
|------|------|
@@ -254,11 +252,11 @@ To choose which which part of the file path is affected some tags can be added t
This is used by adding the tag into the transform name like this: `--name-transform file,prefix=ABC` or `--name-transform dir,prefix=DEF`.
For some conversions using all is more likely to be useful, for example `--name-transform all,nfc`
For some conversions using all is more likely to be useful, for example `--name-transform all,nfc`.
Note that `--name-transform` may not add path separators `/` to the name. This will cause an error.
# Ordering and Conflicts ##
# Ordering and Conflicts
* Transformations will be applied in the order specified by the user.
* If the `file` tag is in use (the default) then only the leaf name of files will be transformed.
@@ -273,19 +271,19 @@ user, allowing for intentional use cases (e.g., trimming one prefix before addin
* Users should be aware that certain combinations may lead to unexpected results and should verify
transformations using `--dry-run` before execution.
# Race Conditions and Non-Deterministic Behavior ##
# Race Conditions and Non-Deterministic Behavior
Some transformations, such as `replace=old:new`, may introduce conflicts where multiple source files map to the same destination name.
This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these.
* If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic.
* Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results.
* To minimize risks, users should:
* Carefully review transformations that may introduce conflicts.
* Use `--dry-run` to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
* Consider disabling concurrency with `--transfers=1` if necessary.
* Certain transformations (e.g. `prefix`) will have a multiplying effect every time they are used. Avoid these when using `bisync`.
To minimize risks, users should:
* Carefully review transformations that may introduce conflicts.
* Use `--dry-run` to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
* Consider disabling concurrency with `--transfers=1` if necessary.
* Certain transformations (e.g. `prefix`) will have a multiplying effect every time they are used. Avoid these when using `bisync`.
+57
View File
@@ -74,6 +74,47 @@ for more info.
**Note**: Use the `--dry-run` or the `--interactive`/`-i` flag to test without copying anything.
# Logger Flags
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths,
one per line, to the file name (or stdout if it is `-`) supplied. What they write is described
in the help below. For example `--differ` will write all paths which are present
on both the source and destination but different.
The `--combined` flag will write a file (or stdout) which contains all
file paths with a symbol and then a space and then the path to tell
you what happened to it. These are reminiscent of diff files.
- `= path` means path was found in source and destination and was identical
- `- path` means path was missing on the source, so only in the destination
- `+ path` means path was missing on the destination, so only in the source
- `* path` means path was present in source and destination but different.
- `! path` means there was an error reading or hashing the source or dest.
The `--dest-after` flag writes a list file using the same format flags
as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
Conceptually it is similar to rsync's `--itemize-changes`, but not identical
-- it should output an accurate list of what will be on the destination
after the command is finished.
When the `--no-traverse` flag is set, all logs involving files that exist only
on the destination will be incomplete or completely missing.
Note that these logger flags have a few limitations, and certain scenarios
are not currently supported:
- `--max-duration` / `CutoffModeHard`
- `--compare-dest` / `--copy-dest`
- server-side moves of an entire dir at once
- High-level retries, because there would be duplicates (use `--retries 1` to disable)
- Possibly some unusual error scenarios
Note also that each file is logged during execution, as opposed to after, so it
is most useful as a predictor of what SHOULD happen to each file
(which may or may not match what actually DID.)
```
rclone copy source:path dest:path [flags]
@@ -82,8 +123,24 @@ rclone copy source:path dest:path [flags]
## Options
```
--absolute Put a leading / in front of path names
--combined string Make a combined report of changes to this file
--create-empty-src-dirs Create empty source dirs on destination after copy
--csv Output in CSV format
--dest-after string Report all files that exist on the dest post-sync
--differ string Report all non-matching files to this file
-d, --dir-slash Append a slash to directory names (default true)
--dirs-only Only list directories
--error string Report all files with errors (hashing or reading) to this file
--files-only Only list files (default true)
-F, --format string Output format - see lsf help for details (default "p")
--hash h Use this hash when h is used in the format MD5|SHA-1|DropboxHash (default "md5")
-h, --help help for copy
--match string Report all matching files to this file
--missing-on-dst string Report all files missing from the destination to this file
--missing-on-src string Report all files missing from the source to this file
-s, --separator string Separator for the items in the format (default ";")
-t, --timeformat string Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)
```
Options shared with other commands are described next.
+58 -1
View File
@@ -40,6 +40,47 @@ the destination.
**Note**: Use the `-P`/`--progress` flag to view real-time transfer statistics
# Logger Flags
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths,
one per line, to the file name (or stdout if it is `-`) supplied. What they write is described
in the help below. For example `--differ` will write all paths which are present
on both the source and destination but different.
The `--combined` flag will write a file (or stdout) which contains all
file paths with a symbol and then a space and then the path to tell
you what happened to it. These are reminiscent of diff files.
- `= path` means path was found in source and destination and was identical
- `- path` means path was missing on the source, so only in the destination
- `+ path` means path was missing on the destination, so only in the source
- `* path` means path was present in source and destination but different.
- `! path` means there was an error reading or hashing the source or dest.
The `--dest-after` flag writes a list file using the same format flags
as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
Conceptually it is similar to rsync's `--itemize-changes`, but not identical
-- it should output an accurate list of what will be on the destination
after the command is finished.
When the `--no-traverse` flag is set, all logs involving files that exist only
on the destination will be incomplete or completely missing.
Note that these logger flags have a few limitations, and certain scenarios
are not currently supported:
- `--max-duration` / `CutoffModeHard`
- `--compare-dest` / `--copy-dest`
- server-side moves of an entire dir at once
- High-level retries, because there would be duplicates (use `--retries 1` to disable)
- Possibly some unusual error scenarios
Note also that each file is logged during execution, as opposed to after, so it
is most useful as a predictor of what SHOULD happen to each file
(which may or may not match what actually DID.)
```
rclone copyto source:path dest:path [flags]
@@ -48,7 +89,23 @@ rclone copyto source:path dest:path [flags]
## Options
```
-h, --help help for copyto
--absolute Put a leading / in front of path names
--combined string Make a combined report of changes to this file
--csv Output in CSV format
--dest-after string Report all files that exist on the dest post-sync
--differ string Report all non-matching files to this file
-d, --dir-slash Append a slash to directory names (default true)
--dirs-only Only list directories
--error string Report all files with errors (hashing or reading) to this file
--files-only Only list files (default true)
-F, --format string Output format - see lsf help for details (default "p")
--hash h Use this hash when h is used in the format MD5|SHA-1|DropboxHash (default "md5")
-h, --help help for copyto
--match string Report all matching files to this file
--missing-on-dst string Report all files missing from the destination to this file
--missing-on-src string Report all files missing from the source to this file
-s, --separator string Separator for the items in the format (default ";")
-t, --timeformat string Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)
```
Options shared with other commands are described next.
+1 -1
View File
@@ -54,7 +54,7 @@ you what happened to it. These are reminiscent of diff files.
- `* path` means path was present in source and destination but different.
- `! path` means there was an error reading or hashing the source or dest.
The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
The default number of parallel checks is 8. See the [--checkers](/docs/#checkers-int)
option for more information.
+3
View File
@@ -37,6 +37,9 @@ Run without a hash to see the list of all supported hashes, e.g.
* crc32
* sha256
* sha512
* blake3
* xxh3
* xxh128
Then
+5 -5
View File
@@ -252,7 +252,7 @@ to run as the SYSTEM account. A third alternative is to use the
Read more in the [install documentation](https://rclone.org/install/).
Note that when running rclone as another user, it will not use
the configuration file from your profile unless you tell it to
with the [`--config`](https://rclone.org/docs/#config-config-file) option.
with the [`--config`](https://rclone.org/docs/#config-string) option.
Note also that it is now the SYSTEM account that will have the owner
permissions, and other accounts will have permissions according to the
group or others scopes. As mentioned above, these will then not get the
@@ -328,10 +328,10 @@ See the [VFS File Caching](#vfs-file-caching) section for more info.
When using NFS mount on macOS, if you don't specify |--vfs-cache-mode|
the mount point will be read-only.
The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2)
do not support the concept of empty directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.
Bucket-based remotes - Azure Blob, Swift, S3, Google Cloud Storage and B2 -
can't store empty directories. Of these, only Azure Blob, Google Cloud Storage
and S3 can preserve them when you add `--xxx-directory_markers`; otherwise,
empty directories will vanish once they drop out of the directory cache.
When `rclone mount` is invoked on Unix with `--daemon` flag, the main rclone
program will wait for the background mount to become ready or until the timeout
+57
View File
@@ -48,6 +48,47 @@ for more info.
**Note**: Use the `-P`/`--progress` flag to view real-time transfer statistics.
# Logger Flags
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths,
one per line, to the file name (or stdout if it is `-`) supplied. What they write is described
in the help below. For example `--differ` will write all paths which are present
on both the source and destination but different.
The `--combined` flag will write a file (or stdout) which contains all
file paths with a symbol and then a space and then the path to tell
you what happened to it. These are reminiscent of diff files.
- `= path` means path was found in source and destination and was identical
- `- path` means path was missing on the source, so only in the destination
- `+ path` means path was missing on the destination, so only in the source
- `* path` means path was present in source and destination but different.
- `! path` means there was an error reading or hashing the source or dest.
The `--dest-after` flag writes a list file using the same format flags
as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
Conceptually it is similar to rsync's `--itemize-changes`, but not identical
-- it should output an accurate list of what will be on the destination
after the command is finished.
When the `--no-traverse` flag is set, all logs involving files that exist only
on the destination will be incomplete or completely missing.
Note that these logger flags have a few limitations, and certain scenarios
are not currently supported:
- `--max-duration` / `CutoffModeHard`
- `--compare-dest` / `--copy-dest`
- server-side moves of an entire dir at once
- High-level retries, because there would be duplicates (use `--retries 1` to disable)
- Possibly some unusual error scenarios
Note also that each file is logged during execution, as opposed to after, so it
is most useful as a predictor of what SHOULD happen to each file
(which may or may not match what actually DID.)
```
rclone move source:path dest:path [flags]
@@ -56,9 +97,25 @@ rclone move source:path dest:path [flags]
## Options
```
--absolute Put a leading / in front of path names
--combined string Make a combined report of changes to this file
--create-empty-src-dirs Create empty source dirs on destination after move
--csv Output in CSV format
--delete-empty-src-dirs Delete empty source dirs after move
--dest-after string Report all files that exist on the dest post-sync
--differ string Report all non-matching files to this file
-d, --dir-slash Append a slash to directory names (default true)
--dirs-only Only list directories
--error string Report all files with errors (hashing or reading) to this file
--files-only Only list files (default true)
-F, --format string Output format - see lsf help for details (default "p")
--hash h Use this hash when h is used in the format MD5|SHA-1|DropboxHash (default "md5")
-h, --help help for move
--match string Report all matching files to this file
--missing-on-dst string Report all files missing from the destination to this file
--missing-on-src string Report all files missing from the source to this file
-s, --separator string Separator for the items in the format (default ";")
-t, --timeformat string Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)
```
Options shared with other commands are described next.
+58 -1
View File
@@ -41,6 +41,47 @@ successful transfer.
**Note**: Use the `-P`/`--progress` flag to view real-time transfer statistics.
# Logger Flags
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths,
one per line, to the file name (or stdout if it is `-`) supplied. What they write is described
in the help below. For example `--differ` will write all paths which are present
on both the source and destination but different.
The `--combined` flag will write a file (or stdout) which contains all
file paths with a symbol and then a space and then the path to tell
you what happened to it. These are reminiscent of diff files.
- `= path` means path was found in source and destination and was identical
- `- path` means path was missing on the source, so only in the destination
- `+ path` means path was missing on the destination, so only in the source
- `* path` means path was present in source and destination but different.
- `! path` means there was an error reading or hashing the source or dest.
The `--dest-after` flag writes a list file using the same format flags
as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
Conceptually it is similar to rsync's `--itemize-changes`, but not identical
-- it should output an accurate list of what will be on the destination
after the command is finished.
When the `--no-traverse` flag is set, all logs involving files that exist only
on the destination will be incomplete or completely missing.
Note that these logger flags have a few limitations, and certain scenarios
are not currently supported:
- `--max-duration` / `CutoffModeHard`
- `--compare-dest` / `--copy-dest`
- server-side moves of an entire dir at once
- High-level retries, because there would be duplicates (use `--retries 1` to disable)
- Possibly some unusual error scenarios
Note also that each file is logged during execution, as opposed to after, so it
is most useful as a predictor of what SHOULD happen to each file
(which may or may not match what actually DID.)
```
rclone moveto source:path dest:path [flags]
@@ -49,7 +90,23 @@ rclone moveto source:path dest:path [flags]
## Options
```
-h, --help help for moveto
--absolute Put a leading / in front of path names
--combined string Make a combined report of changes to this file
--csv Output in CSV format
--dest-after string Report all files that exist on the dest post-sync
--differ string Report all non-matching files to this file
-d, --dir-slash Append a slash to directory names (default true)
--dirs-only Only list directories
--error string Report all files with errors (hashing or reading) to this file
--files-only Only list files (default true)
-F, --format string Output format - see lsf help for details (default "p")
--hash h Use this hash when h is used in the format MD5|SHA-1|DropboxHash (default "md5")
-h, --help help for moveto
--match string Report all matching files to this file
--missing-on-dst string Report all files missing from the destination to this file
--missing-on-src string Report all files missing from the source to this file
-s, --separator string Separator for the items in the format (default ";")
-t, --timeformat string Specify a custom time format, or 'max' for max precision supported by remote (default: 2006-01-02 15:04:05)
```
Options shared with other commands are described next.
+5 -5
View File
@@ -253,7 +253,7 @@ to run as the SYSTEM account. A third alternative is to use the
Read more in the [install documentation](https://rclone.org/install/).
Note that when running rclone as another user, it will not use
the configuration file from your profile unless you tell it to
with the [`--config`](https://rclone.org/docs/#config-config-file) option.
with the [`--config`](https://rclone.org/docs/#config-string) option.
Note also that it is now the SYSTEM account that will have the owner
permissions, and other accounts will have permissions according to the
group or others scopes. As mentioned above, these will then not get the
@@ -329,10 +329,10 @@ See the [VFS File Caching](#vfs-file-caching) section for more info.
When using NFS mount on macOS, if you don't specify |--vfs-cache-mode|
the mount point will be read-only.
The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2)
do not support the concept of empty directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.
Bucket-based remotes - Azure Blob, Swift, S3, Google Cloud Storage and B2 -
can't store empty directories. Of these, only Azure Blob, Google Cloud Storage
and S3 can preserve them when you add `--xxx-directory_markers`; otherwise,
empty directories will vanish once they drop out of the directory cache.
When `rclone mount` is invoked on Unix with `--daemon` flag, the main rclone
program will wait for the background mount to become ready or until the timeout
+16 -10
View File
@@ -18,13 +18,15 @@ want to delete files from destination, use the
[copy](/commands/rclone_copy/) command instead.
**Important**: Since this can cause data loss, test first with the
`--dry-run` or the `--interactive`/`-i` flag.
`--dry-run` or the `--interactive`/`i` flag.
rclone sync --interactive SOURCE remote:DESTINATION
Note that files in the destination won't be deleted if there were any
errors at any point. Duplicate objects (files with the same name, on
those providers that support it) are also not yet handled.
Files in the destination won't be deleted if there were any errors at any
point. Duplicate objects (files with the same name, on those providers that
support it) are not yet handled. Files that are excluded won't be deleted
unless `--delete-excluded` is used. Symlinks won't be transferred or
deleted from local file systems unless `--links` is used.
It is always the contents of the directory that is synced, not the
directory itself. So when source:path is a directory, it's the contents of
@@ -54,10 +56,10 @@ See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14
# Logger Flags
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths, one per line, to the file name (or
stdout if it is `-`) supplied. What they write is described in the
help below. For example `--differ` will write all paths which are
present on both the source and destination but different.
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths,
one per line, to the file name (or stdout if it is `-`) supplied. What they write is described
in the help below. For example `--differ` will write all paths which are present
on both the source and destination but different.
The `--combined` flag will write a file (or stdout) which contains all
file paths with a symbol and then a space and then the path to tell
@@ -74,7 +76,10 @@ as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
Conceptually it is similar to rsync's `--itemize-changes`, but not identical
-- it should output an accurate list of what will be on the destination
after the sync.
after the command is finished.
When the `--no-traverse` flag is set, all logs involving files that exist only
on the destination will be incomplete or completely missing.
Note that these logger flags have a few limitations, and certain scenarios
are not currently supported:
@@ -85,11 +90,12 @@ are not currently supported:
- High-level retries, because there would be duplicates (use `--retries 1` to disable)
- Possibly some unusual error scenarios
Note also that each file is logged during the sync, as opposed to after, so it
Note also that each file is logged during execution, as opposed to after, so it
is most useful as a predictor of what SHOULD happen to each file
(which may or may not match what actually DID.)
```
rclone sync source:path dest:path [flags]
```
+15 -1
View File
@@ -437,7 +437,7 @@ Do not use single character names on Windows as it creates ambiguity with Window
drives' names, e.g.: remote called `C` is indistinguishable from `C` drive. Rclone
will always assume that single letter name refers to a drive.
## Adding global configuration to a remote
## Adding global configuration to a remote {#globalconfig}
It is possible to add global configuration to the remote configuration which
will be applied just before the remote is created.
@@ -3000,6 +3000,20 @@ The `--client-key` flag is required too when using this.
This loads the PEM encoded client side private key used for mutual TLS
authentication. Used in conjunction with `--client-cert`.
Supported types are:
- Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY")
- Unencrypted PKCS#8 ("BEGIN PRIVATE KEY")
- Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY")
- Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected.
### --client-pass string
This can be used to supply an optional password to decrypt the client key file.
**NB** the password should be obscured so it should be the output of
`rclone obscure YOURPASSWORD`.
### --no-check-certificate
`--no-check-certificate` controls whether a client verifies the
+2
View File
@@ -699,6 +699,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+2
View File
@@ -305,6 +305,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+23 -5
View File
@@ -104,6 +104,7 @@ Flags for general networking and HTTP stuff.
--ca-cert stringArray CA certificate used to verify servers
--client-cert string Client SSL certificate (PEM) for mutual TLS auth
--client-key string Client SSL private key (PEM) for mutual TLS auth
--client-pass string Password for client SSL private key (PEM) for mutual TLS auth (obscured) (obscured)
--contimeout Duration Connect timeout (default 1m0s)
--disable-http-keep-alives Disable HTTP keep-alives and use each connection once
--disable-http2 Disable HTTP/2 in the global transport
@@ -112,6 +113,7 @@ Flags for general networking and HTTP stuff.
--header stringArray Set HTTP header for all transactions
--header-download stringArray Set HTTP header for download transactions
--header-upload stringArray Set HTTP header for upload transactions
--http-proxy string HTTP proxy URL
--max-connections int Maximum number of simultaneous backend API connections, 0 for unlimited
--no-check-certificate Do not verify the server SSL certificate (insecure)
--no-gzip-encoding Don't set Accept-Encoding: gzip
@@ -119,7 +121,7 @@ Flags for general networking and HTTP stuff.
--tpslimit float Limit HTTP transactions per second to this
--tpslimit-burst int Max burst of transactions for --tpslimit (default 1)
--use-cookies Enable session cookiejar
--user-agent string Set the user-agent to a specified string (default "rclone/v1.70.0")
--user-agent string Set the user-agent to a specified string (default "rclone/v1.71.0")
```
@@ -226,6 +228,10 @@ Flags for logging and statistics.
```
--log-file string Log everything to this file
--log-file-compress If set, compress rotated log files using gzip
--log-file-max-age Duration Maximum duration to retain old log files (eg "7d") (default 0s)
--log-file-max-backups int Maximum number of old log files to retain
--log-file-max-size SizeSuffix Maximum size of the log file before it's rotated (eg "10M") (default off)
--log-format Bits Comma separated list of log format options (default date,time)
--log-level LogLevel Log level DEBUG|INFO|NOTICE|ERROR (default NOTICE)
--log-systemd Activate systemd integration for the logger
@@ -589,6 +595,7 @@ Backend-only flags (these can be set in the config file also).
--filescom-password string The password used to authenticate with Files.com (obscured)
--filescom-site string Your site subdomain (e.g. mysite) or custom domain (e.g. myfiles.customdomain.com)
--filescom-username string The username used to authenticate with Files.com
--ftp-allow-insecure-tls-ciphers Allow insecure TLS ciphers
--ftp-ask-password Allow asking for FTP password when needed
--ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s)
--ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited
@@ -601,6 +608,7 @@ Backend-only flags (these can be set in the config file also).
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
--ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
--ftp-host string FTP host to connect to
--ftp-http-proxy string URL for HTTP CONNECT proxy
--ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s)
--ftp-no-check-certificate Do not verify the TLS certificate of the server
--ftp-no-check-upload Don't check the upload is OK
@@ -740,6 +748,7 @@ Backend-only flags (these can be set in the config file also).
--local-case-sensitive Force the filesystem to report itself as case sensitive
--local-description string Description of the remote
--local-encoding Encoding The encoding for the backend (default Slash,Dot)
--local-hashes CommaSepList Comma separated list of supported checksum types
--local-links Translate symlinks to/from regular files with a '.rclonelink' extension for the local backend
--local-no-check-updated Don't check to see if the files change during upload
--local-no-clone Disable reflink cloning for server-side copies
@@ -859,7 +868,8 @@ Backend-only flags (these can be set in the config file also).
--pikpak-pass string Pikpak password (obscured)
--pikpak-root-folder-id string ID of the root folder
--pikpak-trashed-only Only show files that are in the trash
--pikpak-upload-concurrency int Concurrency for multipart uploads (default 5)
--pikpak-upload-concurrency int Concurrency for multipart uploads (default 4)
--pikpak-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi)
--pikpak-use-trash Send files to the trash instead of deleting permanently (default true)
--pikpak-user string Pikpak username
--pikpak-user-agent string HTTP user agent for pikpak (default "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0")
@@ -963,6 +973,7 @@ Backend-only flags (these can be set in the config file also).
--s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint
--s3-use-accept-encoding-gzip Accept-Encoding: gzip Whether to send Accept-Encoding: gzip header (default unset)
--s3-use-already-exists Tristate Set if rclone should report BucketAlreadyExists errors on bucket creation (default unset)
--s3-use-arn-region If true, enables arn region support for the service
--s3-use-dual-stack If true use AWS S3 dual-stack endpoint (IPv6 support)
--s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset)
--s3-use-multipart-uploads Tristate Set if rclone should use multipart uploads (default unset)
@@ -976,22 +987,25 @@ Backend-only flags (these can be set in the config file also).
--seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled)
--seafile-create-library Should rclone create a library if it doesn't exist
--seafile-description string Description of the remote
--seafile-encoding Encoding The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
--seafile-encoding Encoding The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8,Dot)
--seafile-library string Name of the library
--seafile-library-key string Library password (for encrypted libraries only) (obscured)
--seafile-pass string Password (obscured)
--seafile-url string URL of seafile host to connect to
--seafile-user string User name (usually email address)
--sftp-ask-password Allow asking for SFTP password when needed
--sftp-blake3sum-command string The command used to read BLAKE3 hashes
--sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki)
--sftp-ciphers SpaceSepList Space separated list of ciphers to be used for session encryption, ordered by preference
--sftp-concurrency int The maximum number of outstanding requests for one file (default 64)
--sftp-connections int Maximum number of SFTP simultaneous connections, 0 for unlimited
--sftp-copy-is-hardlink Set to enable server side copies using hardlinks
--sftp-crc32sum-command string The command used to read CRC-32 hashes
--sftp-description string Description of the remote
--sftp-disable-concurrent-reads If set don't use concurrent reads
--sftp-disable-concurrent-writes If set don't use concurrent writes
--sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available
--sftp-hashes CommaSepList Comma separated list of supported checksum types
--sftp-host string SSH host to connect to
--sftp-host-key-algorithms SpaceSepList Space separated list of host key algorithms, ordered by preference
--sftp-http-proxy string URL for HTTP CONNECT proxy
@@ -1003,7 +1017,7 @@ Backend-only flags (these can be set in the config file also).
--sftp-key-use-agent When set forces the usage of the ssh-agent
--sftp-known-hosts-file string Optional path to known_hosts file
--sftp-macs SpaceSepList Space separated list of MACs (message authentication code) algorithms, ordered by preference
--sftp-md5sum-command string The command used to read md5 hashes
--sftp-md5sum-command string The command used to read MD5 hashes
--sftp-pass string SSH password, leave blank to use ssh-agent (obscured)
--sftp-path-override string Override path used by SSH shell commands
--sftp-port int SSH port number (default 22)
@@ -1012,7 +1026,8 @@ Backend-only flags (these can be set in the config file also).
--sftp-server-command string Specifies the path or command to run a sftp server on the remote host
--sftp-set-env SpaceSepList Environment variables to pass to sftp and commands
--sftp-set-modtime Set the modified time on the remote if set (default true)
--sftp-sha1sum-command string The command used to read sha1 hashes
--sftp-sha1sum-command string The command used to read SHA-1 hashes
--sftp-sha256sum-command string The command used to read SHA-256 hashes
--sftp-shell-type string The type of SSH shell on remote server, if any
--sftp-skip-links Set to skip any symlinks and any other non regular files
--sftp-socks-proxy string Socks 5 proxy host
@@ -1021,6 +1036,8 @@ Backend-only flags (these can be set in the config file also).
--sftp-use-fstat If set use fstat instead of stat
--sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods
--sftp-user string SSH username (default "$USER")
--sftp-xxh128sum-command string The command used to read XXH128 hashes
--sftp-xxh3sum-command string The command used to read XXH3 hashes
--sharefile-auth-url string Auth server URL
--sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi)
--sharefile-client-credentials Use client credentials OAuth flow
@@ -1046,6 +1063,7 @@ Backend-only flags (these can be set in the config file also).
--smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true)
--smb-host string SMB server hostname to connect to
--smb-idle-timeout Duration Max time before closing idle connections (default 1m0s)
--smb-kerberos-ccache string Path to the Kerberos credential cache (krb5cc)
--smb-pass string SMB password (obscured)
--smb-port int SMB port number (default 445)
--smb-spn string Service principal name
+59
View File
@@ -134,6 +134,35 @@ be enabled in the FTP backend config for the remote, or with
[`--ftp-tls`](#ftp-tls). The default FTPS port is `990`, not `21` and
can be set with [`--ftp-port`](#ftp-port).
## TLS Options
TLS options for Implicit and Explicit TLS can be set using the
following flags which are specific to the FTP backend:
```
--ftp-no-check-certificate Do not verify the TLS certificate of the server
--ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
--ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32)
```
However any of the global TLS flags can also be used such as:
```
--ca-cert stringArray CA certificate used to verify servers
--client-cert string Client SSL certificate (PEM) for mutual TLS auth
--client-key string Client SSL private key (PEM) for mutual TLS auth
--no-check-certificate Do not verify the server SSL certificate (insecure)
```
If these need to be put in the config file so they apply to just the
FTP backend then use the `override` syntax, eg
```
override.ca_cert = XXX
override.client_cert = XXX
override.client_key = XXX
```
### Restricted filename characters
In addition to the [default restricted characters set](/overview/#restricted-characters)
@@ -390,6 +419,22 @@ Properties:
- Type: bool
- Default: false
#### --ftp-allow-insecure-tls-ciphers
Allow insecure TLS ciphers
Setting this flag will allow the usage of the following TLS ciphers in addition to the secure defaults:
- TLS_RSA_WITH_AES_128_GCM_SHA256
Properties:
- Config: allow_insecure_tls_ciphers
- Env Var: RCLONE_FTP_ALLOW_INSECURE_TLS_CIPHERS
- Type: bool
- Default: false
#### --ftp-shut-timeout
Maximum time to wait for data connection closing status.
@@ -433,6 +478,20 @@ Properties:
- Type: string
- Required: false
#### --ftp-http-proxy
URL for HTTP CONNECT proxy
Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
Properties:
- Config: http_proxy
- Env Var: RCLONE_FTP_HTTP_PROXY
- Type: string
- Required: false
#### --ftp-no-check-upload
Don't check the upload is OK
+2
View File
@@ -679,6 +679,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+2
View File
@@ -324,6 +324,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+2
View File
@@ -288,6 +288,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+2
View File
@@ -383,6 +383,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+11
View File
@@ -630,6 +630,17 @@ Properties:
- "ctime"
- The last status change time.
#### --local-hashes
Comma separated list of supported checksum types.
Properties:
- Config: hashes
- Env Var: RCLONE_LOCAL_HASHES
- Type: CommaSepList
- Default:
#### --local-encoding
The encoding for the backend.
+2
View File
@@ -300,6 +300,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+2
View File
@@ -386,6 +386,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
@@ -720,6 +720,23 @@ Properties:
- Type: string
- Required: false
### Metadata
User metadata is stored as opc-meta- keys.
Here are the possible system metadata items for the oracleobjectstorage backend.
| Name | Help | Type | Example | Read Only |
|------|------|------|---------|-----------|
| opc-meta-atime | Time of last access | ISO 8601 | 2025-06-30T22:27:43-04:00 | N |
| opc-meta-btime | Time of file birth (creation) | ISO 8601 | 2025-06-30T22:27:43-04:00 | N |
| opc-meta-gid | Group ID of owner | decimal number | 500 | N |
| opc-meta-mode | File type and mode | octal, unix style | 0100664 | N |
| opc-meta-mtime | Time of last modification | ISO 8601 | 2025-06-30T22:27:43-04:00 | N |
| opc-meta-uid | User ID of owner | decimal number | 500 | N |
See the [metadata](/docs/#metadata) docs for more info.
## Backend commands
Here are the commands specific to the oracleobjectstorage backend.
+2
View File
@@ -230,6 +230,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+15 -1
View File
@@ -201,6 +201,20 @@ Properties:
- Type: SizeSuffix
- Default: 10Mi
#### --pikpak-upload-cutoff
Cutoff for switching to chunked upload.
Any files larger than this will be uploaded in chunks of chunk_size.
The minimum is 0 and the maximum is 5 GiB.
Properties:
- Config: upload_cutoff
- Env Var: RCLONE_PIKPAK_UPLOAD_CUTOFF
- Type: SizeSuffix
- Default: 200Mi
#### --pikpak-chunk-size
Chunk size for multipart uploads.
@@ -247,7 +261,7 @@ Properties:
- Config: upload_concurrency
- Env Var: RCLONE_PIKPAK_UPLOAD_CONCURRENCY
- Type: int
- Default: 5
- Default: 4
#### --pikpak-encoding
+2
View File
@@ -195,6 +195,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+2
View File
@@ -192,6 +192,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+15 -3
View File
@@ -783,6 +783,18 @@ Parameters:
**Authentication is required for this call.**
### config/unlock: Unlock the config file. {#config-unlock}
Unlocks the config file if it is locked.
Parameters:
- 'config_password' - password to unlock the config file
A good idea is to disable AskPassword before making this call
**Authentication is required for this call.**
### config/update: update the config for a remote. {#config-update}
This takes the following parameters:
@@ -1066,6 +1078,7 @@ Returns the following values:
"size": size of the file in bytes,
"bytes": total transferred bytes for this file,
"checked": if the transfer is only checked (skipped, deleted),
"what": the purpose of the transfer (transferring, deleting, checking, importing, hashing, merging, listing, moving, renaming),
"timestamp": integer representing millisecond unix epoch,
"error": string description of the error (empty if successful),
"jobid": id of the job that this transfer belongs to
@@ -1783,7 +1796,7 @@ Parameters:
Note that these are the global options which are unaffected by use of
the _config and _filter parameters. If you wish to read the parameters
set in _config then use options/config and for _filter use options/filter.
set in _config or _filter use options/local.
This shows the internal names of the option within rclone which should
map to the external options very easily with a few exceptions.
@@ -2115,8 +2128,7 @@ This takes the following parameters
- removeEmptyDirs - remove empty directories at the final cleanup step
- filtersFile - read filtering patterns from a file
- ignoreListingChecksum - Do not use checksums for listings
- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync.
Use at your own risk!
- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync.
- workdir - server directory for history files (default: `~/.cache/rclone/bisync`)
- backupdir1 - --backup-dir for Path1. Must be a non-overlapping path on the same remote.
- backupdir2 - --backup-dir for Path2. Must be a non-overlapping path on the same remote.
+18 -3
View File
@@ -795,7 +795,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
### Standard options
Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, Exaba, FlashBlade, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Mega, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu and others).
Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, Exaba, FlashBlade, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Mega, Minio, Netease, Outscale, OVHcloud, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu, Zata and others).
#### --s3-provider
@@ -856,6 +856,8 @@ Properties:
- Netease Object Storage (NOS)
- "Outscale"
- OUTSCALE Object Storage (OOS)
- "OVHcloud"
- OVHcloud Object Storage
- "Petabox"
- Petabox Object Storage
- "RackCorp"
@@ -880,6 +882,8 @@ Properties:
- Wasabi Object Storage
- "Qiniu"
- Qiniu Object Storage (Kodo)
- "Zata"
- Zata (S3 compatible Gateway)
- "Other"
- Any other S3 compatible provider
@@ -1257,7 +1261,7 @@ Properties:
### Advanced options
Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, Exaba, FlashBlade, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Mega, Minio, Netease, Outscale, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu and others).
Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, ChinaMobile, Cloudflare, DigitalOcean, Dreamhost, Exaba, FlashBlade, GCS, HuaweiOBS, IBMCOS, IDrive, IONOS, LyveCloud, Leviia, Liara, Linode, Magalu, Mega, Minio, Netease, Outscale, OVHcloud, Petabox, RackCorp, Rclone, Scaleway, SeaweedFS, Selectel, StackPath, Storj, Synology, TencentCOS, Wasabi, Qiniu, Zata and others).
#### --s3-bucket-acl
@@ -1615,6 +1619,17 @@ Properties:
- Type: bool
- Default: false
#### --s3-use-arn-region
If true, enables arn region support for the service.
Properties:
- Config: use_arn_region
- Env Var: RCLONE_S3_USE_ARN_REGION
- Type: bool
- Default: false
#### --s3-leave-parts-on-error
If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery.
@@ -1913,7 +1928,7 @@ The parameter should be a date, "2006-01-02", datetime "2006-01-02
Note that when using this no file write operations are permitted,
so you can't upload files or delete them.
See [the time option docs](/docs/#time-option) for valid formats.
See [the time option docs](/docs/#time-options) for valid formats.
Properties:
+1 -1
View File
@@ -387,7 +387,7 @@ Properties:
- Config: encoding
- Env Var: RCLONE_SEAFILE_ENCODING
- Type: Encoding
- Default: Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8
- Default: Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8,Dot
#### --seafile-description
+78 -2
View File
@@ -725,9 +725,20 @@ Properties:
- "cmd"
- Windows Command Prompt
#### --sftp-hashes
Comma separated list of supported checksum types.
Properties:
- Config: hashes
- Env Var: RCLONE_SFTP_HASHES
- Type: CommaSepList
- Default:
#### --sftp-md5sum-command
The command used to read md5 hashes.
The command used to read MD5 hashes.
Leave blank for autodetect.
@@ -740,7 +751,7 @@ Properties:
#### --sftp-sha1sum-command
The command used to read sha1 hashes.
The command used to read SHA-1 hashes.
Leave blank for autodetect.
@@ -751,6 +762,71 @@ Properties:
- Type: string
- Required: false
#### --sftp-crc32sum-command
The command used to read CRC-32 hashes.
Leave blank for autodetect.
Properties:
- Config: crc32sum_command
- Env Var: RCLONE_SFTP_CRC32SUM_COMMAND
- Type: string
- Required: false
#### --sftp-sha256sum-command
The command used to read SHA-256 hashes.
Leave blank for autodetect.
Properties:
- Config: sha256sum_command
- Env Var: RCLONE_SFTP_SHA256SUM_COMMAND
- Type: string
- Required: false
#### --sftp-blake3sum-command
The command used to read BLAKE3 hashes.
Leave blank for autodetect.
Properties:
- Config: blake3sum_command
- Env Var: RCLONE_SFTP_BLAKE3SUM_COMMAND
- Type: string
- Required: false
#### --sftp-xxh3sum-command
The command used to read XXH3 hashes.
Leave blank for autodetect.
Properties:
- Config: xxh3sum_command
- Env Var: RCLONE_SFTP_XXH3SUM_COMMAND
- Type: string
- Required: false
#### --sftp-xxh128sum-command
The command used to read XXH128 hashes.
Leave blank for autodetect.
Properties:
- Config: xxh128sum_command
- Env Var: RCLONE_SFTP_XXH128SUM_COMMAND
- Type: string
- Required: false
#### --sftp-skip-links
Set to skip any symlinks and any other non regular files.
+2
View File
@@ -252,6 +252,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+21
View File
@@ -252,6 +252,27 @@ Properties:
- Type: bool
- Default: true
#### --smb-kerberos-ccache
Path to the Kerberos credential cache (krb5cc).
Overrides the default KRB5CCNAME environment variable and allows this
instance of the SMB backend to use a different Kerberos cache file.
This is useful when mounting multiple SMB with different credentials
or running in multi-user environments.
Supported formats:
- FILE:/path/to/ccache – Use the specified file.
- DIR:/path/to/ccachedir – Use the primary file inside the specified directory.
- /path/to/ccache – Interpreted as a file path.
Properties:
- Config: kerberos_ccache
- Env Var: RCLONE_SMB_KERBEROS_CCACHE
- Type: string
- Required: false
#### --smb-encoding
The encoding for the backend.
+2
View File
@@ -192,6 +192,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+2
View File
@@ -230,6 +230,8 @@ Use client credentials OAuth flow.
This will use the OAUTH2 client Credentials Flow as described in RFC 6749.
Note that this option is NOT supported by all backends.
Properties:
- Config: client_credentials
+7
View File
@@ -438,6 +438,12 @@ var ConfigOptionsInfo = Options{{
Default: "",
Help: "Client SSL private key (PEM) for mutual TLS auth",
Groups: "Networking",
}, {
Name: "client_pass",
Default: "",
Help: "Password for client SSL private key (PEM) for mutual TLS auth (obscured)",
Groups: "Networking",
IsPassword: true,
}, {
Name: "multi_thread_cutoff",
Default: SizeSuffix(256 * 1024 * 1024),
@@ -644,6 +650,7 @@ type ConfigInfo struct {
CaCert []string `config:"ca_cert"` // Client Side CA
ClientCert string `config:"client_cert"` // Client Side Cert
ClientKey string `config:"client_key"` // Client Side Key
ClientPass string `config:"client_pass"` // Client Side Key Password (obscured)
MultiThreadCutoff SizeSuffix `config:"multi_thread_cutoff"`
MultiThreadStreams int `config:"multi_thread_streams"`
MultiThreadSet bool `config:"multi_thread_set"` // whether MultiThreadStreams was set (set in fs/config/configflags)
+166 -32
View File
@@ -6,6 +6,8 @@ import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"net"
"net/http"
@@ -18,7 +20,9 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/lib/structs"
"github.com/youmark/pkcs8"
"golang.org/x/net/publicsuffix"
)
@@ -28,7 +32,7 @@ const (
)
var (
transport http.RoundTripper
transport *Transport
noTransport = new(sync.Once)
cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
logMutex sync.Mutex
@@ -48,10 +52,160 @@ func ResetTransport() {
noTransport = new(sync.Once)
}
// LoadKeyPair loads a TLS certificate and private key from PEM-encoded files,
// with extended support for encrypted private keys.
//
// This function is designed as a robust replacement for tls.X509KeyPair,
// providing the same core functionality but adding support for
// password-protected private keys.
//
// The certificate file (certFile) must contain one or more PEM-encoded
// certificates. The first certificate is treated as the leaf certificate, and
// any subsequent certificates are treated as its chain.
//
// The key file (keyFile) must contain a PEM-encoded private key. Supported
// formats are:
//
// - Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY")
// - Unencrypted PKCS#8 ("BEGIN PRIVATE KEY")
// - Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY")
// - Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected.
//
// The password parameter is used to decrypt the private key. If the
// key is not encrypted, this parameter is ignored and can be an empty
// string. The password should be an obscured string.
//
// On success, it returns a fully populated tls.Certificate struct, including the
// Leaf certificate field.
func LoadKeyPair(certFile, keyFile, password string) (cert tls.Certificate, err error) {
certPEM, err := os.ReadFile(certFile)
if err != nil {
return cert, fmt.Errorf("read cert: %w", err)
}
keyPEM, err := os.ReadFile(keyFile)
if err != nil {
return cert, fmt.Errorf("read key: %w", err)
}
if password != "" {
password, err = obscure.Reveal(password)
if err != nil {
return cert, fmt.Errorf("reveal key password: %w", err)
}
}
// Fast path: unencrypted PKCS#1/PKCS#8
cert, err = tls.X509KeyPair(certPEM, keyPEM)
if err == nil {
if len(cert.Certificate) == 0 {
return cert, errors.New("no certificates parsed")
}
leaf, err := x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return cert, fmt.Errorf("parse leaf: %w", err)
}
cert.Leaf = leaf
return cert, nil
}
// Decrypt / parse key manually
block, rest := pem.Decode(keyPEM)
if block == nil {
return cert, errors.New("no PEM block in key")
}
if len(rest) != 0 {
fs.Debugf(nil, "Trailing data (%d bytes) in key PEM loaded from %q", len(rest), keyFile)
}
var privKey any
switch {
case block.Type == "ENCRYPTED PRIVATE KEY":
if password == "" {
return cert, errors.New("key is encrypted but no --client-pass provided")
}
privKey, err = pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(password))
if err != nil {
return cert, fmt.Errorf("parse encrypted PKCS#8: %w", err)
}
case x509.IsEncryptedPEMBlock(block): //nolint:staticcheck // this is Legacy and insecure
if password == "" {
return cert, errors.New("key is encrypted but no --client-pass provided")
}
der, err := x509.DecryptPEMBlock(block, []byte(password)) //nolint:staticcheck // this is Legacy and insecure
if err != nil {
return cert, fmt.Errorf("decrypt PEM key: %w", err)
}
// Try PKCS#8, then RSA PKCS#1, then EC
if k, kerr1 := x509.ParsePKCS8PrivateKey(der); kerr1 == nil {
privKey = k
} else if k, kerr2 := x509.ParsePKCS1PrivateKey(der); kerr2 == nil {
privKey = k
} else if k, kerr3 := x509.ParseECPrivateKey(der); kerr3 == nil {
privKey = k
} else {
return cert, fmt.Errorf("parse decrypted key: pkcs8: %v, pkcs1: %v, ec: %v", kerr1, kerr2, kerr3)
}
default:
// Unencrypted specific types
switch block.Type {
case "PRIVATE KEY":
k, kerr := x509.ParsePKCS8PrivateKey(block.Bytes)
if kerr != nil {
return cert, fmt.Errorf("parse PKCS#8: %w", kerr)
}
privKey = k
case "RSA PRIVATE KEY":
k, kerr := x509.ParsePKCS1PrivateKey(block.Bytes)
if kerr != nil {
return cert, fmt.Errorf("parse PKCS#1 RSA: %w", kerr)
}
privKey = k
case "EC PRIVATE KEY":
k, kerr := x509.ParseECPrivateKey(block.Bytes)
if kerr != nil {
return cert, fmt.Errorf("parse EC: %w", kerr)
}
privKey = k
default:
return cert, fmt.Errorf("unsupported key type %q", block.Type)
}
}
// Build cert chain from PEM
var certDERs [][]byte
for rest := certPEM; ; {
var b *pem.Block
b, rest = pem.Decode(rest)
if b == nil {
break
}
if b.Type == "CERTIFICATE" {
certDERs = append(certDERs, b.Bytes)
}
}
if len(certDERs) == 0 {
return cert, fmt.Errorf("no CERTIFICATE blocks in %s", certFile)
}
cert = tls.Certificate{
Certificate: certDERs,
PrivateKey: privKey,
}
// Leaf is always the first certificate
cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return cert, fmt.Errorf("parse leaf: %w", err)
}
return cert, nil
}
// NewTransportCustom returns an http.RoundTripper with the correct timeouts.
// The customize function is called if set to give the caller an opportunity to
// customize any defaults in the Transport.
func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) http.RoundTripper {
func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) *Transport {
ci := fs.GetConfig(ctx)
// Start with a sensible set of defaults then override.
// This also means we get new stuff when it gets added to go
@@ -85,17 +239,10 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
if ci.ClientCert == "" || ci.ClientKey == "" {
fs.Fatalf(nil, "Both --client-cert and --client-key must be set")
}
cert, err := tls.LoadX509KeyPair(ci.ClientCert, ci.ClientKey)
cert, err := LoadKeyPair(ci.ClientCert, ci.ClientKey, ci.ClientPass)
if err != nil {
fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err)
}
if cert.Leaf == nil {
// Leaf is always the first certificate
cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
fs.Fatalf(nil, "Failed to parse the certificate")
}
}
t.TLSClientConfig.Certificates = []tls.Certificate{cert}
}
@@ -145,7 +292,7 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
}
// NewTransport returns an http.RoundTripper with the correct timeouts
func NewTransport(ctx context.Context) http.RoundTripper {
func NewTransport(ctx context.Context) *Transport {
(*noTransport).Do(func() {
transport = NewTransportCustom(ctx, nil)
})
@@ -187,14 +334,12 @@ func NewClientWithUnixSocket(ctx context.Context, path string) *http.Client {
// * Updates metrics
type Transport struct {
*http.Transport
ci *fs.ConfigInfo
dump fs.DumpFlags
filterRequest func(req *http.Request)
userAgent string
headers []*fs.HTTPOption
metrics *Metrics
// Filename of the client cert in case we need to reload it
clientCert string
clientKey string
// Mutex for serializing attempts at reloading the certificates
reloadMutex sync.Mutex
}
@@ -203,13 +348,12 @@ type Transport struct {
// roundtrips including the body if logBody is set.
func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport {
return &Transport{
Transport: transport,
dump: ci.Dump,
userAgent: ci.UserAgent,
headers: ci.Headers,
metrics: DefaultMetrics,
clientCert: ci.ClientCert,
clientKey: ci.ClientKey,
Transport: transport,
ci: ci,
dump: ci.Dump,
userAgent: ci.UserAgent,
headers: ci.Headers,
metrics: DefaultMetrics,
}
}
@@ -309,20 +453,10 @@ func (t *Transport) reloadCertificates() {
if !isCertificateExpired(t.TLSClientConfig) {
return
}
cert, err := tls.LoadX509KeyPair(t.clientCert, t.clientKey)
cert, err := LoadKeyPair(t.ci.ClientCert, t.ci.ClientKey, t.ci.ClientPass)
if err != nil {
fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err)
}
// Check if we need to parse the certificate again, we need it
// for checking the expiration date
if cert.Leaf == nil {
// Leaf is always the first certificate
cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
fs.Fatalf(nil, "Failed to parse the certificate")
}
}
t.TLSClientConfig.Certificates = []tls.Certificate{cert}
}
+4 -16
View File
@@ -7,7 +7,6 @@ package mmap
import (
"fmt"
"reflect"
"unsafe"
"golang.org/x/sys/windows"
@@ -21,21 +20,10 @@ func Alloc(size int) ([]byte, error) {
if err != nil {
return nil, fmt.Errorf("mmap: failed to allocate memory for buffer: %w", err)
}
// SliceHeader is deprecated...
var mem []byte
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem)) // nolint:staticcheck
sh.Data = p
sh.Len = size
sh.Cap = size
return mem, nil
// ...However the correct code gives a go vet warning
// "possible misuse of unsafe.Pointer"
//
// Maybe there is a different way of writing this, but none of
// the allowed uses of unsafe.Pointer seemed to cover it other
// than using SliceHeader (use 6 of unsafe.Pointer).
//
// return unsafe.Slice((*byte)(unsafe.Pointer(p)), size), nil
pp := unsafe.Pointer(&p)
up := *(*unsafe.Pointer)(pp)
return unsafe.Slice((*byte)(up), size), nil
}
// Free frees buffers allocated by Alloc. Note it should be passed
Generated
+11550 -9335
View File
File diff suppressed because it is too large. Load diff