Compare commits
105
Commits
v1.70.2
...
encoder-nfc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
24835344eb | ||
|
|
bfdd5e2c22 | ||
|
|
f3f16cd2b9 | ||
|
|
d84ea2ec52 | ||
|
|
b259241c07 | ||
|
|
a8ab0730a7 | ||
|
|
cef207cf94 | ||
|
|
e728ea32d1 | ||
|
|
ccdee0420f | ||
|
|
8a51e11d23 | ||
|
|
9083f1ff15 | ||
|
|
2964b1a169 | ||
|
|
b6767820de | ||
|
|
821e7fce45 | ||
|
|
b7c6268d3e | ||
|
|
521d6b88d4 | ||
|
|
cf767b0856 | ||
|
|
25f7809822 | ||
|
|
74c0b1ea3b | ||
|
|
f4dcb1e9cf | ||
|
|
90f1d023ff | ||
|
|
e9c5f2d4e8 | ||
|
|
1249e9b5ac | ||
|
|
d47bc5f6c4 | ||
|
|
efb1794135 | ||
|
|
71b98a03a9 | ||
|
|
8e625c6593 | ||
|
|
6b2cd7c631 | ||
|
|
aa4aead63c | ||
|
|
c491d12cd0 | ||
|
|
9e4d703a56 | ||
|
|
fc0c0a7771 | ||
|
|
d5cc0d83b0 | ||
|
|
52762dc866 | ||
|
|
3c092cfc17 | ||
|
|
7f3f1af541 | ||
|
|
f885c481f0 | ||
|
|
865d4b2bda | ||
|
|
3cb1e65eb6 | ||
|
|
f667346718 | ||
|
|
c6e1f59415 | ||
|
|
f353c92852 | ||
|
|
1e88c6a18b | ||
|
|
7242aed1c3 | ||
|
|
81e63785fe | ||
|
|
c7937f53d4 | ||
|
|
58fa1c975f | ||
|
|
da49fc1b6d | ||
|
|
df9c921dd5 | ||
|
|
d9c227eff6 | ||
|
|
524c285d88 | ||
|
|
4107246335 | ||
|
|
87a65ec6a5 | ||
|
|
c6d0b61982 | ||
|
|
88e30eecbf | ||
|
|
f904378c4d | ||
|
|
24eb8dcde0 | ||
|
|
a97425d9cb | ||
|
|
c51878f9a9 | ||
|
|
92f0a73ac6 | ||
|
|
163c149f3f | ||
|
|
224ca0ae8e | ||
|
|
5bf6cd1f4f | ||
|
|
555739eec5 | ||
|
|
c036ce90fe | ||
|
|
6163ae7cc7 | ||
|
|
cd950e30cb | ||
|
|
89dfae96ad | ||
|
|
c0a2d730a6 | ||
|
|
592407230b | ||
|
|
a7c3ddb482 | ||
|
|
7a1813c531 | ||
|
|
16e3d1becd | ||
|
|
c0f6b910ae | ||
|
|
e3bf8dc122 | ||
|
|
086a835131 | ||
|
|
d0668de192 | ||
|
|
4df974ccc4 | ||
|
|
a50c903a82 | ||
|
|
97a8092c14 | ||
|
|
526565b810 | ||
|
|
64804b81bd | ||
|
|
e10f516a5e | ||
|
|
fe62a2bb4e | ||
|
|
d6ecb949ca | ||
|
|
a845a96538 | ||
|
|
92f30fda8d | ||
|
|
559ef2eba8 | ||
|
|
17b25d7ce2 | ||
|
|
fe3253eefd | ||
|
|
c38ca6b2d1 | ||
|
|
5aa9811084 | ||
|
|
3cae373064 | ||
|
|
b6b8526fb4 | ||
|
|
6f86143176 | ||
|
|
beffef2882 | ||
|
|
96f5bbcdd7 | ||
|
|
27ce78bee4 | ||
|
|
898a59062b | ||
|
|
c5f55243e1 | ||
|
|
62a9727ab5 | ||
|
|
16f1e08b73 | ||
|
|
4280ec75cc | ||
|
|
b064cc2116 | ||
|
|
f8b50f8d8f |
No files matched your search
@@ -23,6 +23,9 @@ jobs:
|
||||
build:
|
||||
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
|
||||
timeout-minutes: 60
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -103,7 +106,6 @@ jobs:
|
||||
check-latest: true
|
||||
|
||||
- name: Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
|
||||
echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
|
||||
@@ -112,7 +114,6 @@ jobs:
|
||||
if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
|
||||
|
||||
- name: Install Libraries on Linux
|
||||
shell: bash
|
||||
run: |
|
||||
sudo modprobe fuse
|
||||
sudo chmod 666 /dev/fuse
|
||||
@@ -122,7 +123,6 @@ jobs:
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
|
||||
- name: Install Libraries on macOS
|
||||
shell: bash
|
||||
run: |
|
||||
# https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
|
||||
# https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
|
||||
@@ -151,7 +151,6 @@ jobs:
|
||||
if: matrix.os == 'windows-latest'
|
||||
|
||||
- name: Print Go version and environment
|
||||
shell: bash
|
||||
run: |
|
||||
printf "Using go at: $(which go)\n"
|
||||
printf "Go version: $(go version)\n"
|
||||
@@ -163,29 +162,24 @@ jobs:
|
||||
env
|
||||
|
||||
- name: Build rclone
|
||||
shell: bash
|
||||
run: |
|
||||
make
|
||||
|
||||
- name: Rclone version
|
||||
shell: bash
|
||||
run: |
|
||||
rclone version
|
||||
|
||||
- name: Run tests
|
||||
shell: bash
|
||||
run: |
|
||||
make quicktest
|
||||
if: matrix.quicktest
|
||||
|
||||
- name: Race test
|
||||
shell: bash
|
||||
run: |
|
||||
make racequicktest
|
||||
if: matrix.racequicktest
|
||||
|
||||
- name: Run librclone tests
|
||||
shell: bash
|
||||
run: |
|
||||
make -C librclone/ctest test
|
||||
make -C librclone/ctest clean
|
||||
@@ -193,14 +187,12 @@ jobs:
|
||||
if: matrix.librclonetest
|
||||
|
||||
- name: Compile all architectures test
|
||||
shell: bash
|
||||
run: |
|
||||
make
|
||||
make compile_all
|
||||
if: matrix.compile_all
|
||||
|
||||
- name: Deploy built binaries
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
|
||||
make ci_beta
|
||||
@@ -219,7 +211,6 @@ jobs:
|
||||
steps:
|
||||
- name: Get runner parameters
|
||||
id: get-runner-parameters
|
||||
shell: bash
|
||||
run: |
|
||||
echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
|
||||
echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
|
||||
@@ -314,7 +305,6 @@ jobs:
|
||||
go-version: '>=1.24.0-rc.1'
|
||||
|
||||
- name: Set global environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "VERSION=$(make version)" >> $GITHUB_ENV
|
||||
|
||||
@@ -333,7 +323,6 @@ jobs:
|
||||
run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
|
||||
|
||||
- name: arm-v7a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
@@ -347,7 +336,6 @@ jobs:
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a .
|
||||
|
||||
- name: arm64-v8a Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
@@ -360,7 +348,6 @@ jobs:
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a .
|
||||
|
||||
- name: x86 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
@@ -373,7 +360,6 @@ jobs:
|
||||
run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 .
|
||||
|
||||
- name: x64 Set environment variables
|
||||
shell: bash
|
||||
run: |
|
||||
echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
|
||||
echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
|
||||
|
||||
@@ -39,6 +39,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
|
||||
* Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
|
||||
* Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
|
||||
* FileLu [:page_facing_up:](https://rclone.org/filelu/)
|
||||
* Files.com [:page_facing_up:](https://rclone.org/filescom/)
|
||||
* FlashBlade [:page_facing_up:](https://rclone.org/s3/#pure-storage-flashblade)
|
||||
* FTP [:page_facing_up:](https://rclone.org/ftp/)
|
||||
@@ -111,6 +112,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
|
||||
* WebDAV [:page_facing_up:](https://rclone.org/webdav/)
|
||||
* Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
|
||||
* Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
|
||||
* Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
|
||||
* The local filesystem [:page_facing_up:](https://rclone.org/local/)
|
||||
|
||||
Please see [the full list of all storage providers and their features](https://rclone.org/overview/)
|
||||
|
||||
+104
-15
@@ -72,6 +72,7 @@ const (
|
||||
emulatorAccount = "devstoreaccount1"
|
||||
emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
||||
emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
|
||||
sasCopyValidity = time.Hour // how long SAS should last when doing server side copy
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -559,6 +560,11 @@ type Fs struct {
|
||||
pacer *fs.Pacer // To pace and retry the API calls
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
publicAccess container.PublicAccessType // Container Public Access Level
|
||||
|
||||
// user delegation cache
|
||||
userDelegationMu sync.Mutex
|
||||
userDelegation *service.UserDelegationCredential
|
||||
userDelegationExpiry time.Time
|
||||
}
|
||||
|
||||
// Object describes an azure object
|
||||
@@ -984,6 +990,38 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
f.cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
case opt.UseAZ:
|
||||
var options = azidentity.AzureCLICredentialOptions{}
|
||||
f.cred, err = azidentity.NewAzureCLICredential(&options)
|
||||
@@ -1688,6 +1726,38 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
return f.deleteContainer(ctx, container)
|
||||
}
|
||||
|
||||
// Get a user delegation which is valid for at least sasCopyValidity
|
||||
//
|
||||
// This value is cached in f
|
||||
func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCredential, error) {
|
||||
f.userDelegationMu.Lock()
|
||||
defer f.userDelegationMu.Unlock()
|
||||
|
||||
if f.userDelegation != nil && time.Until(f.userDelegationExpiry) > sasCopyValidity {
|
||||
return f.userDelegation, nil
|
||||
}
|
||||
|
||||
// Validity window
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(2 * sasCopyValidity)
|
||||
startStr := start.Format(time.RFC3339)
|
||||
expiryStr := expiry.Format(time.RFC3339)
|
||||
|
||||
// Acquire user delegation key from the service client
|
||||
info := service.KeyInfo{
|
||||
Start: &startStr,
|
||||
Expiry: &expiryStr,
|
||||
}
|
||||
userDelegationKey, err := f.svc.GetUserDelegationCredential(ctx, info, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get user delegation key: %w", err)
|
||||
}
|
||||
|
||||
f.userDelegation = userDelegationKey
|
||||
f.userDelegationExpiry = expiry
|
||||
return f.userDelegation, nil
|
||||
}
|
||||
|
||||
// getAuth gets auth to copy o.
|
||||
//
|
||||
// tokenOK is used to signal that token based auth (Microsoft Entra
|
||||
@@ -1699,7 +1769,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
|
||||
// URL (not a SAS) and token will be empty.
|
||||
//
|
||||
// If tokenOK is true it may also return a token for the auth.
|
||||
func (o *Object) getAuth(ctx context.Context, tokenOK bool, noAuth bool) (srcURL string, token *string, err error) {
|
||||
func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err error) {
|
||||
f := o.fs
|
||||
srcBlobSVC := o.getBlobSVC()
|
||||
srcURL = srcBlobSVC.URL()
|
||||
@@ -1708,29 +1778,47 @@ func (o *Object) getAuth(ctx context.Context, tokenOK bool, noAuth bool) (srcURL
|
||||
case noAuth:
|
||||
// If same storage account then no auth needed
|
||||
case f.cred != nil:
|
||||
if !tokenOK {
|
||||
return srcURL, token, errors.New("not supported: Microsoft Entra ID")
|
||||
}
|
||||
options := policy.TokenRequestOptions{}
|
||||
accessToken, err := f.cred.GetToken(ctx, options)
|
||||
// Generate a User Delegation SAS URL using Azure AD credentials
|
||||
userDelegationKey, err := f.getUserDelegation(ctx)
|
||||
if err != nil {
|
||||
return srcURL, token, fmt.Errorf("failed to create access token: %w", err)
|
||||
return "", fmt.Errorf("sas creation: %w", err)
|
||||
}
|
||||
token = &accessToken.Token
|
||||
|
||||
// Build the SAS values
|
||||
perms := sas.BlobPermissions{Read: true}
|
||||
container, containerPath := o.split()
|
||||
start := time.Now().UTC()
|
||||
expiry := start.Add(sasCopyValidity)
|
||||
vals := sas.BlobSignatureValues{
|
||||
StartTime: start,
|
||||
ExpiryTime: expiry,
|
||||
Permissions: perms.String(),
|
||||
ContainerName: container,
|
||||
BlobName: containerPath,
|
||||
}
|
||||
|
||||
// Sign with the delegation key
|
||||
queryParameters, err := vals.SignWithUserDelegation(userDelegationKey)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("signing SAS with user delegation failed: %w", err)
|
||||
}
|
||||
|
||||
// Append the SAS to the URL
|
||||
srcURL = srcBlobSVC.URL() + "?" + queryParameters.Encode()
|
||||
case f.sharedKeyCred != nil:
|
||||
// Generate a short lived SAS URL if using shared key credentials
|
||||
expiry := time.Now().Add(time.Hour)
|
||||
expiry := time.Now().Add(sasCopyValidity)
|
||||
sasOptions := blob.GetSASURLOptions{}
|
||||
srcURL, err = srcBlobSVC.GetSASURL(sas.BlobPermissions{Read: true}, expiry, &sasOptions)
|
||||
if err != nil {
|
||||
return srcURL, token, fmt.Errorf("failed to create SAS URL: %w", err)
|
||||
return srcURL, fmt.Errorf("failed to create SAS URL: %w", err)
|
||||
}
|
||||
case f.anonymous || f.opt.SASURL != "":
|
||||
// If using a SASURL or anonymous, no need for any extra auth
|
||||
default:
|
||||
return srcURL, token, errors.New("unknown authentication type")
|
||||
return srcURL, errors.New("unknown authentication type")
|
||||
}
|
||||
return srcURL, token, nil
|
||||
return srcURL, nil
|
||||
}
|
||||
|
||||
// Do multipart parallel copy.
|
||||
@@ -1751,7 +1839,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
|
||||
o.fs = f
|
||||
o.remote = remote
|
||||
|
||||
srcURL, token, err := src.getAuth(ctx, true, false)
|
||||
srcURL, err := src.getAuth(ctx, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("multipart copy: %w", err)
|
||||
}
|
||||
@@ -1795,7 +1883,8 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
|
||||
Count: partSize,
|
||||
},
|
||||
// Specifies the authorization scheme and signature for the copy source.
|
||||
CopySourceAuthorization: token,
|
||||
// We use SAS URLs as this doesn't seem to work always
|
||||
// CopySourceAuthorization: token,
|
||||
// CPKInfo *blob.CPKInfo
|
||||
// CPKScopeInfo *blob.CPKScopeInfo
|
||||
}
|
||||
@@ -1865,7 +1954,7 @@ func (f *Fs) copySinglepart(ctx context.Context, remote, dstContainer, dstPath s
|
||||
dstBlobSVC := f.getBlobSVC(dstContainer, dstPath)
|
||||
|
||||
// Get the source auth - none needed for same storage account
|
||||
srcURL, _, err := src.getAuth(ctx, false, f == src.fs)
|
||||
srcURL, err := src.getAuth(ctx, f == src.fs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("single part copy: source auth: %w", err)
|
||||
}
|
||||
|
||||
@@ -569,6 +569,38 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
|
||||
// Workload Identity based authentication
|
||||
var options azidentity.ManagedIdentityCredentialOptions
|
||||
options.ID = azidentity.ClientID(opt.MSIClientID)
|
||||
|
||||
msiCred, err := azidentity.NewManagedIdentityCredential(&options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
getClientAssertions := func(context.Context) (string, error) {
|
||||
token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
|
||||
Scopes: []string{"api://AzureADTokenExchange"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to acquire MSI token: %w", err)
|
||||
}
|
||||
|
||||
return token.Token, nil
|
||||
}
|
||||
|
||||
assertOpts := &azidentity.ClientAssertionCredentialOptions{}
|
||||
cred, err = azidentity.NewClientAssertionCredential(
|
||||
opt.Tenant,
|
||||
opt.ClientID,
|
||||
getClientAssertions,
|
||||
assertOpts)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
|
||||
}
|
||||
default:
|
||||
return nil, errors.New("no authentication method configured")
|
||||
}
|
||||
@@ -922,7 +954,7 @@ func (o *Object) setMetadata(resp *file.GetPropertiesResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
// readMetaData gets the metadata if it hasn't already been fetched
|
||||
// getMetadata gets the metadata if it hasn't already been fetched
|
||||
func (o *Object) getMetadata(ctx context.Context) error {
|
||||
resp, err := o.fileClient().GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -1673,6 +1673,21 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) {
|
||||
return o.getMetaDataListing(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// If using versionAt we need to list the find the correct version.
|
||||
if o.fs.opt.VersionAt.IsSet() {
|
||||
info, err := o.getMetaDataListing(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.Action == "hide" {
|
||||
// Rerturn object not found error if the current version is deleted.
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
_, info, err = o.getOrHead(ctx, "HEAD", nil)
|
||||
return info, err
|
||||
}
|
||||
|
||||
@@ -446,14 +446,14 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
|
||||
t.Run("List", func(t *testing.T) {
|
||||
fstest.CheckListing(t, f, test.want)
|
||||
})
|
||||
// b2 NewObject doesn't work with VersionAt
|
||||
//t.Run("NewObject", func(t *testing.T) {
|
||||
// gotObj, gotErr := f.NewObject(ctx, fileName)
|
||||
// assert.Equal(t, test.wantErr, gotErr)
|
||||
// if gotErr == nil {
|
||||
// assert.Equal(t, test.wantSize, gotObj.Size())
|
||||
// }
|
||||
//})
|
||||
|
||||
t.Run("NewObject", func(t *testing.T) {
|
||||
gotObj, gotErr := f.NewObject(ctx, fileName)
|
||||
assert.Equal(t, test.wantErr, gotErr)
|
||||
if gotErr == nil {
|
||||
assert.Equal(t, test.wantSize, gotObj.Size())
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -858,7 +858,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
}
|
||||
return wrappedCallback(entries)
|
||||
}
|
||||
return listP(ctx, dir, wrappedCallback)
|
||||
return listP(ctx, uRemote, wrappedCallback)
|
||||
}
|
||||
|
||||
// ListR lists the objects and directories of the Fs starting
|
||||
|
||||
@@ -52,7 +52,7 @@ func (f *Fs) testUploadTimeout(t *testing.T) {
|
||||
ci.Timeout = saveTimeout
|
||||
}()
|
||||
ci.LowLevelRetries = 1
|
||||
ci.Timeout = idleTimeout
|
||||
ci.Timeout = fs.Duration(idleTimeout)
|
||||
|
||||
upload := func(concurrency int, shutTimeout time.Duration) (obj fs.Object, err error) {
|
||||
fixFs := deriveFs(ctx, t, f, settings{
|
||||
|
||||
@@ -421,6 +421,9 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
// will return the object and the error, otherwise will return
|
||||
// nil and the error
|
||||
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
|
||||
if src.Size() == 0 {
|
||||
return nil, fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
return uploadFile(ctx, f, in, src.Remote(), options...)
|
||||
}
|
||||
|
||||
@@ -659,6 +662,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
|
||||
// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
|
||||
// return an error or update the object properly (rather than e.g. calling panic).
|
||||
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
|
||||
if src.Size() == 0 {
|
||||
return fs.ErrorCantUploadEmptyFiles
|
||||
}
|
||||
|
||||
srcRemote := o.Remote()
|
||||
|
||||
@@ -670,7 +676,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
var resp *client.UploadResult
|
||||
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
var res *http.Response
|
||||
res, resp, err = o.fs.ik.Upload(ctx, in, client.UploadParam{
|
||||
FileName: fileName,
|
||||
@@ -725,7 +731,7 @@ func uploadFile(ctx context.Context, f *Fs, in io.Reader, srcRemote string, opti
|
||||
UseUniqueFileName := new(bool)
|
||||
*UseUniqueFileName = false
|
||||
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
err := f.pacer.CallNoRetry(func() (bool, error) {
|
||||
var res *http.Response
|
||||
var err error
|
||||
res, _, err = f.ik.Upload(ctx, in, client.UploadParam{
|
||||
@@ -794,35 +800,10 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// Copy src to this remote using server-side move operations.
|
||||
//
|
||||
// This is stored with the remote path given.
|
||||
//
|
||||
// It returns the destination Object and a possible error.
|
||||
//
|
||||
// Will only be called if src.Fs().Name() == f.Name()
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantMove
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
file, err := srcObj.Open(ctx)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return uploadFile(ctx, f, file, remote)
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied.
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
_ fs.Purger = &Fs{}
|
||||
_ fs.PublicLinker = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
_ fs.Copier = &Fs{}
|
||||
)
|
||||
@@ -461,7 +461,7 @@ func translateErrorsDir(err error) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
|
||||
// translateErrorsObject translates Koofr errors to rclone errors (for an object operation)
|
||||
func translateErrorsObject(err error) error {
|
||||
switch err := err.(type) {
|
||||
case httpclient.InvalidStatusError:
|
||||
|
||||
@@ -617,16 +617,36 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
case 1:
|
||||
// upload file using link from first step
|
||||
var res *http.Response
|
||||
var location string
|
||||
|
||||
// Check to see if we are being redirected
|
||||
opts := &rest.Opts{
|
||||
Method: "HEAD",
|
||||
RootURL: getFirstStepResult.Data.SignURL,
|
||||
Options: options,
|
||||
NoRedirect: true,
|
||||
}
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
res, err = o.fs.srv.Call(ctx, opts)
|
||||
return o.fs.shouldRetry(ctx, res, err)
|
||||
})
|
||||
if res != nil {
|
||||
location = res.Header.Get("Location")
|
||||
if location != "" {
|
||||
// set the URL to the new Location
|
||||
opts.RootURL = location
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("head upload URL: %w", err)
|
||||
}
|
||||
|
||||
file := io.MultiReader(bytes.NewReader(first10mBytes), in)
|
||||
|
||||
opts := &rest.Opts{
|
||||
Method: "PUT",
|
||||
RootURL: getFirstStepResult.Data.SignURL,
|
||||
Options: options,
|
||||
Body: file,
|
||||
ContentLength: &size,
|
||||
}
|
||||
opts.Method = "PUT"
|
||||
opts.Body = file
|
||||
opts.ContentLength = &size
|
||||
|
||||
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
|
||||
res, err = o.fs.srv.Call(ctx, opts)
|
||||
|
||||
+29
-1
@@ -305,6 +305,12 @@ only useful for reading.
|
||||
Help: "The last status change time.",
|
||||
}},
|
||||
},
|
||||
{
|
||||
Name: "hashes",
|
||||
Help: `Comma separated list of supported checksum types.`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
},
|
||||
{
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -331,6 +337,7 @@ type Options struct {
|
||||
NoSparse bool `config:"no_sparse"`
|
||||
NoSetModTime bool `config:"no_set_modtime"`
|
||||
TimeType timeType `config:"time_type"`
|
||||
Hashes fs.CommaSepList `config:"hashes"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
NoClone bool `config:"no_clone"`
|
||||
}
|
||||
@@ -1021,6 +1028,19 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
func (f *Fs) Hashes() hash.Set {
|
||||
if len(f.opt.Hashes) > 0 {
|
||||
// Return only configured hashes.
|
||||
// Note: Could have used hash.SupportOnly to limit supported hashes for all hash related features.
|
||||
var supported hash.Set
|
||||
for _, hashName := range f.opt.Hashes {
|
||||
var ht hash.Type
|
||||
if err := ht.Set(hashName); err != nil {
|
||||
fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
|
||||
}
|
||||
supported.Add(ht)
|
||||
}
|
||||
return supported
|
||||
}
|
||||
return hash.Supported()
|
||||
}
|
||||
|
||||
@@ -1201,7 +1221,15 @@ func (o *Object) Storable() bool {
|
||||
o.fs.objectMetaMu.RLock()
|
||||
mode := o.mode
|
||||
o.fs.objectMetaMu.RUnlock()
|
||||
if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
|
||||
|
||||
// On Windows items with os.ModeIrregular are likely Junction
|
||||
// points so we treat them as symlinks for the purpose of ignoring them.
|
||||
// https://github.com/golang/go/issues/73827
|
||||
symlinkFlag := os.ModeSymlink
|
||||
if runtime.GOOS == "windows" {
|
||||
symlinkFlag |= os.ModeIrregular
|
||||
}
|
||||
if mode&symlinkFlag != 0 && !o.fs.opt.TranslateSymlinks {
|
||||
if !o.fs.opt.SkipSymlinks {
|
||||
fs.Logf(o, "Can't follow symlink without -L/--copy-links")
|
||||
}
|
||||
|
||||
@@ -634,7 +634,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
|
||||
return
|
||||
}
|
||||
|
||||
// itemToEntry converts API item to rclone directory entry
|
||||
// itemToDirEntry converts API item to rclone directory entry
|
||||
// The dirSize return value is:
|
||||
//
|
||||
// <0 - for a file or in case of error
|
||||
|
||||
+21
-1
@@ -17,9 +17,11 @@ Improvements:
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -216,7 +218,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
defer megaCacheMu.Unlock()
|
||||
srv := megaCache[opt.User]
|
||||
if srv == nil {
|
||||
srv = mega.New().SetClient(fshttp.NewClient(ctx))
|
||||
// srv = mega.New().SetClient(fshttp.NewClient(ctx))
|
||||
|
||||
// Workaround for Mega's use of insecure cipher suites which are no longer supported by default since Go 1.22.
|
||||
// Relevant issues:
|
||||
// https://github.com/rclone/rclone/issues/8565
|
||||
// https://github.com/meganz/webclient/issues/103
|
||||
clt := fshttp.NewClient(ctx)
|
||||
clt.Transport = fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
|
||||
var ids []uint16
|
||||
// Read default ciphers
|
||||
for _, cs := range tls.CipherSuites() {
|
||||
ids = append(ids, cs.ID)
|
||||
}
|
||||
// Insecure but Mega uses TLS_RSA_WITH_AES_128_GCM_SHA256 for storage endpoints
|
||||
// (e.g. https://gfs302n114.userstorage.mega.co.nz) as of June 18, 2025.
|
||||
t.TLSClientConfig.CipherSuites = append(ids, tls.TLS_RSA_WITH_AES_128_GCM_SHA256)
|
||||
})
|
||||
srv = mega.New().SetClient(clt)
|
||||
|
||||
srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
|
||||
srv.SetHTTPS(opt.UseHTTPS)
|
||||
srv.SetLogger(func(format string, v ...any) {
|
||||
|
||||
@@ -155,6 +155,7 @@ func (f *Fs) getFile(ctx context.Context, ID string) (info *api.File, err error)
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
|
||||
if err == nil && !info.Links.ApplicationOctetStream.Valid() {
|
||||
time.Sleep(5 * time.Second)
|
||||
return true, errors.New("no link")
|
||||
}
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/rclone/rclone/backend/pikpak/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
bufferSize = 1024 * 1024 // default size of the pages used in the reader
|
||||
bufferCacheSize = 64 // max number of buffers to keep in cache
|
||||
bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long
|
||||
)
|
||||
|
||||
// bufferPool is a global pool of buffers
|
||||
var (
|
||||
bufferPool *pool.Pool
|
||||
bufferPoolOnce sync.Once
|
||||
)
|
||||
|
||||
// get a buffer pool
|
||||
func getPool() *pool.Pool {
|
||||
bufferPoolOnce.Do(func() {
|
||||
ci := fs.GetConfig(context.Background())
|
||||
// Initialise the buffer pool when used
|
||||
bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)
|
||||
})
|
||||
return bufferPool
|
||||
}
|
||||
|
||||
// NewRW gets a pool.RW using the multipart pool
|
||||
func NewRW() *pool.RW {
|
||||
return pool.NewRW(getPool())
|
||||
}
|
||||
|
||||
// Upload does a multipart upload in parallel
|
||||
func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
|
||||
// make concurrency machinery
|
||||
tokens := pacer.NewTokenDispenser(w.con)
|
||||
|
||||
uploadCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
defer atexit.OnError(&err, func() {
|
||||
cancel()
|
||||
fs.Debugf(w.o, "multipart upload: Cancelling...")
|
||||
errCancel := w.Abort(ctx)
|
||||
if errCancel != nil {
|
||||
fs.Debugf(w.o, "multipart upload: failed to cancel: %v", errCancel)
|
||||
}
|
||||
})()
|
||||
|
||||
var (
|
||||
g, gCtx = errgroup.WithContext(uploadCtx)
|
||||
finished = false
|
||||
off int64
|
||||
size = w.size
|
||||
chunkSize = w.chunkSize
|
||||
)
|
||||
|
||||
// Do the accounting manually
|
||||
in, acc := accounting.UnWrapAccounting(w.in)
|
||||
|
||||
for partNum := int64(0); !finished; partNum++ {
|
||||
// Get a block of memory from the pool and token which limits concurrency.
|
||||
tokens.Get()
|
||||
rw := NewRW()
|
||||
if acc != nil {
|
||||
rw.SetAccounting(acc.AccountRead)
|
||||
}
|
||||
|
||||
free := func() {
|
||||
// return the memory and token
|
||||
_ = rw.Close() // Can't return an error
|
||||
tokens.Put()
|
||||
}
|
||||
|
||||
// Fail fast, in case an errgroup managed function returns an error
|
||||
// gCtx is cancelled. There is no point in uploading all the other parts.
|
||||
if gCtx.Err() != nil {
|
||||
free()
|
||||
break
|
||||
}
|
||||
|
||||
// Read the chunk
|
||||
var n int64
|
||||
n, err = io.CopyN(rw, in, chunkSize)
|
||||
if err == io.EOF {
|
||||
if n == 0 && partNum != 0 { // end if no data and if not first chunk
|
||||
free()
|
||||
break
|
||||
}
|
||||
finished = true
|
||||
} else if err != nil {
|
||||
free()
|
||||
return fmt.Errorf("multipart upload: failed to read source: %w", err)
|
||||
}
|
||||
|
||||
partNum := partNum
|
||||
partOff := off
|
||||
off += n
|
||||
g.Go(func() (err error) {
|
||||
defer free()
|
||||
fs.Debugf(w.o, "multipart upload: starting chunk %d size %v offset %v/%v", partNum, fs.SizeSuffix(n), fs.SizeSuffix(partOff), fs.SizeSuffix(size))
|
||||
_, err = w.WriteChunk(gCtx, int32(partNum), rw)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
err = g.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = w.Close(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multipart upload: failed to finalise: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var warnStreamUpload sync.Once
|
||||
|
||||
// state of ChunkWriter
|
||||
type pikpakChunkWriter struct {
|
||||
chunkSize int64
|
||||
size int64
|
||||
con int
|
||||
f *Fs
|
||||
o *Object
|
||||
in io.Reader
|
||||
mu sync.Mutex
|
||||
completedParts []types.CompletedPart
|
||||
client *s3.Client
|
||||
mOut *s3.CreateMultipartUploadOutput
|
||||
}
|
||||
|
||||
func (f *Fs) newChunkWriter(ctx context.Context, remote string, size int64, p *api.ResumableParams, in io.Reader, options ...fs.OpenOption) (w *pikpakChunkWriter, err error) {
|
||||
// Temporary Object under construction
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
|
||||
// calculate size of parts
|
||||
chunkSize := f.opt.ChunkSize
|
||||
|
||||
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
|
||||
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
|
||||
// 48 GiB which seems like a not too unreasonable limit.
|
||||
if size == -1 {
|
||||
warnStreamUpload.Do(func() {
|
||||
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
|
||||
f.opt.ChunkSize, fs.SizeSuffix(int64(chunkSize)*int64(maxUploadParts)))
|
||||
})
|
||||
} else {
|
||||
chunkSize = chunksize.Calculator(o, size, maxUploadParts, chunkSize)
|
||||
}
|
||||
|
||||
client, err := f.newS3Client(ctx, p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create upload client: %w", err)
|
||||
}
|
||||
w = &pikpakChunkWriter{
|
||||
chunkSize: int64(chunkSize),
|
||||
size: size,
|
||||
con: max(1, f.opt.UploadConcurrency),
|
||||
f: f,
|
||||
o: o,
|
||||
in: in,
|
||||
completedParts: make([]types.CompletedPart, 0),
|
||||
client: client,
|
||||
}
|
||||
|
||||
req := &s3.CreateMultipartUploadInput{
|
||||
Bucket: &p.Bucket,
|
||||
Key: &p.Key,
|
||||
}
|
||||
// Apply upload options
|
||||
for _, option := range options {
|
||||
key, value := option.Header()
|
||||
lowerKey := strings.ToLower(key)
|
||||
switch lowerKey {
|
||||
case "":
|
||||
// ignore
|
||||
case "cache-control":
|
||||
req.CacheControl = aws.String(value)
|
||||
case "content-disposition":
|
||||
req.ContentDisposition = aws.String(value)
|
||||
case "content-encoding":
|
||||
req.ContentEncoding = aws.String(value)
|
||||
case "content-type":
|
||||
req.ContentType = aws.String(value)
|
||||
}
|
||||
}
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
w.mOut, err = w.client.CreateMultipartUpload(ctx, req)
|
||||
return w.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create multipart upload failed: %w", err)
|
||||
}
|
||||
fs.Debugf(w.o, "multipart upload: %q initiated", *w.mOut.UploadId)
|
||||
return
|
||||
}
|
||||
|
||||
// shouldRetry returns a boolean as to whether this err
|
||||
// deserve to be retried. It returns the err as a convenience
|
||||
func (w *pikpakChunkWriter) shouldRetry(ctx context.Context, err error) (bool, error) {
|
||||
if fserrors.ContextError(ctx, &err) {
|
||||
return false, err
|
||||
}
|
||||
if fserrors.ShouldRetry(err) {
|
||||
return true, err
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// add a part number and etag to the completed parts
|
||||
func (w *pikpakChunkWriter) addCompletedPart(part types.CompletedPart) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.completedParts = append(w.completedParts, part)
|
||||
}
|
||||
|
||||
// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
|
||||
func (w *pikpakChunkWriter) WriteChunk(ctx context.Context, chunkNumber int32, reader io.ReadSeeker) (currentChunkSize int64, err error) {
|
||||
if chunkNumber < 0 {
|
||||
err := fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
|
||||
return -1, err
|
||||
}
|
||||
|
||||
partNumber := chunkNumber + 1
|
||||
var res *s3.UploadPartOutput
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
// Discover the size by seeking to the end
|
||||
currentChunkSize, err = reader.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// rewind the reader on retry and after reading md5
|
||||
_, err := reader.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
res, err = w.client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: w.mOut.Bucket,
|
||||
Key: w.mOut.Key,
|
||||
UploadId: w.mOut.UploadId,
|
||||
PartNumber: &partNumber,
|
||||
Body: reader,
|
||||
})
|
||||
if err != nil {
|
||||
if chunkNumber <= 8 {
|
||||
return w.shouldRetry(ctx, err)
|
||||
}
|
||||
// retry all chunks once have done the first few
|
||||
return true, err
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return -1, fmt.Errorf("failed to upload chunk %d with %v bytes: %w", partNumber, currentChunkSize, err)
|
||||
}
|
||||
|
||||
w.addCompletedPart(types.CompletedPart{
|
||||
PartNumber: &partNumber,
|
||||
ETag: res.ETag,
|
||||
})
|
||||
|
||||
fs.Debugf(w.o, "multipart upload: wrote chunk %d with %v bytes", partNumber, currentChunkSize)
|
||||
return currentChunkSize, err
|
||||
}
|
||||
|
||||
// Abort the multipart upload
|
||||
func (w *pikpakChunkWriter) Abort(ctx context.Context) (err error) {
|
||||
// Abort the upload session
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
_, err = w.client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: w.mOut.Bucket,
|
||||
Key: w.mOut.Key,
|
||||
UploadId: w.mOut.UploadId,
|
||||
})
|
||||
return w.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to abort multipart upload %q: %w", *w.mOut.UploadId, err)
|
||||
}
|
||||
fs.Debugf(w.o, "multipart upload: %q aborted", *w.mOut.UploadId)
|
||||
return
|
||||
}
|
||||
|
||||
// Close and finalise the multipart upload
|
||||
func (w *pikpakChunkWriter) Close(ctx context.Context) (err error) {
|
||||
// sort the completed parts by part number
|
||||
sort.Slice(w.completedParts, func(i, j int) bool {
|
||||
return *w.completedParts[i].PartNumber < *w.completedParts[j].PartNumber
|
||||
})
|
||||
// Finalise the upload session
|
||||
err = w.f.pacer.Call(func() (bool, error) {
|
||||
_, err = w.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: w.mOut.Bucket,
|
||||
Key: w.mOut.Key,
|
||||
UploadId: w.mOut.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: w.completedParts,
|
||||
},
|
||||
})
|
||||
return w.shouldRetry(ctx, err)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to complete multipart upload: %w", err)
|
||||
}
|
||||
fs.Debugf(w.o, "multipart upload: %q finished", *w.mOut.UploadId)
|
||||
return
|
||||
}
|
||||
+127
-31
@@ -41,12 +41,10 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/rclone/rclone/backend/pikpak/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/chunksize"
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
@@ -66,17 +64,22 @@ import (
|
||||
|
||||
// Constants
|
||||
const (
|
||||
clientID = "YUMx5nI8ZU8Ap8pm"
|
||||
clientVersion = "2.0.0"
|
||||
packageName = "mypikpak.com"
|
||||
defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
taskWaitTime = 500 * time.Millisecond
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
rootURL = "https://api-drive.mypikpak.com"
|
||||
minChunkSize = fs.SizeSuffix(manager.MinUploadPartSize)
|
||||
defaultUploadConcurrency = manager.DefaultUploadConcurrency
|
||||
clientID = "YUMx5nI8ZU8Ap8pm"
|
||||
clientVersion = "2.0.0"
|
||||
packageName = "mypikpak.com"
|
||||
defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
|
||||
minSleep = 100 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
taskWaitTime = 500 * time.Millisecond
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
rootURL = "https://api-drive.mypikpak.com"
|
||||
|
||||
maxUploadParts = 10000 // Part number must be an integer between 1 and 10000, inclusive.
|
||||
defaultChunkSize = fs.SizeSuffix(1024 * 1024 * 5) // Part size should be in [100KB, 5GB]
|
||||
minChunkSize = 100 * fs.Kibi
|
||||
maxChunkSize = 5 * fs.Gibi
|
||||
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
|
||||
maxUploadCutoff = 5 * fs.Gibi // maximum allowed size for singlepart uploads
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -223,6 +226,14 @@ Fill in for rclone to use a non root folder as its starting point.
|
||||
Help: "Files bigger than this will be cached on disk to calculate hash if required.",
|
||||
Default: fs.SizeSuffix(10 * 1024 * 1024),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: `Cutoff for switching to chunked upload.
|
||||
|
||||
Any files larger than this will be uploaded in chunks of chunk_size.
|
||||
The minimum is 0 and the maximum is 5 GiB.`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Chunk size for multipart uploads.
|
||||
@@ -241,7 +252,7 @@ large file of known size to stay below the 10,000 chunks limit.
|
||||
|
||||
Increasing the chunk size decreases the accuracy of the progress
|
||||
statistics displayed with "-P" flag.`,
|
||||
Default: minChunkSize,
|
||||
Default: defaultChunkSize,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_concurrency",
|
||||
@@ -257,7 +268,7 @@ in memory.
|
||||
If you are uploading small numbers of large files over high-speed links
|
||||
and these uploads do not fully utilize your bandwidth, then increasing
|
||||
this may help to speed up the transfers.`,
|
||||
Default: defaultUploadConcurrency,
|
||||
Default: 4,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
@@ -294,6 +305,7 @@ type Options struct {
|
||||
NoMediaLink bool `config:"no_media_link"`
|
||||
HashMemoryThreshold fs.SizeSuffix `config:"hash_memory_limit"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
@@ -467,6 +479,11 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
|
||||
// when a zero-byte file was uploaded with an invalid captcha token
|
||||
f.rst.captcha.Invalidate()
|
||||
return true, err
|
||||
} else if strings.Contains(apiErr.Reason, "idx.shub.mypikpak.com") && apiErr.Code == 500 {
|
||||
// internal server error: Post "http://idx.shub.mypikpak.com": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
|
||||
// This typically happens when trying to retrieve a gcid for which no record exists.
|
||||
// No retry is needed in this case.
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -524,6 +541,39 @@ func (f *Fs) newClientWithPacer(ctx context.Context) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkUploadChunkSize(cs fs.SizeSuffix) error {
|
||||
if cs < minChunkSize {
|
||||
return fmt.Errorf("%s is less than %s", cs, minChunkSize)
|
||||
}
|
||||
if cs > maxChunkSize {
|
||||
return fmt.Errorf("%s is greater than %s", cs, maxChunkSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
||||
err = checkUploadChunkSize(cs)
|
||||
if err == nil {
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func checkUploadCutoff(cs fs.SizeSuffix) error {
|
||||
if cs > maxUploadCutoff {
|
||||
return fmt.Errorf("%s is greater than %s", cs, maxUploadCutoff)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
|
||||
err = checkUploadCutoff(cs)
|
||||
if err == nil {
|
||||
old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// newFs partially constructs Fs from the path
|
||||
//
|
||||
// It constructs a valid Fs but doesn't attempt to figure out whether
|
||||
@@ -531,11 +581,17 @@ func (f *Fs) newClientWithPacer(ctx context.Context) (err error) {
|
||||
func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, error) {
|
||||
// Parse config into Options struct
|
||||
opt := new(Options)
|
||||
if err := configstruct.Set(m, opt); err != nil {
|
||||
err := configstruct.Set(m, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if opt.ChunkSize < minChunkSize {
|
||||
return nil, fmt.Errorf("chunk size must be at least %s", minChunkSize)
|
||||
err = checkUploadChunkSize(opt.ChunkSize)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pikpak: chunk size: %w", err)
|
||||
}
|
||||
err = checkUploadCutoff(opt.UploadCutoff)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pikpak: upload cutoff: %w", err)
|
||||
}
|
||||
|
||||
root := parsePath(path)
|
||||
@@ -1260,9 +1316,7 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable) (err error) {
|
||||
p := resumable.Params
|
||||
|
||||
func (f *Fs) newS3Client(ctx context.Context, p *api.ResumableParams) (s3Client *s3.Client, err error) {
|
||||
// Create a credentials provider
|
||||
creds := credentials.NewStaticCredentialsProvider(p.AccessKeyID, p.AccessKeySecret, p.SecurityToken)
|
||||
|
||||
@@ -1272,22 +1326,64 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, s
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ci := fs.GetConfig(ctx)
|
||||
cfg.RetryMaxAttempts = ci.LowLevelRetries
|
||||
cfg.HTTPClient = getClient(ctx, &f.opt)
|
||||
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String("https://mypikpak.com/")
|
||||
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
|
||||
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
|
||||
})
|
||||
partSize := chunksize.Calculator(name, size, int(manager.MaxUploadParts), f.opt.ChunkSize)
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// Create an uploader with custom options
|
||||
uploader := manager.NewUploader(client, func(u *manager.Uploader) {
|
||||
u.PartSize = int64(partSize)
|
||||
u.Concurrency = f.opt.UploadConcurrency
|
||||
})
|
||||
// Perform an upload
|
||||
_, err = uploader.Upload(ctx, &s3.PutObjectInput{
|
||||
func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable, options ...fs.OpenOption) (err error) {
|
||||
p := resumable.Params
|
||||
|
||||
if size < 0 || size >= int64(f.opt.UploadCutoff) {
|
||||
mu, err := f.newChunkWriter(ctx, name, size, p, in, options...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multipart upload failed to initialise: %w", err)
|
||||
}
|
||||
return mu.Upload(ctx)
|
||||
}
|
||||
|
||||
// upload singlepart
|
||||
client, err := f.newS3Client(ctx, p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create upload client: %w", err)
|
||||
}
|
||||
req := &s3.PutObjectInput{
|
||||
Bucket: &p.Bucket,
|
||||
Key: &p.Key,
|
||||
Body: in,
|
||||
Body: io.NopCloser(in),
|
||||
}
|
||||
// Apply upload options
|
||||
for _, option := range options {
|
||||
key, value := option.Header()
|
||||
lowerKey := strings.ToLower(key)
|
||||
switch lowerKey {
|
||||
case "":
|
||||
// ignore
|
||||
case "cache-control":
|
||||
req.CacheControl = aws.String(value)
|
||||
case "content-disposition":
|
||||
req.ContentDisposition = aws.String(value)
|
||||
case "content-encoding":
|
||||
req.ContentEncoding = aws.String(value)
|
||||
case "content-type":
|
||||
req.ContentType = aws.String(value)
|
||||
}
|
||||
}
|
||||
var s3opts = []func(*s3.Options){}
|
||||
// Can't retry single part uploads as only have an io.Reader
|
||||
s3opts = append(s3opts, func(o *s3.Options) {
|
||||
o.RetryMaxAttempts = 1
|
||||
})
|
||||
err = f.pacer.CallNoRetry(func() (bool, error) {
|
||||
_, err = client.PutObject(ctx, req, s3opts...)
|
||||
return f.shouldRetry(ctx, nil, err)
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -1345,7 +1441,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, gcid string,
|
||||
if uploadType == api.UploadTypeForm && new.Form != nil {
|
||||
err = f.uploadByForm(ctx, in, req.Name, size, new.Form, options...)
|
||||
} else if uploadType == api.UploadTypeResumable && new.Resumable != nil {
|
||||
err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable)
|
||||
err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable, options...)
|
||||
} else {
|
||||
err = fmt.Errorf("no method available for uploading: %+v", new)
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Test PikPak filesystem interface
|
||||
package pikpak_test
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/pikpak"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
@@ -12,6 +12,23 @@ import (
|
||||
func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestPikPak:",
|
||||
NilObject: (*pikpak.Object)(nil),
|
||||
NilObject: (*Object)(nil),
|
||||
ChunkedUpload: fstests.ChunkedUploadConfig{
|
||||
MinChunkSize: minChunkSize,
|
||||
MaxChunkSize: maxChunkSize,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
return f.setUploadChunkSize(cs)
|
||||
}
|
||||
|
||||
func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
|
||||
return f.setUploadCutoff(cs)
|
||||
}
|
||||
|
||||
var (
|
||||
_ fstests.SetUploadChunkSizer = (*Fs)(nil)
|
||||
_ fstests.SetUploadCutoffer = (*Fs)(nil)
|
||||
)
|
||||
+27
-3
@@ -185,6 +185,9 @@ var providerOption = fs.Option{
|
||||
}, {
|
||||
Value: "Qiniu",
|
||||
Help: "Qiniu Object Storage (Kodo)",
|
||||
}, {
|
||||
Value: "Zata",
|
||||
Help: "Zata (S3 compatible Gateway)",
|
||||
}, {
|
||||
Value: "Other",
|
||||
Help: "Any other S3 compatible provider",
|
||||
@@ -490,6 +493,14 @@ func init() {
|
||||
Value: "ap-northeast-1",
|
||||
Help: "Northeast Asia Region 1.\nNeeds location constraint ap-northeast-1.",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region where you can connect with.\n",
|
||||
Provider: "Zata",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "us-east-1",
|
||||
Help: "Indore, Madhya Pradesh, India",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region where your bucket will be created and your data stored.\n",
|
||||
@@ -576,7 +587,7 @@ func init() {
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
|
||||
Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega",
|
||||
Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega,Zata",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
|
||||
@@ -1380,6 +1391,14 @@ func init() {
|
||||
Value: "s3-ap-northeast-1.qiniucs.com",
|
||||
Help: "Northeast Asia Endpoint 1",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for Zata Object Storage.",
|
||||
Provider: "Zata",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "idr01.zata.ai",
|
||||
Help: "South Asia Endpoint",
|
||||
}},
|
||||
}, {
|
||||
// Selectel endpoints: https://docs.selectel.ru/en/cloud/object-storage/manage/domains/#s3-api-domains
|
||||
Name: "endpoint",
|
||||
@@ -1392,7 +1411,7 @@ func init() {
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
|
||||
Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
|
||||
Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox,Zata",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "objects-us-east-1.dream.io",
|
||||
Help: "Dream Objects endpoint",
|
||||
@@ -2616,7 +2635,7 @@ The parameter should be a date, "2006-01-02", datetime "2006-01-02
|
||||
Note that when using this no file write operations are permitted,
|
||||
so you can't upload files or delete them.
|
||||
|
||||
See [the time option docs](/docs/#time-option) for valid formats.
|
||||
See [the time option docs](/docs/#time-options) for valid formats.
|
||||
`,
|
||||
Default: fs.Time{},
|
||||
Advanced: true,
|
||||
@@ -3631,6 +3650,11 @@ func setQuirks(opt *Options) {
|
||||
urlEncodeListings = false
|
||||
virtualHostStyle = false
|
||||
useAlreadyExists = false // untested
|
||||
case "Zata":
|
||||
useMultipartEtag = false
|
||||
mightGzip = false
|
||||
useUnsignedPayload = false
|
||||
useAlreadyExists = false
|
||||
case "Exaba":
|
||||
virtualHostStyle = false
|
||||
case "GCS":
|
||||
|
||||
@@ -111,7 +111,8 @@ func init() {
|
||||
encoder.EncodeSlash |
|
||||
encoder.EncodeBackSlash |
|
||||
encoder.EncodeDoubleQuote |
|
||||
encoder.EncodeInvalidUtf8),
|
||||
encoder.EncodeInvalidUtf8 |
|
||||
encoder.EncodeDot),
|
||||
}},
|
||||
})
|
||||
}
|
||||
|
||||
+214
-63
@@ -222,15 +222,45 @@ E.g. the second example above should be rewritten as:
|
||||
Help: "Windows Command Prompt",
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: "hashes",
|
||||
Help: `Comma separated list of supported checksum types.`,
|
||||
Default: fs.CommaSepList{},
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "md5sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read md5 hashes.\n\nLeave blank for autodetect.",
|
||||
Help: "The command used to read MD5 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "sha1sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read sha1 hashes.\n\nLeave blank for autodetect.",
|
||||
Help: "The command used to read SHA-1 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "crc32sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read CRC-32 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "sha256sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read SHA-256 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "blake3sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read BLAKE3 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "xxh3sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read XXH3 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "xxh128sum_command",
|
||||
Default: "",
|
||||
Help: "The command used to read XXH128 hashes.\n\nLeave blank for autodetect.",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "skip_links",
|
||||
@@ -535,8 +565,14 @@ type Options struct {
|
||||
PathOverride string `config:"path_override"`
|
||||
SetModTime bool `config:"set_modtime"`
|
||||
ShellType string `config:"shell_type"`
|
||||
Hashes fs.CommaSepList `config:"hashes"`
|
||||
Md5sumCommand string `config:"md5sum_command"`
|
||||
Sha1sumCommand string `config:"sha1sum_command"`
|
||||
Crc32sumCommand string `config:"crc32sum_command"`
|
||||
Sha256sumCommand string `config:"sha256sum_command"`
|
||||
Blake3sumCommand string `config:"blake3sum_command"`
|
||||
Xxh3sumCommand string `config:"xxh3sum_command"`
|
||||
Xxh128sumCommand string `config:"xxh128sum_command"`
|
||||
SkipLinks bool `config:"skip_links"`
|
||||
Subsystem string `config:"subsystem"`
|
||||
ServerCommand string `config:"server_command"`
|
||||
@@ -585,13 +621,18 @@ type Fs struct {
|
||||
|
||||
// Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
|
||||
type Object struct {
|
||||
fs *Fs
|
||||
remote string
|
||||
size int64 // size of the object
|
||||
modTime uint32 // modification time of the object as unix time
|
||||
mode os.FileMode // mode bits from the file
|
||||
md5sum *string // Cached MD5 checksum
|
||||
sha1sum *string // Cached SHA1 checksum
|
||||
fs *Fs
|
||||
remote string
|
||||
size int64 // size of the object
|
||||
modTime uint32 // modification time of the object as unix time
|
||||
mode os.FileMode // mode bits from the file
|
||||
md5sum *string // Cached MD5 checksum
|
||||
sha1sum *string // Cached SHA-1 checksum
|
||||
crc32sum *string // Cached CRC-32 checksum
|
||||
sha256sum *string // Cached SHA-256 checksum
|
||||
blake3sum *string // Cached BLAKE3 checksum
|
||||
xxh3sum *string // Cached XXH3 checksum
|
||||
xxh128sum *string // Cached XXH128 checksum
|
||||
}
|
||||
|
||||
// conn encapsulates an ssh client and corresponding sftp client
|
||||
@@ -891,7 +932,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
User: opt.User,
|
||||
Auth: []ssh.AuthMethod{},
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: f.ci.ConnectTimeout,
|
||||
Timeout: time.Duration(f.ci.ConnectTimeout),
|
||||
ClientVersion: "SSH-2.0-" + f.ci.UserAgent,
|
||||
}
|
||||
|
||||
@@ -1643,14 +1684,112 @@ func (f *Fs) Hashes() hash.Set {
|
||||
return *f.cachedHashes
|
||||
}
|
||||
|
||||
hashSet := hash.NewHashSet()
|
||||
f.cachedHashes = &hashSet
|
||||
hashTypesSupported := hash.NewHashSet()
|
||||
f.cachedHashes = &hashTypesSupported
|
||||
|
||||
if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported {
|
||||
return hashSet
|
||||
return hashTypesSupported
|
||||
}
|
||||
|
||||
hashTypes := hash.NewHashSet()
|
||||
if len(f.opt.Hashes) > 0 {
|
||||
for _, hashName := range f.opt.Hashes {
|
||||
var hashType hash.Type
|
||||
if err := hashType.Set(hashName); err != nil {
|
||||
fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
|
||||
}
|
||||
hashTypes.Add(hashType)
|
||||
}
|
||||
} else {
|
||||
hashTypes.Add(hash.MD5, hash.SHA1)
|
||||
}
|
||||
|
||||
hashCommands := map[hash.Type]struct {
|
||||
option *string
|
||||
emptyHash string
|
||||
hashCommands []struct{ hashFile, hashEmpty string }
|
||||
}{
|
||||
hash.MD5: {
|
||||
&f.opt.Md5sumCommand,
|
||||
"d41d8cd98f00b204e9800998ecf8427e",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"md5sum", "md5sum"},
|
||||
{"md5 -r", "md5 -r"},
|
||||
{"rclone md5sum", "rclone md5sum"},
|
||||
},
|
||||
},
|
||||
hash.SHA1: {
|
||||
&f.opt.Sha1sumCommand,
|
||||
"da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"sha1sum", "sha1sum"},
|
||||
{"sha1 -r", "sha1 -r"},
|
||||
{"rclone sha1sum", "rclone sha1sum"},
|
||||
},
|
||||
},
|
||||
hash.CRC32: {
|
||||
&f.opt.Sha1sumCommand,
|
||||
"00000000",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"crc32", "crc32"},
|
||||
{"rclone hashsum crc32", "rclone hashsum crc32"},
|
||||
},
|
||||
},
|
||||
hash.SHA256: {
|
||||
&f.opt.Sha256sumCommand,
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"sha256sum", "sha1sum"},
|
||||
{"sha256 -r", "sha1 -r"},
|
||||
{"rclone hashsum sha256", "rclone hashsum sha256"},
|
||||
},
|
||||
},
|
||||
hash.BLAKE3: {
|
||||
&f.opt.Blake3sumCommand,
|
||||
"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"b3sum", "b3sum"},
|
||||
{"rclone hashsum blake3", "rclone hashsum blake3"},
|
||||
},
|
||||
},
|
||||
hash.XXH3: {
|
||||
&f.opt.Xxh3sumCommand,
|
||||
"2d06800538d394c2",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
// The xxhsum tool uses a non-standard prefix "XXH3_" preceding the hash output for the 64-bit variant
|
||||
// of XXH3, to avoid confusion with the older 64-bit algorithm XXH64. This was introduced in version
|
||||
// 0.8.3 released Dec 30, 2024. Older versions only supported the alternative BSD style output format,
|
||||
// otherwise optional with argument --tag. We are currently not expecting these output formats and can
|
||||
// therefore not use the "xxhsum -H3" command or its xxh3sum alias directly.
|
||||
//{"xxh3sum", "xxh3sum"},
|
||||
//{"xxhsum -H3", "xxhsum -H3"},
|
||||
{"rclone hashsum xxh3", "rclone hashsum xxh3"},
|
||||
},
|
||||
},
|
||||
hash.XXH128: {
|
||||
&f.opt.Xxh128sumCommand,
|
||||
"99aa06d3014798d86001c324468d497f",
|
||||
[]struct{ hashFile, hashEmpty string }{
|
||||
{"xxh128sum", "xxh128sum"},
|
||||
{"xxhsum -H2", "xxhsum -H2"},
|
||||
{"rclone hashsum xxh128", "rclone hashsum xxh128"},
|
||||
},
|
||||
},
|
||||
}
|
||||
if f.shellType == "powershell" {
|
||||
for _, hashType := range []hash.Type{hash.MD5, hash.SHA1, hash.SHA256} {
|
||||
if entry, ok := hashCommands[hashType]; ok {
|
||||
entry.hashCommands = append(hashCommands[hashType].hashCommands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
fmt.Sprintf("&{param($Path);Get-FileHash -Algorithm %v -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}", hashType),
|
||||
fmt.Sprintf("Get-FileHash -Algorithm %v -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", hashType),
|
||||
})
|
||||
hashCommands[hashType] = entry
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// look for a hash command which works
|
||||
checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool {
|
||||
if *hashCommand == hashCommandNotSupported {
|
||||
return false
|
||||
@@ -1679,55 +1818,25 @@ func (f *Fs) Hashes() hash.Set {
|
||||
}
|
||||
|
||||
changed := false
|
||||
md5Commands := []struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
{"md5sum", "md5sum"},
|
||||
{"md5 -r", "md5 -r"},
|
||||
{"rclone md5sum", "rclone md5sum"},
|
||||
for _, hashType := range hashTypes.Array() {
|
||||
if entry, ok := hashCommands[hashType]; ok {
|
||||
if works := checkHash(hashType, entry.hashCommands, entry.emptyHash, entry.option, &changed); works {
|
||||
hashTypesSupported.Add(hashType)
|
||||
}
|
||||
}
|
||||
}
|
||||
sha1Commands := []struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
{"sha1sum", "sha1sum"},
|
||||
{"sha1 -r", "sha1 -r"},
|
||||
{"rclone sha1sum", "rclone sha1sum"},
|
||||
}
|
||||
if f.shellType == "powershell" {
|
||||
md5Commands = append(md5Commands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
"&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
|
||||
"Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
|
||||
})
|
||||
|
||||
sha1Commands = append(sha1Commands, struct {
|
||||
hashFile, hashEmpty string
|
||||
}{
|
||||
"&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}",
|
||||
"Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
|
||||
})
|
||||
}
|
||||
|
||||
md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
|
||||
sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
|
||||
|
||||
if changed {
|
||||
// Save permanently in config to avoid the extra work next time
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand)
|
||||
f.m.Set("md5sum_command", f.opt.Md5sumCommand)
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand)
|
||||
f.m.Set("sha1sum_command", f.opt.Sha1sumCommand)
|
||||
for _, hashType := range hashTypes.Array() {
|
||||
if entry, ok := hashCommands[hashType]; ok {
|
||||
fs.Debugf(f, "Setting hash command for %v to %q (set %vsum_command to override)", hashType, *entry.option, hashType)
|
||||
f.m.Set(fmt.Sprintf("%vsum_command", hashType), *entry.option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sha1Works {
|
||||
hashSet.Add(hash.SHA1)
|
||||
}
|
||||
if md5Works {
|
||||
hashSet.Add(hash.MD5)
|
||||
}
|
||||
|
||||
return hashSet
|
||||
return hashTypesSupported
|
||||
}
|
||||
|
||||
// About gets usage stats
|
||||
@@ -1862,17 +1971,43 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
_ = o.fs.Hashes()
|
||||
|
||||
var hashCmd string
|
||||
if r == hash.MD5 {
|
||||
switch r {
|
||||
case hash.MD5:
|
||||
if o.md5sum != nil {
|
||||
return *o.md5sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Md5sumCommand
|
||||
} else if r == hash.SHA1 {
|
||||
case hash.SHA1:
|
||||
if o.sha1sum != nil {
|
||||
return *o.sha1sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Sha1sumCommand
|
||||
} else {
|
||||
case hash.CRC32:
|
||||
if o.crc32sum != nil {
|
||||
return *o.crc32sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Crc32sumCommand
|
||||
case hash.SHA256:
|
||||
if o.sha256sum != nil {
|
||||
return *o.sha256sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Sha256sumCommand
|
||||
case hash.BLAKE3:
|
||||
if o.blake3sum != nil {
|
||||
return *o.blake3sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Blake3sumCommand
|
||||
case hash.XXH3:
|
||||
if o.xxh3sum != nil {
|
||||
return *o.xxh3sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Xxh3sumCommand
|
||||
case hash.XXH128:
|
||||
if o.xxh128sum != nil {
|
||||
return *o.xxh128sum, nil
|
||||
}
|
||||
hashCmd = o.fs.opt.Xxh128sumCommand
|
||||
default:
|
||||
return "", hash.ErrUnsupported
|
||||
}
|
||||
if hashCmd == "" || hashCmd == hashCommandNotSupported {
|
||||
@@ -1889,10 +2024,21 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
}
|
||||
hashString := parseHash(outBytes)
|
||||
fs.Debugf(o, "Parsed hash: %s", hashString)
|
||||
if r == hash.MD5 {
|
||||
switch r {
|
||||
case hash.MD5:
|
||||
o.md5sum = &hashString
|
||||
} else if r == hash.SHA1 {
|
||||
case hash.SHA1:
|
||||
o.sha1sum = &hashString
|
||||
case hash.CRC32:
|
||||
o.crc32sum = &hashString
|
||||
case hash.SHA256:
|
||||
o.sha256sum = &hashString
|
||||
case hash.BLAKE3:
|
||||
o.blake3sum = &hashString
|
||||
case hash.XXH3:
|
||||
o.xxh3sum = &hashString
|
||||
case hash.XXH128:
|
||||
o.xxh128sum = &hashString
|
||||
}
|
||||
return hashString, nil
|
||||
}
|
||||
@@ -1957,7 +2103,7 @@ func (f *Fs) remoteShellPath(remote string) string {
|
||||
}
|
||||
|
||||
// Converts a byte array from the SSH session returned by
|
||||
// an invocation of md5sum/sha1sum to a hash string
|
||||
// an invocation of hash command to a hash string
|
||||
// as expected by the rest of this application
|
||||
func parseHash(bytes []byte) string {
|
||||
// For strings with backslash *sum writes a leading \
|
||||
@@ -2186,6 +2332,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
// Clear the hash cache since we are about to update the object
|
||||
o.md5sum = nil
|
||||
o.sha1sum = nil
|
||||
o.crc32sum = nil
|
||||
o.sha256sum = nil
|
||||
o.blake3sum = nil
|
||||
o.xxh3sum = nil
|
||||
o.xxh128sum = nil
|
||||
c, err := o.fs.getSftpConnection(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Update: %w", err)
|
||||
|
||||
@@ -38,7 +38,7 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
|
||||
|
||||
d := &smb2.Dialer{}
|
||||
if f.opt.UseKerberos {
|
||||
cl, err := getKerberosClient()
|
||||
cl, err := createKerberosClient(f.opt.KerberosCCache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+54
-38
@@ -14,65 +14,81 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
kerberosClient *client.Client
|
||||
kerberosErr error
|
||||
kerberosOnce sync.Once
|
||||
kerberosClient sync.Map // map[string]*client.Client
|
||||
kerberosErr sync.Map // map[string]error
|
||||
)
|
||||
|
||||
// getKerberosClient returns a Kerberos client that can be used to authenticate.
|
||||
func getKerberosClient() (*client.Client, error) {
|
||||
if kerberosClient == nil || kerberosErr == nil {
|
||||
kerberosOnce.Do(func() {
|
||||
kerberosClient, kerberosErr = createKerberosClient()
|
||||
})
|
||||
func resolveCcachePath(ccachePath string) (string, error) {
|
||||
if ccachePath == "" {
|
||||
ccachePath = os.Getenv("KRB5CCNAME")
|
||||
}
|
||||
|
||||
return kerberosClient, kerberosErr
|
||||
}
|
||||
|
||||
// createKerberosClient creates a new Kerberos client.
|
||||
func createKerberosClient() (*client.Client, error) {
|
||||
cfgPath := os.Getenv("KRB5_CONFIG")
|
||||
if cfgPath == "" {
|
||||
cfgPath = "/etc/krb5.conf"
|
||||
}
|
||||
|
||||
cfg, err := config.Load(cfgPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Determine the ccache location from the environment, falling back to the
|
||||
// default location.
|
||||
ccachePath := os.Getenv("KRB5CCNAME")
|
||||
switch {
|
||||
case strings.Contains(ccachePath, ":"):
|
||||
parts := strings.SplitN(ccachePath, ":", 2)
|
||||
switch parts[0] {
|
||||
prefix, path := parts[0], parts[1]
|
||||
switch prefix {
|
||||
case "FILE":
|
||||
ccachePath = parts[1]
|
||||
return path, nil
|
||||
case "DIR":
|
||||
primary, err := os.ReadFile(filepath.Join(parts[1], "primary"))
|
||||
primary, err := os.ReadFile(filepath.Join(path, "primary"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return "", err
|
||||
}
|
||||
ccachePath = filepath.Join(parts[1], strings.TrimSpace(string(primary)))
|
||||
return filepath.Join(path, strings.TrimSpace(string(primary))), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
|
||||
return "", fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
|
||||
}
|
||||
case ccachePath == "":
|
||||
u, err := user.Current()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return "", err
|
||||
}
|
||||
|
||||
ccachePath = "/tmp/krb5cc_" + u.Uid
|
||||
return "/tmp/krb5cc_" + u.Uid, nil
|
||||
default:
|
||||
return ccachePath, nil
|
||||
}
|
||||
}
|
||||
|
||||
ccache, err := credentials.LoadCCache(ccachePath)
|
||||
func loadKerberosConfig() (*config.Config, error) {
|
||||
cfgPath := os.Getenv("KRB5_CONFIG")
|
||||
if cfgPath == "" {
|
||||
cfgPath = "/etc/krb5.conf"
|
||||
}
|
||||
return config.Load(cfgPath)
|
||||
}
|
||||
|
||||
// createKerberosClient creates a new Kerberos client.
|
||||
func createKerberosClient(ccachePath string) (*client.Client, error) {
|
||||
ccachePath, err := resolveCcachePath(ccachePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return client.NewFromCCache(ccache, cfg)
|
||||
// check if we already have a client or an error for this ccache path
|
||||
if errVal, ok := kerberosErr.Load(ccachePath); ok {
|
||||
return nil, errVal.(error)
|
||||
}
|
||||
if clientVal, ok := kerberosClient.Load(ccachePath); ok {
|
||||
return clientVal.(*client.Client), nil
|
||||
}
|
||||
|
||||
// create a new client if not found in the map
|
||||
cfg, err := loadKerberosConfig()
|
||||
if err != nil {
|
||||
kerberosErr.Store(ccachePath, err)
|
||||
return nil, err
|
||||
}
|
||||
ccache, err := credentials.LoadCCache(ccachePath)
|
||||
if err != nil {
|
||||
kerberosErr.Store(ccachePath, err)
|
||||
return nil, err
|
||||
}
|
||||
cl, err := client.NewFromCCache(ccache, cfg)
|
||||
if err != nil {
|
||||
kerberosErr.Store(ccachePath, err)
|
||||
return nil, err
|
||||
}
|
||||
kerberosClient.Store(ccachePath, cl)
|
||||
return cl, nil
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package smb
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestResolveCcachePath(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
// Setup: files for FILE and DIR modes
|
||||
fileCcache := filepath.Join(tmpDir, "file_ccache")
|
||||
err := os.WriteFile(fileCcache, []byte{}, 0600)
|
||||
assert.NoError(t, err)
|
||||
|
||||
dirCcache := filepath.Join(tmpDir, "dir_ccache")
|
||||
err = os.Mkdir(dirCcache, 0755)
|
||||
assert.NoError(t, err)
|
||||
err = os.WriteFile(filepath.Join(dirCcache, "primary"), []byte("ticket"), 0600)
|
||||
assert.NoError(t, err)
|
||||
dirCcacheTicket := filepath.Join(dirCcache, "ticket")
|
||||
err = os.WriteFile(dirCcacheTicket, []byte{}, 0600)
|
||||
assert.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ccachePath string
|
||||
envKRB5CCNAME string
|
||||
expected string
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "FILE: prefix from env",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "FILE:" + fileCcache,
|
||||
expected: fileCcache,
|
||||
},
|
||||
{
|
||||
name: "DIR: prefix from env",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "DIR:" + dirCcache,
|
||||
expected: dirCcacheTicket,
|
||||
},
|
||||
{
|
||||
name: "Unsupported prefix",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "MEMORY:/bad/path",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "Direct file path (no prefix)",
|
||||
ccachePath: "/tmp/myccache",
|
||||
expected: "/tmp/myccache",
|
||||
},
|
||||
{
|
||||
name: "Default to /tmp/krb5cc_<uid>",
|
||||
ccachePath: "",
|
||||
envKRB5CCNAME: "",
|
||||
expected: "/tmp/krb5cc_",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Setenv("KRB5CCNAME", tt.envKRB5CCNAME)
|
||||
result, err := resolveCcachePath(tt.ccachePath)
|
||||
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.Contains(t, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -107,6 +107,20 @@ Set to 0 to keep connections indefinitely.
|
||||
Help: "Whether the server is configured to be case-insensitive.\n\nAlways true on Windows shares.",
|
||||
Default: true,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "kerberos_ccache",
|
||||
Help: `Path to the Kerberos credential cache (krb5cc).
|
||||
|
||||
Overrides the default KRB5CCNAME environment variable and allows this
|
||||
instance of the SMB backend to use a different Kerberos cache file.
|
||||
This is useful when mounting multiple SMB with different credentials
|
||||
or running in multi-user environments.
|
||||
|
||||
Supported formats:
|
||||
- FILE:/path/to/ccache – Use the specified file.
|
||||
- DIR:/path/to/ccachedir – Use the primary file inside the specified directory.
|
||||
- /path/to/ccache – Interpreted as a file path.`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -137,6 +151,7 @@ type Options struct {
|
||||
Domain string `config:"domain"`
|
||||
SPN string `config:"spn"`
|
||||
UseKerberos bool `config:"use_kerberos"`
|
||||
KerberosCCache string `config:"kerberos_ccache"`
|
||||
HideSpecial bool `config:"hide_special_share"`
|
||||
CaseInsensitive bool `config:"case_insensitive"`
|
||||
IdleTimeout fs.Duration `config:"idle_timeout"`
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/backend/smb"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
@@ -18,6 +19,9 @@ func TestIntegration(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIntegration2(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("skipping as -remote is set")
|
||||
}
|
||||
krb5Dir := t.TempDir()
|
||||
t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
|
||||
t.Setenv("KRB5CCNAME", filepath.Join(krb5Dir, "ccache"))
|
||||
@@ -26,3 +30,24 @@ func TestIntegration2(t *testing.T) {
|
||||
NilObject: (*smb.Object)(nil),
|
||||
})
|
||||
}
|
||||
|
||||
func TestIntegration3(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("skipping as -remote is set")
|
||||
}
|
||||
|
||||
krb5Dir := t.TempDir()
|
||||
t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
|
||||
ccache := filepath.Join(krb5Dir, "ccache")
|
||||
t.Setenv("RCLONE_TEST_CUSTOM_CCACHE_LOCATION", ccache)
|
||||
|
||||
name := "TestSMBKerberosCcache"
|
||||
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":rclone",
|
||||
NilObject: (*smb.Object)(nil),
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "kerberos_ccache", Value: ccache},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -491,8 +491,8 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
|
||||
ApplicationCredentialName: opt.ApplicationCredentialName,
|
||||
ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
|
||||
EndpointType: swift.EndpointType(opt.EndpointType),
|
||||
ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
|
||||
Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
|
||||
ConnectTimeout: time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transport
|
||||
Timeout: time.Duration(10 * ci.Timeout), // Use the timeouts in the transport
|
||||
Transport: fshttp.NewTransport(ctx),
|
||||
FetchUntilEmptyPage: opt.FetchUntilEmptyPage,
|
||||
PartialPageFetchThreshold: opt.PartialPageFetchThreshold,
|
||||
|
||||
@@ -1550,7 +1550,7 @@ func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string
|
||||
extraHeaders := map[string]string{}
|
||||
if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
|
||||
if o.fs.useOCMtime {
|
||||
extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", o.modTime.Unix())
|
||||
extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
|
||||
}
|
||||
// Set one upload checksum
|
||||
// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
|
||||
|
||||
+2
-1
@@ -12,4 +12,5 @@
|
||||
<seb•ɑƬ•chezwam•ɖɵʈ•org>
|
||||
<allllaboutyou@gmail.com>
|
||||
<psycho@feltzv.fr>
|
||||
<afw5059@gmail.com>
|
||||
<afw5059@gmail.com>
|
||||
<piyushgarg80>
|
||||
@@ -278,7 +278,7 @@ func testBisync(t *testing.T, path1, path2 string) {
|
||||
}
|
||||
bisync.Colors = true
|
||||
time.Local = bisync.TZ
|
||||
ci.FsCacheExpireDuration = 5 * time.Hour
|
||||
ci.FsCacheExpireDuration = fs.Duration(5 * time.Hour)
|
||||
|
||||
baseDir, err := os.Getwd()
|
||||
require.NoError(t, err, "get current directory")
|
||||
|
||||
+2
-2
@@ -55,7 +55,7 @@ type Options struct {
|
||||
Compare CompareOpt
|
||||
CompareFlag string
|
||||
DebugName string
|
||||
MaxLock time.Duration
|
||||
MaxLock fs.Duration
|
||||
ConflictResolve Prefer
|
||||
ConflictLoser ConflictLoserAction
|
||||
ConflictSuffixFlag string
|
||||
@@ -146,7 +146,7 @@ func init() {
|
||||
flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")
|
||||
flags.BoolVarP(cmdFlags, &Opt.Compare.DownloadHash, "download-hash", "", Opt.Compare.DownloadHash, "Compute hash by downloading when otherwise unavailable. (warning: may be slow and use lots of data!)", "")
|
||||
flags.DurationVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", Opt.MaxLock, "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
|
||||
flags.FVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
|
||||
flags.FVarP(cmdFlags, &Opt.ConflictResolve, "conflict-resolve", "", "Automatically resolve conflicts by preferring the version that is: "+ConflictResolveList+" (default: none)", "")
|
||||
flags.FVarP(cmdFlags, &Opt.ConflictLoser, "conflict-loser", "", "Action to take on the loser of a sync conflict (when there is a winner) or on both files (when there is no winner): "+ConflictLoserList+" (default: num)", "")
|
||||
flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/terminal"
|
||||
)
|
||||
|
||||
const basicallyforever = 200 * 365 * 24 * time.Hour
|
||||
const basicallyforever = fs.Duration(200 * 365 * 24 * time.Hour)
|
||||
|
||||
var stopRenewal func()
|
||||
|
||||
@@ -66,9 +66,9 @@ func (b *bisyncRun) removeLockFile() {
|
||||
}
|
||||
|
||||
func (b *bisyncRun) setLockFileExpiration() {
|
||||
if b.opt.MaxLock > 0 && b.opt.MaxLock < 2*time.Minute {
|
||||
if b.opt.MaxLock > 0 && b.opt.MaxLock < fs.Duration(2*time.Minute) {
|
||||
fs.Logf(nil, Color(terminal.YellowFg, "--max-lock cannot be shorter than 2 minutes (unless 0.) Changing --max-lock from %v to %v"), b.opt.MaxLock, 2*time.Minute)
|
||||
b.opt.MaxLock = 2 * time.Minute
|
||||
b.opt.MaxLock = fs.Duration(2 * time.Minute)
|
||||
} else if b.opt.MaxLock <= 0 {
|
||||
b.opt.MaxLock = basicallyforever
|
||||
}
|
||||
@@ -80,7 +80,7 @@ func (b *bisyncRun) renewLockFile() {
|
||||
data.Session = b.basePath
|
||||
data.PID = strconv.Itoa(os.Getpid())
|
||||
data.TimeRenewed = time.Now()
|
||||
data.TimeExpires = time.Now().Add(b.opt.MaxLock)
|
||||
data.TimeExpires = time.Now().Add(time.Duration(b.opt.MaxLock))
|
||||
|
||||
// save data file
|
||||
df, err := os.Create(b.lockFile)
|
||||
@@ -131,7 +131,7 @@ func (b *bisyncRun) startLockRenewal() func() {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ticker := time.NewTicker(b.opt.MaxLock - time.Minute)
|
||||
ticker := time.NewTicker(time.Duration(b.opt.MaxLock) - time.Minute)
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
|
||||
@@ -138,7 +138,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
|
||||
if b.SyncCI != nil {
|
||||
fs.Infoc(nil, Color(terminal.YellowFg, "Telling Sync to wrap up early."))
|
||||
b.SyncCI.MaxTransfer = 1
|
||||
b.SyncCI.MaxDuration = 1 * time.Second
|
||||
b.SyncCI.MaxDuration = fs.Duration(1 * time.Second)
|
||||
b.SyncCI.CutoffMode = fs.CutoffModeSoft
|
||||
gracePeriod := 30 * time.Second // TODO: flag to customize this?
|
||||
if !waitFor("Canceling Sync if not done in", gracePeriod, func() bool { return b.CleanupCompleted }) {
|
||||
|
||||
+2
-2
@@ -376,8 +376,8 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
|
||||
return files.Save(queueFile)
|
||||
}
|
||||
|
||||
func naptime(totalWait time.Duration) {
|
||||
expireTime := time.Now().Add(totalWait)
|
||||
func naptime(totalWait fs.Duration) {
|
||||
expireTime := time.Now().Add(time.Duration(totalWait))
|
||||
fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST"))
|
||||
for i := 0; time.Until(expireTime) > 0; i++ {
|
||||
if i > 0 && i%10 == 0 {
|
||||
|
||||
+1
-1
@@ -73,7 +73,7 @@ you what happened to it. These are reminiscent of diff files.
|
||||
- |* path| means path was present in source and destination but different.
|
||||
- |! path| means there was an error reading or hashing the source or dest.
|
||||
|
||||
The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
|
||||
The default number of parallel checks is 8. See the [--checkers](/docs/#checkers-int)
|
||||
option for more information.
|
||||
`, "|", "`")
|
||||
|
||||
|
||||
+1
-1
@@ -288,7 +288,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
|
||||
accounting.GlobalStats().ResetErrors()
|
||||
}
|
||||
if ci.RetriesInterval > 0 {
|
||||
time.Sleep(ci.RetriesInterval)
|
||||
time.Sleep(time.Duration(ci.RetriesInterval))
|
||||
}
|
||||
}
|
||||
stopStats()
|
||||
|
||||
+12
-14
@@ -34,17 +34,15 @@ var commandDefinition = &cobra.Command{
|
||||
Long: strings.ReplaceAll(`
|
||||
convmv supports advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations.
|
||||
|
||||
`+transform.SprintList()+`
|
||||
|
||||
Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
|
||||
`+transform.Help()+`Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
|
||||
|
||||
The ¡--name-transform¡ flag is also available in ¡sync¡, ¡copy¡, and ¡move¡.
|
||||
|
||||
## Files vs Directories ##
|
||||
## Files vs Directories
|
||||
|
||||
By default ¡--name-transform¡ will only apply to file names. The means only the leaf file name will be transformed.
|
||||
However some of the transforms would be better applied to the whole path or just directories.
|
||||
To choose which which part of the file path is affected some tags can be added to the ¡--name-transform¡
|
||||
To choose which which part of the file path is affected some tags can be added to the ¡--name-transform¡.
|
||||
|
||||
| Tag | Effect |
|
||||
|------|------|
|
||||
@@ -54,11 +52,11 @@ To choose which which part of the file path is affected some tags can be added t
|
||||
|
||||
This is used by adding the tag into the transform name like this: ¡--name-transform file,prefix=ABC¡ or ¡--name-transform dir,prefix=DEF¡.
|
||||
|
||||
For some conversions using all is more likely to be useful, for example ¡--name-transform all,nfc¡
|
||||
For some conversions using all is more likely to be useful, for example ¡--name-transform all,nfc¡.
|
||||
|
||||
Note that ¡--name-transform¡ may not add path separators ¡/¡ to the name. This will cause an error.
|
||||
|
||||
## Ordering and Conflicts ##
|
||||
## Ordering and Conflicts
|
||||
|
||||
* Transformations will be applied in the order specified by the user.
|
||||
* If the ¡file¡ tag is in use (the default) then only the leaf name of files will be transformed.
|
||||
@@ -73,19 +71,19 @@ user, allowing for intentional use cases (e.g., trimming one prefix before addin
|
||||
* Users should be aware that certain combinations may lead to unexpected results and should verify
|
||||
transformations using ¡--dry-run¡ before execution.
|
||||
|
||||
## Race Conditions and Non-Deterministic Behavior ##
|
||||
## Race Conditions and Non-Deterministic Behavior
|
||||
|
||||
Some transformations, such as ¡replace=old:new¡, may introduce conflicts where multiple source files map to the same destination name.
|
||||
This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these.
|
||||
* If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic.
|
||||
* Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results.
|
||||
|
||||
* To minimize risks, users should:
|
||||
* Carefully review transformations that may introduce conflicts.
|
||||
* Use ¡--dry-run¡ to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
|
||||
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
|
||||
* Consider disabling concurrency with ¡--transfers=1¡ if necessary.
|
||||
* Certain transformations (e.g. ¡prefix¡) will have a multiplying effect every time they are used. Avoid these when using ¡bisync¡.
|
||||
To minimize risks, users should:
|
||||
* Carefully review transformations that may introduce conflicts.
|
||||
* Use ¡--dry-run¡ to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
|
||||
* Avoid transformations that cause multiple distinct source files to map to the same destination name.
|
||||
* Consider disabling concurrency with ¡--transfers=1¡ if necessary.
|
||||
* Certain transformations (e.g. ¡prefix¡) will have a multiplying effect every time they are used. Avoid these when using ¡bisync¡.
|
||||
|
||||
`, "¡", "`"),
|
||||
Annotations: map[string]string{
|
||||
|
||||
@@ -141,7 +141,7 @@ func TestTransform(t *testing.T) {
|
||||
}
|
||||
|
||||
// const alphabet = "ƀɀɠʀҠԀڀڠݠހ߀ကႠᄀᄠᅀᆀᇠሀሠበዠጠᎠᏀᐠᑀᑠᒀᒠᓀᓠᔀᔠᕀᕠᖀᖠᗀᗠᘀᘠᙀᚠᛀកᠠᡀᣀᦀ᧠ᨠᯀᰀᴀ⇠⋀⍀⍠⎀⎠⏀␀─┠╀╠▀■◀◠☀☠♀♠⚀⚠⛀⛠✀✠❀➀➠⠀⠠⡀⡠⢀⢠⣀⣠⤀⤠⥀⥠⦠⨠⩀⪀⪠⫠⬀⬠⭀ⰀⲀⲠⳀⴀⵀ⺠⻀㇀㐀㐠㑀㑠㒀㒠㓀㓠㔀㔠㕀㕠㖀㖠㗀㗠㘀㘠㙀㙠㚀㚠㛀㛠㜀㜠㝀㝠㞀㞠㟀㟠㠀㠠㡀㡠㢀㢠㣀㣠㤀㤠㥀㥠㦀㦠㧀㧠㨀㨠㩀㩠㪀㪠㫀㫠㬀㬠㭀㭠㮀㮠㯀㯠㰀㰠㱀㱠㲀㲠㳀㳠㴀㴠㵀㵠㶀㶠㷀㷠㸀㸠㹀㹠㺀㺠㻀㻠㼀㼠㽀㽠㾀㾠㿀㿠䀀䀠䁀䁠䂀䂠䃀䃠䄀䄠䅀䅠䆀䆠䇀䇠䈀䈠䉀䉠䊀䊠䋀䋠䌀䌠䍀䍠䎀䎠䏀䏠䐀䐠䑀䑠䒀䒠䓀䓠䔀䔠䕀䕠䖀䖠䗀䗠䘀䘠䙀䙠䚀䚠䛀䛠䜀䜠䝀䝠䞀䞠䟀䟠䠀䠠䡀䡠䢀䢠䣀䣠䤀䤠䥀䥠䦀䦠䧀䧠䨀䨠䩀䩠䪀䪠䫀䫠䬀䬠䭀䭠䮀䮠䯀䯠䰀䰠䱀䱠䲀䲠䳀䳠䴀䴠䵀䵠䶀䷀䷠一丠乀习亀亠什仠伀传佀你侀侠俀俠倀倠偀偠傀傠僀僠儀儠兀兠冀冠净几刀删剀剠劀加勀勠匀匠區占厀厠叀叠吀吠呀呠咀咠哀哠唀唠啀啠喀喠嗀嗠嘀嘠噀噠嚀嚠囀因圀圠址坠垀垠埀埠堀堠塀塠墀墠壀壠夀夠奀奠妀妠姀姠娀娠婀婠媀媠嫀嫠嬀嬠孀孠宀宠寀寠尀尠局屠岀岠峀峠崀崠嵀嵠嶀嶠巀巠帀帠幀幠庀庠廀廠开张彀彠往徠忀忠怀怠恀恠悀悠惀惠愀愠慀慠憀憠懀懠戀戠所扠技抠拀拠挀挠捀捠掀掠揀揠搀搠摀摠撀撠擀擠攀攠敀敠斀斠旀无昀映晀晠暀暠曀曠最朠杀杠枀枠柀柠栀栠桀桠梀梠检棠椀椠楀楠榀榠槀槠樀樠橀橠檀檠櫀櫠欀欠歀歠殀殠毀毠氀氠汀池沀沠泀泠洀洠浀浠涀涠淀淠渀渠湀湠満溠滀滠漀漠潀潠澀澠激濠瀀瀠灀灠炀炠烀烠焀焠煀煠熀熠燀燠爀爠牀牠犀犠狀狠猀猠獀獠玀玠珀珠琀琠瑀瑠璀璠瓀瓠甀甠畀畠疀疠痀痠瘀瘠癀癠皀皠盀盠眀眠着睠瞀瞠矀矠砀砠础硠碀碠磀磠礀礠祀祠禀禠秀秠稀稠穀穠窀窠竀章笀笠筀筠简箠節篠簀簠籀籠粀粠糀糠紀素絀絠綀綠緀締縀縠繀繠纀纠绀绠缀缠罀罠羀羠翀翠耀耠聀聠肀肠胀胠脀脠腀腠膀膠臀臠舀舠艀艠芀芠苀苠茀茠荀荠莀莠菀菠萀萠葀葠蒀蒠蓀蓠蔀蔠蕀蕠薀薠藀藠蘀蘠虀虠蚀蚠蛀蛠蜀蜠蝀蝠螀螠蟀蟠蠀蠠血衠袀袠裀裠褀褠襀襠覀覠觀觠言訠詀詠誀誠諀諠謀謠譀譠讀讠诀诠谀谠豀豠貀負賀賠贀贠赀赠趀趠跀跠踀踠蹀蹠躀躠軀軠輀輠轀轠辀辠迀迠退造遀遠邀邠郀郠鄀鄠酀酠醀醠釀釠鈀鈠鉀鉠銀銠鋀鋠錀錠鍀鍠鎀鎠鏀鏠鐀鐠鑀鑠钀钠铀铠销锠镀镠門閠闀闠阀阠陀陠隀隠雀雠需霠靀靠鞀鞠韀韠頀頠顀顠颀颠飀飠餀餠饀饠馀馠駀駠騀騠驀驠骀骠髀髠鬀鬠魀魠鮀鮠鯀鯠鰀鰠鱀鱠鲀鲠鳀鳠鴀鴠鵀鵠鶀鶠鷀鷠鸀鸠鹀鹠麀麠黀黠鼀鼠齀齠龀龠ꀀꀠꁀꁠꂀꂠꃀꃠꄀꄠꅀꅠꆀꆠꇀꇠꈀꈠꉀꉠꊀꊠꋀꋠꌀꌠꍀꍠꎀꎠꏀꏠꐀꐠꑀꑠ꒠ꔀꔠꕀꕠꖀꖠꗀꗠꙀꚠꛀ꜀꜠ꝀꞀꡀ測試_Русский___ě_áñ"
|
||||
const alphabet = "abcdefg123456789"
|
||||
const alphabet = "abcdefg123456789Ü"
|
||||
|
||||
var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"}
|
||||
|
||||
@@ -251,3 +251,25 @@ func detectEncoding(s string) string {
|
||||
}
|
||||
return "OTHER"
|
||||
}
|
||||
|
||||
func TestUnicodeEquivalence(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
ctx := context.Background()
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
const remote = "Über"
|
||||
item := r.WriteObject(ctx, remote, "", t1)
|
||||
|
||||
obj, err := r.Fremote.NewObject(ctx, remote) // can't use r.CheckRemoteListing here as it forces NFC
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, obj)
|
||||
|
||||
err = transform.SetOptions(ctx, "all,nfc")
|
||||
require.NoError(t, err)
|
||||
|
||||
err = sync.Transform(ctx, r.Fremote, true, true)
|
||||
assert.NoError(t, err)
|
||||
item.Path = norm.NFC.String(item.Path)
|
||||
r.CheckRemoteListing(t, []fstest.Item{item}, nil)
|
||||
}
|
||||
+21
-5
@@ -8,18 +8,23 @@ import (
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
createEmptySrcDirs = false
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -90,19 +95,30 @@ for more info.
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
|
||||
|
||||
**Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
|
||||
`, "|", "`"),
|
||||
|
||||
`, "|", "`") + operationsflags.Help(),
|
||||
Annotations: map[string]string{
|
||||
"groups": "Copy,Filter,Listing,Important",
|
||||
},
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(context.Background(), fdst, fsrc, createEmptySrcDirs)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(ctx, fdst, fsrc, createEmptySrcDirs)
|
||||
}
|
||||
return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
+25
-4
@@ -6,12 +6,21 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -46,7 +55,8 @@ the destination.
|
||||
*If you are looking to copy just a byte range of a file, please see 'rclone cat --offset X --count Y'*
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
`,
|
||||
|
||||
` + operationsflags.Help(),
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.35",
|
||||
"groups": "Copy,Filter,Listing,Important",
|
||||
@@ -55,10 +65,21 @@ the destination.
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(context.Background(), fdst, fsrc, false)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.CopyFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.CopyDir(ctx, fdst, fsrc, false)
|
||||
}
|
||||
return operations.CopyFile(ctx, fdst, fsrc, dstFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
@@ -240,7 +240,7 @@ to run as the SYSTEM account. A third alternative is to use the
|
||||
Read more in the [install documentation](https://rclone.org/install/).
|
||||
Note that when running rclone as another user, it will not use
|
||||
the configuration file from your profile unless you tell it to
|
||||
with the [`--config`](https://rclone.org/docs/#config-config-file) option.
|
||||
with the [`--config`](https://rclone.org/docs/#config-string) option.
|
||||
Note also that it is now the SYSTEM account that will have the owner
|
||||
permissions, and other accounts will have permissions according to the
|
||||
group or others scopes. As mentioned above, these will then not get the
|
||||
|
||||
+21
-4
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -16,6 +17,8 @@ import (
|
||||
var (
|
||||
deleteEmptySrcDirs = false
|
||||
createEmptySrcDirs = false
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -23,6 +26,8 @@ func init() {
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "")
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -66,7 +71,8 @@ for more info.
|
||||
|--dry-run| or the |--interactive|/|-i| flag.
|
||||
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
|
||||
`, "|", "`"),
|
||||
|
||||
`, "|", "`") + operationsflags.Help(),
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.19",
|
||||
"groups": "Filter,Listing,Important,Copy",
|
||||
@@ -75,10 +81,21 @@ for more info.
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(context.Background(), fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.MoveFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
|
||||
}
|
||||
return operations.MoveFile(ctx, fdst, fsrc, srcFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
+25
-4
@@ -6,12 +6,21 @@ import (
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
"github.com/rclone/rclone/fs/sync"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -47,7 +56,8 @@ successful transfer.
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
|
||||
`,
|
||||
|
||||
` + operationsflags.Help(),
|
||||
Annotations: map[string]string{
|
||||
"versionIntroduced": "v1.35",
|
||||
"groups": "Filter,Listing,Important,Copy",
|
||||
@@ -57,10 +67,21 @@ successful transfer.
|
||||
fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
|
||||
|
||||
cmd.Run(true, true, command, func() error {
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(context.Background(), fdst, fsrc, false, false)
|
||||
ctx := context.Background()
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return operations.MoveFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
|
||||
defer close()
|
||||
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
return sync.MoveDir(ctx, fdst, fsrc, false, false)
|
||||
}
|
||||
return operations.MoveFile(ctx, fdst, fsrc, dstFileName, srcFileName)
|
||||
})
|
||||
},
|
||||
}
|
||||
+100
-49
@@ -95,61 +95,44 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
|
||||
if err != nil {
|
||||
return fmt.Errorf("send output failed: %w", err)
|
||||
}
|
||||
case "md5sum", "sha1sum":
|
||||
ht := hash.MD5
|
||||
if binary == "sha1sum" {
|
||||
ht = hash.SHA1
|
||||
case "md5sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.MD5, args)
|
||||
case "sha1sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.SHA1, args)
|
||||
case "crc32":
|
||||
return c.handleHashsumCommand(ctx, out, hash.CRC32, args)
|
||||
case "sha256sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.SHA256, args)
|
||||
case "b3sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.BLAKE3, args)
|
||||
case "xxh128sum":
|
||||
return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
|
||||
case "xxhsum":
|
||||
argv := strings.SplitN(args, " ", 2)
|
||||
if len(argv) == 0 || argv[0] != "-H2" {
|
||||
return fmt.Errorf("%q not implemented", command)
|
||||
}
|
||||
if !c.vfs.Fs().Hashes().Contains(ht) {
|
||||
return fmt.Errorf("%v hash not supported", ht)
|
||||
}
|
||||
var hashSum string
|
||||
if args == "" {
|
||||
// empty hash for no input
|
||||
if ht == hash.MD5 {
|
||||
hashSum = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
} else {
|
||||
hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
}
|
||||
args = "-"
|
||||
if len(argv) > 1 {
|
||||
args = argv[1]
|
||||
} else {
|
||||
node, err := c.vfs.Stat(args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed finding file %q: %w", args, err)
|
||||
args = ""
|
||||
}
|
||||
return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
|
||||
case "rclone":
|
||||
argv := strings.SplitN(args, " ", 3)
|
||||
if len(argv) > 1 && argv[0] == "hashsum" {
|
||||
var ht hash.Type
|
||||
if err := ht.Set(argv[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
if node.IsDir() {
|
||||
return errors.New("can't hash directory")
|
||||
}
|
||||
o, ok := node.DirEntry().(fs.ObjectInfo)
|
||||
if !ok {
|
||||
fs.Debugf(args, "File uploading - reading hash from VFS cache")
|
||||
in, err := node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs open failed: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = in.Close()
|
||||
}()
|
||||
h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
|
||||
}
|
||||
_, err = io.Copy(h, in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs copy failed: %w", err)
|
||||
}
|
||||
hashSum = h.Sums()[ht]
|
||||
if len(argv) > 2 {
|
||||
args = argv[2]
|
||||
} else {
|
||||
hashSum, err = o.Hash(ctx, ht)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed: %w", err)
|
||||
}
|
||||
args = ""
|
||||
}
|
||||
return c.handleHashsumCommand(ctx, out, ht, args)
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "%s %s\n", hashSum, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("send output failed: %w", err)
|
||||
}
|
||||
return fmt.Errorf("%q not implemented", command)
|
||||
case "echo":
|
||||
// Special cases for legacy rclone command detection.
|
||||
// Before rclone v1.49.0 the sftp backend used "echo 'abc' | md5sum" when
|
||||
@@ -189,6 +172,74 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleHashsumCommand is a helper to execCommand for common functionality of hashsum related commands
|
||||
func (c *conn) handleHashsumCommand(ctx context.Context, out io.Writer, ht hash.Type, args string) (err error) {
|
||||
if !c.vfs.Fs().Hashes().Contains(ht) {
|
||||
return fmt.Errorf("%v hash not supported", ht)
|
||||
}
|
||||
var hashSum string
|
||||
if args == "" {
|
||||
// empty hash for no input
|
||||
switch ht {
|
||||
case hash.MD5:
|
||||
hashSum = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
case hash.SHA1:
|
||||
hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
case hash.CRC32:
|
||||
hashSum = "00000000"
|
||||
case hash.SHA256:
|
||||
hashSum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
case hash.BLAKE3:
|
||||
hashSum = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"
|
||||
case hash.XXH3:
|
||||
hashSum = "2d06800538d394c2"
|
||||
case hash.XXH128:
|
||||
hashSum = "99aa06d3014798d86001c324468d497f"
|
||||
default:
|
||||
return fmt.Errorf("%v hash not implemented", ht)
|
||||
}
|
||||
args = "-"
|
||||
} else {
|
||||
node, err := c.vfs.Stat(args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed finding file %q: %w", args, err)
|
||||
}
|
||||
if node.IsDir() {
|
||||
return errors.New("can't hash directory")
|
||||
}
|
||||
o, ok := node.DirEntry().(fs.ObjectInfo)
|
||||
if !ok {
|
||||
fs.Debugf(args, "File uploading - reading hash from VFS cache")
|
||||
in, err := node.Open(os.O_RDONLY)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs open failed: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = in.Close()
|
||||
}()
|
||||
h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
|
||||
}
|
||||
_, err = io.Copy(h, in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash vfs copy failed: %w", err)
|
||||
}
|
||||
hashSum = h.Sums()[ht]
|
||||
} else {
|
||||
hashSum, err = o.Hash(ctx, ht)
|
||||
if err != nil {
|
||||
return fmt.Errorf("hash failed: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
_, err = fmt.Fprintf(out, "%s %s\n", hashSum, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("send output failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// handle a new incoming channel request
|
||||
func (c *conn) handleChannel(newChannel ssh.NewChannel) {
|
||||
fs.Debugf(c.what, "Incoming channel: %s\n", newChannel.ChannelType())
|
||||
|
||||
+14
-160
@@ -3,13 +3,9 @@ package sync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
mutex "sync" // renamed as "sync" already in use
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/operations/operationsflags"
|
||||
@@ -19,7 +15,7 @@ import (
|
||||
|
||||
var (
|
||||
createEmptySrcDirs = false
|
||||
opt = operations.LoggerOpt{}
|
||||
loggerOpt = operations.LoggerOpt{}
|
||||
loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
|
||||
)
|
||||
|
||||
@@ -27,119 +23,15 @@ func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)
|
||||
// TODO: add same flags to move and copy
|
||||
}
|
||||
|
||||
var lock mutex.Mutex
|
||||
|
||||
func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, srcOk := src.(fs.Object)
|
||||
_, dstOk := dst.(fs.Object)
|
||||
var filename string
|
||||
if !srcOk && !dstOk {
|
||||
return
|
||||
} else if srcOk && !dstOk {
|
||||
filename = src.String()
|
||||
} else {
|
||||
filename = dst.String()
|
||||
}
|
||||
|
||||
if sigil.Writer(opt) != nil {
|
||||
operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)
|
||||
}
|
||||
if opt.Combined != nil {
|
||||
operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
|
||||
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
|
||||
}
|
||||
if opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
}
|
||||
}
|
||||
|
||||
// GetSyncLoggerOpt gets the options corresponding to the logger flags
|
||||
func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
opt.LoggerFn = syncLoggerFn
|
||||
if opt.TimeFormat == "max" {
|
||||
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
|
||||
}
|
||||
opt.SetListFormat(ctx, command.Flags())
|
||||
opt.NewListJSON(ctx, fdst, "")
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {
|
||||
return opt, nil, err
|
||||
}
|
||||
|
||||
close := func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return opt, close, nil
|
||||
}
|
||||
|
||||
func anyNotBlank(s ...string) bool {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
|
||||
loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "sync source:path dest:path",
|
||||
Short: `Make source and dest identical, modifying destination only.`,
|
||||
Long: `Sync the source to the destination, changing the destination
|
||||
// Warning! "|" will be replaced by backticks below
|
||||
Long: strings.ReplaceAll(`Sync the source to the destination, changing the destination
|
||||
only. Doesn't transfer files that are identical on source and
|
||||
destination, testing by size and modification time or MD5SUM.
|
||||
Destination is updated to match source, including deleting files
|
||||
@@ -148,7 +40,7 @@ want to delete files from destination, use the
|
||||
[copy](/commands/rclone_copy/) command instead.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
|--dry-run| or the |--interactive|/|i| flag.
|
||||
|
||||
rclone sync --interactive SOURCE remote:DESTINATION
|
||||
|
||||
@@ -171,55 +63,18 @@ destination that is inside the source directory.
|
||||
|
||||
Rclone will sync the modification times of files and directories if
|
||||
the backend supports it. If metadata syncing is required then use the
|
||||
` + "`--metadata`" + ` flag.
|
||||
|--metadata| flag.
|
||||
|
||||
Note that the modification time and metadata for the root directory
|
||||
will **not** be synced. See https://github.com/rclone/rclone/issues/7652
|
||||
for more info.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics
|
||||
|
||||
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
**Note**: Use the |rclone dedupe| command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
|
||||
See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
|
||||
|
||||
## Logger Flags
|
||||
|
||||
The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` +
|
||||
"`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or
|
||||
stdout if it is ` + "`-`" + `) supplied. What they write is described in the
|
||||
help below. For example ` + "`--differ`" + ` will write all paths which are
|
||||
present on both the source and destination but different.
|
||||
|
||||
The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- ` + "`= path`" + ` means path was found in source and destination and was identical
|
||||
- ` + "`- path`" + ` means path was missing on the source, so only in the destination
|
||||
- ` + "`+ path`" + ` means path was missing on the destination, so only in the source
|
||||
- ` + "`* path`" + ` means path was present in source and destination but different.
|
||||
- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.
|
||||
|
||||
The ` + "`--dest-after`" + ` flag writes a list file using the same format flags
|
||||
as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the sync.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `
|
||||
- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during the sync, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
`,
|
||||
`, "|", "`") + operationsflags.Help(),
|
||||
Annotations: map[string]string{
|
||||
"groups": "Sync,Copy,Filter,Listing,Important",
|
||||
},
|
||||
@@ -228,15 +83,14 @@ is most useful as a predictor of what SHOULD happen to each file
|
||||
fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
|
||||
cmd.Run(true, true, command, func() error {
|
||||
ctx := context.Background()
|
||||
opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)
|
||||
close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,
|
||||
loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {
|
||||
ctx = operations.WithSyncLogger(ctx, opt)
|
||||
if loggerFlagsOpt.AnySet() {
|
||||
ctx = operations.WithSyncLogger(ctx, loggerOpt)
|
||||
}
|
||||
|
||||
if srcFileName == "" {
|
||||
|
||||
@@ -14,13 +14,13 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
pollInterval = 10 * time.Second
|
||||
pollInterval = fs.Duration(10 * time.Second)
|
||||
)
|
||||
|
||||
func init() {
|
||||
test.Command.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.DurationVarP(cmdFlags, &pollInterval, "poll-interval", "", pollInterval, "Time to wait between polling for changes", "")
|
||||
flags.FVarP(cmdFlags, &pollInterval, "poll-interval", "", "Time to wait between polling for changes", "")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -39,7 +39,7 @@ var commandDefinition = &cobra.Command{
|
||||
if do := features.ChangeNotify; do != nil {
|
||||
pollChan := make(chan time.Duration)
|
||||
do(ctx, changeNotify, pollChan)
|
||||
pollChan <- pollInterval
|
||||
pollChan <- time.Duration(pollInterval)
|
||||
fs.Logf(nil, "Waiting for changes, polling every %v", pollInterval)
|
||||
} else {
|
||||
return errors.New("poll-interval is not supported by this remote")
|
||||
|
||||
@@ -40,7 +40,7 @@ var (
|
||||
checkStreaming bool
|
||||
checkBase32768 bool
|
||||
all bool
|
||||
uploadWait time.Duration
|
||||
uploadWait fs.Duration
|
||||
positionLeftRe = regexp.MustCompile(`(?s)^(.*)-position-left-([[:xdigit:]]+)$`)
|
||||
positionMiddleRe = regexp.MustCompile(`(?s)^position-middle-([[:xdigit:]]+)-(.*)-$`)
|
||||
positionRightRe = regexp.MustCompile(`(?s)^position-right-([[:xdigit:]]+)-(.*)$`)
|
||||
@@ -52,7 +52,7 @@ func init() {
|
||||
flags.StringVarP(cmdFlags, &writeJSON, "write-json", "", "", "Write results to file", "")
|
||||
flags.BoolVarP(cmdFlags, &checkNormalization, "check-normalization", "", false, "Check UTF-8 Normalization", "")
|
||||
flags.BoolVarP(cmdFlags, &checkControl, "check-control", "", false, "Check control characters", "")
|
||||
flags.DurationVarP(cmdFlags, &uploadWait, "upload-wait", "", 0, "Wait after writing a file", "")
|
||||
flags.FVarP(cmdFlags, &uploadWait, "upload-wait", "", "Wait after writing a file", "")
|
||||
flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", false, "Check max filename length", "")
|
||||
flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", false, "Check uploads with indeterminate file size", "")
|
||||
flags.BoolVarP(cmdFlags, &checkBase32768, "check-base32768", "", false, "Check can store all possible base32768 characters", "")
|
||||
@@ -204,7 +204,7 @@ func (r *results) writeFile(path string) (fs.Object, error) {
|
||||
src := object.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
|
||||
obj, err := r.f.Put(r.ctx, bytes.NewBufferString(contents), src)
|
||||
if uploadWait > 0 {
|
||||
time.Sleep(uploadWait)
|
||||
time.Sleep(time.Duration(uploadWait))
|
||||
}
|
||||
return obj, err
|
||||
}
|
||||
|
||||
@@ -123,8 +123,8 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
|
||||
{{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
|
||||
{{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
|
||||
{{< provider name="Files.com" home="https://www.files.com/" config="/filescom/" >}}
|
||||
{{< provider name="FileLu Cloud Storage" home="https://filelu.com/" config="/filelu/" >}}
|
||||
{{< provider name="Files.com" home="https://www.files.com/" config="/filescom/" >}}
|
||||
{{< provider name="FlashBlade" home="https://www.purestorage.com/products/unstructured-data-storage.html" config="/s3/#pure-storage-flashblade" >}}
|
||||
{{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
|
||||
{{< provider name="Gofile" home="https://gofile.io/" config="/gofile/" >}}
|
||||
@@ -197,6 +197,7 @@ WebDAV or S3, that work out of the box.)
|
||||
{{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
|
||||
{{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}
|
||||
{{< provider name="Zoho WorkDrive" home="https://www.zoho.com/workdrive/" config="/zoho/" >}}
|
||||
{{< provider name="Zata" home="https://zata.ai/" config="/s3/#Zata" end="true" >}}
|
||||
{{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}}
|
||||
{{< /provider_list >}}
|
||||
|
||||
|
||||
+10
-1
@@ -788,7 +788,6 @@ put them back in again.` >}}
|
||||
* Adithya Kumar <akumar42@protonmail.com>
|
||||
* Tayo-pasedaRJ <138471223+Tayo-pasedaRJ@users.noreply.github.com>
|
||||
* Peter Kreuser <logo@kreuser.name>
|
||||
* Piyush <piyushgarg80>
|
||||
* fotile96 <fotile96@users.noreply.github.com>
|
||||
* Luc Ritchie <luc.ritchie@gmail.com>
|
||||
* cynful <cynful@users.noreply.github.com>
|
||||
@@ -982,3 +981,13 @@ put them back in again.` >}}
|
||||
* Flora Thiebaut <johann.thiebaut@gmail.com>
|
||||
* kingston125 <support@filelu.com>
|
||||
* Ser-Bul <30335009+Ser-Bul@users.noreply.github.com>
|
||||
* jinjingroad <jinjingroad@sina.com>
|
||||
* necaran <55765083+necaran@users.noreply.github.com>
|
||||
* Marvin Rösch <marvinroesch99@gmail.com>
|
||||
* curlwget <curlwget@icloud.com>
|
||||
* $@M@RTH_ <165886349+Samarth1015@users.noreply.github.com>
|
||||
* Ali Zein Yousuf <azy5030@gmail.com>
|
||||
* Davide Bizzarri <davide.bizzarri@willhaben.at>
|
||||
* Ross Smith II <ross@smithii.com>
|
||||
* Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com>
|
||||
* Sudipto Baral <sudiptobaral.me@gmail.com>
|
||||
@@ -297,6 +297,16 @@ be explicitly specified using exactly one of the `msi_object_id`,
|
||||
If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
|
||||
set, this is is equivalent to using `env_auth`.
|
||||
|
||||
#### Fedrated Identity Credentials
|
||||
|
||||
If these variables are set, rclone will authenticate with fedrated identity.
|
||||
|
||||
- `tenant_id`: tenant_id to authenticate in storage
|
||||
- `client_id`: client ID of the application the user will authenticate to storage
|
||||
- `msi_client_id`: managed identity client ID of the application the user will authenticate to
|
||||
|
||||
By default "api://AzureADTokenExchange" is used as scope for token retrieval over MSI. This token is then exchanged for actual storage token using 'tenant_id' and 'client_id'.
|
||||
|
||||
#### Azure CLI tool `az` {#use_az}
|
||||
|
||||
Set to use the [Azure CLI tool `az`](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
|
||||
@@ -295,6 +295,16 @@ be explicitly specified using exactly one of the `msi_object_id`,
|
||||
|
||||
If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
|
||||
set, this is is equivalent to using `env_auth`.
|
||||
|
||||
#### Fedrated Identity Credentials
|
||||
|
||||
If these variables are set, rclone will authenticate with fedrated identity.
|
||||
|
||||
- `tenant_id`: tenant_id to authenticate in storage
|
||||
- `client_id`: client ID of the application the user will authenticate to storage
|
||||
- `msi_client_id`: managed identity client ID of the application the user will authenticate to
|
||||
|
||||
By default "api://AzureADTokenExchange" is used as scope for token retrieval over MSI. This token is then exchanged for actual storage token using 'tenant_id' and 'client_id'.
|
||||
|
||||
#### Azure CLI tool `az` {#use_az}
|
||||
Set to use the [Azure CLI tool `az`](https://learn.microsoft.com/en-us/cli/azure/)
|
||||
|
||||
@@ -635,7 +635,7 @@ directory separators.) To address this particular issue, an additional
|
||||
`2006-01-02 0304PM`.
|
||||
|
||||
Note that `--conflict-suffix` is entirely separate from rclone's main
|
||||
[`--sufix`](/docs/#suffix-suffix) flag. This is intentional, as users may wish
|
||||
[`--sufix`](/docs/#suffix-string) flag. This is intentional, as users may wish
|
||||
to use both flags simultaneously, if also using
|
||||
[`--backup-dir`](#backup-dir1-and-backup-dir2).
|
||||
|
||||
@@ -810,7 +810,7 @@ without requiring the user to get involved and run a `--resync`. (See also:
|
||||
|
||||
### --backup-dir1 and --backup-dir2
|
||||
|
||||
As of `v1.66`, [`--backup-dir`](/docs/#backup-dir-dir) is supported in bisync.
|
||||
As of `v1.66`, [`--backup-dir`](/docs/#backup-dir-string) is supported in bisync.
|
||||
Because `--backup-dir` must be a non-overlapping path on the same remote,
|
||||
Bisync has introduced new `--backup-dir1` and `--backup-dir2` flags to support
|
||||
separate backup-dirs for `Path1` and `Path2` (bisyncing between different
|
||||
@@ -841,7 +841,7 @@ In the event of a [rename due to a sync conflict](#conflict-loser), the
|
||||
rename is not considered a delete, unless a previous conflict with the same
|
||||
name already exists and would get overwritten.
|
||||
|
||||
See also: [`--suffix`](/docs/#suffix-suffix),
|
||||
See also: [`--suffix`](/docs/#suffix-string),
|
||||
[`--suffix-keep-extension`](/docs/#suffix-keep-extension)
|
||||
|
||||
## Operation
|
||||
@@ -1831,7 +1831,7 @@ about _Unison_ and synchronization in general.
|
||||
* Final listings are now generated from sync results, to avoid needing to re-list
|
||||
* Bisync is now much more resilient to changes that happen during a bisync run, and far less prone to critical errors / undetected changes
|
||||
* Bisync is now capable of rolling a file listing back in cases of uncertainty, essentially marking the file as needing to be rechecked next time.
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`)
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color) (`AUTO`|`NEVER`|`ALWAYS`)
|
||||
* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`,
|
||||
for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors).
|
||||
* Fixed handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)
|
||||
|
||||
@@ -5,6 +5,54 @@ description: "Rclone Changelog"
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.70.3 - 2025-07-09
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.2...v1.70.3)
|
||||
|
||||
* Bug Fixes
|
||||
* check: Fix difference report (was reporting error counts) (albertony)
|
||||
* march: Fix deadlock when using `--no-traverse` (Nick Craig-Wood)
|
||||
* doc fixes (albertony, Nick Craig-Wood)
|
||||
* Azure Blob
|
||||
* Fix server side copy error "requires exactly one scope" (Nick Craig-Wood)
|
||||
* B2
|
||||
* Fix finding objects when using `--b2-version-at` (Davide Bizzarri)
|
||||
* Linkbox
|
||||
* Fix upload error "user upload file not exist" (Nick Craig-Wood)
|
||||
* Pikpak
|
||||
* Improve error handling for missing links and unrecoverable 500s (wiserain)
|
||||
* WebDAV
|
||||
* Fix setting modtime to that of local object instead of remote (WeidiDeng)
|
||||
|
||||
## v1.70.2 - 2025-06-27
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.1...v1.70.2)
|
||||
|
||||
* Bug Fixes
|
||||
* convmv: Make --dry-run logs less noisy (nielash)
|
||||
* sync: Avoid copying dir metadata to itself (nielash)
|
||||
* build: Bump github.com/go-chi/chi/v5 from 5.2.1 to 5.2.2 to fix GHSA-vrw8-fxc6-2r93 (dependabot[bot])
|
||||
* convmv: Fix moving to unicode-equivalent name (nielash)
|
||||
* log: Fix deadlock when using systemd logging (Nick Craig-Wood)
|
||||
* pacer: Fix nil pointer deref in RetryError (Nick Craig-Wood)
|
||||
* doc fixes (Ali Zein Yousuf, Nick Craig-Wood)
|
||||
* Local
|
||||
* Fix --skip-links on Windows when skipping Junction points (Nick Craig-Wood)
|
||||
* Combine
|
||||
* Fix directory not found errors with ListP interface (Nick Craig-Wood)
|
||||
* Mega
|
||||
* Fix tls handshake failure (necaran)
|
||||
* Pikpak
|
||||
* Fix uploads fail with "aws-chunked encoding is not supported" error (Nick Craig-Wood)
|
||||
|
||||
## v1.70.1 - 2025-06-19
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.70.0...v1.70.1)
|
||||
|
||||
* Bug Fixes
|
||||
* convmv: Fix spurious "error running command echo" on Windows (Nick Craig-Wood)
|
||||
* doc fixes (albertony, Ed Craig-Wood, jinjingroad)
|
||||
|
||||
## v1.70.0 - 2025-06-17
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.70.0)
|
||||
@@ -20,7 +68,7 @@ description: "Rclone Changelog"
|
||||
* New Features
|
||||
* Add [`--max-connections`](/docs/#max-connections-n) to control maximum backend concurrency (Nick Craig-Wood)
|
||||
* Add [`--max-buffer-memory`](/docs/#max-buffer-memory) to limit total buffer memory usage (Nick Craig-Wood)
|
||||
* Add transform library and [`--name-transform`](/docs/#name-transform-command-xxxx) flag (nielash)
|
||||
* Add transform library and [`--name-transform`](/docs/#name-transform-stringarray) flag (nielash)
|
||||
* sync: Implement [`--list-cutoff`](/docs/#list-cutoff) to allow on disk sorting for reduced memory use (Nick Craig-Wood)
|
||||
* accounting: Add listed stat for number of directory entries listed (Nick Craig-Wood)
|
||||
* backend: Skip hash calculation when the hashType is None (Oleksiy Stashok)
|
||||
@@ -729,7 +777,7 @@ description: "Rclone Changelog"
|
||||
* Final listings are now generated from sync results, to avoid needing to re-list (nielash)
|
||||
* Bisync is now much more resilient to changes that happen during a bisync run, and far less prone to critical errors / undetected changes (nielash)
|
||||
* Bisync is now capable of rolling a file listing back in cases of uncertainty, essentially marking the file as needing to be rechecked next time. (nielash)
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`) (nielash)
|
||||
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-autoneveralways) (`AUTO`|`NEVER`|`ALWAYS`) (nielash)
|
||||
* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`, for performance improvements and less risk of error. (nielash)
|
||||
* `--resync` is now much more efficient (especially for users of `--create-empty-src-dirs`) (nielash)
|
||||
* Google Docs (and other files of unknown size) are now supported (with the same options as in `sync`) (nielash)
|
||||
@@ -741,7 +789,7 @@ instead of of `--size-only`, when `check` is not available.
|
||||
* A new `--max-lock` setting allows lock files to automatically renew and expire, for better automatic recovery when a run is interrupted. (nielash)
|
||||
* Bisync now supports auto-resolving sync conflicts and customizing rename behavior with new [`--conflict-resolve`](#conflict-resolve), [`--conflict-loser`](#conflict-loser), and [`--conflict-suffix`](#conflict-suffix) flags. (nielash)
|
||||
* A new [`--resync-mode`](#resync-mode) flag allows more control over which version of a file gets kept during a `--resync`. (nielash)
|
||||
* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time) (when [`--resilient`](#resilient) is set.) (nielash)
|
||||
* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-duration) (when [`--resilient`](#resilient) is set.) (nielash)
|
||||
* Clarify file operation directions in dry-run logs (Kyle Reynolds)
|
||||
* Local
|
||||
* Fix cleanRootPath on Windows after go1.21.4 stdlib update (nielash)
|
||||
@@ -2992,7 +3040,7 @@ instead of of `--size-only`, when `check` is not available.
|
||||
* New Features
|
||||
* The [VFS layer](/commands/rclone_mount/#vfs-virtual-file-system) was heavily reworked for this release - see below for more details
|
||||
* Interactive mode [-i/--interactive](/docs/#interactive) for destructive operations (fishbullet)
|
||||
* Add [--bwlimit-file](/docs/#bwlimit-file-bandwidth-spec) flag to limit speeds of individual file transfers (Nick Craig-Wood)
|
||||
* Add [--bwlimit-file](/docs/#bwlimit-file-bwtimetable) flag to limit speeds of individual file transfers (Nick Craig-Wood)
|
||||
* Transfers are sorted by start time in the stats and progress output (Max Sum)
|
||||
* Make sure backends expand `~` and environment vars in file names they use (Nick Craig-Wood)
|
||||
* Add [--refresh-times](/docs/#refresh-times) flag to set modtimes on hashless backends (Nick Craig-Wood)
|
||||
|
||||
+521
-497
File diff suppressed because it is too large.
Load diff
+1
-1
@@ -8,7 +8,7 @@ versionIntroduced: "?"
|
||||
|
||||
The DOI remote is a read only remote for reading files from digital object identifiers (DOI).
|
||||
|
||||
Currently, the DOI backend supports supports DOIs hosted with:
|
||||
Currently, the DOI backend supports DOIs hosted with:
|
||||
- [InvenioRDM](https://inveniosoftware.org/products/rdm/)
|
||||
- [Zenodo](https://zenodo.org)
|
||||
- [CaltechDATA](https://data.caltech.edu)
|
||||
|
||||
@@ -9,6 +9,9 @@ type: page
|
||||
Rclone is single executable (`rclone`, or `rclone.exe` on Windows) that you can
|
||||
simply download as a zip archive and extract into a location of your choosing.
|
||||
See the [install](https://rclone.org/install/) documentation for more details.
|
||||
We also offer a secure [enterprise-grade, zero CVE docker
|
||||
image](https://securebuild.com/images/rclone) through our partner
|
||||
[SecureBuild](https://securebuild.com/blog/introducing-securebuild).
|
||||
|
||||
## Release {{% version %}} OS requirements {#osrequirements}
|
||||
|
||||
|
||||
+1
-1
@@ -233,7 +233,7 @@ Restart-Service hns
|
||||
It is likely you have more than 10,000 files that need to be
|
||||
synced. By default, rclone only gets 10,000 files ahead in a sync so as
|
||||
not to use up too much memory. You can change this default with the
|
||||
[--max-backlog](/docs/#max-backlog-n) flag.
|
||||
[--max-backlog](/docs/#max-backlog-int) flag.
|
||||
|
||||
### Rclone is using too much memory or appears to have a memory leak
|
||||
|
||||
|
||||
@@ -99,6 +99,21 @@ excess files in the directory.
|
||||
|
||||
rclone sync --interactive /home/local/directory remote:dir
|
||||
|
||||
### Hashes
|
||||
|
||||
In December 2024 `files.com` started [supporting more checksums](https://www.files.com/blog/2024/11/01/new-modern-checksum-options-now-available-with-opt).
|
||||
|
||||
However if you want hashes you **must** enable them in your site by
|
||||
visiting the "File Integrity" section in "Data Governance" to
|
||||
configure your settings and select the checksum algorithms that meet
|
||||
your requirements.
|
||||
|
||||
Rclone currently only support CRC32 and MD5 for use in syncing but
|
||||
selecting more checksums will not affect rclone's operations.
|
||||
|
||||
For use with rclone, selecting at least MD5 is recommended so rclone
|
||||
can do an end to end integrity check.
|
||||
|
||||
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/filescom/filescom.go then run make backenddocs" >}}
|
||||
### Standard options
|
||||
|
||||
|
||||
@@ -683,7 +683,7 @@ Default units are `KiB` but abbreviations `B`, `K`, `M`, `G`, `T` or `P` are val
|
||||
E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50 KiB
|
||||
size or larger.
|
||||
|
||||
See [the size option docs](/docs/#size-option) for more info.
|
||||
See [the size option docs](/docs/#size-options) for more info.
|
||||
|
||||
### `--max-size` - Don't transfer any file larger than this
|
||||
|
||||
@@ -693,7 +693,7 @@ Default units are `KiB` but abbreviations `B`, `K`, `M`, `G`, `T` or `P` are val
|
||||
E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1 GiB
|
||||
size or smaller.
|
||||
|
||||
See [the size option docs](/docs/#size-option) for more info.
|
||||
See [the size option docs](/docs/#size-options) for more info.
|
||||
|
||||
### `--max-age` - Don't transfer any file older than this
|
||||
|
||||
@@ -704,7 +704,7 @@ Controls the maximum age of files within the scope of an rclone command.
|
||||
E.g. `rclone ls remote: --max-age 2d` lists files on `remote:` of 2 days
|
||||
old or less.
|
||||
|
||||
See [the time option docs](/docs/#time-option) for valid formats.
|
||||
See [the time option docs](/docs/#time-options) for valid formats.
|
||||
|
||||
### `--min-age` - Don't transfer any file younger than this
|
||||
|
||||
@@ -716,7 +716,7 @@ Controls the minimum age of files within the scope of an rclone command.
|
||||
E.g. `rclone ls remote: --min-age 2d` lists files on `remote:` of 2 days
|
||||
old or more.
|
||||
|
||||
See [the time option docs](/docs/#time-option) for valid formats.
|
||||
See [the time option docs](/docs/#time-options) for valid formats.
|
||||
|
||||
### `--hash-filter` - Deterministically select a subset of files {#hash-filter}
|
||||
|
||||
|
||||
@@ -630,3 +630,22 @@ Rclone cannot delete files anywhere except under `album`.
|
||||
### Deleting albums
|
||||
|
||||
The Google Photos API does not support deleting albums - see [bug #135714733](https://issuetracker.google.com/issues/135714733).
|
||||
|
||||
## Making your own client_id
|
||||
|
||||
When you use rclone with Google photos in its default configuration you
|
||||
are using rclone's client_id. This is shared between all the rclone
|
||||
users. There is a global rate limit on the number of queries per
|
||||
second that each client_id can do set by Google.
|
||||
|
||||
If there is a problem with this client_id (eg quota too low or the
|
||||
client_id stops working) then you can make your own.
|
||||
|
||||
Please follow the steps in [the google drive docs](https://rclone.org/drive/#making-your-own-client-id).
|
||||
You will need these scopes instead of the drive ones detailed:
|
||||
|
||||
```
|
||||
https://www.googleapis.com/auth/photoslibrary.appendonly
|
||||
https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata
|
||||
https://www.googleapis.com/auth/photoslibrary.edit.appcreateddata
|
||||
```
|
||||
+10
-4
@@ -225,6 +225,12 @@ package is here.
|
||||
|
||||
The rclone developers maintain a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
|
||||
|
||||
**Note:** We also now offer a paid version of rclone with
|
||||
enterprise-grade security and zero CVEs through our partner
|
||||
[SecureBuild](https://securebuild.com/blog/introducing-securebuild).
|
||||
If you are interested, check out their website and the [Rclone
|
||||
SecureBuild Image](https://securebuild.com/images/rclone).
|
||||
|
||||
These images are built as part of the release process based on a
|
||||
minimal Alpine Linux.
|
||||
|
||||
@@ -485,9 +491,9 @@ the locations that rclone will use.
|
||||
|
||||
To override them set the corresponding options (as command-line arguments, or as
|
||||
[environment variables](https://rclone.org/docs/#environment-variables)):
|
||||
- [--config](https://rclone.org/docs/#config-config-file)
|
||||
- [--cache-dir](https://rclone.org/docs/#cache-dir-dir)
|
||||
- [--temp-dir](https://rclone.org/docs/#temp-dir-dir)
|
||||
- [--config](https://rclone.org/docs/#config-string)
|
||||
- [--cache-dir](https://rclone.org/docs/#cache-dir-string)
|
||||
- [--temp-dir](https://rclone.org/docs/#temp-dir-string)
|
||||
|
||||
## Autostart
|
||||
|
||||
@@ -540,7 +546,7 @@ NOTE: Remember that when rclone runs as the `SYSTEM` user, the user profile
|
||||
that it sees will not be yours. This means that if you normally run rclone with
|
||||
configuration file in the default location, to be able to use the same configuration
|
||||
when running as the system user you must explicitly tell rclone where to find
|
||||
it with the [`--config`](https://rclone.org/docs/#config-config-file) option,
|
||||
it with the [`--config`](https://rclone.org/docs/#config-string) option,
|
||||
or else it will look in the system users profile path (`C:\Windows\System32\config\systemprofile`).
|
||||
To test your command manually from a Command Prompt, you can run it with
|
||||
the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
|
||||
|
||||
@@ -257,7 +257,7 @@ flag.
|
||||
Note that Jottacloud requires the MD5 hash before upload so if the
|
||||
source does not have an MD5 checksum then the file will be cached
|
||||
temporarily on disk (in location given by
|
||||
[--temp-dir](/docs/#temp-dir-dir)) before it is uploaded.
|
||||
[--temp-dir](/docs/#temp-dir-string)) before it is uploaded.
|
||||
Small files will be cached in memory - see the
|
||||
[--jottacloud-md5-memory-limit](#jottacloud-md5-memory-limit) flag.
|
||||
When uploading from local disk the source checksum is always available,
|
||||
|
||||
@@ -132,7 +132,8 @@ For example, you might see throttling.
|
||||
|
||||
To create your own Client ID, please follow these steps:
|
||||
|
||||
1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`.
|
||||
1. Open https://portal.azure.com/?quickstart=true#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/Overview and then under the `Add` menu click `App registration`.
|
||||
* If you have not created an Azure account, you will be prompted to. This is free, but you need to provide a phone number, address, and credit card for identity verification.
|
||||
2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI`, then type (do not copy and paste) `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use.
|
||||
3. Under `manage` select `Certificates & secrets`, click `New client secret`. Enter a description (can be anything) and set `Expires` to 24 months. Copy and keep that secret _Value_ for later use (you _won't_ be able to see this value afterwards).
|
||||
4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`.
|
||||
|
||||
@@ -377,6 +377,8 @@ will show you the defaults for the backends.
|
||||
| Exclamation | `!` | `!` |
|
||||
| Hash | `#` | `#` |
|
||||
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | `�` |
|
||||
| ForceNFC | All invalid NFC characters | Their valid NFC equivalents |
|
||||
| ForceNFD | All invalid NFD characters | Their valid NFD equivalents |
|
||||
| LeftCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string | `␍`, `␊`, `␉`, `␋` |
|
||||
| LeftPeriod | `.` on the left of a string | `.` |
|
||||
| LeftSpace | SPACE on the left of a string | `␠` |
|
||||
@@ -516,7 +518,7 @@ upon backend-specific capabilities.
|
||||
| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | No | Yes |
|
||||
| HTTP | No | No | No | No | No | No | No | No | No | No | Yes |
|
||||
| iCloud Drive | Yes | Yes | Yes | Yes | No | No | No | No | No | No | Yes |
|
||||
| ImageKit | Yes | Yes | Yes | No | No | No | No | No | No | No | Yes |
|
||||
| ImageKit | Yes | No | Yes | No | No | No | No | No | No | No | Yes |
|
||||
| Internet Archive | No | Yes | No | No | Yes | Yes | No | No | Yes | Yes | No |
|
||||
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
|
||||
| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | No | Yes | Yes | Yes |
|
||||
|
||||
+4
-4
@@ -546,8 +546,8 @@ And this is equivalent to `/tmp/dir`
|
||||
|
||||
```
|
||||
{
|
||||
type = "local",
|
||||
_root = "/tmp/dir"
|
||||
"type": "local",
|
||||
"_root": "/tmp/dir"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -2359,8 +2359,8 @@ e.g.
|
||||
"fs": "/tmp",
|
||||
"remote": 3
|
||||
},
|
||||
"status": 400
|
||||
"path": "operations/rmdir",
|
||||
"status": 400,
|
||||
"path": "operations/rmdir"
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ The S3 backend can be used with a number of different providers:
|
||||
{{< provider name="Synology C2 Object Storage" home="https://c2.synology.com/en-global/object-storage/overview" config="/s3/#synology-c2" >}}
|
||||
{{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
|
||||
{{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}}
|
||||
{{< provider name="Zata" home="https://zata.ai/" config="/s3/#Zata" end="true" >}}
|
||||
{{< /provider_list >}}
|
||||
|
||||
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
|
||||
@@ -6040,3 +6041,155 @@ d) Delete this remote
|
||||
|
||||
y/e/d> y
|
||||
```
|
||||
### Zata Object Storage {#Zata}
|
||||
|
||||
[Zata Object Storage](https://zata.ai/) provides a secure, S3-compatible cloud storage solution designed for scalability and performance, ideal for a variety of data storage needs.
|
||||
|
||||
First run:
|
||||
|
||||
```
|
||||
rclone config
|
||||
```
|
||||
|
||||
```
|
||||
This will guide you through an interactive setup process:
|
||||
|
||||
e) Edit existing remote
|
||||
n) New remote
|
||||
d) Delete remote
|
||||
r) Rename remote
|
||||
c) Copy remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
e/n/d/r/c/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> my zata storage
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, ...
|
||||
\ (s3)
|
||||
|
||||
Storage> s3
|
||||
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
|
||||
XX / Zata (S3 compatible Gateway)
|
||||
\ (Zata)
|
||||
|
||||
provider> Zata
|
||||
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth>
|
||||
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> "your key"
|
||||
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> "your secret key"
|
||||
|
||||
Option region.
|
||||
Region where you can connect with.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / Indore, Madhya Pradesh, India
|
||||
\ (us-east-1)
|
||||
region> 1
|
||||
|
||||
Option endpoint.
|
||||
Endpoint for Zata Object Storage.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
1 / South Asia Endpoint
|
||||
\ (idr01.zata.ai)
|
||||
endpoint> 1
|
||||
|
||||
Option location_constraint.
|
||||
Location constraint - must be set to match the Region.
|
||||
Leave blank if not sure. Used when creating buckets only.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
location_constraint>
|
||||
|
||||
Option acl.
|
||||
Canned ACL used when creating buckets and storing or copying objects.
|
||||
This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
|
||||
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
Note that this ACL is applied when server-side copying objects as S3
|
||||
doesn't copy the ACL from the source but rather writes a fresh one.
|
||||
If the acl is an empty string then no X-Amz-Acl: header is added and
|
||||
the default (private) will be used.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
/ Owner gets FULL_CONTROL.
|
||||
1 | No one else has access rights (default).
|
||||
\ (private)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
2 | The AllUsers group gets READ access.
|
||||
\ (public-read)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
3 | The AllUsers group gets READ and WRITE access.
|
||||
| Granting this on a bucket is generally not recommended.
|
||||
\ (public-read-write)
|
||||
/ Owner gets FULL_CONTROL.
|
||||
4 | The AuthenticatedUsers group gets READ access.
|
||||
\ (authenticated-read)
|
||||
/ Object owner gets FULL_CONTROL.
|
||||
5 | Bucket owner gets READ access.
|
||||
| If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
\ (bucket-owner-read)
|
||||
/ Both the object owner and the bucket owner get FULL_CONTROL over the object.
|
||||
6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
|
||||
\ (bucket-owner-full-control)
|
||||
acl>
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n>
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: Zata
|
||||
- access_key_id: xxx
|
||||
- secret_access_key: xxx
|
||||
- region: us-east-1
|
||||
- endpoint: idr01.zata.ai
|
||||
Keep this "my zata storage" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d>
|
||||
|
||||
```
|
||||
This will leave the config file looking like this.
|
||||
```
|
||||
[my zata storage]
|
||||
type = s3
|
||||
provider = Zata
|
||||
access_key_id = xxx
|
||||
secret_access_key = xxx
|
||||
region = us-east-1
|
||||
endpoint = idr01.zata.ai
|
||||
|
||||
```
|
||||
+24
-20
@@ -318,29 +318,30 @@ is able to use checksumming if the same login has shell access,
|
||||
and can execute remote commands. If there is a command that can
|
||||
calculate compatible checksums on the remote system, Rclone can
|
||||
then be configured to execute this whenever a checksum is needed,
|
||||
and read back the results. Currently MD5 and SHA-1 are supported.
|
||||
and read back the results. By default MD5 and SHA-1 are considered,
|
||||
but also CRC32, SHA-256, BLAKE3, XXH3 and XXH128 are supported,
|
||||
option `hashes` can be set to specify which to consider.
|
||||
|
||||
Normally this requires an external utility being available on
|
||||
the server. By default rclone will try commands `md5sum`, `md5`
|
||||
and `rclone md5sum` for MD5 checksums, and the first one found usable
|
||||
will be picked. Same with `sha1sum`, `sha1` and `rclone sha1sum`
|
||||
commands for SHA-1 checksums. These utilities normally need to
|
||||
be in the remote's PATH to be found.
|
||||
the server. E.g. for MD5 checksums, by default rclone will try commands
|
||||
`md5sum`, `md5` and `rclone md5sum`, and the first one found
|
||||
usable will be picked. These utilities normally need to be in the
|
||||
remote's PATH to be found.
|
||||
|
||||
In some cases the shell itself is capable of calculating checksums.
|
||||
PowerShell is an example of such a shell. If rclone detects that the
|
||||
remote shell is PowerShell, which means it most probably is a
|
||||
Windows OpenSSH server, rclone will use a predefined script block
|
||||
to produce the checksums when no external checksum commands are found
|
||||
(see [shell access](#shell-access)). This assumes PowerShell version
|
||||
4.0 or newer.
|
||||
to produce the checksums for MD5, SHA-1 and SHA-256 when no external
|
||||
checksum commands are found (see [shell access](#shell-access)). This
|
||||
assumes PowerShell version 4.0 or newer.
|
||||
|
||||
The options `md5sum_command` and `sha1_command` can be used to customize
|
||||
the command to be executed for calculation of checksums. You can for
|
||||
example set a specific path to where md5sum and sha1sum executables
|
||||
are located, or use them to specify some other tools that print checksums
|
||||
in compatible format. The value can include command-line arguments,
|
||||
or even shell script blocks as with PowerShell. Rclone has subcommands
|
||||
The options `md5sum_command`, `sha1_command`, etc. can be used to customize
|
||||
the commands to be executed for calculation of checksums. You can for
|
||||
example set a specific path to where the md5sum executable are located,
|
||||
or specify some other tool that print checksums in compatible format.
|
||||
The value can include command-line arguments, or even shell script blocks
|
||||
as with PowerShell. Rclone has subcommands [hashsum](/commands/rclone_hashsum/),
|
||||
[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/)
|
||||
that use compatible format, which means if you have an rclone executable
|
||||
on the server it can be used. As mentioned above, they will be automatically
|
||||
@@ -356,11 +357,14 @@ configuration, so next time it will use the same. Value `none`
|
||||
will be set if none of the default commands could be used for a specific
|
||||
algorithm, and this algorithm will not be supported by the remote.
|
||||
|
||||
Disabling the checksumming may be required if you are connecting to SFTP servers
|
||||
which are not under your control, and to which the execution of remote shell
|
||||
commands is prohibited. Set the configuration option `disable_hashcheck`
|
||||
to `true` to disable checksumming entirely, or set `shell_type` to `none`
|
||||
to disable all functionality based on remote shell command execution.
|
||||
Disabling the checksumming completely may be required if you are connecting to
|
||||
SFTP servers which are not under your control, and to which the execution of
|
||||
remote shell commands is prohibited. Set the configuration option `disable_hashcheck`
|
||||
to `true` to disable checksumming entirely (you get the same effect by setting
|
||||
option `hashes` to `none` or options `md5sum_command`, `sha1_command` etc.
|
||||
to `none`). Set option `shell_type` to `none` to not only disable checksumming,
|
||||
but also disable all other functionality that are based on remote shell command
|
||||
execution.
|
||||
|
||||
### Modification times and hashes
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ off donation.
|
||||
Thank you very much to our sponsors:
|
||||
|
||||
{{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}}
|
||||
{{< sponsor src="/img/logos/warp.svg" width="285" height="200" title="Visit our sponsor warp.dev" link="https://www.warp.dev/?utm_source=rclone&utm_medium=referral&utm_campaign=rclone_20231103">}}
|
||||
{{< sponsor src="/img/logos/filescom-enterprise-grade-workflows.png" width="300" height="200" title="Start Your Free Trial Today" link="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone">}}
|
||||
{{< sponsor src="/img/logos/sia.svg" width="200" height="200" title="Visit our sponsor sia" link="https://sia.tech">}}
|
||||
{{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}}
|
||||
{{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}}
|
||||
|
||||
@@ -19,6 +19,15 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="card-header" style="padding: 5px 15px;">
|
||||
Gold Sponsor
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<a href="https://files.com/?utm_source=rclone&utm_medium=referral&utm_campaign=banner&utm_term=rclone" target="_blank" rel="noopener" title="Start Your Free Trial Today"><img style="max-width: 100%; height: auto;" src="/img/logos/filescom-enterprise-grade-workflows.png"></a><br />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{if .IsHome}}
|
||||
<div class="card">
|
||||
<div class="card-header" style="padding: 5px 15px;">
|
||||
|
||||
@@ -66,8 +66,8 @@
|
||||
<a class="dropdown-item" href="/koofr/#digi-storage"><i class="fa fa-cloud fa-fw"></i> Digi Storage</a>
|
||||
<a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a>
|
||||
<a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a>
|
||||
<a class="dropdown-item" href="/filelu/"><i class="fa fa-brands fa-files-pinwheel fa-fw"></i> Files.com</a>
|
||||
<a class="dropdown-item" href="/filescom/"><i class="fa fa-cloud fa-fw"></i> FileLu Cloud Storage</a>
|
||||
<a class="dropdown-item" href="/filelu/"><i class="fa fa-folder"></i> FileLu Cloud Storage</a>
|
||||
<a class="dropdown-item" href="/filescom/"><i class="fa fa-brands fa-files-pinwheel fa-fw"></i> Files.com</a>
|
||||
<a class="dropdown-item" href="/ftp/"><i class="fa fa-file fa-fw"></i> FTP</a>
|
||||
<a class="dropdown-item" href="/gofile/"><i class="fa fa-folder fa-fw"></i> Gofile</a>
|
||||
<a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google fa-fw"></i> Google Cloud Storage</a>
|
||||
|
||||
@@ -1 +1 @@
|
||||
v1.70.0
|
||||
v1.71.0
|
||||
@@ -304,7 +304,7 @@ func (acc *Account) ServerSideTransferEnd(n int64) {
|
||||
acc.stats.Bytes(n)
|
||||
}
|
||||
|
||||
// serverSideEnd accounts for non specific server side data
|
||||
// serverSideEnd accounts for non specific server-side data
|
||||
func (acc *Account) serverSideEnd(n int64) {
|
||||
// Account for bytes unless we are checking
|
||||
if !acc.checking {
|
||||
@@ -312,13 +312,13 @@ func (acc *Account) serverSideEnd(n int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// ServerSideCopyEnd accounts for a read of n bytes in a sever side copy
|
||||
// ServerSideCopyEnd accounts for a read of n bytes in a server-side copy
|
||||
func (acc *Account) ServerSideCopyEnd(n int64) {
|
||||
acc.stats.AddServerSideCopy(n)
|
||||
acc.serverSideEnd(n)
|
||||
}
|
||||
|
||||
// ServerSideMoveEnd accounts for a read of n bytes in a sever side move
|
||||
// ServerSideMoveEnd accounts for a read of n bytes in a server-side move
|
||||
func (acc *Account) ServerSideMoveEnd(n int64) {
|
||||
acc.stats.AddServerSideMove(n)
|
||||
acc.serverSideEnd(n)
|
||||
|
||||
Vendored
+3
-2
@@ -6,6 +6,7 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
@@ -25,8 +26,8 @@ func createOnFirstUse() {
|
||||
once.Do(func() {
|
||||
ci := fs.GetConfig(context.Background())
|
||||
c = cache.New()
|
||||
c.SetExpireDuration(ci.FsCacheExpireDuration)
|
||||
c.SetExpireInterval(ci.FsCacheExpireInterval)
|
||||
c.SetExpireDuration(time.Duration(ci.FsCacheExpireDuration))
|
||||
c.SetExpireInterval(time.Duration(ci.FsCacheExpireInterval))
|
||||
c.SetFinalizer(func(value any) {
|
||||
if s, ok := value.(fs.Shutdowner); ok {
|
||||
_ = fs.CountError(context.Background(), s.Shutdown(context.Background()))
|
||||
|
||||
+10
-10
@@ -570,12 +570,12 @@ type ConfigInfo struct {
|
||||
IgnoreTimes bool `config:"ignore_times"`
|
||||
IgnoreExisting bool `config:"ignore_existing"`
|
||||
IgnoreErrors bool `config:"ignore_errors"`
|
||||
ModifyWindow time.Duration `config:"modify_window"`
|
||||
ModifyWindow Duration `config:"modify_window"`
|
||||
Checkers int `config:"checkers"`
|
||||
Transfers int `config:"transfers"`
|
||||
ConnectTimeout time.Duration `config:"contimeout"` // Connect timeout
|
||||
Timeout time.Duration `config:"timeout"` // Data channel timeout
|
||||
ExpectContinueTimeout time.Duration `config:"expect_continue_timeout"`
|
||||
ConnectTimeout Duration `config:"contimeout"` // Connect timeout
|
||||
Timeout Duration `config:"timeout"` // Data channel timeout
|
||||
ExpectContinueTimeout Duration `config:"expect_continue_timeout"`
|
||||
Dump DumpFlags `config:"dump"`
|
||||
InsecureSkipVerify bool `config:"no_check_certificate"` // Skip server certificate verification
|
||||
DeleteMode DeleteMode `config:"delete_mode"`
|
||||
@@ -584,7 +584,7 @@ type ConfigInfo struct {
|
||||
TrackRenames bool `config:"track_renames"` // Track file renames.
|
||||
TrackRenamesStrategy string `config:"track_renames_strategy"` // Comma separated list of strategies used to track renames
|
||||
Retries int `config:"retries"` // High-level retries
|
||||
RetriesInterval time.Duration `config:"retries_sleep"`
|
||||
RetriesInterval Duration `config:"retries_sleep"`
|
||||
LowLevelRetries int `config:"low_level_retries"`
|
||||
UpdateOlder bool `config:"update"` // Skip files that are newer on the destination
|
||||
NoGzip bool `config:"no_gzip_encoding"` // Disable compression
|
||||
@@ -623,7 +623,7 @@ type ConfigInfo struct {
|
||||
PasswordCommand SpaceSepList `config:"password_command"`
|
||||
UseServerModTime bool `config:"use_server_modtime"`
|
||||
MaxTransfer SizeSuffix `config:"max_transfer"`
|
||||
MaxDuration time.Duration `config:"max_duration"`
|
||||
MaxDuration Duration `config:"max_duration"`
|
||||
CutoffMode CutoffMode `config:"cutoff_mode"`
|
||||
MaxBacklog int `config:"max_backlog"`
|
||||
MaxStatsGroups int `config:"max_stats_groups"`
|
||||
@@ -652,11 +652,11 @@ type ConfigInfo struct {
|
||||
RefreshTimes bool `config:"refresh_times"`
|
||||
NoConsole bool `config:"no_console"`
|
||||
TrafficClass uint8 `config:"traffic_class"`
|
||||
FsCacheExpireDuration time.Duration `config:"fs_cache_expire_duration"`
|
||||
FsCacheExpireInterval time.Duration `config:"fs_cache_expire_interval"`
|
||||
FsCacheExpireDuration Duration `config:"fs_cache_expire_duration"`
|
||||
FsCacheExpireInterval Duration `config:"fs_cache_expire_interval"`
|
||||
DisableHTTP2 bool `config:"disable_http2"`
|
||||
HumanReadable bool `config:"human_readable"`
|
||||
KvLockTime time.Duration `config:"kv_lock_time"` // maximum time to keep key-value database locked by process
|
||||
KvLockTime Duration `config:"kv_lock_time"` // maximum time to keep key-value database locked by process
|
||||
DisableHTTPKeepAlives bool `config:"disable_http_keep_alives"`
|
||||
Metadata bool `config:"metadata"`
|
||||
ServerSideAcrossConfigs bool `config:"server_side_across_configs"`
|
||||
@@ -766,7 +766,7 @@ func InitialLogLevel() LogLevel {
|
||||
// TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise
|
||||
func (ci *ConfigInfo) TimeoutOrInfinite() time.Duration {
|
||||
if ci.Timeout > 0 {
|
||||
return ci.Timeout
|
||||
return time.Duration(ci.Timeout)
|
||||
}
|
||||
return ModTimeNotSupported
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ func FileExists(ctx context.Context, fs Fs, remote string) (bool, error) {
|
||||
// GetModifyWindow calculates the maximum modify window between the given Fses
|
||||
// and the Config.ModifyWindow parameter.
|
||||
func GetModifyWindow(ctx context.Context, fss ...Info) time.Duration {
|
||||
window := GetConfig(ctx).ModifyWindow
|
||||
window := time.Duration(GetConfig(ctx).ModifyWindow)
|
||||
for _, f := range fss {
|
||||
if f != nil {
|
||||
precision := f.Precision()
|
||||
|
||||
+2
-2
@@ -27,10 +27,10 @@ func NewDialer(ctx context.Context) *Dialer {
|
||||
ci := fs.GetConfig(ctx)
|
||||
dialer := &Dialer{
|
||||
Dialer: net.Dialer{
|
||||
Timeout: ci.ConnectTimeout,
|
||||
Timeout: time.Duration(ci.ConnectTimeout),
|
||||
KeepAlive: 30 * time.Second,
|
||||
},
|
||||
timeout: ci.Timeout,
|
||||
timeout: time.Duration(ci.Timeout),
|
||||
tclass: int(ci.TrafficClass),
|
||||
}
|
||||
if ci.BindAddr != nil {
|
||||
|
||||
+3
-3
@@ -58,8 +58,8 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
|
||||
t.Proxy = http.ProxyFromEnvironment
|
||||
t.MaxIdleConnsPerHost = 2 * (ci.Checkers + ci.Transfers + 1)
|
||||
t.MaxIdleConns = 2 * t.MaxIdleConnsPerHost
|
||||
t.TLSHandshakeTimeout = ci.ConnectTimeout
|
||||
t.ResponseHeaderTimeout = ci.Timeout
|
||||
t.TLSHandshakeTimeout = time.Duration(ci.ConnectTimeout)
|
||||
t.ResponseHeaderTimeout = time.Duration(ci.Timeout)
|
||||
t.DisableKeepAlives = ci.DisableHTTPKeepAlives
|
||||
|
||||
// TLS Config
|
||||
@@ -109,7 +109,7 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
|
||||
return NewDialer(ctx).DialContext(reqCtx, network, addr)
|
||||
}
|
||||
t.IdleConnTimeout = 60 * time.Second
|
||||
t.ExpectContinueTimeout = ci.ExpectContinueTimeout
|
||||
t.ExpectContinueTimeout = time.Duration(ci.ExpectContinueTimeout)
|
||||
|
||||
if ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
|
||||
fs.Debugf(nil, "You have specified to dump information. Please be noted that the "+
|
||||
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/jzelinskie/whirlpool"
|
||||
"github.com/zeebo/blake3"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
// Type indicates a standard hashing algorithm
|
||||
@@ -91,8 +93,27 @@ var (
|
||||
|
||||
// SHA512 indicates SHA-512 support
|
||||
SHA512 Type
|
||||
|
||||
// BLAKE3 indicates BLAKE3 support
|
||||
BLAKE3 Type
|
||||
|
||||
// XXH3 indicates XXH3 support, also known as XXH3-64, a variant of xxHash
|
||||
XXH3 Type
|
||||
|
||||
// XXH128 indicates XXH128 support, also known as XXH3-128, a variant of xxHash
|
||||
XXH128 Type
|
||||
)
|
||||
|
||||
type xxh128Hasher struct {
|
||||
xxh3.Hasher
|
||||
}
|
||||
|
||||
// Sum overrides xxh3.Sum to return value based on Sum128 instead of the default Sum64.
|
||||
func (h *xxh128Hasher) Sum(b []byte) []byte {
|
||||
buf := h.Sum128().Bytes()
|
||||
return buf[:]
|
||||
}
|
||||
|
||||
func init() {
|
||||
MD5 = RegisterHash("md5", "MD5", 32, md5.New)
|
||||
SHA1 = RegisterHash("sha1", "SHA-1", 40, sha1.New)
|
||||
@@ -100,6 +121,9 @@ func init() {
|
||||
CRC32 = RegisterHash("crc32", "CRC-32", 8, func() hash.Hash { return crc32.NewIEEE() })
|
||||
SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New)
|
||||
SHA512 = RegisterHash("sha512", "SHA-512", 128, sha512.New)
|
||||
BLAKE3 = RegisterHash("blake3", "BLAKE3", 64, func() hash.Hash { return blake3.New() })
|
||||
XXH3 = RegisterHash("xxh3", "XXH3", 16, func() hash.Hash { return xxh3.New() })
|
||||
XXH128 = RegisterHash("xxh128", "XXH128", 32, func() hash.Hash { return &xxh128Hasher{} })
|
||||
}
|
||||
|
||||
// Supported returns a set of all the supported hashes by
|
||||
|
||||
@@ -78,6 +78,9 @@ var hashTestSet = []hashTest{
|
||||
hash.CRC32: "a6041d7e",
|
||||
hash.SHA256: "c839e57675862af5c21bd0a15413c3ec579e0d5522dab600bc6c3489b05b8f54",
|
||||
hash.SHA512: "008e7e9b5d94d37bf5e07c955890f730f137a41b8b0db16cb535a9b4cb5632c2bccff31685ec470130fe10e2258a0ab50ab587472258f3132ccf7d7d59fb91db",
|
||||
hash.BLAKE3: "0a7276a407a3be1b4d31488318ee05a335aad5a3b82c4420e592a8178c9e86bb",
|
||||
hash.XXH3: "4b83b0c51c543525",
|
||||
hash.XXH128: "438de241a57d684214f67657f7aad93b",
|
||||
},
|
||||
},
|
||||
// Empty data set
|
||||
@@ -90,6 +93,9 @@ var hashTestSet = []hashTest{
|
||||
hash.CRC32: "00000000",
|
||||
hash.SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
hash.SHA512: "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
|
||||
hash.BLAKE3: "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
|
||||
hash.XXH3: "2d06800538d394c2",
|
||||
hash.XXH128: "99aa06d3014798d86001c324468d497f",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -5,10 +5,7 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
|
||||
"github.com/coreos/go-systemd/v22/journal"
|
||||
"github.com/rclone/rclone/fs"
|
||||
@@ -18,10 +15,8 @@ import (
|
||||
func startSystemdLog(handler *OutputHandler) bool {
|
||||
handler.clearFormatFlags(logFormatDate | logFormatTime | logFormatMicroseconds | logFormatUTC | logFormatLongFile | logFormatShortFile | logFormatPid)
|
||||
handler.setFormatFlags(logFormatNoLevel)
|
||||
// TODO: Use the native journal.Print approach rather than a custom implementation
|
||||
handler.SetOutput(func(level slog.Level, text string) {
|
||||
text = fmt.Sprintf("<%s>%-6s: %s", systemdLogPrefix(level), level, text)
|
||||
_ = log.Output(4, text)
|
||||
_ = journal.Print(slogLevelToSystemdPriority(level), "%-6s: %s\n", level, text)
|
||||
})
|
||||
return true
|
||||
}
|
||||
@@ -37,12 +32,12 @@ var slogLevelToSystemdPrefix = map[slog.Level]journal.Priority{
|
||||
slog.LevelDebug: journal.PriDebug,
|
||||
}
|
||||
|
||||
func systemdLogPrefix(l slog.Level) string {
|
||||
func slogLevelToSystemdPriority(l slog.Level) journal.Priority {
|
||||
prio, ok := slogLevelToSystemdPrefix[l]
|
||||
if !ok {
|
||||
return ""
|
||||
return journal.PriInfo
|
||||
}
|
||||
return strconv.Itoa(int(prio))
|
||||
return prio
|
||||
}
|
||||
|
||||
func isJournalStream() bool {
|
||||
|
||||
+56
-29
@@ -16,7 +16,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
@@ -291,6 +290,7 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
|
||||
srcPrev, dstPrev fs.DirEntry
|
||||
srcPrevName, dstPrevName string
|
||||
src, dst fs.DirEntry
|
||||
srcHasMore, dstHasMore = true, true
|
||||
srcName, dstName string
|
||||
)
|
||||
srcDone := func() {
|
||||
@@ -311,14 +311,14 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
|
||||
}
|
||||
// Reload src and dst if needed - we set them to nil if used
|
||||
if src == nil {
|
||||
src = <-srcChan
|
||||
src, srcHasMore = <-srcChan
|
||||
srcName = m.srcKey(src)
|
||||
}
|
||||
if dst == nil {
|
||||
dst = <-dstChan
|
||||
dst, dstHasMore = <-dstChan
|
||||
dstName = m.dstKey(dst)
|
||||
}
|
||||
if src == nil && dst == nil {
|
||||
if !srcHasMore && !dstHasMore {
|
||||
break
|
||||
}
|
||||
if src != nil && srcPrev != nil {
|
||||
@@ -419,38 +419,65 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
|
||||
// If NoTraverse is set, then try to find a matching object
|
||||
// for each item in the srcList to head dst object
|
||||
if m.NoTraverse && !m.NoCheckDest {
|
||||
startedDst = true
|
||||
workers := ci.Checkers
|
||||
originalSrcChan := srcChan
|
||||
srcChan = make(chan fs.DirEntry, 100)
|
||||
ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.dstKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
type matchTask struct {
|
||||
src fs.DirEntry // src object to find in destination
|
||||
dstMatch chan<- fs.DirEntry // channel to receive matching dst object or nil
|
||||
}
|
||||
matchTasks := make(chan matchTask, workers)
|
||||
dstMatches := make(chan (<-chan fs.DirEntry), workers)
|
||||
|
||||
// Create the tasks from the originalSrcChan. These are put into matchTasks for
|
||||
// processing and dstMatches so they can be retrieved in order.
|
||||
go func() {
|
||||
for src := range originalSrcChan {
|
||||
srcChan <- src
|
||||
dstMatch := make(chan fs.DirEntry, 1)
|
||||
matchTasks <- matchTask{
|
||||
src: src,
|
||||
dstMatch: dstMatch,
|
||||
}
|
||||
dstMatches <- dstMatch
|
||||
}
|
||||
close(matchTasks)
|
||||
}()
|
||||
|
||||
// Get the tasks from the queue and find a matching object.
|
||||
var workerWg sync.WaitGroup
|
||||
for range workers {
|
||||
workerWg.Add(1)
|
||||
go func() {
|
||||
defer workerWg.Done()
|
||||
for t := range matchTasks {
|
||||
leaf := path.Base(t.src.Remote())
|
||||
dst, err := m.Fdst.NewObject(m.Ctx, path.Join(job.dstRemote, leaf))
|
||||
if err != nil {
|
||||
dst = nil
|
||||
}
|
||||
t.dstMatch <- dst
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
startedDst = true
|
||||
// Close dstResults when all the workers have finished
|
||||
go func() {
|
||||
workerWg.Wait()
|
||||
close(dstMatches)
|
||||
}()
|
||||
|
||||
// Read the matches in order and send them to dstChan if found.
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer ls.CleanUp()
|
||||
|
||||
g, gCtx := errgroup.WithContext(m.Ctx)
|
||||
g.SetLimit(ci.Checkers)
|
||||
for src := range originalSrcChan {
|
||||
srcChan <- src
|
||||
if srcObj, ok := src.(fs.Object); ok {
|
||||
g.Go(func() error {
|
||||
leaf := path.Base(srcObj.Remote())
|
||||
dstObj, err := m.Fdst.NewObject(gCtx, path.Join(job.dstRemote, leaf))
|
||||
if err == nil {
|
||||
_ = ls.Add(fs.DirEntries{dstObj}) // ignore errors
|
||||
}
|
||||
return nil // ignore errors
|
||||
})
|
||||
}
|
||||
}
|
||||
dstListErr = g.Wait()
|
||||
sendErr := ls.Send()
|
||||
if dstListErr == nil {
|
||||
dstListErr = sendErr
|
||||
for dstMatch := range dstMatches {
|
||||
dst := <-dstMatch
|
||||
// Note that dst may be nil here
|
||||
// We send these on so we don't deadlock the reader
|
||||
dstChan <- dst
|
||||
}
|
||||
close(srcChan)
|
||||
close(dstChan)
|
||||
|
||||
@@ -249,7 +249,7 @@ func (c *checkMarch) reportResults(ctx context.Context, err error) error {
|
||||
fs.Logf(c.opt.Fsrc, "%d %s missing", c.srcFilesMissing.Load(), entity)
|
||||
}
|
||||
|
||||
fs.Logf(c.opt.Fdst, "%d differences found", accounting.Stats(ctx).GetErrors())
|
||||
fs.Logf(c.opt.Fdst, "%d differences found", c.differences.Load())
|
||||
if errs := accounting.Stats(ctx).GetErrors(); errs > 0 {
|
||||
fs.Logf(c.opt.Fdst, "%d errors while checking", errs)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
mutex "sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
@@ -106,6 +107,43 @@ type LoggerOpt struct {
|
||||
Absolute bool
|
||||
}
|
||||
|
||||
// NewDefaultLoggerFn creates a logger function that writes the sigil and path to configured files that match the sigil
|
||||
func NewDefaultLoggerFn(opt *LoggerOpt) LoggerFn {
|
||||
var lock mutex.Mutex
|
||||
|
||||
return func(ctx context.Context, sigil Sigil, src, dst fs.DirEntry, err error) {
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
return
|
||||
}
|
||||
|
||||
_, srcOk := src.(fs.Object)
|
||||
_, dstOk := dst.(fs.Object)
|
||||
var filename string
|
||||
if !srcOk && !dstOk {
|
||||
return
|
||||
} else if srcOk && !dstOk {
|
||||
filename = src.String()
|
||||
} else {
|
||||
filename = dst.String()
|
||||
}
|
||||
|
||||
if sigil.Writer(*opt) != nil {
|
||||
SyncFprintf(sigil.Writer(*opt), "%s\n", filename)
|
||||
}
|
||||
if opt.Combined != nil {
|
||||
SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
|
||||
fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
|
||||
}
|
||||
if opt.DestAfter != nil {
|
||||
opt.PrintDestAfter(ctx, sigil, src, dst, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithLogger stores logger in ctx and returns a copy of ctx in which loggerKey = logger
|
||||
func WithLogger(ctx context.Context, logger LoggerFn) context.Context {
|
||||
return context.WithValue(ctx, loggerKey, logger)
|
||||
|
||||
@@ -428,6 +428,10 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
|
||||
origRemote := remote // avoid double-transform on fallback to copy
|
||||
remote = transform.Path(ctx, remote, false)
|
||||
ci := fs.GetConfig(ctx)
|
||||
newDst = dst
|
||||
if ci.DryRun && dst != nil && SameObject(src, dst) && src.Remote() == transform.Path(ctx, dst.Remote(), false) {
|
||||
return // avoid SkipDestructive log for objects that won't really be moved
|
||||
}
|
||||
var tr *accounting.Transfer
|
||||
if isTransfer {
|
||||
tr = accounting.Stats(ctx).NewTransfer(src, fdst)
|
||||
@@ -440,8 +444,11 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
|
||||
}
|
||||
tr.Done(ctx, err)
|
||||
}()
|
||||
newDst = dst
|
||||
if SkipDestructive(ctx, src, "move") {
|
||||
action := "move"
|
||||
if remote != src.Remote() {
|
||||
action += " to " + remote
|
||||
}
|
||||
if SkipDestructive(ctx, src, action) {
|
||||
in := tr.Account(ctx, nil)
|
||||
in.DryRun(src.Size())
|
||||
return newDst, nil
|
||||
@@ -1939,6 +1946,9 @@ func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err err
|
||||
func needsMoveCaseInsensitive(fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) bool {
|
||||
dstFilePath := path.Join(fdst.Root(), dstFileName)
|
||||
srcFilePath := path.Join(fsrc.Root(), srcFileName)
|
||||
if !cp && fdst.Name() == fsrc.Name() && dstFileName != srcFileName && norm.NFC.String(dstFilePath) == norm.NFC.String(srcFilePath) {
|
||||
return true
|
||||
}
|
||||
return !cp && fdst.Name() == fsrc.Name() && fdst.Features().CaseInsensitive && dstFileName != srcFileName && strings.EqualFold(dstFilePath, srcFilePath)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,12 +3,28 @@
|
||||
package operationsflags
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
//go:embed operationsflags.md
|
||||
var help string
|
||||
|
||||
// Help returns the help string cleaned up to simplify appending
|
||||
func Help() string {
|
||||
return strings.TrimSpace(help) + "\n\n"
|
||||
}
|
||||
|
||||
// AddLoggerFlagsOptions contains options for the Logger Flags
|
||||
type AddLoggerFlagsOptions struct {
|
||||
Combined string // a file with file names with leading sigils
|
||||
@@ -20,6 +36,20 @@ type AddLoggerFlagsOptions struct {
|
||||
DestAfter string // files that exist on the destination post-sync
|
||||
}
|
||||
|
||||
// AnySet checks if any of the logger flags have a non-blank value
|
||||
func (o AddLoggerFlagsOptions) AnySet() bool {
|
||||
return anyNotBlank(o.Combined, o.MissingOnSrc, o.MissingOnDst, o.Match, o.Differ, o.ErrFile, o.DestAfter)
|
||||
}
|
||||
|
||||
func anyNotBlank(s ...string) bool {
|
||||
for _, x := range s {
|
||||
if x != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AddLoggerFlags adds the logger flags to the cmdFlags command
|
||||
func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt *AddLoggerFlagsOptions) {
|
||||
flags.StringVarP(cmdFlags, &flagsOpt.Combined, "combined", "", flagsOpt.Combined, "Make a combined report of changes to this file", "Sync")
|
||||
@@ -43,3 +73,75 @@ func AddLoggerFlags(cmdFlags *pflag.FlagSet, opt *operations.LoggerOpt, flagsOpt
|
||||
flags.BoolVarP(cmdFlags, &opt.Absolute, "absolute", "", false, "Put a leading / in front of path names", "Sync")
|
||||
// flags.BoolVarP(cmdFlags, &recurse, "recursive", "R", false, "Recurse into the listing", "")
|
||||
}
|
||||
|
||||
// ConfigureLoggers verifies and sets up writers for log files requested via CLI flags
|
||||
func ConfigureLoggers(ctx context.Context, fdst fs.Fs, command *cobra.Command, opt *operations.LoggerOpt, flagsOpt AddLoggerFlagsOptions) (func(), error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
if opt.TimeFormat == "max" {
|
||||
opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
|
||||
}
|
||||
opt.SetListFormat(ctx, command.Flags())
|
||||
opt.NewListJSON(ctx, fdst, "")
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := open(flagsOpt.Combined, &opt.Combined); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.Match, &opt.Match); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.Differ, &opt.Differ); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.ErrFile, &opt.Error); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := open(flagsOpt.DestAfter, &opt.DestAfter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
close := func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.NoTraverse && opt.Combined != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse does not list any deletes (-) in --combined output\n")
|
||||
}
|
||||
if ci.NoTraverse && opt.MissingOnSrc != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --missing-on-src produce empty output\n")
|
||||
}
|
||||
if ci.NoTraverse && opt.DestAfter != nil {
|
||||
fs.LogPrintf(fs.LogLevelWarning, nil, "--no-traverse makes --dest-after produce incomplete output\n")
|
||||
}
|
||||
|
||||
return close, nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
## Logger Flags
|
||||
|
||||
The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match` and `--error` flags write paths,
|
||||
one per line, to the file name (or stdout if it is `-`) supplied. What they write is described
|
||||
in the help below. For example `--differ` will write all paths which are present
|
||||
on both the source and destination but different.
|
||||
|
||||
The `--combined` flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- `= path` means path was found in source and destination and was identical
|
||||
- `- path` means path was missing on the source, so only in the destination
|
||||
- `+ path` means path was missing on the destination, so only in the source
|
||||
- `* path` means path was present in source and destination but different.
|
||||
- `! path` means there was an error reading or hashing the source or dest.
|
||||
|
||||
The `--dest-after` flag writes a list file using the same format flags
|
||||
as [`lsf`](/commands/rclone_lsf/#synopsis) (including [customizable options
|
||||
for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
|
||||
Conceptually it is similar to rsync's `--itemize-changes`, but not identical
|
||||
-- it should output an accurate list of what will be on the destination
|
||||
after the command is finished.
|
||||
|
||||
When the `--no-traverse` flag is set, all logs involving files that exist only
|
||||
on the destination will be incomplete or completely missing.
|
||||
|
||||
Note that these logger flags have a few limitations, and certain scenarios
|
||||
are not currently supported:
|
||||
|
||||
- `--max-duration` / `CutoffModeHard`
|
||||
- `--compare-dest` / `--copy-dest`
|
||||
- server-side moves of an entire dir at once
|
||||
- High-level retries, because there would be duplicates (use `--retries 1` to disable)
|
||||
- Possibly some unusual error scenarios
|
||||
|
||||
Note also that each file is logged during execution, as opposed to after, so it
|
||||
is most useful as a predictor of what SHOULD happen to each file
|
||||
(which may or may not match what actually DID.)
|
||||
+3
-3
@@ -148,7 +148,7 @@ func (jobs *Jobs) kickExpire() {
|
||||
jobs.mu.Lock()
|
||||
defer jobs.mu.Unlock()
|
||||
if !jobs.expireRunning {
|
||||
time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire)
|
||||
time.AfterFunc(time.Duration(jobs.opt.JobExpireInterval), jobs.Expire)
|
||||
jobs.expireRunning = true
|
||||
}
|
||||
}
|
||||
@@ -160,13 +160,13 @@ func (jobs *Jobs) Expire() {
|
||||
now := time.Now()
|
||||
for ID, job := range jobs.jobs {
|
||||
job.mu.Lock()
|
||||
if job.Finished && now.Sub(job.EndTime) > jobs.opt.JobExpireDuration {
|
||||
if job.Finished && now.Sub(job.EndTime) > time.Duration(jobs.opt.JobExpireDuration) {
|
||||
delete(jobs.jobs, ID)
|
||||
}
|
||||
job.mu.Unlock()
|
||||
}
|
||||
if len(jobs.jobs) != 0 {
|
||||
time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire)
|
||||
time.AfterFunc(time.Duration(jobs.opt.JobExpireInterval), jobs.Expire)
|
||||
jobs.expireRunning = true
|
||||
} else {
|
||||
jobs.expireRunning = false
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestNewJobs(t *testing.T) {
|
||||
func TestJobsKickExpire(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
jobs := newJobs()
|
||||
jobs.opt.JobExpireInterval = time.Millisecond
|
||||
jobs.opt.JobExpireInterval = fs.Duration(time.Millisecond)
|
||||
assert.Equal(t, false, jobs.expireRunning)
|
||||
jobs.kickExpire()
|
||||
jobs.mu.Lock()
|
||||
@@ -41,7 +41,7 @@ func TestJobsExpire(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
wait := make(chan struct{})
|
||||
jobs := newJobs()
|
||||
jobs.opt.JobExpireInterval = time.Millisecond
|
||||
jobs.opt.JobExpireInterval = fs.Duration(time.Millisecond)
|
||||
assert.Equal(t, false, jobs.expireRunning)
|
||||
var gotJobID int64
|
||||
var gotJob *Job
|
||||
@@ -64,7 +64,7 @@ func TestJobsExpire(t *testing.T) {
|
||||
assert.Equal(t, 1, len(jobs.jobs))
|
||||
jobs.mu.Lock()
|
||||
job.mu.Lock()
|
||||
job.EndTime = time.Now().Add(-rc.Opt.JobExpireDuration - 60*time.Second)
|
||||
job.EndTime = time.Now().Add(-time.Duration(rc.Opt.JobExpireDuration) - 60*time.Second)
|
||||
assert.Equal(t, true, jobs.expireRunning)
|
||||
job.mu.Unlock()
|
||||
jobs.mu.Unlock()
|
||||
|
||||
+4
-4
@@ -75,12 +75,12 @@ var OptionsInfo = fs.Options{{
|
||||
Groups: "RC,Metrics",
|
||||
}, {
|
||||
Name: "rc_job_expire_duration",
|
||||
Default: 60 * time.Second,
|
||||
Default: fs.Duration(60 * time.Second),
|
||||
Help: "Expire finished async jobs older than this value",
|
||||
Groups: "RC",
|
||||
}, {
|
||||
Name: "rc_job_expire_interval",
|
||||
Default: 10 * time.Second,
|
||||
Default: fs.Duration(10 * time.Second),
|
||||
Help: "Interval to check for expired async jobs",
|
||||
Groups: "RC",
|
||||
}, {
|
||||
@@ -120,8 +120,8 @@ type Options struct {
|
||||
MetricsHTTP libhttp.Config `config:"metrics"`
|
||||
MetricsAuth libhttp.AuthConfig `config:"metrics"`
|
||||
MetricsTemplate libhttp.TemplateConfig `config:"metrics"`
|
||||
JobExpireDuration time.Duration `config:"rc_job_expire_duration"`
|
||||
JobExpireInterval time.Duration `config:"rc_job_expire_interval"`
|
||||
JobExpireDuration fs.Duration `config:"rc_job_expire_duration"`
|
||||
JobExpireInterval fs.Duration `config:"rc_job_expire_interval"`
|
||||
}
|
||||
|
||||
// Opt is the default values used for Options
|
||||
|
||||
+4
-1
@@ -194,7 +194,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
|
||||
return nil, err
|
||||
}
|
||||
if ci.MaxDuration > 0 {
|
||||
s.maxDurationEndTime = time.Now().Add(ci.MaxDuration)
|
||||
s.maxDurationEndTime = time.Now().Add(time.Duration(ci.MaxDuration))
|
||||
fs.Infof(s.fdst, "Transfer session %v deadline: %s", ci.CutoffMode, s.maxDurationEndTime.Format("2006/01/02 15:04:05"))
|
||||
}
|
||||
// If a max session duration has been defined add a deadline
|
||||
@@ -1109,6 +1109,9 @@ func (s *syncCopyMove) markDirModifiedObject(o fs.Object) {
|
||||
// be nil.
|
||||
func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) {
|
||||
var err error
|
||||
if dst != nil && src.Remote() == dst.Remote() && operations.OverlappingFilterCheck(ctx, s.fdst, s.fsrc) {
|
||||
return nil // src and dst can be the same in convmv
|
||||
}
|
||||
equal := operations.DirsEqual(ctx, src, dst, operations.DirsEqualOpt{ModifyWindow: s.modifyWindow, SetDirModtime: s.setDirModTime, SetDirMetadata: s.setDirMetadata})
|
||||
if !s.setDirModTimeAfter && equal {
|
||||
return nil
|
||||
|
||||
+32
-3
@@ -216,6 +216,35 @@ func TestCopyNoTraverse(t *testing.T) {
|
||||
r.CheckRemoteItems(t, file1)
|
||||
}
|
||||
|
||||
func TestCopyNoTraverseDeadlock(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
if !r.Fremote.Features().IsLocal {
|
||||
t.Skip("Only runs on local")
|
||||
}
|
||||
const nFiles = 200
|
||||
t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
|
||||
|
||||
// Create lots of source files.
|
||||
items := make([]fstest.Item, nFiles)
|
||||
for i := range items {
|
||||
name := fmt.Sprintf("file%d.txt", i)
|
||||
items[i] = r.WriteFile(name, fmt.Sprintf("content%d", i), t1)
|
||||
}
|
||||
r.CheckLocalItems(t, items...)
|
||||
|
||||
// Set --no-traverse
|
||||
ctx, ci := fs.AddConfig(context.Background())
|
||||
ci.NoTraverse = true
|
||||
|
||||
// Initial copy to establish destination.
|
||||
require.NoError(t, CopyDir(ctx, r.Fremote, r.Flocal, false))
|
||||
r.CheckRemoteItems(t, items...)
|
||||
|
||||
// Second copy which shouldn't deadlock
|
||||
require.NoError(t, CopyDir(ctx, r.Flocal, r.Fremote, false))
|
||||
r.CheckRemoteItems(t, items...)
|
||||
}
|
||||
|
||||
// Now with --check-first
|
||||
func TestCopyCheckFirst(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
@@ -1438,7 +1467,7 @@ func TestSyncWithUpdateOlder(t *testing.T) {
|
||||
r.CheckRemoteItems(t, oneO, twoO, threeO, fourO)
|
||||
|
||||
ci.UpdateOlder = true
|
||||
ci.ModifyWindow = fs.ModTimeNotSupported
|
||||
ci.ModifyWindow = fs.Duration(fs.ModTimeNotSupported)
|
||||
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err := Sync(ctx, r.Fremote, r.Flocal, false)
|
||||
@@ -1468,7 +1497,7 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {
|
||||
}
|
||||
r := fstest.NewRun(t)
|
||||
|
||||
maxDuration := 250 * time.Millisecond
|
||||
maxDuration := fs.Duration(250 * time.Millisecond)
|
||||
ci.MaxDuration = maxDuration
|
||||
ci.CutoffMode = cutoffMode
|
||||
ci.CheckFirst = true
|
||||
@@ -1510,7 +1539,7 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {
|
||||
const maxTransferTime = 20 * time.Second
|
||||
|
||||
what := fmt.Sprintf("expecting elapsed time %v between %v and %v", elapsed, maxDuration, maxTransferTime)
|
||||
assert.True(t, elapsed >= maxDuration, what)
|
||||
assert.True(t, elapsed >= time.Duration(maxDuration), what)
|
||||
assert.True(t, elapsed < maxTransferTime, what)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package fs
|
||||
|
||||
// VersionTag of rclone
|
||||
var VersionTag = "v1.70.0"
|
||||
var VersionTag = "v1.71.0"
|
||||
@@ -39,6 +39,7 @@ type Backend struct {
|
||||
IgnoreTests []string // paths of tests not to run, blank for none
|
||||
ListRetries int // -list-retries if > 0
|
||||
ExtraTime float64 // factor to multiply the timeout by
|
||||
Env []string // environment variables to set in form KEY=VALUE
|
||||
}
|
||||
|
||||
// includeTest returns true if this backend should be included in this
|
||||
@@ -94,6 +95,7 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
|
||||
Ignore: ignore,
|
||||
ListRetries: b.ListRetries,
|
||||
ExtraTime: b.ExtraTime,
|
||||
Env: b.Env,
|
||||
}
|
||||
if t.AddBackend {
|
||||
run.Path = path.Join(run.Path, b.Backend)
|
||||
|
||||
+34
-16
@@ -34,9 +34,9 @@ backends:
|
||||
fastlist: true
|
||||
ignore:
|
||||
- TestCopyFileMaxTransfer
|
||||
- backend: "crypt"
|
||||
remote: "TestCryptSwift:"
|
||||
fastlist: false
|
||||
# - backend: "crypt"
|
||||
# remote: "TestCryptSwift:"
|
||||
# fastlist: false
|
||||
## chunker
|
||||
- backend: "chunker"
|
||||
remote: "TestChunkerLocal:"
|
||||
@@ -71,12 +71,15 @@ backends:
|
||||
remote: "TestChunkerChunk50bYandex:"
|
||||
fastlist: true
|
||||
maxfile: 1k
|
||||
- backend: "chunker"
|
||||
remote: "TestChunkerChunk50bBox:"
|
||||
fastlist: true
|
||||
maxfile: 1k
|
||||
ignore:
|
||||
- TestIntegration/FsMkdir/FsChangeNotify
|
||||
# Needs investigation
|
||||
- TestDeduplicateNewestByHash
|
||||
# - backend: "chunker"
|
||||
# remote: "TestChunkerChunk50bBox:"
|
||||
# fastlist: true
|
||||
# maxfile: 1k
|
||||
# ignore:
|
||||
# - TestIntegration/FsMkdir/FsChangeNotify
|
||||
- backend: "chunker"
|
||||
remote: "TestChunkerS3:"
|
||||
fastlist: true
|
||||
@@ -132,9 +135,9 @@ backends:
|
||||
- backend: "compress"
|
||||
remote: "TestCompress:"
|
||||
fastlist: false
|
||||
- backend: "compress"
|
||||
remote: "TestCompressSwift:"
|
||||
fastlist: false
|
||||
# - backend: "compress"
|
||||
# remote: "TestCompressSwift:"
|
||||
# fastlist: false
|
||||
- backend: "compress"
|
||||
remote: "TestCompressDrive:"
|
||||
fastlist: false
|
||||
@@ -362,11 +365,11 @@ backends:
|
||||
fastlist: true
|
||||
ignoretests:
|
||||
- cmd/gitannex
|
||||
- backend: "swift"
|
||||
remote: "TestSwift:"
|
||||
fastlist: true
|
||||
ignoretests:
|
||||
- cmd/bisync
|
||||
# - backend: "swift"
|
||||
# remote: "TestSwift:"
|
||||
# fastlist: true
|
||||
# ignoretests:
|
||||
# - cmd/bisync
|
||||
# - backend: "swift"
|
||||
# remote: "TestSwiftCeph:"
|
||||
# fastlist: true
|
||||
@@ -481,6 +484,8 @@ backends:
|
||||
ignore:
|
||||
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
||||
- TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
|
||||
env:
|
||||
- RCLONE_NO_CHECK_CERTIFICATE=true
|
||||
fastlist: false
|
||||
ignoretests:
|
||||
- cmd/bisync
|
||||
@@ -503,6 +508,7 @@ backends:
|
||||
ignore:
|
||||
- TestIntegration/FsMkdir/FsPutFiles/PublicLink
|
||||
- TestDirRename
|
||||
- TestFileRename
|
||||
ignoretests:
|
||||
- cmd/bisync
|
||||
- backend: "opendrive"
|
||||
@@ -591,6 +597,18 @@ backends:
|
||||
- backend: "smb"
|
||||
remote: "TestSMBKerberos:rclone"
|
||||
fastlist: false
|
||||
env:
|
||||
- KRB5_CONFIG=/tmp/rclone_krb5/krb5.conf
|
||||
- KRB5CCNAME=/tmp/rclone_krb5/ccache
|
||||
ignoretests:
|
||||
- cmd/gitannex
|
||||
- backend: "smb"
|
||||
remote: "TestSMBKerberosCcache:rclone"
|
||||
fastlist: false
|
||||
env:
|
||||
- KRB5_CONFIG=/tmp/rclone_krb5_ccache/krb5.conf
|
||||
ignoretests:
|
||||
- cmd/gitannex
|
||||
- backend: "storj"
|
||||
remote: "TestStorj:"
|
||||
fastlist: true
|
||||
|
||||
@@ -45,8 +45,9 @@ type Run struct {
|
||||
NoBinary bool // set to not build a binary
|
||||
SizeLimit int64 // maximum test file size
|
||||
Ignore map[string]struct{}
|
||||
ListRetries int // -list-retries if > 0
|
||||
ExtraTime float64 // multiply the timeout by this
|
||||
ListRetries int // -list-retries if > 0
|
||||
ExtraTime float64 // multiply the timeout by this
|
||||
Env []string // environment variables in form KEY=VALUE
|
||||
// Internals
|
||||
CmdLine []string
|
||||
CmdString string
|
||||
@@ -252,6 +253,7 @@ func (r *Run) trial() {
|
||||
cmd.Stderr = multiOut
|
||||
cmd.Stdout = multiOut
|
||||
cmd.Dir = r.Path
|
||||
cmd.Env = append(os.Environ(), r.Env...)
|
||||
start := time.Now()
|
||||
r.err = cmd.Run()
|
||||
r.output = b.Bytes()
|
||||
|
||||
@@ -30,6 +30,8 @@ They should be bound to localhost so they are not accessible externally.
|
||||
| 28634 | TestSMBKerberos |
|
||||
| 28635 | TestS3Exaba |
|
||||
| 28636 | TestS3Exaba |
|
||||
| 28637 | TestSMBKerberosCcache |
|
||||
| 28638 | TestSMBKerberosCcache |
|
||||
| 38081 | TestWebdavOwncloud |
|
||||
|
||||
## Non localhost tests
|
||||
|
||||
@@ -12,10 +12,15 @@ required.
|
||||
|
||||
When start is run it should output config parameters for that remote.
|
||||
If a `_connect` parameter is output then that will be used for a
|
||||
connection test. For example if `_connect=127.0.0.1:80` then a TCP
|
||||
connection test. For example if `_connect=127.0.0.1:80` then a TCP
|
||||
connection will be made to `127.0.0.1:80` and only when that succeeds
|
||||
will the test continue.
|
||||
|
||||
If in addition to `_connect`, `_connect_delay=5s` is also present then
|
||||
after the connection succeeds rclone will wait `5s` before continuing.
|
||||
This is for servers that aren't quite ready even though they have
|
||||
opened their TCP ports.
|
||||
|
||||
`run.bash` contains boilerplate to be included in a bash script for
|
||||
interpreting the command line parameters.
|
||||
|
||||
|
||||
@@ -10,10 +10,17 @@ REALM=RCLONE.LOCAL
|
||||
SMB_PORT=28633
|
||||
KRB5_PORT=28634
|
||||
|
||||
# KRB5_CONFIG and KRB5CCNAME should be set by the caller but default
|
||||
# them here for the integration tests
|
||||
export TEMP_DIR=/tmp/rclone_krb5
|
||||
mkdir -p "${TEMP_DIR}"
|
||||
export KRB5_CONFIG=${KRB5_CONFIG:-${TEMP_DIR}/krb5.conf}
|
||||
export KRB5CCNAME=${KRB5CCNAME:-${TEMP_DIR}/ccache}
|
||||
|
||||
. $(dirname "$0")/docker.bash
|
||||
|
||||
start() {
|
||||
docker build -t ${IMAGE} - <<EOF
|
||||
docker build -t ${IMAGE} --load - <<EOF
|
||||
FROM alpine:3.21
|
||||
RUN apk add --no-cache samba-dc
|
||||
RUN rm -rf /etc/samba/smb.conf /var/lib/samba \
|
||||
@@ -27,8 +34,11 @@ RUN rm -rf /etc/samba/smb.conf /var/lib/samba \
|
||||
--dns-backend=SAMBA_INTERNAL \
|
||||
--host-name=localhost \
|
||||
&& samba-tool user add --random-password $USER \
|
||||
&& samba-tool user setexpiry $USER --noexpiry \
|
||||
&& mkdir -m 777 /share /rclone \
|
||||
&& cat <<EOS >> /etc/samba/smb.conf
|
||||
[global]
|
||||
server signing = auto
|
||||
[public]
|
||||
path = /share
|
||||
browseable = yes
|
||||
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
# Set default location for Kerberos config and ccache. Can be overridden by the caller
|
||||
# using environment variables RCLONE_TEST_CUSTOM_CCACHE_LOCATION and KRB5_CONFIG.
|
||||
export TEMP_DIR=/tmp/rclone_krb5_ccache
|
||||
mkdir -p "${TEMP_DIR}"
|
||||
export KRB5_CONFIG=${KRB5_CONFIG:-${TEMP_DIR}/krb5.conf}
|
||||
export RCLONE_TEST_CUSTOM_CCACHE_LOCATION=${RCLONE_TEST_CUSTOM_CCACHE_LOCATION:-${TEMP_DIR}/ccache}
|
||||
|
||||
IMAGE=rclone/test-smb-kerberos-ccache
|
||||
NAME=smb-kerberos-ccache
|
||||
USER=rclone
|
||||
DOMAIN=RCLONE
|
||||
REALM=RCLONE.LOCAL
|
||||
SMB_PORT=28637
|
||||
KRB5_PORT=28638
|
||||
|
||||
. $(dirname "$0")/docker.bash
|
||||
|
||||
start() {
|
||||
docker build -t ${IMAGE} --load - <<EOF
|
||||
FROM alpine:3.21
|
||||
RUN apk add --no-cache samba-dc
|
||||
RUN rm -rf /etc/samba/smb.conf /var/lib/samba \
|
||||
&& mkdir -p /var/lib/samba/private \
|
||||
&& samba-tool domain provision \
|
||||
--use-rfc2307 \
|
||||
--option acl_xattr:security_acl_name=user.NTACL \
|
||||
--realm=$REALM \
|
||||
--domain=$DOMAIN \
|
||||
--server-role=dc \
|
||||
--dns-backend=SAMBA_INTERNAL \
|
||||
--host-name=localhost \
|
||||
&& samba-tool user add --random-password $USER \
|
||||
&& samba-tool user setexpiry $USER --noexpiry \
|
||||
&& mkdir -m 777 /share /rclone \
|
||||
&& cat <<EOS >> /etc/samba/smb.conf
|
||||
[global]
|
||||
server signing = auto
|
||||
[public]
|
||||
path = /share
|
||||
browseable = yes
|
||||
read only = yes
|
||||
guest ok = yes
|
||||
[rclone]
|
||||
path = /rclone
|
||||
browseable = yes
|
||||
read only = no
|
||||
guest ok = no
|
||||
valid users = rclone
|
||||
EOS
|
||||
CMD ["samba", "-i"]
|
||||
EOF
|
||||
|
||||
docker run --rm -d --name ${NAME} \
|
||||
-p 127.0.0.1:${SMB_PORT}:445 \
|
||||
-p 127.0.0.1:${SMB_PORT}:445/udp \
|
||||
-p 127.0.0.1:${KRB5_PORT}:88 \
|
||||
${IMAGE}
|
||||
|
||||
cat > "${KRB5_CONFIG}" <<EOF
|
||||
[libdefaults]
|
||||
default_realm = ${REALM}
|
||||
[realms]
|
||||
${REALM} = {
|
||||
kdc = localhost
|
||||
}
|
||||
EOF
|
||||
|
||||
docker cp "${KRB5_CONFIG}" ${NAME}:/etc/krb5.conf
|
||||
docker exec ${NAME} samba-tool user get-kerberos-ticket rclone --output-krb5-ccache=/tmp/ccache
|
||||
docker cp ${NAME}:/tmp/ccache "${RCLONE_TEST_CUSTOM_CCACHE_LOCATION}"
|
||||
sed -i -e "s/localhost/localhost:${KRB5_PORT}/" "${KRB5_CONFIG}"
|
||||
|
||||
echo type=smb
|
||||
echo host=localhost
|
||||
echo port=$SMB_PORT
|
||||
echo use_kerberos=true
|
||||
echo kerberos_ccache=${RCLONE_TEST_CUSTOM_CCACHE_LOCATION}
|
||||
echo _connect=127.0.0.1:${SMB_PORT}
|
||||
}
|
||||
|
||||
. $(dirname "$0")/run.bash
|
||||
Loaded 100 of 119 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user